From ba28beba51ff7d495e5165471b96f7496bf585c2 Mon Sep 17 00:00:00 2001 From: Alberto Sartori Date: Thu, 13 Jul 2023 13:20:59 +0200 Subject: [PATCH 01/97] bytestream_server: store temporary files under the local-buil-root The bytestream server implementation (deployed by just execute) now stores the temporary files under $local_build_root/protocol-depenedent/generation-0 so that they can be garbage collected if "just exectue" is terminated before they are cleaned up. --- CHANGELOG.md | 2 ++ .../execution_service/bytestream_server.cpp | 20 ++++++++++--------- src/buildtool/storage/local_cas.hpp | 4 +++- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffa17d976..761fe55ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,8 @@ A feature release on top of `1.1.0`, backwards compatible. all entries in a Git repository. Previously it was possible for entries to be skipped inadvertently in, e.g., imported archives if `gitignore` files were present. +- Temporary files generated by `just execute` are now created inside + the local build root. ## Release `1.1.0` (2023-05-19) diff --git a/src/buildtool/execution_api/execution_service/bytestream_server.cpp b/src/buildtool/execution_api/execution_service/bytestream_server.cpp index 234c4adc7..b450c6972 100644 --- a/src/buildtool/execution_api/execution_service/bytestream_server.cpp +++ b/src/buildtool/execution_api/execution_service/bytestream_server.cpp @@ -113,7 +113,14 @@ auto BytestreamServiceImpl::Write( *hash, request.write_offset(), request.finish_write()); - auto tmp_dir = TmpDir::Create("execution-service"); + auto lock = GarbageCollector::SharedLock(); + if (!lock) { + auto str = fmt::format("Could not acquire SharedLock"); + logger_.Emit(LogLevel::Error, str); + return grpc::Status{grpc::StatusCode::INTERNAL, str}; + } + auto tmp_dir = TmpDir::Create(StorageConfig::GenerationCacheRoot(0) / + "execution-service"); if (!tmp_dir) { return ::grpc::Status{::grpc::StatusCode::INTERNAL, "could not create TmpDir"}; @@ -129,21 +136,16 @@ auto BytestreamServiceImpl::Write( of.write(request.data().data(), static_cast(request.data().size())); } - auto lock = GarbageCollector::SharedLock(); - if (!lock) { - auto str = fmt::format("Could not acquire SharedLock"); - logger_.Emit(LogLevel::Error, str); - return grpc::Status{grpc::StatusCode::INTERNAL, str}; - } if (NativeSupport::IsTree(*hash)) { - if (not storage_->CAS().StoreTree(tmp)) { + if (not storage_->CAS().StoreTree(tmp)) { auto str = fmt::format("could not store tree {}", *hash); logger_.Emit(LogLevel::Error, str); return ::grpc::Status{::grpc::StatusCode::INVALID_ARGUMENT, str}; } } else { - if (not storage_->CAS().StoreBlob(tmp, /*is_executable=*/false)) { + if (not storage_->CAS().StoreBlob( + tmp, /*is_executable=*/false)) { auto str = fmt::format("could not store blob {}", *hash); logger_.Emit(LogLevel::Error, str); return ::grpc::Status{::grpc::StatusCode::INVALID_ARGUMENT, str}; diff --git a/src/buildtool/storage/local_cas.hpp b/src/buildtool/storage/local_cas.hpp index af613b1da..4726f8b8c 100644 --- a/src/buildtool/storage/local_cas.hpp +++ b/src/buildtool/storage/local_cas.hpp @@ -70,11 +70,13 @@ class LocalCAS { } /// \brief Store tree from file path. + /// \tparam kOwner Indicates ownership for optimization (hardlink). /// \param file_path The path of the file to store as tree. /// \returns Digest of the stored tree or nullopt otherwise. + template [[nodiscard]] auto StoreTree(std::filesystem::path const& file_path) const noexcept -> std::optional { - return cas_tree_.StoreBlobFromFile(file_path); + return cas_tree_.StoreBlobFromFile(file_path, kOwner); } /// \brief Store tree from bytes. From e60c1168d4d0b43bff7acf19e35f44c786d4fff9 Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Thu, 13 Jul 2023 11:13:27 +0200 Subject: [PATCH 02/97] ["shell/test", "script"]: support target architecture ... provided it is specified on how to obtain a remote-execution endpoint for the given architecture; in this case, the test (script and) dependencies will be cross compiled for and run on that architecture. The actual building (as well as summarizing multiple test runs) happens on the default execution end point. --- rules/shell/test/EXPRESSIONS | 4 ++++ rules/shell/test/RULES | 40 ++++++++++++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/rules/shell/test/EXPRESSIONS b/rules/shell/test/EXPRESSIONS index 3ab3605bc..80e6cdb52 100644 --- a/rules/shell/test/EXPRESSIONS +++ b/rules/shell/test/EXPRESSIONS @@ -9,6 +9,7 @@ , "runner" , "deps-fieldname" , "deps-transition" + , "target properties" ] , "imports": { "artifacts_list": ["./", "../..", "field_artifacts_list"] @@ -135,6 +136,7 @@ } , "timeout scaling": {"type": "var", "name": "TIMEOUT_SCALE", "default": 1.0} + , "execution properties": {"type": "var", "name": "target properties"} } , "else": { "type": "ACTION" @@ -156,6 +158,7 @@ } , "timeout scaling": {"type": "var", "name": "TIMEOUT_SCALE", "default": 1.0} + , "execution properties": {"type": "var", "name": "target properties"} } } } @@ -170,6 +173,7 @@ , "runner" , "deps-fieldname" , "deps-transition" + , "target properties" ] , "imports": {"action": "test-action"} , "expression": diff --git a/rules/shell/test/RULES b/rules/shell/test/RULES index d6baa9ca4..117a060d3 100644 --- a/rules/shell/test/RULES +++ b/rules/shell/test/RULES @@ -3,7 +3,14 @@ , "target_fields": ["deps", "test"] , "string_fields": ["keep", "name"] , "config_vars": - ["ARCH", "HOST_ARCH", "RUNS_PER_TEST", "TEST_ENV", "TIMEOUT_SCALE"] + [ "ARCH" + , "HOST_ARCH" + , "RUNS_PER_TEST" + , "TEST_ENV" + , "TIMEOUT_SCALE" + , "TARGET_ARCH" + , "ARCH_DISPATCH" + ] , "field_doc": { "test": [ "The shell script for the test, launched with sh." @@ -36,6 +43,19 @@ , "TEST_ENV": ["The environment for executing the test runner."] , "TIMEOUT_SCALE": ["Factor on how to scale the timeout for this test. Defaults to 1.0."] + , "TARGET_ARCH": + [ "The architecture to build the test for." + , "" + , "Will only be honored, if that architecture is available in the" + , "ARCH_DISPATCH map. Otherwise, the test will be built for and run" + , "on the host architecture." + ] + , "ARCH_DISPATCH": + [ "Map of architectures to execution properties that ensure execution" + , "on that architecture. Only the actual test binary will be run with" + , "the specified execution properties (i.e., on the target architecture);" + , "all building will be done on the host architecture." + ] } , "tainted": ["test"] , "artifacts_doc": @@ -58,10 +78,13 @@ { "test-result": "test-result" , "action": "test-action" , "stage": ["./", "../..", "stage_singleton_field"] - , "host transition": ["transitions", "for host"] + , "host transition": ["transitions", "maybe for host"] + , "target properties": ["transitions", "target properties"] } , "config_transitions": - {"deps": [{"type": "CALL_EXPRESSION", "name": "host transition"}]} + { "deps": [{"type": "CALL_EXPRESSION", "name": "host transition"}] + , "test": [{"type": "CALL_EXPRESSION", "name": "host transition"}] + } , "expression": { "type": "let*" , "bindings": @@ -70,7 +93,13 @@ , "msg": "Expecting 'test' to specify precisely one file containing a shell script" , "$1": { "type": "let*" - , "bindings": [["fieldname", "test"], ["location", "test.sh"]] + , "bindings": + [ ["fieldname", "test"] + , ["location", "test.sh"] + , [ "transition" + , {"type": "CALL_EXPRESSION", "name": "host transition"} + ] + ] , "body": {"type": "CALL_EXPRESSION", "name": "stage"} } } @@ -87,6 +116,9 @@ , [ "deps-transition" , {"type": "CALL_EXPRESSION", "name": "host transition"} ] + , [ "target properties" + , {"type": "CALL_EXPRESSION", "name": "target properties"} + ] ] , "body": { "type": "if" From c87d2ac032f682f489e23d5950cd3166f3e303a2 Mon Sep 17 00:00:00 2001 From: Michael Thies Date: Sun, 16 Jul 2023 00:23:00 +0000 Subject: [PATCH 03/97] operation_cache: add missing include --- .../execution_api/execution_service/operation_cache.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/buildtool/execution_api/execution_service/operation_cache.hpp b/src/buildtool/execution_api/execution_service/operation_cache.hpp index 97b137fb9..25ca49a70 100644 --- a/src/buildtool/execution_api/execution_service/operation_cache.hpp +++ b/src/buildtool/execution_api/execution_service/operation_cache.hpp @@ -16,6 +16,7 @@ #define OPERATION_CACHE_HPP #include +#include #include #include #include From cdd2d5fcf8e4287ebbf45a8cc0a74a15bca4ddad Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Mon, 17 Jul 2023 14:20:33 +0200 Subject: [PATCH 04/97] Add a tutorial on cross compiling --- README.md | 1 + doc/tutorial/cross-compiling.md | 276 ++++++++++++++++++++++++++++++++ 2 files changed, 277 insertions(+) create mode 100644 doc/tutorial/cross-compiling.md diff --git a/README.md b/README.md index 8ba12b2a4..74acea147 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ taken from user-defined rules described by functional expressions. - [Ensuring reproducibility](doc/tutorial/rebuild.md) - [Using protobuf](doc/tutorial/proto.md) - [How to create a single-node remote execution service](doc/tutorial/just-execute.org) + - [Cross compiling and testing cross-compiled targets](doc/tutorial/cross-compiling.org) ## Documentation diff --git a/doc/tutorial/cross-compiling.md b/doc/tutorial/cross-compiling.md new file mode 100644 index 000000000..88f930214 --- /dev/null +++ b/doc/tutorial/cross-compiling.md @@ -0,0 +1,276 @@ +Cros compiling and testing cross-compiled targets +================================================== + +So far, we were always building for the platform on which we were +building. There are, however, good reasons to build on a different +one. For example, the other platform could be faster (common theme +when developing for embedded devices), cheaper, or simply available +in larger quantities. The standard solution for these kind of +situations is cross compiling: the binary is completely built on +one platform, while being intended to run on a different one. + +Cross compiling using the CC rules +---------------------------------- + +The `C` and `C++` rules that come with the `justbuild` repository +already have cross-compiling already built into the tool-chain +definitions for the `"gnu"` and `"clang"` compiler families; as +different compilers expect different ways to be told about the target +architecture, cross compiling is not supported for the `"unknown"` +compiler family. + +First, ensure that the required tools and libraries (which also have +to be available for the target architecture) for cross compiling +are installed. Depending on the distribution used, this can be +spread over several packages, often shared by the architecture to +cross compile to. + +Once the required packages are installed, usage is quite forward. +Let's start a new project. +``` sh +$ touch ROOT +``` + +We create a file `reposs.template.json` specifying the one local repository. +``` {.jsonc srcname="repos.template.json"} +{ "repositories": + { "": + { "repository": {"type": "file", "path": "."} + , "bindings": {"rules": "rules-cc"} + } + } +} +``` + +The actual `rules-cc` are imported via `just-import-git`. +``` sh +$ just-import-git -C repos.template.json -b master --as rules-cc https://github.com/just-buildsystem/rules-cc > repos.json +``` + +Let's have `main.cpp` be the canonical Hello-World program. +``` {.cpp srcname="main.cpp"} +#include + +int main() { + std::cout << "Hello world!\n"; + return 0; +} +``` + +Then, our `TARGETS` file describe a simple binary, as usual. +``` {.jsonc srcname="TARGETS"} +{ "helloworld": + { "type": ["@", "rules", "CC", "binary"] + , "name": ["helloworld"] + , "srcs": ["main.cpp"] + } +} +``` + +As mentioned in the introduction, we need to specify `COMPILER_FAMILY`, +`OS`, and `ARCH`. So the canonical building for host looks something like +the following. +``` sh +$ just-mr build -D '{"COMPILER_FAMILY": "gnu", "OS": "linux", "ARCH": "x86_64"}' +INFO: Performing repositories setup +INFO: Found 21 repositories to set up +INFO: Setup finished, exec ["just","build","-C","...","-D","{\"COMPILER_FAMILY\": \"gnu\", \"OS\": \"linux\", \"ARCH\": \"x86_64\"}"] +INFO: Requested target is [["@","","","helloworld"],{"ARCH":"x86_64","COMPILER_FAMILY":"gnu","OS":"linux"}] +INFO: Analysed target [["@","","","helloworld"],{"ARCH":"x86_64","COMPILER_FAMILY":"gnu","OS":"linux"}] +INFO: Discovered 2 actions, 1 trees, 0 blobs +INFO: Building [["@","","","helloworld"],{"ARCH":"x86_64","COMPILER_FAMILY":"gnu","OS":"linux"}]. +INFO: Processed 2 actions, 0 cache hits. +INFO: Artifacts built, logical paths are: + helloworld [0d5754a83c7c787b1c4dd717c8588ecef203fb72:16992:x] +$ +``` + +To cross compile, we simply add `TARGET_ARCH`. +``` +$ just-mr build -D '{"COMPILER_FAMILY": "gnu", "OS": "linux", "ARCH": "x86_64", "TARGET_ARCH": "arm64"}' +INFO: Performing repositories setup +INFO: Found 21 repositories to set up +INFO: Setup finished, exec ["just","build","-C","/home/aehlig/.cache/just/protocol-dependent/generation-0/git-sha1/casf/8c/c5d925c6d230e233f0e89107c8edb22d699063","-D","{\"COMPILER_FAMILY\": \"gnu\", \"OS\": \"linux\", \"ARCH\": \"x86_64\", \"TARGET_ARCH\": \"arm64\"}"] +INFO: Requested target is [["@","","","helloworld"],{"ARCH":"x86_64","COMPILER_FAMILY":"gnu","OS":"linux","TARGET_ARCH":"arm64"}] +INFO: Analysed target [["@","","","helloworld"],{"ARCH":"x86_64","COMPILER_FAMILY":"gnu","OS":"linux","TARGET_ARCH":"arm64"}] +INFO: Discovered 2 actions, 1 trees, 0 blobs +INFO: Building [["@","","","helloworld"],{"ARCH":"x86_64","COMPILER_FAMILY":"gnu","OS":"linux","TARGET_ARCH":"arm64"}]. +INFO: Processed 2 actions, 0 cache hits. +INFO: Artifacts built, logical paths are: + helloworld [b45459ea3dd36c7531756a4de9aaefd6af30e417:9856:x] +$ +``` + +To inspect the different command lines for native and cross compilation, +we can use `just analyse`. +``` sh +$ just-mr analyse -D '{"COMPILER_FAMILY": "gnu", "OS": "linux", "ARCH": "x86_64"}' --dump-actions - +$ just-mr analyse -D '{"COMPILER_FAMILY": "gnu", "OS": "linux", "ARCH": "x86_64", "TARGET_ARCH": "arm64"}' --dump-actions - +$ just-mr analyse -D '{"COMPILER_FAMILY": "clang", "OS": "linux", "ARCH": "x86_64"}' --dump-actions - +$ just-mr analyse -D '{"COMPILER_FAMILY": "clang", "OS": "linux", "ARCH": "x86_64", "TARGET_ARCH": "arm64"}' --dump-actions - +``` + + +Testing in the presence of cross compiling +------------------------------------------ + +To understand testing the in the presence of cross compiling, let's +walk through a simple example. We create a basic test by adding +a file `test/TARGETS`. +``` {.jsonc srcname="test/TARGETS"} +{ "basic": + { "type": ["@", "rules", "shell/test", "script"] + , "name": ["basic"] + , "test": ["basic.sh"] + , "deps": [["", "helloworld"]] + } +, "basic.sh": + { "type": "file_gen" + , "name": "basic.sh" + , "data": "./helloworld | grep -i world" + } +} +``` + +Now, if we try to run the test by simply specifying the target architecture, +we find that the binary to be tested is still only built for host. +``` sh +$ just-mr analyse --dump-targets - -D '{"COMPILER_FAMILY": "gnu", "OS": "linux", "ARCH": "x86_64", "TARGET_ARCH": "arm64"}' test basic +INFO: Performing repositories setup +INFO: Found 21 repositories to set up +INFO: Setup finished, exec ["just","analyse","-C","...","--dump-targets","-","-D","{\"COMPILER_FAMILY\": \"gnu\", \"OS\": \"linux\", \"ARCH\": \"x86_64\", \"TARGET_ARCH\": \"arm64\"}","test","basic"] +INFO: Requested target is [["@","","test","basic"],{"ARCH":"x86_64","COMPILER_FAMILY":"gnu","OS":"linux","TARGET_ARCH":"arm64"}] +INFO: Result of target [["@","","test","basic"],{"ARCH":"x86_64","COMPILER_FAMILY":"gnu","OS":"linux","TARGET_ARCH":"arm64"}]: { + "artifacts": { + "result": {"data":{"id":"33eb2ebd2ea0d6d335dfc1f948d14d506a19f693","path":"result"},"type":"ACTION"}, + "stderr": {"data":{"id":"33eb2ebd2ea0d6d335dfc1f948d14d506a19f693","path":"stderr"},"type":"ACTION"}, + "stdout": {"data":{"id":"33eb2ebd2ea0d6d335dfc1f948d14d506a19f693","path":"stdout"},"type":"ACTION"}, + "time-start": {"data":{"id":"33eb2ebd2ea0d6d335dfc1f948d14d506a19f693","path":"time-start"},"type":"ACTION"}, + "time-stop": {"data":{"id":"33eb2ebd2ea0d6d335dfc1f948d14d506a19f693","path":"time-stop"},"type":"ACTION"} + }, + "provides": { + }, + "runfiles": { + "basic": {"data":{"id":"c57ed1d00bb6c800ff7ebd1c89519c8481885eda"},"type":"TREE"} + } + } +INFO: List of analysed targets: +{ + "@": { + "": { + "": { + "helloworld": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"ADD_LDFLAGS":null,"ARCH":"x86_64","BUILD_POSITION_INDEPENDENT":null,"CC":null,"CFLAGS":null,"COMPILER_FAMILY":"gnu","CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"LDFLAGS":null,"OS":"linux","TARGET_ARCH":"x86_64"}] + }, + "test": { + "basic": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"ADD_LDFLAGS":null,"ARCH":"x86_64","ARCH_DISPATCH":null,"BUILD_POSITION_INDEPENDENT":null,"CC":null,"CFLAGS":null,"COMPILER_FAMILY":"gnu","CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"LDFLAGS":null,"OS":"linux","RUNS_PER_TEST":null,"TARGET_ARCH":"arm64","TEST_ENV":null,"TIMEOUT_SCALE":null}], + "basic.sh": [{}] + } + }, + "rules-cc": { + "CC": { + "defaults": [{"ARCH":"x86_64","COMPILER_FAMILY":"gnu","DEBUG":null,"HOST_ARCH":null,"OS":"linux","TARGET_ARCH":"x86_64"}] + } + }, + "rules-cc/just/rules": { + "CC": { + "defaults": [{"ARCH":"x86_64","COMPILER_FAMILY":"gnu","DEBUG":null,"HOST_ARCH":null,"OS":"linux","TARGET_ARCH":"x86_64"}], + "gcc": [{"ARCH":"x86_64","HOST_ARCH":null,"OS":"linux","TARGET_ARCH":"x86_64"}], + "toolchain": [{"ARCH":"x86_64","COMPILER_FAMILY":"gnu","HOST_ARCH":null,"OS":"linux","TARGET_ARCH":"x86_64"}] + } + } + } +} +INFO: Target tainted ["test"]. +$ +``` + +The reason is that a test actually has to run the created binary +and that requires a build enviroment of the target architecture. +So, if not being told how to obtain such an environemt, they carry +out the test in the best mannor they can, i.e., by transitioning +everything to host. So, in order to properly test the cross-compiled +binary, we need to do two things. + +- We need to setup remote execution on the correct architecture, + either by buying the apropriate hardware, or by running an emulator. +- We need to tell `justbuild` on how to reach that endpoint. + +To continue the example, let's say we set up an `arm64` machine, +e.g., a Raspberry Pi, in the local network. On that machine, we can +simply run a single-node execution service using `just execute`; +note that the `just` binary used there has to be an `arm64` binary, +e.g., obtained by cross compiling. + +The next step is to tell `justbuild` how to reach that machine; +as we only want to use it for certain actions we can't simply +set it as (default) remote-execution endpoint (specified by the +`-r` option). Instead we crate a file `dispatch.json`. + +``` {.jsonc srcname="dispatch.json"} +[[{"runner": "arm64-worker"}, "10.204.62.67:9999"]] +``` +This file contains a list of lists pairs (lists of length 2) with the +first entry a map of remote-execution properties and the second entry +a remote-execution endpoint. For each action, if the remote-execution +properties of that action match the first component of a pair the +second component of the first matching pair is taken as remote-execution +endpoint instead of the default endpoint specified on the command +line. Here "matching" means that all values in the domain of the +map have the same value in the remote-execution properties of the +action (in particular, `{}` matches everything); so more specific +dispatches have to be specified earlier in the list. In our case, +we have a single endpoint in our private network that we should +use whenever the propterty `"runnner"` has value `"arm64-worker"`. +The IP/port specification might differ in your setup. The path to +this file is passed by the `--endpoint-configuration` option. + +Next, we have to tell the test rule, that we actually do have a +runner for the `arm64` architecture. The rule expects this in the +`ARCH_DISPATCH` configuration variable that you might have seen in +the list of analysed targets earlier. It is expected to be a map +from architectures to remote-execution properties ensuring the action +will be run on that architecture. If the rules finds a non-empty +entry for the specified target architecture, it assumes a runner is +available and will run the test action with those properties; for +the compilation steps, still cross compiling is used. In our case, +we set `ARCH_DISPATCH` to `{"arm64": {"runner": "arm64-worker"}}`. + +Finally, we have to provide the credentials needed for mutual +authentication with the remote-execution endpoint by setting `--tls-*` +options appropriately. `just` assumes that the same credentials +can be used all remote-execution endpoints involved. In our example +we're building locally (where the build process starts the actions +itself) which does not require any credentials; nevertheless, it +still accepts any credentials provided. + +``` sh +$ just-mr build -D '{"COMPILER_FAMILY": "gnu", "OS": "linux", "ARCH": "x86_64", "TARGET_ARCH": "arm64", "ARCH_DISPATCH": {"arm64": {"runner": "arm64-worker"}}}' --endpoint-configuration dispatch.json --tls-ca-cert ca.crt --tls-client-cert client.crt --tls-client-key client.key test basic +INFO: Performing repositories setup +INFO: Found 21 repositories to set up +INFO: Setup finished, exec ["just","build","-C","...","-D","{\"COMPILER_FAMILY\": \"gnu\", \"OS\": \"linux\", \"ARCH\": \"x86_64\", \"TARGET_ARCH\": \"arm64\", \"ARCH_DISPATCH\": {\"arm64\": {\"runner\": \"arm64-worker\"}}}","--endpoint-configuration","dispatch.json","--tls-ca-cert","ca.crt","--tls-client-cert","client.crt","--tls-client-key","client.key","test","basic"] +INFO: Requested target is [["@","","test","basic"],{"ARCH":"x86_64","ARCH_DISPATCH":{"arm64":{"runner":"arm64-worker"}},"COMPILER_FAMILY":"gnu","OS":"linux","TARGET_ARCH":"arm64"}] +INFO: Analysed target [["@","","test","basic"],{"ARCH":"x86_64","ARCH_DISPATCH":{"arm64":{"runner":"arm64-worker"}},"COMPILER_FAMILY":"gnu","OS":"linux","TARGET_ARCH":"arm64"}] +INFO: Target tainted ["test"]. +INFO: Discovered 3 actions, 3 trees, 1 blobs +INFO: Building [["@","","test","basic"],{"ARCH":"x86_64","ARCH_DISPATCH":{"arm64":{"runner":"arm64-worker"}},"COMPILER_FAMILY":"gnu","OS":"linux","TARGET_ARCH":"arm64"}]. +INFO: Processed 3 actions, 2 cache hits. +INFO: Artifacts built, logical paths are: + result [7ef22e9a431ad0272713b71fdc8794016c8ef12f:5:f] + stderr [e69de29bb2d1d6434b8b29ae775ad8c2e48c5391:0:f] + stdout [cd0875583aabe89ee197ea133980a9085d08e497:13:f] + time-start [298a717bc8ba9b7e681a6d789104a1204ebe75b8:11:f] + time-stop [298a717bc8ba9b7e681a6d789104a1204ebe75b8:11:f] + (1 runfiles omitted.) +INFO: Target tainted ["test"]. +$ +``` + +The resulting command line might look complicated, but the +authentication-related options, as well as the dispatch-related +otpions (including setting `ARGUMENTS_DISPATCH` via `-D`) can simply +be set in the `"just args"` entry of the `.just-mrrc` file. + +When inspecting the result, we can use `just install-cas` as usual, +without any special arguments. Whenever dispatching an action to +a non-default end point, `just` will take care of syncing back the +artifacts to the default CAS. From 5548997571612cb8a78dfa9ac50ed690a28341ca Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Wed, 19 Jul 2023 14:25:51 +0200 Subject: [PATCH 05/97] README: fix typo in reference --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 74acea147..7dda2731f 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ taken from user-defined rules described by functional expressions. - [Ensuring reproducibility](doc/tutorial/rebuild.md) - [Using protobuf](doc/tutorial/proto.md) - [How to create a single-node remote execution service](doc/tutorial/just-execute.org) - - [Cross compiling and testing cross-compiled targets](doc/tutorial/cross-compiling.org) + - [Cross compiling and testing cross-compiled targets](doc/tutorial/cross-compiling.md) ## Documentation From 629dc072c7c81693e55cafaa90e63ff1128b0efe Mon Sep 17 00:00:00 2001 From: Alberto Sartori Date: Fri, 21 Jul 2023 11:26:52 +0200 Subject: [PATCH 06/97] bugfix: stage symlinks as symlinks when creating an action directory Before this patch, when creating an action directory, symlinks were staged as regular files. --- .../execution_api/local/local_action.cpp | 12 ++++ .../executor/executor_api.test.hpp | 6 +- test/end-to-end/TARGETS | 1 + test/end-to-end/symlinks/TARGETS | 8 +++ test/end-to-end/symlinks/stage-links.sh | 55 +++++++++++++++++++ 5 files changed, 78 insertions(+), 4 deletions(-) create mode 100644 test/end-to-end/symlinks/TARGETS create mode 100644 test/end-to-end/symlinks/stage-links.sh diff --git a/src/buildtool/execution_api/local/local_action.cpp b/src/buildtool/execution_api/local/local_action.cpp index d42dcdd61..d3ad7fd9c 100644 --- a/src/buildtool/execution_api/local/local_action.cpp +++ b/src/buildtool/execution_api/local/local_action.cpp @@ -185,6 +185,18 @@ auto LocalAction::StageFile(std::filesystem::path const& target_path, return false; } + if (info.type == ObjectType::Symlink) { + auto to = + FileSystemManager::ReadContentAtPath(*blob_path, ObjectType::File); + if (not to) { + logger_.Emit(LogLevel::Error, + "could not read content of symlink {}", + (*blob_path).string()); + return false; + } + return FileSystemManager::CreateSymlink(*to, target_path); + } + return FileSystemManager::CreateDirectory(target_path.parent_path()) and FileSystemManager::CreateFileHardlink(*blob_path, target_path); } diff --git a/test/buildtool/execution_engine/executor/executor_api.test.hpp b/test/buildtool/execution_engine/executor/executor_api.test.hpp index 30d724eaf..ffee51846 100755 --- a/test/buildtool/execution_engine/executor/executor_api.test.hpp +++ b/test/buildtool/execution_engine/executor/executor_api.test.hpp @@ -398,11 +398,9 @@ static inline void TestUploadAndDownloadTrees(ApiFactory const& factory, CHECK(FileSystemManager::IsDirectory(tmpdir)); CHECK(FileSystemManager::IsDirectory(tmpdir / "b")); CHECK(FileSystemManager::IsFile(tmpdir / "a")); - // echo command returns regular file instead of symlink - CHECK(FileSystemManager::IsFile(tmpdir / "b" / "a")); + CHECK(FileSystemManager::IsNonUpwardsSymlink(tmpdir / "b" / "a")); CHECK(*FileSystemManager::ReadFile(tmpdir / "a") == "foo"); - // echo command returns regular file instead of symlink - CHECK(*FileSystemManager::ReadFile(tmpdir / "b" / "a") == "bar"); + CHECK(*FileSystemManager::ReadSymlink(tmpdir / "b" / "a") == "bar"); REQUIRE(FileSystemManager::RemoveDirectory(tmpdir, true)); } diff --git a/test/end-to-end/TARGETS b/test/end-to-end/TARGETS index 0e81d7495..be1f642f1 100644 --- a/test/end-to-end/TARGETS +++ b/test/end-to-end/TARGETS @@ -58,6 +58,7 @@ , [["./", "git-import", "TESTS"], "git-import"] , [["./", "gc", "TESTS"], "gc"] , [["./", "execution-service", "TESTS"], "execution-service"] + , [["./", "symlinks", "TESTS"], "symlinks"] ] } } diff --git a/test/end-to-end/symlinks/TARGETS b/test/end-to-end/symlinks/TARGETS new file mode 100644 index 000000000..12add4515 --- /dev/null +++ b/test/end-to-end/symlinks/TARGETS @@ -0,0 +1,8 @@ +{ "stage-links": + { "type": ["end-to-end", "with remote"] + , "name": ["stage-links"] + , "test": ["stage-links.sh"] + , "deps": [["end-to-end", "tool-under-test"]] + } +, "TESTS": {"type": "install", "tainted": ["test"], "deps": ["stage-links"]} +} diff --git a/test/end-to-end/symlinks/stage-links.sh b/test/end-to-end/symlinks/stage-links.sh new file mode 100644 index 000000000..b8234d2a5 --- /dev/null +++ b/test/end-to-end/symlinks/stage-links.sh @@ -0,0 +1,55 @@ +#!/bin/sh +# Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +readonly JUST="${PWD}/bin/tool-under-test" + +touch ROOT +cat > TARGETS < Date: Fri, 21 Jul 2023 11:34:10 +0200 Subject: [PATCH 07/97] Update libgit2 to 1.7.0 --- etc/import/TARGETS.git2 | 10 +- etc/import/deps/xdiff/TARGETS.git2 | 5 + etc/import/include/git2/TARGETS.git2 | 158 +---------- etc/import/src/libgit2/TARGETS.git2 | 268 ++---------------- etc/import/src/libgit2/streams/TARGETS.git2 | 11 + .../src/libgit2/transports/TARGETS.git2 | 11 + etc/import/src/util/TARGETS.git2 | 172 ++++++++++- etc/repos.json | 10 +- 8 files changed, 231 insertions(+), 414 deletions(-) create mode 100644 etc/import/deps/xdiff/TARGETS.git2 create mode 100644 etc/import/src/libgit2/streams/TARGETS.git2 create mode 100644 etc/import/src/libgit2/transports/TARGETS.git2 diff --git a/etc/import/TARGETS.git2 b/etc/import/TARGETS.git2 index 0d15117f7..486eb0e07 100644 --- a/etc/import/TARGETS.git2 +++ b/etc/import/TARGETS.git2 @@ -18,7 +18,7 @@ , "ENV" , "USE_SYSTEM_LIBS" , "DEBUG_POOL" - , "THREADSAFE" + , "USE_THREADS" , "REGEX_BACKEND" , "WINHTTP" , "USE_ICONV" @@ -88,7 +88,7 @@ , "TARGET_ARCH" , "USE_SYSTEM_LIBS" , "DEBUG_POOL" - , "THREADSAFE" + , "USE_THREADS" , "REGEX_BACKEND" , "WINHTTP" , "USE_ICONV" @@ -124,7 +124,9 @@ {"type": "fail", "msg": "Required variable 'OS' is not set."} } ] - , ["THREADSAFE", {"type": "var", "name": "THREADSAFE", "default": true}] + , [ "USE_THREADS" + , {"type": "var", "name": "USE_THREADS", "default": true} + ] , ["USE_SSH", {"type": "var", "name": "USE_SSH", "default": true}] , ["USE_GSSAPI", {"type": "var", "name": "USE_GSSAPI", "default": true}] , ["USE_SHA1", {"type": "var", "name": "USE_SHA1", "default": true}] @@ -226,7 +228,7 @@ , "vars": [ "USE_SYSTEM_LIBS" , "DEBUG_POOL" - , "THREADSAFE" + , "USE_THREADS" , "REGEX_BACKEND" , "USE_ICONV" , "USE_NSEC" diff --git a/etc/import/deps/xdiff/TARGETS.git2 b/etc/import/deps/xdiff/TARGETS.git2 new file mode 100644 index 000000000..c204aec69 --- /dev/null +++ b/etc/import/deps/xdiff/TARGETS.git2 @@ -0,0 +1,5 @@ +{ "srcs": + {"type": ["@", "rules", "data", "staged"], "srcs": [["GLOB", null, "*.c"]]} +, "hdrs": + {"type": ["@", "rules", "data", "staged"], "srcs": [["GLOB", null, "*.h"]]} +} diff --git a/etc/import/include/git2/TARGETS.git2 b/etc/import/include/git2/TARGETS.git2 index f0ab360de..5f16be192 100644 --- a/etc/import/include/git2/TARGETS.git2 +++ b/etc/import/include/git2/TARGETS.git2 @@ -1,158 +1,4 @@ -{ "sys/features.h": - { "type": "configure" - , "arguments_config": - [ "OS" - , "ARCH" - , "TARGET_ARCH" - , "DEBUG_POOL" - , "THREADSAFE" - , "REGEX_BACKEND" - , "USE_ICONV" - , "USE_NSEC" - , "USE_SSH" - , "USE_NTLMCLIENT" - , "USE_GSSAPI" - , "USE_SHA1" - , "USE_SHA256" - , "USE_HTTPS" - ] - , "target": "feature_header" - , "config": - { "type": "let*" - , "bindings": - [ [ "TARGET_ARCH" - , { "type": "var" - , "name": "TARGET_ARCH" - , "default": {"type": "var", "name": "ARCH"} - } - ] - , [ "IS_BSD" - , { "type": "case" - , "expr": {"type": "var", "name": "OS"} - , "case": {"darwin": true, "bsd": true} - } - ] - , [ "IS_32BIT" - , { "type": "case" - , "expr": - { "type": "var" - , "name": "TARGET_ARCH" - , "default": {"type": "var", "name": "ARCH"} - } - , "case": {"x86": true, "arm": true} - } - ] - , [ "defines1" - , { "type": "++" - , "$1": - [ [ ["GIT_DEBUG_POOL", {"type": "var", "name": "DEBUG_POOL"}] - , ["GIT_THREADS", {"type": "var", "name": "THREADSAFE"}] - , { "type": "if" - , "cond": {"type": "var", "name": "IS_32BIT"} - , "then": ["GIT_ARCH_32", 1] - , "else": ["GIT_ARCH_64", 1] - } - , ["GIT_USE_ICONV", {"type": "var", "name": "USE_ICONV"}] - , ["GIT_SSH", {"type": "var", "name": "USE_SSH"}] - , [ "GIT_SSH_MEMORY_CREDENTIALS" - , {"type": "var", "name": "USE_SSH"} - ] - , ["GIT_NTLM", {"type": "var", "name": "USE_NTLMCLIENT"}] - ] - , { "type": "if" - , "cond": {"type": "var", "name": "USE_NSEC"} - , "then": - [ ["GIT_USE_NSEC", 1] - , ["GIT_USE_FUTIMENS", 1] - , ["GIT_USE_STAT_MTIME_NSEC", 1] - , { "type": "if" - , "cond": {"type": "var", "name": "IS_BSD"} - , "then": ["GIT_USE_STAT_MTIMESPEC", 1] - , "else": ["GIT_USE_STAT_MTIM", 1] - } - ] - } - , [ { "type": "case*" - , "expr": {"type": "var", "name": "REGEX_BACKEND"} - , "case": - [ ["regcomp_l", ["GIT_REGEX_REGCOMP_L", 1]] - , ["regcomp", ["GIT_REGEX_REGCOMP", 1]] - , ["pcre", ["GIT_REGEX_PCRE", 1]] - , ["pcre2", ["GIT_REGEX_PCRE2", 1]] - , ["builtin", ["GIT_REGEX_BUILTIN", 1]] - , [null, ["GIT_REGEX_BUILTIN", 1]] - ] - , "default": - { "type": "fail" - , "msg": "The REGEX_BACKEND option provided is not supported" - } - } - ] - , { "type": "case*" - , "expr": {"type": "var", "name": "USE_GSSAPI"} - , "case": - [ ["GSS.framework", [["GIT_GSSFRAMEWORK", 1]]] - , ["gssapi", [["GIT_GSSAPI", 1]]] - , [false, []] - ] - , "default": - { "type": "fail" - , "msg": "Backend asked for in USE_GSSAPI is not supported" - } - } - , { "type": "if" - , "cond": {"type": "var", "name": "USE_HTTPS"} - , "then": - [ ["GIT_HTTPS", 1] - , { "type": "case*" - , "expr": {"type": "var", "name": "USE_HTTPS"} - , "case": - [ ["SecureTransport", ["GIT_SECURE_TRANSPORT", 1]] - , ["OpenSSL", ["GIT_OPENSSL", 1]] - , ["mbedTLS", ["GIT_MBEDTLS", 1]] - , ["WinHTTP", ["GIT_WINHTTP", 1]] - ] - , "default": - { "type": "fail" - , "msg": "Backend asked for in USE_HTTPS is not supported" - } - } - ] - } - , { "type": "case*" - , "expr": {"type": "var", "name": "USE_SHA1"} - , "case": - [ ["CollisionDetection", [["GIT_SHA1_COLLISIONDETECT", 1]]] - , ["Win32", [["GIT_SHA1_WIN32", 1]]] - , ["CommonCrypto", [["GIT_SHA1_COMMON_CRYPTO", 1]]] - , ["OpenSSL", [["GIT_SHA1_OPENSSL", 1]]] - , ["mbedTLS", [["GIT_SHA1_MBEDTLS", 1]]] - ] - } - , { "type": "case*" - , "expr": {"type": "var", "name": "USE_SHA256"} - , "case": - [ ["Builtin", [["GIT_SHA256_BUILTIN", 1]]] - , ["Win32", [["GIT_SHA256_WIN32", 1]]] - , ["CommonCrypto", [["GIT_SHA256_COMMON_CRYPTO", 1]]] - , ["OpenSSL", [["GIT_SHA256_OPENSSL", 1]]] - , ["mbedTLS", [["GIT_SHA256_MBEDTLS", 1]]] - ] - } - ] - } - ] - ] - , "body": {"type": "env", "vars": ["defines1"]} - } - } -, "feature_header": - { "type": ["@", "rules", "CC/auto", "config"] - , "name": ["features.h"] - , "guard": ["INCLUDE_features_h__"] - , "stage": ["sys"] - } -, "TREE": +{ "TREE": { "type": "tree" , "name": "git2" , "deps": @@ -247,7 +93,7 @@ , "sys/repository.h" , "sys/stream.h" , "sys/transport.h" - , "sys/features.h" + , ["src/libgit2", "experimental.h"] ] } } diff --git a/etc/import/src/libgit2/TARGETS.git2 b/etc/import/src/libgit2/TARGETS.git2 index 67d4b7727..075f3f126 100644 --- a/etc/import/src/libgit2/TARGETS.git2 +++ b/etc/import/src/libgit2/TARGETS.git2 @@ -1,259 +1,33 @@ { "libgit2_private_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": - [ "annotated_commit.h" - , "apply.h" - , "attr.h" - , "attr_file.h" - , "attrcache.h" - , "blame.h" - , "blame_git.h" - , "blob.h" - , "branch.h" - , "buf.h" - , "cache.h" - , "checkout.h" - , "clone.h" - , "commit.h" - , "commit_graph.h" - , "commit_list.h" - , "common.h" - , "config.h" - , "config_backend.h" - , "config_entries.h" - , "config_parse.h" - , "delta.h" - , "diff.h" - , "diff_driver.h" - , "diff_file.h" - , "diff_generate.h" - , "diff_parse.h" - , "diff_stats.h" - , "diff_tform.h" - , "diff_xdiff.h" - , "email.h" - , "errors.h" - , "fetch.h" - , "fetchhead.h" - , "filter.h" - , "idxmap.h" - , "ignore.h" - , "index.h" - , "indexer.h" - , "iterator.h" - , "libgit2.h" - , "mailmap.h" - , "merge.h" - , "merge_driver.h" - , "midx.h" - , "mwindow.h" - , "netops.h" - , "notes.h" - , "object.h" - , "odb.h" - , "offmap.h" - , "oid.h" - , "oidarray.h" - , "oidmap.h" - , "pack-objects.h" - , "pack.h" - , "parse.h" - , "patch.h" - , "patch_generate.h" - , "patch_parse.h" - , "path.h" - , "pathspec.h" - , "proxy.h" - , "push.h" - , "reader.h" - , "refdb.h" - , "reflog.h" - , "refs.h" - , "refspec.h" - , "remote.h" - , "repo_template.h" - , "repository.h" - , "revwalk.h" - , "settings.h" - , "signature.h" - , "status.h" - , "stream.h" - , "submodule.h" - , "sysdir.h" - , "tag.h" - , "threadstate.h" - , "trace.h" - , "transaction.h" - , "tree-cache.h" - , "tree.h" - , "userdiff.h" - , "worktree.h" - , "streams/mbedtls.h" - , "streams/openssl.h" - , "streams/openssl_dynamic.h" - , "streams/openssl_legacy.h" - , "streams/registry.h" - , "streams/socket.h" - , "streams/stransport.h" - , "streams/tls.h" - , "transports/auth.h" - , "transports/auth_negotiate.h" - , "transports/auth_ntlm.h" - , "transports/http.h" - , "transports/httpclient.h" - , "transports/smart.h" - , "transports/ssh.h" - , "xdiff/git-xdiff.h" - , "xdiff/xdiff.h" - , "xdiff/xdiffi.h" - , "xdiff/xemit.h" - , "xdiff/xinclude.h" - , "xdiff/xmacros.h" - , "xdiff/xprepare.h" - , "xdiff/xtypes.h" - , "xdiff/xutils.h" + [ ["GLOB", null, "*.h"] + , ["./", "streams", "hdrs"] + , ["./", "transports", "hdrs"] , ["src/util", "util_private_headers"] + , "experimental.h" ] } , "libgit2_sources": { "type": ["@", "rules", "data", "staged"] , "srcs": - [ "annotated_commit.c" - , "apply.c" - , "attr.c" - , "attr_file.c" - , "attrcache.c" - , "blame.c" - , "blame_git.c" - , "blob.c" - , "branch.c" - , "buf.c" - , "cache.c" - , "checkout.c" - , "cherrypick.c" - , "clone.c" - , "commit.c" - , "commit_graph.c" - , "commit_list.c" - , "config.c" - , "config_cache.c" - , "config_entries.c" - , "config_file.c" - , "config_mem.c" - , "config_parse.c" - , "config_snapshot.c" - , "crlf.c" - , "delta.c" - , "describe.c" - , "diff.c" - , "diff_driver.c" - , "diff_file.c" - , "diff_generate.c" - , "diff_parse.c" - , "diff_print.c" - , "diff_stats.c" - , "diff_tform.c" - , "diff_xdiff.c" - , "email.c" - , "errors.c" - , "fetch.c" - , "fetchhead.c" - , "filter.c" - , "graph.c" - , "hashsig.c" - , "ident.c" - , "idxmap.c" - , "ignore.c" - , "index.c" - , "indexer.c" - , "iterator.c" - , "libgit2.c" - , "mailmap.c" - , "merge.c" - , "merge_driver.c" - , "merge_file.c" - , "message.c" - , "midx.c" - , "mwindow.c" - , "netops.c" - , "notes.c" - , "object.c" - , "object_api.c" - , "odb.c" - , "odb_loose.c" - , "odb_mempack.c" - , "odb_pack.c" - , "offmap.c" - , "oid.c" - , "oidarray.c" - , "oidmap.c" - , "pack-objects.c" - , "pack.c" - , "parse.c" - , "patch.c" - , "patch_generate.c" - , "patch_parse.c" - , "path.c" - , "pathspec.c" - , "proxy.c" - , "push.c" - , "reader.c" - , "rebase.c" - , "refdb.c" - , "refdb_fs.c" - , "reflog.c" - , "refs.c" - , "refspec.c" - , "remote.c" - , "repository.c" - , "reset.c" - , "revert.c" - , "revparse.c" - , "revwalk.c" - , "signature.c" - , "stash.c" - , "status.c" - , "strarray.c" - , "submodule.c" - , "sysdir.c" - , "tag.c" - , "threadstate.c" - , "trace.c" - , "trailer.c" - , "transaction.c" - , "transport.c" - , "tree-cache.c" - , "tree.c" - , "worktree.c" - , "streams/mbedtls.c" - , "streams/openssl.c" - , "streams/openssl_dynamic.c" - , "streams/openssl_legacy.c" - , "streams/registry.c" - , "streams/socket.c" - , "streams/stransport.c" - , "streams/tls.c" - , "transports/auth.c" - , "transports/auth_negotiate.c" - , "transports/auth_ntlm.c" - , "transports/credential.c" - , "transports/credential_helpers.c" - , "transports/git.c" - , "transports/http.c" - , "transports/httpclient.c" - , "transports/local.c" - , "transports/smart.c" - , "transports/smart_pkt.c" - , "transports/smart_protocol.c" - , "transports/ssh.c" - , "transports/winhttp.c" - , "xdiff/xdiffi.c" - , "xdiff/xemit.c" - , "xdiff/xhistogram.c" - , "xdiff/xmerge.c" - , "xdiff/xpatience.c" - , "xdiff/xprepare.c" - , "xdiff/xutils.c" + [ ["GLOB", null, "*.c"] + , ["./", "streams", "srcs"] + , ["./", "transports", "srcs"] ] } +, "experimental.h": + { "type": "configure" + , "target": "experimental" + , "config": + { "type": "let*" + , "bindings": [["defines1", [["GIT_EXPERIMENTAL_SHA256", 0]]]] + , "body": {"type": "env", "vars": []} + } + } +, "experimental": + { "type": ["@", "rules", "CC/auto", "config"] + , "name": ["experimental.h"] + , "guard": ["INCLUDE_experimental_h__"] + } } diff --git a/etc/import/src/libgit2/streams/TARGETS.git2 b/etc/import/src/libgit2/streams/TARGETS.git2 new file mode 100644 index 000000000..d30ccddae --- /dev/null +++ b/etc/import/src/libgit2/streams/TARGETS.git2 @@ -0,0 +1,11 @@ +{ "srcs": + { "type": ["@", "rules", "data", "staged"] + , "stage": ["streams"] + , "srcs": [["GLOB", null, "*.c"]] + } +, "hdrs": + { "type": ["@", "rules", "data", "staged"] + , "stage": ["streams"] + , "srcs": [["GLOB", null, "*.h"]] + } +} diff --git a/etc/import/src/libgit2/transports/TARGETS.git2 b/etc/import/src/libgit2/transports/TARGETS.git2 new file mode 100644 index 000000000..3685401db --- /dev/null +++ b/etc/import/src/libgit2/transports/TARGETS.git2 @@ -0,0 +1,11 @@ +{ "srcs": + { "type": ["@", "rules", "data", "staged"] + , "stage": ["transports"] + , "srcs": [["GLOB", null, "*.c"]] + } +, "hdrs": + { "type": ["@", "rules", "data", "staged"] + , "stage": ["transports"] + , "srcs": [["GLOB", null, "*.h"]] + } +} diff --git a/etc/import/src/util/TARGETS.git2 b/etc/import/src/util/TARGETS.git2 index c03fd06e8..cac756493 100644 --- a/etc/import/src/util/TARGETS.git2 +++ b/etc/import/src/util/TARGETS.git2 @@ -23,6 +23,7 @@ , "regexp.h" , "runtime.h" , "sortedcache.h" + , "staticstr.h" , "str.h" , "strmap.h" , "strnlen.h" @@ -65,11 +66,13 @@ , "win32/w32_util.h" , "win32/win32-compat.h" , ["include", "git2_public_headers"] + , "git2_features.h" + , ["deps/xdiff", "hdrs"] ] } , "util_os_unix": { "type": ["@", "rules", "CC", "library"] - , "arguments_config": ["OS", "THREADSAFE"] + , "arguments_config": ["OS", "USE_THREADS"] , "name": ["util_os_unix"] , "pure C": ["yes"] , "srcs": ["unix/map.c", "unix/realpath.c"] @@ -78,7 +81,7 @@ { "type": "++" , "$1": [ { "type": "if" - , "cond": {"type": "var", "name": "THREADSAFE"} + , "cond": {"type": "var", "name": "USE_THREADS"} , "then": ["-pthread"] } , { "type": "case" @@ -465,6 +468,171 @@ , "allocators/failalloc.c" , "allocators/stdalloc.c" , "allocators/win32_leakcheck.c" + , ["deps/xdiff", "srcs"] ] } +, "git2_features.h": + { "type": "configure" + , "arguments_config": + [ "OS" + , "ARCH" + , "TARGET_ARCH" + , "DEBUG_POOL" + , "USE_THREADS" + , "REGEX_BACKEND" + , "USE_ICONV" + , "USE_NSEC" + , "USE_SSH" + , "USE_NTLMCLIENT" + , "USE_GSSAPI" + , "USE_SHA1" + , "USE_SHA256" + , "USE_HTTPS" + ] + , "target": "feature_header" + , "config": + { "type": "let*" + , "bindings": + [ [ "TARGET_ARCH" + , { "type": "var" + , "name": "TARGET_ARCH" + , "default": {"type": "var", "name": "ARCH"} + } + ] + , [ "IS_BSD" + , { "type": "case" + , "expr": {"type": "var", "name": "OS"} + , "case": {"darwin": true, "bsd": true} + } + ] + , [ "IS_32BIT" + , { "type": "case" + , "expr": + { "type": "var" + , "name": "TARGET_ARCH" + , "default": {"type": "var", "name": "ARCH"} + } + , "case": {"x86": true, "arm": true} + } + ] + , [ "defines1" + , { "type": "++" + , "$1": + [ [ ["GIT_DEBUG_POOL", {"type": "var", "name": "DEBUG_POOL"}] + , ["GIT_THREADS", {"type": "var", "name": "USE_THREADS"}] + , { "type": "if" + , "cond": {"type": "var", "name": "IS_32BIT"} + , "then": ["GIT_ARCH_32", 1] + , "else": ["GIT_ARCH_64", 1] + } + , ["GIT_USE_ICONV", {"type": "var", "name": "USE_ICONV"}] + , ["GIT_SSH", {"type": "var", "name": "USE_SSH"}] + , [ "GIT_SSH_MEMORY_CREDENTIALS" + , {"type": "var", "name": "USE_SSH"} + ] + , ["GIT_NTLM", {"type": "var", "name": "USE_NTLMCLIENT"}] + , [ "GIT_IO_WSAPOLL" + , { "type": "==" + , "$1": {"type": "var", "name": "OS"} + , "$2": "windows" + } + ] + ] + , { "type": "if" + , "cond": {"type": "var", "name": "USE_NSEC"} + , "then": + [ ["GIT_USE_NSEC", 1] + , ["GIT_USE_FUTIMENS", 1] + , ["GIT_USE_STAT_MTIME_NSEC", 1] + , { "type": "if" + , "cond": {"type": "var", "name": "IS_BSD"} + , "then": ["GIT_USE_STAT_MTIMESPEC", 1] + , "else": ["GIT_USE_STAT_MTIM", 1] + } + ] + } + , [ { "type": "case*" + , "expr": {"type": "var", "name": "REGEX_BACKEND"} + , "case": + [ ["regcomp_l", ["GIT_REGEX_REGCOMP_L", 1]] + , ["regcomp", ["GIT_REGEX_REGCOMP", 1]] + , ["pcre", ["GIT_REGEX_PCRE", 1]] + , ["pcre2", ["GIT_REGEX_PCRE2", 1]] + , ["builtin", ["GIT_REGEX_BUILTIN", 1]] + , [null, ["GIT_REGEX_BUILTIN", 1]] + ] + , "default": + { "type": "fail" + , "msg": "The REGEX_BACKEND option provided is not supported" + } + } + ] + , { "type": "case*" + , "expr": {"type": "var", "name": "USE_GSSAPI"} + , "case": + [ ["GSS.framework", [["GIT_GSSFRAMEWORK", 1]]] + , ["gssapi", [["GIT_GSSAPI", 1]]] + , [false, []] + ] + , "default": + { "type": "fail" + , "msg": "Backend asked for in USE_GSSAPI is not supported" + } + } + , { "type": "if" + , "cond": {"type": "var", "name": "USE_HTTPS"} + , "then": + [ ["GIT_HTTPS", 1] + , { "type": "case*" + , "expr": {"type": "var", "name": "USE_HTTPS"} + , "case": + [ ["SecureTransport", ["GIT_SECURE_TRANSPORT", 1]] + , ["OpenSSL", ["GIT_OPENSSL", 1]] + , ["mbedTLS", ["GIT_MBEDTLS", 1]] + , ["WinHTTP", ["GIT_WINHTTP", 1]] + ] + , "default": + { "type": "fail" + , "msg": "Backend asked for in USE_HTTPS is not supported" + } + } + ] + } + , { "type": "case*" + , "expr": {"type": "var", "name": "USE_SHA1"} + , "case": + [ ["CollisionDetection", [["GIT_SHA1_COLLISIONDETECT", 1]]] + , ["Win32", [["GIT_SHA1_WIN32", 1]]] + , ["CommonCrypto", [["GIT_SHA1_COMMON_CRYPTO", 1]]] + , ["OpenSSL", [["GIT_SHA1_OPENSSL", 1]]] + , ["mbedTLS", [["GIT_SHA1_MBEDTLS", 1]]] + ] + } + , { "type": "case*" + , "expr": {"type": "var", "name": "USE_SHA256"} + , "case": + [ ["Builtin", [["GIT_SHA256_BUILTIN", 1]]] + , ["Win32", [["GIT_SHA256_WIN32", 1]]] + , ["CommonCrypto", [["GIT_SHA256_COMMON_CRYPTO", 1]]] + , ["OpenSSL", [["GIT_SHA256_OPENSSL", 1]]] + , ["mbedTLS", [["GIT_SHA256_MBEDTLS", 1]]] + ] + } + ] + } + ] + , [ "have_csymbol" + , [ ["GIT_IO_POLL", ["poll", ["poll.h"]]] + , ["GIT_IO_SELECT", ["select", ["sys/select.h"]]] + ] + ] + ] + , "body": {"type": "env", "vars": ["defines1", "have_csymbol"]} + } + } +, "feature_header": + { "type": ["@", "rules", "CC/auto", "config"] + , "name": ["git2_features.h"] + , "guard": ["INCLUDE_features_h__"] + } } diff --git a/etc/repos.json b/etc/repos.json index c28134ff2..1b00ea63a 100644 --- a/etc/repos.json +++ b/etc/repos.json @@ -407,11 +407,11 @@ , "com_github_libgit2_libgit2": { "repository": { "type": "archive" - , "content": "0c7c845401ae9ad63c5617da6df7e84145b699a0" - , "fetch": "https://github.com/libgit2/libgit2/archive/refs/tags/v1.5.2.tar.gz" - , "sha256": "57638ac0e319078f56a7e17570be754515e5b1276d3750904b4214c92e8fa196" - , "sha512": "0de5e82953482bf57e9cd221406581a12c6acfbeeea0cebbb9b701d288804ec085b7c0902ba1f04ad7cdaaead9d472be21ffe8d730050db5c77101a93448fb3a" - , "subdir": "libgit2-1.5.2" + , "content": "02793bbe15088985989d1ff636a23958f4a62c57" + , "fetch": "https://github.com/libgit2/libgit2/archive/refs/tags/v1.7.0.tar.gz" + , "sha256": "d9d0f84a86bf98b73e68997f5c1543cc5067d0ca9c7a5acaba3e8d117ecefef3" + , "sha512": "68c8ed289de7daccaec17ea2ac49f4610325595cf90cddef763a31546a0a1c6bd400bf6180b68e2d3a8bdc3d031328efbbbaf3b61467dfc1b944e8cf3efcfd69" + , "subdir": "libgit2-1.7.0" } , "target_root": "import targets" , "target_file_name": "TARGETS.git2" From 46046066c6004c6cca363c1c11e13de650e9e101 Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Mon, 24 Jul 2023 17:25:25 +0200 Subject: [PATCH 08/97] test_server: avoid races ... by creating a file only once the server is ready and waiting for that file instead of waiting a fixed period of time. --- test/end-to-end/just-mr/just-mr.test.sh | 8 ++++++-- test/other_tools/utils/curl_usage_test.sh | 8 ++++++-- test/utils/run_test_server.py | 3 ++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/test/end-to-end/just-mr/just-mr.test.sh b/test/end-to-end/just-mr/just-mr.test.sh index 3532f8a74..cdd0ace41 100644 --- a/test/end-to-end/just-mr/just-mr.test.sh +++ b/test/end-to-end/just-mr/just-mr.test.sh @@ -88,10 +88,14 @@ readonly GIT_TREE_ID="$(cd "${GIT_ROOT}" && git ls-tree "${GIT_REPO_COMMIT}" foo echo "Publish remote repos to HTTP server" # start Python server as remote repos location port_file="$(mktemp -t port_XXXXXX -p "${TEST_TMPDIR}")" -python3 -u "${ROOT}/utils/run_test_server.py" >${port_file} & server_pid=$! -sleep 1s # give some time to set up properly +python3 -u "${ROOT}/utils/run_test_server.py" "${port_file}" & server_pid=$! # set up cleanup of http server trap "server_cleanup ${server_pid}" INT TERM EXIT +# wait for the server to be available +while [ -z "$(cat "${port_file}")" ] +do + sleep 1s +done # get port number as variable port_num="$(cat ${port_file})" diff --git a/test/other_tools/utils/curl_usage_test.sh b/test/other_tools/utils/curl_usage_test.sh index 4da397dec..8db9e6757 100644 --- a/test/other_tools/utils/curl_usage_test.sh +++ b/test/other_tools/utils/curl_usage_test.sh @@ -39,10 +39,14 @@ echo "Publish test file as local HTTP server" # define location to store port number port_file="${ROOT}/port.txt" # start Python server as remote -python3 -u "${ROOT}/utils/run_test_server.py" >"${port_file}" & server_pid=$! -sleep 1s # give some time to set up properly +python3 -u "${ROOT}/utils/run_test_server.py" "${port_file}" & server_pid=$! # set up cleanup of http server trap "server_cleanup ${server_pid}" INT TERM EXIT +# wait for the server to be available +while [ -z "$(cat "${port_file}")" ] +do + sleep 1s +done cd "${ROOT}" diff --git a/test/utils/run_test_server.py b/test/utils/run_test_server.py index 23bfb5236..a23207fe6 100644 --- a/test/utils/run_test_server.py +++ b/test/utils/run_test_server.py @@ -41,6 +41,7 @@ def RecvSig(*_) -> None: with HTTPServer((hostname, 0), HTTPHandler) as httpd: # print port number socket_info = httpd.socket.getsockname() - print(socket_info[1]) + with open(sys.argv[1], "w") as f: + f.write("%d" % (socket_info[1],)) # run server httpd.serve_forever() From 57d3222a4ff1ab3cc64e0d33d721a30827b684e3 Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Wed, 26 Jul 2023 14:43:52 +0200 Subject: [PATCH 09/97] remote api: honor the --raw-tree option --- CHANGELOG.md | 2 ++ .../execution_api/remote/bazel/bazel_api.cpp | 4 +-- .../remote/bazel/bazel_network.cpp | 26 ++++++++++++++----- .../remote/bazel/bazel_network.hpp | 6 ++--- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 761fe55ca..97b2f7308 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,8 @@ A feature release on top of `1.1.0`, backwards compatible. if `gitignore` files were present. - Temporary files generated by `just execute` are now created inside the local build root. +- `just install-cas` now correctly handles `--raw-tree` also for + remote-execution endpoints. ## Release `1.1.0` (2023-05-19) diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp index f4fd3b1fd..c4f170a31 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp @@ -142,7 +142,7 @@ auto BazelApi::CreateAction( [[nodiscard]] auto BazelApi::RetrieveToFds( std::vector const& artifacts_info, std::vector const& fds, - bool /*raw_tree*/) noexcept -> bool { + bool raw_tree) noexcept -> bool { if (artifacts_info.size() != fds.size()) { Logger::Log(LogLevel::Error, "different number of digests and file descriptors."); @@ -154,7 +154,7 @@ auto BazelApi::CreateAction( auto const& info = artifacts_info[i]; if (gsl::owner out = fdopen(fd, "wb")) { // NOLINT - auto const success = network_->DumpToStream(info, out); + auto const success = network_->DumpToStream(info, out, raw_tree); std::fclose(out); if (not success) { Logger::Log(LogLevel::Error, diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network.cpp b/src/buildtool/execution_api/remote/bazel/bazel_network.cpp index 60ed22e15..538f26333 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_network.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_network.cpp @@ -79,7 +79,20 @@ namespace { [[nodiscard]] auto TreeToStream( gsl::not_null const& network, bazel_re::Digest const& tree_digest, - gsl::not_null const& stream) noexcept -> bool { + gsl::not_null const& stream, + bool raw_tree) noexcept -> bool { + if (raw_tree) { + auto blobs = network->ReadBlobs({tree_digest}).Next(); + if (blobs.size() != 1) { + Logger::Log(LogLevel::Error, + "Object {} not found in CAS", + NativeSupport::Unprefix(tree_digest.hash())); + return false; + } + auto const& str = blobs.at(0).data; + std::fwrite(str.data(), 1, str.size(), stream); + return true; + } if (Compatibility::IsCompatible()) { if (auto dir = ReadDirectory(network, tree_digest)) { if (auto data = BazelMsgFactory::DirectoryToString(*dir)) { @@ -423,9 +436,10 @@ auto BazelNetwork::ReadObjectInfosRecursively( return false; } -auto BazelNetwork::DumpToStream( - Artifact::ObjectInfo const& info, - gsl::not_null const& stream) const noexcept -> bool { - return IsTreeObject(info.type) ? TreeToStream(this, info.digest, stream) - : BlobToStream(this, info.digest, stream); +auto BazelNetwork::DumpToStream(Artifact::ObjectInfo const& info, + gsl::not_null const& stream, + bool raw_tree) const noexcept -> bool { + return IsTreeObject(info.type) + ? TreeToStream(this, info.digest, stream, raw_tree) + : BlobToStream(this, info.digest, stream); } diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network.hpp b/src/buildtool/execution_api/remote/bazel/bazel_network.hpp index c1ae34add..e513b23ef 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_network.hpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_network.hpp @@ -119,9 +119,9 @@ class BazelNetwork { -> std::optional, std::vector>>; - [[nodiscard]] auto DumpToStream( - Artifact::ObjectInfo const& info, - gsl::not_null const& stream) const noexcept -> bool; + [[nodiscard]] auto DumpToStream(Artifact::ObjectInfo const& info, + gsl::not_null const& stream, + bool raw_tree) const noexcept -> bool; private: using DirectoryMap = From e0663fd415934287ed338784abdf8181c6c77e24 Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Wed, 26 Jul 2023 15:12:33 +0200 Subject: [PATCH 10/97] Add basic test for install-cas ... verifying the consistency of the representation of trees between local and remote CAS. --- test/end-to-end/remote-execution/TARGETS | 12 ++++ .../remote-execution/install-cas.sh | 69 +++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 test/end-to-end/remote-execution/install-cas.sh diff --git a/test/end-to-end/remote-execution/TARGETS b/test/end-to-end/remote-execution/TARGETS index b6071b251..951499d1a 100644 --- a/test/end-to-end/remote-execution/TARGETS +++ b/test/end-to-end/remote-execution/TARGETS @@ -23,6 +23,18 @@ , "test": ["install.sh"] , "deps": [["end-to-end", "tool-under-test"]] } +, "install-cas": + { "type": ["end-to-end", "with remote"] + , "name": ["install-cas"] + , "test": ["install-cas.sh"] + , "deps": [["end-to-end", "tool-under-test"]] + , "keep": + [ "out/stdout/remote" + , "out/stdout/remote-raw" + , "out/stdout/local" + , "out/stdout/local-raw" + ] + } , "dispatch": { "type": ["end-to-end", "with remote"] , "name": ["dispatch"] diff --git a/test/end-to-end/remote-execution/install-cas.sh b/test/end-to-end/remote-execution/install-cas.sh new file mode 100644 index 000000000..e39ff41e2 --- /dev/null +++ b/test/end-to-end/remote-execution/install-cas.sh @@ -0,0 +1,69 @@ +#!/bin/sh +# Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -eu + +readonly JUST="${PWD}/bin/tool-under-test" +readonly LBRDIR="${TEST_TMPDIR}/local-build-root" +readonly OUTDIR="${PWD}/out" + +LOCAL_ARGS="--local-build-root ${LBRDIR}" +REMOTE_ARGS="${LOCAL_ARGS} -r ${REMOTE_EXECUTION_ADDRESS}" +if [ "${REMOTE_EXECUTION_PROPERTIES:-}" != "" ] +then + REMOTE_ARGS="${REMOTE_EXECUTION_ARGS} --remote-execution-property ${REMOTE_EXECUTION_PROPERTIES}" +fi +if [ -n "${COMPATIBLE:-}" ] +then + REMOTE_ARGS="${REMOTE_ARGS} --compatible" + LOCAL_ARGS="${LOCAL_ARGS} --compatible" +fi + +# Build a tree remotely and get its identifier +mkdir src +cd src +touch ROOT +cat > TARGETS <<'EOF' +{ "": + { "type": "generic" + , "out_dirs": ["out"] + , "cmds": + [ "mkdir -p out/deep/inside/path" + , "echo Hello World > out/deep/inside/path/hello.txt" + ] + } +} +EOF + +"${JUST}" build ${REMOTE_ARGS} --dump-artifacts artifacts.json 2>&1 +echo +cat artifacts.json +OUT="$(jq -r '.out.id' artifacts.json)::t" +echo $OUT +echo + +# install to stdout in all possible ways +mkdir -p "${OUTDIR}/stdout" +"${JUST}" install-cas ${REMOTE_ARGS} --raw-tree "${OUT}" > "${OUTDIR}/stdout/remote-raw" +"${JUST}" install-cas ${REMOTE_ARGS} --remember "${OUT}" > "${OUTDIR}/stdout/remote" +"${JUST}" install-cas ${LOCAL_ARGS} "${OUT}" --raw-tree > "${OUTDIR}/stdout/local-raw" +"${JUST}" install-cas ${LOCAL_ARGS} "${OUT}" > "${OUTDIR}/stdout/local" + +# verify consistency between local and remote values +cmp "${OUTDIR}/stdout/local-raw" "${OUTDIR}/stdout/remote-raw" +cmp "${OUTDIR}/stdout/local" "${OUTDIR}/stdout/remote" + +echo OK From 106ad30405914d20c05a32b32157f17da57c570e Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Thu, 27 Jul 2023 16:36:53 +0200 Subject: [PATCH 11/97] link install-cas test to test suite While there, fix a typo in the setup. --- test/end-to-end/remote-execution/TARGETS | 3 ++- test/end-to-end/remote-execution/install-cas.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/end-to-end/remote-execution/TARGETS b/test/end-to-end/remote-execution/TARGETS index 951499d1a..01c801847 100644 --- a/test/end-to-end/remote-execution/TARGETS +++ b/test/end-to-end/remote-execution/TARGETS @@ -44,6 +44,7 @@ , "TESTS": { "type": "install" , "tainted": ["test"] - , "deps": ["native-protocol", "large-blobs", "upload-test", "install"] + , "deps": + ["native-protocol", "large-blobs", "upload-test", "install", "install-cas"] } } diff --git a/test/end-to-end/remote-execution/install-cas.sh b/test/end-to-end/remote-execution/install-cas.sh index e39ff41e2..0a2f16a25 100644 --- a/test/end-to-end/remote-execution/install-cas.sh +++ b/test/end-to-end/remote-execution/install-cas.sh @@ -24,7 +24,7 @@ LOCAL_ARGS="--local-build-root ${LBRDIR}" REMOTE_ARGS="${LOCAL_ARGS} -r ${REMOTE_EXECUTION_ADDRESS}" if [ "${REMOTE_EXECUTION_PROPERTIES:-}" != "" ] then - REMOTE_ARGS="${REMOTE_EXECUTION_ARGS} --remote-execution-property ${REMOTE_EXECUTION_PROPERTIES}" + REMOTE_ARGS="${REMOTE_ARGS} --remote-execution-property ${REMOTE_EXECUTION_PROPERTIES}" fi if [ -n "${COMPATIBLE:-}" ] then From 26090ce503f40502a62d7ebabcb4ee060b28d05b Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Thu, 27 Jul 2023 13:59:11 +0200 Subject: [PATCH 12/97] Execution API: support fetching a single artifact to memory --- .../execution_api/common/execution_api.hpp | 7 ++++++ src/buildtool/execution_api/git/git_api.hpp | 7 ++++++ src/buildtool/execution_api/local/TARGETS | 1 + .../execution_api/local/local_api.hpp | 22 +++++++++++++++++++ .../execution_api/remote/bazel/bazel_api.cpp | 9 ++++++++ .../execution_api/remote/bazel/bazel_api.hpp | 4 ++++ .../executor/executor.test.cpp | 4 ++++ 7 files changed, 54 insertions(+) diff --git a/src/buildtool/execution_api/common/execution_api.hpp b/src/buildtool/execution_api/common/execution_api.hpp index 7a301f716..a8ba80091 100644 --- a/src/buildtool/execution_api/common/execution_api.hpp +++ b/src/buildtool/execution_api/common/execution_api.hpp @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -86,6 +87,12 @@ class IExecutionApi { std::vector const& artifacts_info, gsl::not_null const& api) noexcept -> bool = 0; + /// \brief Retrieve one artifact from CAS and make it available for + /// furter in-memory processing + [[nodiscard]] virtual auto RetrieveToMemory( + Artifact::ObjectInfo const& artifact_info) + -> std::optional = 0; + /// \brief Upload blobs to CAS. Uploads only the blobs that are not yet /// available in CAS, unless `skip_find_missing` is specified. /// \param blobs Container of blobs to upload. diff --git a/src/buildtool/execution_api/git/git_api.hpp b/src/buildtool/execution_api/git/git_api.hpp index 69179fe1c..5b5522209 100644 --- a/src/buildtool/execution_api/git/git_api.hpp +++ b/src/buildtool/execution_api/git/git_api.hpp @@ -168,6 +168,13 @@ class GitApi final : public IExecutionApi { return false; } + [[nodiscard]] auto RetrieveToMemory( + Artifact::ObjectInfo const& artifact_info) + -> std::optional override { + return RepositoryConfig::Instance().ReadBlobFromGitCAS( + artifact_info.digest.hash()); + } + /// NOLINTNEXTLINE(google-default-arguments) [[nodiscard]] auto Upload(BlobContainer const& /*blobs*/, bool /*skip_find_missing*/ = false) noexcept diff --git a/src/buildtool/execution_api/local/TARGETS b/src/buildtool/execution_api/local/TARGETS index b9e056b8b..1ea94d695 100644 --- a/src/buildtool/execution_api/local/TARGETS +++ b/src/buildtool/execution_api/local/TARGETS @@ -43,6 +43,7 @@ , "config" , ["src/buildtool/common", "common"] , ["src/buildtool/common", "bazel_types"] + , ["src/buildtool/file_system", "file_system_manager"] ] } } diff --git a/src/buildtool/execution_api/local/local_api.hpp b/src/buildtool/execution_api/local/local_api.hpp index d699ad4fd..e0de15985 100644 --- a/src/buildtool/execution_api/local/local_api.hpp +++ b/src/buildtool/execution_api/local/local_api.hpp @@ -31,6 +31,7 @@ #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/git/git_api.hpp" #include "src/buildtool/execution_api/local/local_action.hpp" +#include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/storage/storage.hpp" @@ -237,6 +238,27 @@ class LocalApi final : public IExecutionApi { return api->Upload(container, /*skip_find_missing=*/true); } + [[nodiscard]] auto RetrieveToMemory( + Artifact::ObjectInfo const& artifact_info) + -> std::optional override { + std::optional location{}; + if (IsTreeObject(artifact_info.type)) { + location = storage_->CAS().TreePath(artifact_info.digest); + } + else { + location = storage_->CAS().BlobPath( + artifact_info.digest, IsExecutableObject(artifact_info.type)); + } + if (not location) { + return std::nullopt; + } + auto const content = FileSystemManager::ReadFile(*location); + if (not content) { + return std::nullopt; + } + return *content; + } + [[nodiscard]] auto Upload(BlobContainer const& blobs, bool /*skip_find_missing*/) noexcept -> bool final { diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp index c4f170a31..e2da01887 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp @@ -251,6 +251,15 @@ auto BazelApi::CreateAction( return api->Upload(container, /*skip_find_missing=*/true); } +[[nodiscard]] auto BazelApi::RetrieveToMemory( + Artifact::ObjectInfo const& artifact_info) -> std::optional { + auto blobs = network_->ReadBlobs({artifact_info.digest}).Next(); + if (blobs.size() == 1) { + return blobs.at(0).data; + } + return std::nullopt; +} + [[nodiscard]] auto BazelApi::Upload(BlobContainer const& blobs, bool skip_find_missing) noexcept -> bool { return network_->UploadBlobs(blobs, skip_find_missing); diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.hpp b/src/buildtool/execution_api/remote/bazel/bazel_api.hpp index 6edfae5f1..310fb3478 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_api.hpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_api.hpp @@ -81,6 +81,10 @@ class BazelApi final : public IExecutionApi { [[nodiscard]] auto IsAvailable(std::vector const& digests) const noexcept -> std::vector final; + [[nodiscard]] auto RetrieveToMemory( + Artifact::ObjectInfo const& artifact_info) + -> std::optional final; + private: std::shared_ptr network_; diff --git a/test/buildtool/execution_engine/executor/executor.test.cpp b/test/buildtool/execution_engine/executor/executor.test.cpp index 685e4d476..5442aa21f 100644 --- a/test/buildtool/execution_engine/executor/executor.test.cpp +++ b/test/buildtool/execution_engine/executor/executor.test.cpp @@ -157,6 +157,10 @@ class TestApi : public IExecutionApi { // irrelevant for testing though. return unused.empty(); // not needed by Executor } + auto RetrieveToMemory(Artifact::ObjectInfo const& /*artifact_info*/) + -> std::optional override { + return std::nullopt; // not needed by Executor + } auto Upload(BlobContainer const& blobs, bool /*unused*/) noexcept -> bool final { return std::all_of( From 396c2edeb5e240f339cc51728a2ae1d83aded5e1 Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Tue, 25 Jul 2023 17:21:54 +0200 Subject: [PATCH 13/97] Add option -P to just-install-cas ... to allow selecting a sub object of a specified tree. --- share/man/just.1.md | 5 ++ src/buildtool/common/cli.hpp | 10 +++ src/buildtool/main/TARGETS | 1 + src/buildtool/main/install_cas.cpp | 98 ++++++++++++++++++++++++++++++ 4 files changed, 114 insertions(+) diff --git a/share/man/just.1.md b/share/man/just.1.md index aaa4e4f5d..e587da7a8 100644 --- a/share/man/just.1.md +++ b/share/man/just.1.md @@ -461,6 +461,11 @@ When installing a tree to stdout, i.e., when no option **`-o`** is given, dump the raw tree rather than a pretty-printed version. Supported by: install-cas. +**`-P`**, **`--sub-object-path`** *`PATH`* +Instead of the specified tree object take the object at the specified +logical path inside. +Supported by: install-cas. + **`--remember`** Ensure that all installed artifacts are available in local CAS as well, even when using remote execution. diff --git a/src/buildtool/common/cli.hpp b/src/buildtool/common/cli.hpp index 74ff0cce0..a49b2f6ba 100644 --- a/src/buildtool/common/cli.hpp +++ b/src/buildtool/common/cli.hpp @@ -119,6 +119,7 @@ struct RebuildArguments { struct FetchArguments { std::string object_id{}; std::optional output_path{}; + std::optional sub_path{}; bool remember{false}; bool raw_tree{}; }; @@ -518,6 +519,15 @@ static inline auto SetupFetchArguments( "Install path for the artifact. (omit to dump to stdout)") ->type_name("PATH"); + app->add_option_function( + "-P,--sub-object-path", + [clargs](auto const& rel_path) { + clargs->sub_path = ToNormalPath(rel_path).relative_path(); + }, + "Select the sub-object at the specified path (if artifact is a " + "tree).") + ->type_name("PATH"); + app->add_flag("--raw-tree", clargs->raw_tree, "Dump raw tree object (omit pretty printing)"); diff --git a/src/buildtool/main/TARGETS b/src/buildtool/main/TARGETS index 01ed747cf..8ff214d0b 100644 --- a/src/buildtool/main/TARGETS +++ b/src/buildtool/main/TARGETS @@ -83,6 +83,7 @@ , "private-deps": [ ["src/buildtool/compatibility", "compatibility"] , ["src/buildtool/crypto", "hash_function"] + , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"] , ["src/buildtool/execution_api/remote", "config"] , ["src/buildtool/logging", "logging"] ] diff --git a/src/buildtool/main/install_cas.cpp b/src/buildtool/main/install_cas.cpp index cdeca553f..1c0290df4 100644 --- a/src/buildtool/main/install_cas.cpp +++ b/src/buildtool/main/install_cas.cpp @@ -16,6 +16,9 @@ #include "src/buildtool/compatibility/compatibility.hpp" #include "src/buildtool/crypto/hash_function.hpp" +#ifndef BOOTSTRAP_BUILD_TOOL +#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" +#endif #include "src/buildtool/execution_api/remote/config.hpp" namespace { @@ -76,6 +79,101 @@ auto FetchAndInstallArtifacts( } } + if (clargs.sub_path) { + std::filesystem::path sofar{}; + for (auto const& segment : *clargs.sub_path) { + if (object_info.type != ObjectType::Tree) { + Logger::Log( + LogLevel::Warning, + "Non-tree found at path '{}', cannot follow to '{}'", + sofar.string(), + segment.string()); + break; + } + auto data = api->RetrieveToMemory(object_info); + if (not data) { + Logger::Log(LogLevel::Error, + "Failed to retrieve artifact {} at path '{}'", + object_info.ToString(), + sofar.string()); + return false; + } + if (Compatibility::IsCompatible()) { + auto directory = + BazelMsgFactory::MessageFromString( + *data); + if (not directory) { + Logger::Log( + LogLevel::Warning, + "Failed to parse directory message at path '{}'", + sofar.string()); + break; + } + std::optional new_object_info{}; + if (not BazelMsgFactory::ReadObjectInfosFromDirectory( + *directory, + [&new_object_info, &segment](auto path, auto info) { + if (path == segment) { + new_object_info = info; + } + return true; + })) { + Logger::Log( + LogLevel::Warning, + "Failed to process directory message at path '{}'", + sofar.string()); + break; + } + if (not new_object_info) { + Logger::Log(LogLevel::Warning, + "Entry {} not found at path '{}'", + segment.string(), + sofar.string()); + break; + } + object_info = *new_object_info; + } + else { + auto entries = GitRepo::ReadTreeData( + *data, + HashFunction::ComputeTreeHash(*data).Bytes(), + [](auto const& /*unused*/) { return true; }, + /*is_hex_id=*/false); + if (not entries) { + Logger::Log(LogLevel::Warning, + "Failed to parse tree {} at path '{}'", + object_info.ToString(), + sofar.string()); + break; + } + std::optional new_object_info{}; + if (not BazelMsgFactory::ReadObjectInfosFromGitTree( + *entries, + [&new_object_info, &segment](auto path, auto info) { + if (path == segment) { + new_object_info = info; + } + return true; + })) { + Logger::Log(LogLevel::Warning, + "Failed to process tree entries at path '{}'", + sofar.string()); + break; + } + + if (not new_object_info) { + Logger::Log(LogLevel::Warning, + "Entry {} not found at path '{}'", + segment.string(), + sofar.string()); + break; + } + object_info = *new_object_info; + } + sofar /= segment; + } + } + if (clargs.output_path) { auto output_path = (*clargs.output_path / "").parent_path(); if (FileSystemManager::IsDirectory(output_path)) { From d89c3e5e49f484c28569fed09f431240f557a677 Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Thu, 27 Jul 2023 12:43:37 +0200 Subject: [PATCH 14/97] install-cas test: also verify the -P option --- test/end-to-end/remote-execution/install-cas.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/end-to-end/remote-execution/install-cas.sh b/test/end-to-end/remote-execution/install-cas.sh index 0a2f16a25..32e346247 100644 --- a/test/end-to-end/remote-execution/install-cas.sh +++ b/test/end-to-end/remote-execution/install-cas.sh @@ -66,4 +66,17 @@ mkdir -p "${OUTDIR}/stdout" cmp "${OUTDIR}/stdout/local-raw" "${OUTDIR}/stdout/remote-raw" cmp "${OUTDIR}/stdout/local" "${OUTDIR}/stdout/remote" +# verify the -P option +"${JUST}" install-cas ${REMOTE_ARGS} -P deep/inside/path/hello.txt \ + -o "${OUTDIR}/path/remote" "${OUT}" 2>&1 +"${JUST}" install-cas ${LOCAL_ARGS} -P deep/inside/path/hello.txt \ + -o "${OUTDIR}/path/local" "${OUT}" 2>&1 +echo +ls -alR "${OUTDIR}/path" +[ -f "${OUTDIR}/path/remote" ] +grep World "${OUTDIR}/path/remote" +[ -f "${OUTDIR}/path/local" ] +grep World "${OUTDIR}/path/local" + + echo OK From 10501d91b4d7268c9a123eebb28e42ac3aff688e Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Fri, 14 Jul 2023 14:03:38 +0200 Subject: [PATCH 15/97] utils: path normalization method should be noexcept --- src/utils/cpp/path.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/cpp/path.hpp b/src/utils/cpp/path.hpp index a9fb08e52..3b44748ae 100644 --- a/src/utils/cpp/path.hpp +++ b/src/utils/cpp/path.hpp @@ -18,8 +18,8 @@ #include #include -[[nodiscard]] static inline auto ToNormalPath(std::filesystem::path const& p) - -> std::filesystem::path { +[[nodiscard]] static inline auto ToNormalPath( + std::filesystem::path const& p) noexcept -> std::filesystem::path { auto n = p.lexically_normal(); if (not n.has_filename()) { n = n.parent_path(); From 6fc79fa140cff9adab664f7fdd8fc81309ee5762 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Wed, 19 Jul 2023 16:32:38 +0200 Subject: [PATCH 16/97] just-mr maps: Clean up remnants from progress reporting rework --- src/other_tools/ops_maps/repo_fetch_map.cpp | 1 - src/other_tools/root_maps/content_git_map.cpp | 2 -- src/other_tools/root_maps/distdir_git_map.cpp | 2 -- 3 files changed, 5 deletions(-) diff --git a/src/other_tools/ops_maps/repo_fetch_map.cpp b/src/other_tools/ops_maps/repo_fetch_map.cpp index 885060ada..1cc663d7b 100644 --- a/src/other_tools/ops_maps/repo_fetch_map.cpp +++ b/src/other_tools/ops_maps/repo_fetch_map.cpp @@ -47,7 +47,6 @@ auto CreateRepoFetchMap(gsl::not_null const& content_cas_map, [fetch_dir, content = key.archive.content, distfile, - origin = key.archive.origin, setter, logger]([[maybe_unused]] auto const& values) { // content is now in CAS, so copy content into fetch_dir diff --git a/src/other_tools/root_maps/content_git_map.cpp b/src/other_tools/root_maps/content_git_map.cpp index b652a5e12..524a59a43 100644 --- a/src/other_tools/root_maps/content_git_map.cpp +++ b/src/other_tools/root_maps/content_git_map.cpp @@ -146,7 +146,6 @@ auto CreateContentGitMap( content_id = key.archive.content, subdir = key.subdir, ignore_special = key.ignore_special, - origin = key.archive.origin, import_to_git_map, ts, setter, @@ -188,7 +187,6 @@ auto CreateContentGitMap( archive_tree_id_file, subdir, ignore_special, - origin, setter, logger](auto const& values) { // check for errors diff --git a/src/other_tools/root_maps/distdir_git_map.cpp b/src/other_tools/root_maps/distdir_git_map.cpp index 78539158d..13a161aa4 100644 --- a/src/other_tools/root_maps/distdir_git_map.cpp +++ b/src/other_tools/root_maps/distdir_git_map.cpp @@ -130,7 +130,6 @@ auto CreateDistdirGitMap( [distdir_tree_id_file, content_id = key.content_id, content_list = key.content_list, - origin = key.origin, ignore_special = key.ignore_special, import_to_git_map, ts, @@ -162,7 +161,6 @@ auto CreateDistdirGitMap( {std::move(c_info)}, [tmp_dir, // keep tmp_dir alive distdir_tree_id_file, - origin, ignore_special, setter, logger](auto const& values) { From 6a3f465d5bdc43fc60c6d3667097a82861212f61 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Tue, 18 Jul 2023 18:16:26 +0200 Subject: [PATCH 17/97] test: Fix typo in script generating test_repo_symlinks bundle --- .../buildtool/file_system/create_fs_test_git_bundle_symlinks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/buildtool/file_system/create_fs_test_git_bundle_symlinks.sh b/test/buildtool/file_system/create_fs_test_git_bundle_symlinks.sh index 0cfd0cca3..bf2ab8c54 100644 --- a/test/buildtool/file_system/create_fs_test_git_bundle_symlinks.sh +++ b/test/buildtool/file_system/create_fs_test_git_bundle_symlinks.sh @@ -27,7 +27,7 @@ set -e # +--baz <--kBazSymId: 1868f82682c290f0b1db3cacd092727eef1fa57f (tree) # | +--bar # | +--foo -# | +--bar_l <-- kBazBarLinkId: 79264abecd108745abb4086427ac988c7df7b639 (blob) +# | +--bar_l <-- kBazBarLinkId: same as kBarId [same content] (blob) # | # --- # From 25bc7b0388b0433cb4f0bfe2e546798cfaa024ab Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Wed, 12 Jul 2023 16:11:39 +0200 Subject: [PATCH 18/97] just-mr: Set ignore-special roots via a pragma... ...instead of a regular field. Also removes the ignore-special logic from distdir repositories, as those will never contain symlinks, so no entries would be ignored. --- src/other_tools/just_mr/main.cpp | 17 ++-- .../repo_map/repos_to_setup_map.cpp | 90 +++++++++++-------- src/other_tools/root_maps/TARGETS | 1 - src/other_tools/root_maps/distdir_git_map.cpp | 26 ++---- src/other_tools/root_maps/distdir_git_map.hpp | 11 +-- test/end-to-end/just-mr/install-roots.sh | 2 +- 6 files changed, 74 insertions(+), 73 deletions(-) diff --git a/src/other_tools/just_mr/main.cpp b/src/other_tools/just_mr/main.cpp index e331a0638..af6589bc8 100644 --- a/src/other_tools/just_mr/main.cpp +++ b/src/other_tools/just_mr/main.cpp @@ -787,9 +787,16 @@ void DefaultReachableRepositories( (*resolved_repo_desc)->Get("sha256", Expression::none_t{}); auto repo_desc_sha512 = (*resolved_repo_desc)->Get("sha512", Expression::none_t{}); - auto repo_desc_ignore_special = - (*resolved_repo_desc) - ->Get("ignore_special", Expression::none_t{}); + // check "special" pragma + auto repo_desc_pragma = (*resolved_repo_desc)->At("pragma"); + auto pragma_special = + repo_desc_pragma ? repo_desc_pragma->get()->At("special") + : std::nullopt; + auto pragma_special_value = + pragma_special and pragma_special->get()->IsString() + ? std::make_optional(pragma_special->get()->String()) + : std::nullopt; + ArchiveRepoInfo archive_info = { .archive = {.content = repo_desc_content->get()->String(), .distfile = @@ -810,9 +817,7 @@ void DefaultReachableRepositories( .origin_from_distdir = false}, .repo_type = repo_type_str, .subdir = subdir.empty() ? "." : subdir.string(), - .ignore_special = repo_desc_ignore_special->IsBool() - ? repo_desc_ignore_special->Bool() - : false}; + .ignore_special = pragma_special_value == "ignore"}; // add to list repos_to_fetch.emplace_back(std::move(archive_info)); } diff --git a/src/other_tools/repo_map/repos_to_setup_map.cpp b/src/other_tools/repo_map/repos_to_setup_map.cpp index d65386a38..19dc75385 100644 --- a/src/other_tools/repo_map/repos_to_setup_map.cpp +++ b/src/other_tools/repo_map/repos_to_setup_map.cpp @@ -91,8 +91,15 @@ void GitCheckout(ExpressionPtr const& repo_desc, ? repo_desc_subdir->String() : "") .lexically_normal(); - auto repo_desc_ignore_special = - repo_desc->Get("ignore_special", Expression::none_t{}); + // check "special" pragma + auto repo_desc_pragma = repo_desc->At("pragma"); + auto pragma_special = repo_desc_pragma + ? repo_desc_pragma->get()->At("special") + : std::nullopt; + auto pragma_special_value = + pragma_special and pragma_special->get()->IsString() + ? std::make_optional(pragma_special->get()->String()) + : std::nullopt; // populate struct GitRepoInfo git_repo_info = { .hash = repo_desc_commit->get()->String(), @@ -100,9 +107,7 @@ void GitCheckout(ExpressionPtr const& repo_desc, .branch = repo_desc_branch->get()->String(), .subdir = subdir.empty() ? "." : subdir.string(), .origin = repo_name, - .ignore_special = repo_desc_ignore_special->IsBool() - ? repo_desc_ignore_special->Bool() - : false}; + .ignore_special = pragma_special_value == "ignore"}; // get the WS root as git tree commit_git_map->ConsumeAfterKeysReady( ts, @@ -174,8 +179,15 @@ void ArchiveCheckout(ExpressionPtr const& repo_desc, auto repo_desc_distfile = repo_desc->Get("distfile", Expression::none_t{}); auto repo_desc_sha256 = repo_desc->Get("sha256", Expression::none_t{}); auto repo_desc_sha512 = repo_desc->Get("sha512", Expression::none_t{}); - auto repo_desc_ignore_special = - repo_desc->Get("ignore_special", Expression::none_t{}); + // check "special" pragma + auto repo_desc_pragma = repo_desc->At("pragma"); + auto pragma_special = repo_desc_pragma + ? repo_desc_pragma->get()->At("special") + : std::nullopt; + auto pragma_special_value = + pragma_special and pragma_special->get()->IsString() + ? std::make_optional(pragma_special->get()->String()) + : std::nullopt; // populate struct ArchiveRepoInfo archive_repo_info = { .archive = @@ -194,9 +206,7 @@ void ArchiveCheckout(ExpressionPtr const& repo_desc, .origin_from_distdir = false}, .repo_type = repo_type, .subdir = subdir.empty() ? "." : subdir.string(), - .ignore_special = repo_desc_ignore_special->IsBool() - ? repo_desc_ignore_special->Bool() - : false}; + .ignore_special = pragma_special_value == "ignore"}; // get the WS root as git tree content_git_map->ConsumeAfterKeysReady( ts, @@ -250,20 +260,23 @@ void FileCheckout(ExpressionPtr const& repo_desc, // get absolute path auto fpath = ToNormalPath( std::filesystem::absolute(repo_desc_path->get()->String())); - // get ignore-special entry - auto repo_desc_ignore_special = - repo_desc->Get("ignore_special", Expression::none_t{}); - bool ignore_special = repo_desc_ignore_special->IsBool() - ? repo_desc_ignore_special->Bool() - : false; - // check to_git pragma + // check "special" pragma auto repo_desc_pragma = repo_desc->At("pragma"); + auto pragma_special = repo_desc_pragma + ? repo_desc_pragma->get()->At("special") + : std::nullopt; + auto pragma_special_value = + pragma_special and pragma_special->get()->IsString() + ? std::make_optional(pragma_special->get()->String()) + : std::nullopt; + // check "to_git" pragma auto pragma_to_git = repo_desc_pragma ? repo_desc_pragma->get()->At("to_git") : std::nullopt; if (pragma_to_git and pragma_to_git->get()->IsBool() and pragma_to_git->get()->Bool()) { // get the WS root as git tree - FpathInfo fpath_info = {fpath, ignore_special}; + FpathInfo fpath_info = { + .fpath = fpath, .ignore_special = pragma_special_value == "ignore"}; fpath_git_map->ConsumeAfterKeysReady( ts, {std::move(fpath_info)}, @@ -287,9 +300,11 @@ void FileCheckout(ExpressionPtr const& repo_desc, else { // get the WS root as filesystem location nlohmann::json cfg({}); - cfg["workspace_root"] = nlohmann::json::array( - {ignore_special ? FileRoot::kFileIgnoreSpecialMarker : "file", - fpath.string()}); // explicit array + cfg["workspace_root"] = + nlohmann::json::array({pragma_special_value == "ignore" + ? FileRoot::kFileIgnoreSpecialMarker + : "file", + fpath.string()}); // explicit array SetReposTakeOver(&cfg, repos, repo_name); (*setter)(std::move(cfg)); // report local path @@ -321,12 +336,6 @@ void DistdirCheckout(ExpressionPtr const& repo_desc, /*fatal=*/true); return; } - // get ignore-special entry - auto repo_desc_ignore_special = - repo_desc->Get("ignore_special", Expression::none_t{}); - bool ignore_special = repo_desc_ignore_special->IsBool() - ? repo_desc_ignore_special->Bool() - : false; // map of distfile to content auto distdir_content = std::make_shared>(); @@ -465,8 +474,7 @@ void DistdirCheckout(ExpressionPtr const& repo_desc, DistdirInfo distdir_info = {.content_id = distdir_content_id, .content_list = distdir_content, .repos_to_fetch = dist_repos_to_fetch, - .origin = repo_name, - .ignore_special = ignore_special}; + .origin = repo_name}; distdir_git_map->ConsumeAfterKeysReady( ts, {std::move(distdir_info)}, @@ -558,17 +566,21 @@ void GitTreeCheckout(ExpressionPtr const& repo_desc, } } } - // get ignore-special entry - auto repo_desc_ignore_special = - repo_desc->Get("ignore_special", Expression::none_t{}); - bool ignore_special = repo_desc_ignore_special->IsBool() - ? repo_desc_ignore_special->Bool() - : false; + // check "special" pragma + auto repo_desc_pragma = repo_desc->At("pragma"); + auto pragma_special = repo_desc_pragma + ? repo_desc_pragma->get()->At("special") + : std::nullopt; + auto pragma_special_value = + pragma_special and pragma_special->get()->IsString() + ? std::make_optional(pragma_special->get()->String()) + : std::nullopt; // populate struct - TreeIdInfo tree_id_info = {.hash = repo_desc_hash->get()->String(), - .env_vars = std::move(env), - .command = std::move(cmd), - .ignore_special = ignore_special}; + TreeIdInfo tree_id_info = { + .hash = repo_desc_hash->get()->String(), + .env_vars = std::move(env), + .command = std::move(cmd), + .ignore_special = pragma_special_value == "ignore"}; // get the WS root as git tree tree_id_git_map->ConsumeAfterKeysReady( ts, diff --git a/src/other_tools/root_maps/TARGETS b/src/other_tools/root_maps/TARGETS index d325daa4f..56342137e 100644 --- a/src/other_tools/root_maps/TARGETS +++ b/src/other_tools/root_maps/TARGETS @@ -7,7 +7,6 @@ [ ["@", "json", "", "json"] , ["src/other_tools/ops_maps", "import_to_git_map"] , ["src/other_tools/ops_maps", "content_cas_map"] - , ["src/utils/cpp", "hash_combine"] ] , "stage": ["src", "other_tools", "root_maps"] , "private-deps": diff --git a/src/other_tools/root_maps/distdir_git_map.cpp b/src/other_tools/root_maps/distdir_git_map.cpp index 13a161aa4..a09714782 100644 --- a/src/other_tools/root_maps/distdir_git_map.cpp +++ b/src/other_tools/root_maps/distdir_git_map.cpp @@ -90,10 +90,8 @@ auto CreateDistdirGitMap( critical_git_op_map->ConsumeAfterKeysReady( ts, {std::move(op_key)}, - [distdir_tree_id = *distdir_tree_id, - ignore_special = key.ignore_special, - setter, - logger](auto const& values) { + [distdir_tree_id = *distdir_tree_id, setter, logger]( + auto const& values) { GitOpValue op_result = *values[0]; // check flag if (not op_result.result) { @@ -103,14 +101,12 @@ auto CreateDistdirGitMap( } // subdir is ".", so no need to deal with the Git cache // set the workspace root - (*setter)(std::pair( - nlohmann::json::array( - {ignore_special - ? FileRoot::kGitTreeIgnoreSpecialMarker - : FileRoot::kGitTreeMarker, - distdir_tree_id, - StorageConfig::GitRoot().string()}), - true)); + (*setter)( + std::pair(nlohmann::json::array( + {FileRoot::kGitTreeMarker, + distdir_tree_id, + StorageConfig::GitRoot().string()}), + true)); }, [logger, target_path = StorageConfig::GitRoot()]( auto const& msg, bool fatal) { @@ -130,7 +126,6 @@ auto CreateDistdirGitMap( [distdir_tree_id_file, content_id = key.content_id, content_list = key.content_list, - ignore_special = key.ignore_special, import_to_git_map, ts, setter, @@ -161,7 +156,6 @@ auto CreateDistdirGitMap( {std::move(c_info)}, [tmp_dir, // keep tmp_dir alive distdir_tree_id_file, - ignore_special, setter, logger](auto const& values) { // check for errors @@ -185,9 +179,7 @@ auto CreateDistdirGitMap( // set the workspace root (*setter)(std::pair( nlohmann::json::array( - {ignore_special - ? FileRoot::kGitTreeIgnoreSpecialMarker - : FileRoot::kGitTreeMarker, + {FileRoot::kGitTreeMarker, distdir_tree_id, StorageConfig::GitRoot().string()}), false)); diff --git a/src/other_tools/root_maps/distdir_git_map.hpp b/src/other_tools/root_maps/distdir_git_map.hpp index 7ede3b959..c440253be 100644 --- a/src/other_tools/root_maps/distdir_git_map.hpp +++ b/src/other_tools/root_maps/distdir_git_map.hpp @@ -20,7 +20,6 @@ #include "nlohmann/json.hpp" #include "src/other_tools/ops_maps/content_cas_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" -#include "src/utils/cpp/hash_combine.hpp" struct DistdirInfo { std::string content_id; /* key */ @@ -28,13 +27,10 @@ struct DistdirInfo { std::shared_ptr> repos_to_fetch; // name of repository for which work is done; used in progress reporting std::string origin; - // create root that ignores symlinks - bool ignore_special; /* key */ [[nodiscard]] auto operator==(const DistdirInfo& other) const noexcept -> bool { - return content_id == other.content_id and - ignore_special == other.ignore_special; + return content_id == other.content_id; } }; @@ -55,10 +51,7 @@ template <> struct hash { [[nodiscard]] auto operator()(const DistdirInfo& dd) const noexcept -> std::size_t { - size_t seed{}; - hash_combine(&seed, dd.content_id); - hash_combine(&seed, dd.ignore_special); - return seed; + return std::hash{}(dd.content_id); } }; } // namespace std diff --git a/test/end-to-end/just-mr/install-roots.sh b/test/end-to-end/just-mr/install-roots.sh index 368e505a2..aab0c90b9 100644 --- a/test/end-to-end/just-mr/install-roots.sh +++ b/test/end-to-end/just-mr/install-roots.sh @@ -54,10 +54,10 @@ cat > repos.json < Date: Wed, 12 Jul 2023 16:56:02 +0200 Subject: [PATCH 19/97] just-mr: Add map for pragma 'special' values Moves away from literal strings in the async maps. --- src/other_tools/just_mr/main.cpp | 10 ++++-- src/other_tools/just_mr/utils.hpp | 13 +++++++ .../repo_map/repos_to_setup_map.cpp | 35 ++++++++++++------- 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/src/other_tools/just_mr/main.cpp b/src/other_tools/just_mr/main.cpp index af6589bc8..cc33f5f3b 100644 --- a/src/other_tools/just_mr/main.cpp +++ b/src/other_tools/just_mr/main.cpp @@ -793,8 +793,11 @@ void DefaultReachableRepositories( repo_desc_pragma ? repo_desc_pragma->get()->At("special") : std::nullopt; auto pragma_special_value = - pragma_special and pragma_special->get()->IsString() - ? std::make_optional(pragma_special->get()->String()) + pragma_special and pragma_special->get()->IsString() and + kPragmaSpecialMap.contains( + pragma_special->get()->String()) + ? std::make_optional(kPragmaSpecialMap.at( + pragma_special->get()->String())) : std::nullopt; ArchiveRepoInfo archive_info = { @@ -817,7 +820,8 @@ void DefaultReachableRepositories( .origin_from_distdir = false}, .repo_type = repo_type_str, .subdir = subdir.empty() ? "." : subdir.string(), - .ignore_special = pragma_special_value == "ignore"}; + .ignore_special = + pragma_special_value == PragmaSpecial::Ignore}; // add to list repos_to_fetch.emplace_back(std::move(archive_info)); } diff --git a/src/other_tools/just_mr/utils.hpp b/src/other_tools/just_mr/utils.hpp index 5caf33d02..678784452 100644 --- a/src/other_tools/just_mr/utils.hpp +++ b/src/other_tools/just_mr/utils.hpp @@ -89,6 +89,19 @@ std::unordered_map const kCheckoutTypeMap = { {"distdir", CheckoutType::Distdir}, {"git tree", CheckoutType::GitTree}}; +/// \brief Pragma "special" value enum +enum class PragmaSpecial : std::uint8_t { + Ignore, + ResolvePartially, + ResolveCompletely +}; + +/// \brief Pragma "special" value map +std::unordered_map const kPragmaSpecialMap = { + {"ignore", PragmaSpecial::Ignore}, + {"resolve-partially", PragmaSpecial::ResolvePartially}, + {"resolve-completely", PragmaSpecial::ResolveCompletely}}; + namespace JustMR { struct Paths { diff --git a/src/other_tools/repo_map/repos_to_setup_map.cpp b/src/other_tools/repo_map/repos_to_setup_map.cpp index 19dc75385..a07ebaa6f 100644 --- a/src/other_tools/repo_map/repos_to_setup_map.cpp +++ b/src/other_tools/repo_map/repos_to_setup_map.cpp @@ -97,8 +97,10 @@ void GitCheckout(ExpressionPtr const& repo_desc, ? repo_desc_pragma->get()->At("special") : std::nullopt; auto pragma_special_value = - pragma_special and pragma_special->get()->IsString() - ? std::make_optional(pragma_special->get()->String()) + pragma_special and pragma_special->get()->IsString() and + kPragmaSpecialMap.contains(pragma_special->get()->String()) + ? std::make_optional( + kPragmaSpecialMap.at(pragma_special->get()->String())) : std::nullopt; // populate struct GitRepoInfo git_repo_info = { @@ -107,7 +109,7 @@ void GitCheckout(ExpressionPtr const& repo_desc, .branch = repo_desc_branch->get()->String(), .subdir = subdir.empty() ? "." : subdir.string(), .origin = repo_name, - .ignore_special = pragma_special_value == "ignore"}; + .ignore_special = pragma_special_value == PragmaSpecial::Ignore}; // get the WS root as git tree commit_git_map->ConsumeAfterKeysReady( ts, @@ -185,8 +187,10 @@ void ArchiveCheckout(ExpressionPtr const& repo_desc, ? repo_desc_pragma->get()->At("special") : std::nullopt; auto pragma_special_value = - pragma_special and pragma_special->get()->IsString() - ? std::make_optional(pragma_special->get()->String()) + pragma_special and pragma_special->get()->IsString() and + kPragmaSpecialMap.contains(pragma_special->get()->String()) + ? std::make_optional( + kPragmaSpecialMap.at(pragma_special->get()->String())) : std::nullopt; // populate struct ArchiveRepoInfo archive_repo_info = { @@ -206,7 +210,7 @@ void ArchiveCheckout(ExpressionPtr const& repo_desc, .origin_from_distdir = false}, .repo_type = repo_type, .subdir = subdir.empty() ? "." : subdir.string(), - .ignore_special = pragma_special_value == "ignore"}; + .ignore_special = pragma_special_value == PragmaSpecial::Ignore}; // get the WS root as git tree content_git_map->ConsumeAfterKeysReady( ts, @@ -266,8 +270,10 @@ void FileCheckout(ExpressionPtr const& repo_desc, ? repo_desc_pragma->get()->At("special") : std::nullopt; auto pragma_special_value = - pragma_special and pragma_special->get()->IsString() - ? std::make_optional(pragma_special->get()->String()) + pragma_special and pragma_special->get()->IsString() and + kPragmaSpecialMap.contains(pragma_special->get()->String()) + ? std::make_optional( + kPragmaSpecialMap.at(pragma_special->get()->String())) : std::nullopt; // check "to_git" pragma auto pragma_to_git = @@ -276,7 +282,8 @@ void FileCheckout(ExpressionPtr const& repo_desc, pragma_to_git->get()->Bool()) { // get the WS root as git tree FpathInfo fpath_info = { - .fpath = fpath, .ignore_special = pragma_special_value == "ignore"}; + .fpath = fpath, + .ignore_special = pragma_special_value == PragmaSpecial::Ignore}; fpath_git_map->ConsumeAfterKeysReady( ts, {std::move(fpath_info)}, @@ -301,7 +308,7 @@ void FileCheckout(ExpressionPtr const& repo_desc, // get the WS root as filesystem location nlohmann::json cfg({}); cfg["workspace_root"] = - nlohmann::json::array({pragma_special_value == "ignore" + nlohmann::json::array({pragma_special_value == PragmaSpecial::Ignore ? FileRoot::kFileIgnoreSpecialMarker : "file", fpath.string()}); // explicit array @@ -572,15 +579,17 @@ void GitTreeCheckout(ExpressionPtr const& repo_desc, ? repo_desc_pragma->get()->At("special") : std::nullopt; auto pragma_special_value = - pragma_special and pragma_special->get()->IsString() - ? std::make_optional(pragma_special->get()->String()) + pragma_special and pragma_special->get()->IsString() and + kPragmaSpecialMap.contains(pragma_special->get()->String()) + ? std::make_optional( + kPragmaSpecialMap.at(pragma_special->get()->String())) : std::nullopt; // populate struct TreeIdInfo tree_id_info = { .hash = repo_desc_hash->get()->String(), .env_vars = std::move(env), .command = std::move(cmd), - .ignore_special = pragma_special_value == "ignore"}; + .ignore_special = pragma_special_value == PragmaSpecial::Ignore}; // get the WS root as git tree tree_id_git_map->ConsumeAfterKeysReady( ts, From 317cb72c64330036f23fd19a7caf65637bac5bc1 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Thu, 13 Jul 2023 11:41:07 +0200 Subject: [PATCH 20/97] just-mr: Update async map keys with the 'special' pragma value For 'file' and 'archive' checkouts we will handle more than just the 'ignore' value. --- src/other_tools/just_mr/main.cpp | 16 ++-------------- src/other_tools/ops_maps/content_cas_map.hpp | 8 ++++---- src/other_tools/repo_map/repos_to_setup_map.cpp | 14 ++++++++------ src/other_tools/root_maps/TARGETS | 4 ++-- src/other_tools/root_maps/content_git_map.cpp | 10 +++++----- src/other_tools/root_maps/fpath_git_map.cpp | 15 ++++++++------- src/other_tools/root_maps/fpath_git_map.hpp | 9 +++++---- 7 files changed, 34 insertions(+), 42 deletions(-) diff --git a/src/other_tools/just_mr/main.cpp b/src/other_tools/just_mr/main.cpp index cc33f5f3b..c15970d23 100644 --- a/src/other_tools/just_mr/main.cpp +++ b/src/other_tools/just_mr/main.cpp @@ -787,18 +787,6 @@ void DefaultReachableRepositories( (*resolved_repo_desc)->Get("sha256", Expression::none_t{}); auto repo_desc_sha512 = (*resolved_repo_desc)->Get("sha512", Expression::none_t{}); - // check "special" pragma - auto repo_desc_pragma = (*resolved_repo_desc)->At("pragma"); - auto pragma_special = - repo_desc_pragma ? repo_desc_pragma->get()->At("special") - : std::nullopt; - auto pragma_special_value = - pragma_special and pragma_special->get()->IsString() and - kPragmaSpecialMap.contains( - pragma_special->get()->String()) - ? std::make_optional(kPragmaSpecialMap.at( - pragma_special->get()->String())) - : std::nullopt; ArchiveRepoInfo archive_info = { .archive = {.content = repo_desc_content->get()->String(), @@ -820,8 +808,8 @@ void DefaultReachableRepositories( .origin_from_distdir = false}, .repo_type = repo_type_str, .subdir = subdir.empty() ? "." : subdir.string(), - .ignore_special = - pragma_special_value == PragmaSpecial::Ignore}; + .pragma_special = std::nullopt // not used + }; // add to list repos_to_fetch.emplace_back(std::move(archive_info)); } diff --git a/src/other_tools/ops_maps/content_cas_map.hpp b/src/other_tools/ops_maps/content_cas_map.hpp index 17fa38f13..3b40010f6 100644 --- a/src/other_tools/ops_maps/content_cas_map.hpp +++ b/src/other_tools/ops_maps/content_cas_map.hpp @@ -43,13 +43,13 @@ struct ArchiveRepoInfo { ArchiveContent archive; /* key */ std::string repo_type; /* key */ std::string subdir; /* key */ - // create root that ignores symlinks - bool ignore_special; /* key */ + // create root based on "special" pragma value + std::optional pragma_special{std::nullopt}; /* key */ [[nodiscard]] auto operator==(const ArchiveRepoInfo& other) const -> bool { return archive == other.archive and repo_type == other.repo_type and subdir == other.subdir and - ignore_special == other.ignore_special; + pragma_special == other.pragma_special; } }; @@ -78,7 +78,7 @@ struct hash { hash_combine(&seed, ct.archive); hash_combine(&seed, ct.repo_type); hash_combine(&seed, ct.subdir); - hash_combine(&seed, ct.ignore_special); + hash_combine>(&seed, ct.pragma_special); return seed; } }; diff --git a/src/other_tools/repo_map/repos_to_setup_map.cpp b/src/other_tools/repo_map/repos_to_setup_map.cpp index a07ebaa6f..69addd882 100644 --- a/src/other_tools/repo_map/repos_to_setup_map.cpp +++ b/src/other_tools/repo_map/repos_to_setup_map.cpp @@ -210,7 +210,7 @@ void ArchiveCheckout(ExpressionPtr const& repo_desc, .origin_from_distdir = false}, .repo_type = repo_type, .subdir = subdir.empty() ? "." : subdir.string(), - .ignore_special = pragma_special_value == PragmaSpecial::Ignore}; + .pragma_special = pragma_special_value}; // get the WS root as git tree content_git_map->ConsumeAfterKeysReady( ts, @@ -278,12 +278,14 @@ void FileCheckout(ExpressionPtr const& repo_desc, // check "to_git" pragma auto pragma_to_git = repo_desc_pragma ? repo_desc_pragma->get()->At("to_git") : std::nullopt; - if (pragma_to_git and pragma_to_git->get()->IsBool() and - pragma_to_git->get()->Bool()) { + // resolving symlinks implies also to_git + if (pragma_special_value == PragmaSpecial::ResolvePartially or + pragma_special_value == PragmaSpecial::ResolveCompletely or + (pragma_to_git and pragma_to_git->get()->IsBool() and + pragma_to_git->get()->Bool())) { // get the WS root as git tree - FpathInfo fpath_info = { - .fpath = fpath, - .ignore_special = pragma_special_value == PragmaSpecial::Ignore}; + FpathInfo fpath_info = {.fpath = fpath, + .pragma_special = pragma_special_value}; fpath_git_map->ConsumeAfterKeysReady( ts, {std::move(fpath_info)}, diff --git a/src/other_tools/root_maps/TARGETS b/src/other_tools/root_maps/TARGETS index 56342137e..469caf493 100644 --- a/src/other_tools/root_maps/TARGETS +++ b/src/other_tools/root_maps/TARGETS @@ -53,11 +53,11 @@ , ["@", "json", "", "json"] , ["src/utils/cpp", "path_hash"] , ["src/utils/cpp", "hash_combine"] + , ["src/other_tools/just_mr", "utils"] ] , "stage": ["src", "other_tools", "root_maps"] , "private-deps": - [ ["src/other_tools/just_mr", "utils"] - , ["src/buildtool/execution_api/local", "config"] + [ ["src/buildtool/execution_api/local", "config"] , ["src/buildtool/storage", "config"] , ["src/utils/cpp", "tmp_dir"] , ["src/buildtool/file_system", "file_root"] diff --git a/src/other_tools/root_maps/content_git_map.cpp b/src/other_tools/root_maps/content_git_map.cpp index 524a59a43..df62fab67 100644 --- a/src/other_tools/root_maps/content_git_map.cpp +++ b/src/other_tools/root_maps/content_git_map.cpp @@ -83,7 +83,7 @@ auto CreateContentGitMap( {std::move(op_key)}, [archive_tree_id = *archive_tree_id, subdir = key.subdir, - ignore_special = key.ignore_special, + pragma_special = key.pragma_special, setter, logger](auto const& values) { GitOpValue op_result = *values[0]; @@ -119,7 +119,7 @@ auto CreateContentGitMap( // set the workspace root (*setter)(std::pair( nlohmann::json::array( - {ignore_special + {pragma_special == PragmaSpecial::Ignore ? FileRoot::kGitTreeIgnoreSpecialMarker : FileRoot::kGitTreeMarker, *subtree_hash, @@ -145,7 +145,7 @@ auto CreateContentGitMap( repo_type = key.repo_type, content_id = key.archive.content, subdir = key.subdir, - ignore_special = key.ignore_special, + pragma_special = key.pragma_special, import_to_git_map, ts, setter, @@ -186,7 +186,7 @@ auto CreateContentGitMap( [tmp_dir, // keep tmp_dir alive archive_tree_id_file, subdir, - ignore_special, + pragma_special, setter, logger](auto const& values) { // check for errors @@ -246,7 +246,7 @@ auto CreateContentGitMap( // set the workspace root (*setter)(std::pair( nlohmann::json::array( - {ignore_special + {pragma_special == PragmaSpecial::Ignore ? FileRoot::kGitTreeIgnoreSpecialMarker : FileRoot::kGitTreeMarker, *subtree_hash, diff --git a/src/other_tools/root_maps/fpath_git_map.cpp b/src/other_tools/root_maps/fpath_git_map.cpp index e7946e71b..e26b3a9d0 100644 --- a/src/other_tools/root_maps/fpath_git_map.cpp +++ b/src/other_tools/root_maps/fpath_git_map.cpp @@ -17,7 +17,6 @@ #include "src/buildtool/execution_api/local/config.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/storage/config.hpp" -#include "src/other_tools/just_mr/utils.hpp" #include "src/utils/cpp/tmp_dir.hpp" auto CreateFilePathGitMap( @@ -73,7 +72,7 @@ auto CreateFilePathGitMap( ts, {std::move(op_key)}, [fpath = key.fpath, - ignore_special = key.ignore_special, + pragma_special = key.pragma_special, git_cas = std::move(git_cas), repo_root = std::move(*repo_root), setter, @@ -111,8 +110,9 @@ auto CreateFilePathGitMap( } // set the workspace root (*setter)(nlohmann::json::array( - {ignore_special ? FileRoot::kGitTreeIgnoreSpecialMarker - : FileRoot::kGitTreeMarker, + {pragma_special == PragmaSpecial::Ignore + ? FileRoot::kGitTreeIgnoreSpecialMarker + : FileRoot::kGitTreeMarker, *tree_hash, repo_root})); }, @@ -159,7 +159,7 @@ auto CreateFilePathGitMap( {std::move(c_info)}, // tmp_dir passed, to ensure folder is not removed until import // to git is done - [tmp_dir, ignore_special = key.ignore_special, setter, logger]( + [tmp_dir, pragma_special = key.pragma_special, setter, logger]( auto const& values) { // check for errors if (not values[0]->second) { @@ -171,8 +171,9 @@ auto CreateFilePathGitMap( std::string tree = values[0]->first; // set the workspace root (*setter)(nlohmann::json::array( - {ignore_special ? FileRoot::kGitTreeIgnoreSpecialMarker - : FileRoot::kGitTreeMarker, + {pragma_special == PragmaSpecial::Ignore + ? FileRoot::kGitTreeIgnoreSpecialMarker + : FileRoot::kGitTreeMarker, tree, StorageConfig::GitRoot()})); }, diff --git a/src/other_tools/root_maps/fpath_git_map.hpp b/src/other_tools/root_maps/fpath_git_map.hpp index 9c78b868f..25721c19f 100644 --- a/src/other_tools/root_maps/fpath_git_map.hpp +++ b/src/other_tools/root_maps/fpath_git_map.hpp @@ -16,18 +16,19 @@ #define INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_FPATH_GIT_MAP_HPP #include "nlohmann/json.hpp" +#include "src/other_tools/just_mr/utils.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" #include "src/utils/cpp/hash_combine.hpp" #include "src/utils/cpp/path_hash.hpp" struct FpathInfo { std::filesystem::path fpath{}; /* key */ - // create root that ignores symlinks - bool ignore_special{}; /* key */ + // create root based on "special" pragma value + std::optional pragma_special{std::nullopt}; /* key */ [[nodiscard]] auto operator==(const FpathInfo& other) const noexcept -> bool { - return fpath == other.fpath and ignore_special == other.ignore_special; + return fpath == other.fpath and pragma_special == other.pragma_special; } }; @@ -47,7 +48,7 @@ struct hash { -> std::size_t { size_t seed{}; hash_combine(&seed, ct.fpath); - hash_combine(&seed, ct.ignore_special); + hash_combine>(&seed, ct.pragma_special); return seed; } }; From c0226e230601972e0b4bf55f74a6514409d47bfe Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Mon, 17 Jul 2023 13:15:27 +0200 Subject: [PATCH 21/97] utils: Add function to check for confined paths --- src/utils/cpp/path.hpp | 17 +++++++++++++++++ test/utils/cpp/path.test.cpp | 8 ++++++++ 2 files changed, 25 insertions(+) diff --git a/src/utils/cpp/path.hpp b/src/utils/cpp/path.hpp index 3b44748ae..bbbbfbf61 100644 --- a/src/utils/cpp/path.hpp +++ b/src/utils/cpp/path.hpp @@ -42,4 +42,21 @@ return *path.lexically_normal().begin() != ".."; } +/// \brief Perform a confined condition check on a path with respect to +/// another path. A path is confined wrt another if it is relative and results +/// in a non-upwards path when applied from the directory of the other path. +/// This models the situation when a symlink is being resolved inside a tree. +/// NOTE: No explicit check is done whether applied_to is actually a relative +/// path, as this is implicit by the non-upwardness condition. +[[nodiscard]] static auto PathIsConfined( + std::filesystem::path const& path, + std::filesystem::path const& applied_to) noexcept -> bool { + if (path.is_absolute()) { + return false; + } + // check confined upwards condition; this call also handles the case when + // applied_to is an absolute path + return PathIsNonUpwards(applied_to.parent_path() / path); +} + #endif diff --git a/test/utils/cpp/path.test.cpp b/test/utils/cpp/path.test.cpp index 5a450fd5d..7a0ac1fe6 100644 --- a/test/utils/cpp/path.test.cpp +++ b/test/utils/cpp/path.test.cpp @@ -40,3 +40,11 @@ TEST_CASE("non-upwards condition", "[path]") { CHECK_FALSE(PathIsNonUpwards( "foo/../bar/../../foo")); // relative with non-upwards indirection } + +TEST_CASE("confined upwards condition", "[path]") { + CHECK_FALSE(PathIsConfined("/foo", "dummy")); // absolute path + CHECK(PathIsConfined("foo", "dummy")); // relative non-upwards + CHECK(PathIsConfined("../foo", "dummy/bar")); // upwards, but confined + CHECK_FALSE(PathIsConfined("foo/../bar/../../../foo", + "dummy")); // upwards, not confined +} From 2ab4fc7b08b47ca04007a56574bb263a1c7324da Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Tue, 1 Aug 2023 11:08:04 +0200 Subject: [PATCH 22/97] GitRepo: Add method for reading object from tree by its path --- src/buildtool/file_system/git_repo.cpp | 113 +++++++++++++++++++++++++ src/buildtool/file_system/git_repo.hpp | 20 ++++- 2 files changed, 131 insertions(+), 2 deletions(-) diff --git a/src/buildtool/file_system/git_repo.cpp b/src/buildtool/file_system/git_repo.cpp index bb60c68bc..a989c6086 100644 --- a/src/buildtool/file_system/git_repo.cpp +++ b/src/buildtool/file_system/git_repo.cpp @@ -1169,6 +1169,119 @@ auto GitRepo::CheckTreeExists(std::string const& tree_id, #endif // BOOTSTRAP_BUILD_TOOL } +auto GitRepo::GetObjectByPathFromTree(std::string const& tree_id, + std::string const& rel_path) noexcept + -> std::optional { +#ifdef BOOTSTRAP_BUILD_TOOL + return std::nullopt; +#else + try { + std::string entry_id{tree_id}; + ObjectType entry_type{ObjectType::Tree}; + + // preferably with a "fake" repository! + if (not IsRepoFake()) { + Logger::Log(LogLevel::Debug, + "Subtree id retrieval from tree called on a real " + "repository"); + } + // check if path is not trivial + if (rel_path != ".") { + // share the odb lock + std::shared_lock lock{GetGitCAS()->mutex_}; + + // get tree object from tree id + git_oid tree_oid; + if (git_oid_fromstr(&tree_oid, tree_id.c_str()) != 0) { + Logger::Log(LogLevel::Trace, + "tree ID parsing in git repository {} failed " + "with:\n{}", + GetGitCAS()->git_path_.string(), + GitLastError()); + return std::nullopt; + } + + git_tree* tree_ptr{nullptr}; + if (git_tree_lookup(&tree_ptr, repo_->Ptr(), &tree_oid) != 0) { + Logger::Log(LogLevel::Trace, + "retrieving tree {} in git repository {} " + "failed with:\n{}", + tree_id, + GetGitCAS()->git_path_.string(), + GitLastError()); + // cleanup resources + git_tree_free(tree_ptr); + return std::nullopt; + } + auto tree = std::unique_ptr( + tree_ptr, tree_closer); + + // get hash for actual entry + git_tree_entry* entry_ptr{nullptr}; + if (git_tree_entry_bypath( + &entry_ptr, tree.get(), rel_path.c_str()) != 0) { + Logger::Log(LogLevel::Trace, + "retrieving entry at {} in git repository {} " + "failed with:\n{}", + rel_path, + GetGitCAS()->git_path_.string(), + GitLastError()); + // cleanup resources + git_tree_entry_free(entry_ptr); + return std::nullopt; + } + + auto entry = + std::unique_ptr( + entry_ptr, tree_entry_closer); + + // get id + entry_id = + std::string(git_oid_tostr_s(git_tree_entry_id(entry.get()))); + + // get type + if (auto e_type = GitFileModeToObjectType( + git_tree_entry_filemode(entry.get()))) { + entry_type = *e_type; + } + else { + Logger::Log(LogLevel::Trace, + "retrieving type of entry {} in git repository " + "{} failed with:\n{}", + entry_id, + GetGitCAS()->git_path_.string(), + GitLastError()); + return std::nullopt; + } + } + + // if symlink, also read target + if (IsSymlinkObject(entry_type)) { + if (auto target = + GetGitCAS()->ReadObject(entry_id, /*is_hex_id=*/true)) { + return TreeEntryInfo{.id = entry_id, + .type = entry_type, + .symlink_content = *target}; + } + Logger::Log( + LogLevel::Trace, + "failed to read target for symlink {} in git repository {}", + entry_id, + GetGitCAS()->git_path_.string()); + return std::nullopt; + } + return TreeEntryInfo{.id = entry_id, + .type = entry_type, + .symlink_content = std::nullopt}; + } catch (std::exception const& ex) { + Logger::Log(LogLevel::Debug, + "get entry by path from tree failed with:\n{}", + ex.what()); + return std::nullopt; + } +#endif // BOOTSTRAP_BUILD_TOOL +} + auto GitRepo::IsRepoFake() const noexcept -> bool { return is_repo_fake_; } diff --git a/src/buildtool/file_system/git_repo.hpp b/src/buildtool/file_system/git_repo.hpp index e96f401b9..b2e129349 100644 --- a/src/buildtool/file_system/git_repo.hpp +++ b/src/buildtool/file_system/git_repo.hpp @@ -49,8 +49,16 @@ class GitRepo { using tree_entries_t = std::unordered_map>; - // Checks whether a list of symlinks given by their hashes are non-upwards, - // based on content read from an actual backend. + // Stores the info of an object read by its path. + struct TreeEntryInfo { + std::string id{}; + ObjectType type; + // if type is symlink, read it in advance + std::optional symlink_content{std::nullopt}; + }; + + // Checks whether a list of symlinks given by their hashes are + // non-upwards, based on content read from an actual backend. using SymlinksCheckFunc = std::function const&)>; @@ -211,6 +219,14 @@ class GitRepo { anon_logger_ptr const& logger) noexcept -> std::optional; + /// \brief Get the object info related to a given path inside a Git tree. + /// Unlike GetSubtreeFromTree, we here ignore errors and only return a value + /// when all is successful. + /// Calling it from a fake repository allows thread-safe use. + [[nodiscard]] auto GetObjectByPathFromTree( + std::string const& tree_id, + std::string const& rel_path) noexcept -> std::optional; + private: /// \brief Wrapped git_repository with guarded destructor. /// Kept privately nested to avoid misuse of its raw pointer members. From b1aacb171e292cbdcb06f428ec1ca37ed01969c2 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Tue, 1 Aug 2023 11:56:37 +0200 Subject: [PATCH 23/97] test: Add check for reading objects by path from a Git tree --- test/buildtool/file_system/git_repo.test.cpp | 32 ++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/test/buildtool/file_system/git_repo.test.cpp b/test/buildtool/file_system/git_repo.test.cpp index 69ab7e6c5..4d8c5b4a2 100644 --- a/test/buildtool/file_system/git_repo.test.cpp +++ b/test/buildtool/file_system/git_repo.test.cpp @@ -29,6 +29,8 @@ auto const kRootCommit = std::string{"3ecce3f5b19ad7941c6354d65d841590662f33ef"}; auto const kRootId = std::string{"18770dacfe14c15d88450c21c16668e13ab0e7f9"}; auto const kBazId = std::string{"1868f82682c290f0b1db3cacd092727eef1fa57f"}; +auto const kFooId = std::string{"19102815663d23f8b75a47e7a01965dcdc96468c"}; +auto const kBarId = std::string{"ba0e162e1c47469e3fe4b393a8bf8c569f302116"}; } // namespace @@ -337,6 +339,36 @@ TEST_CASE("Single-threaded fake repository operations", "[git_repo]") { CHECK_FALSE(*result_non_bare); } } + + SECTION("Read object from tree by relative path") { + SECTION("Non-existing") { + auto obj_info = + repo->GetObjectByPathFromTree(kRootId, "does_not_exist"); + CHECK_FALSE(obj_info); + } + SECTION("File") { + auto obj_info = repo->GetObjectByPathFromTree(kRootId, "foo"); + REQUIRE(obj_info); + CHECK(obj_info->id == kFooId); + CHECK(obj_info->type == ObjectType::File); + CHECK_FALSE(obj_info->symlink_content); + } + SECTION("Tree") { + auto obj_info = repo->GetObjectByPathFromTree(kRootId, "baz"); + REQUIRE(obj_info); + CHECK(obj_info->id == kBazId); + CHECK(obj_info->type == ObjectType::Tree); + CHECK_FALSE(obj_info->symlink_content); + } + SECTION("Symlink") { + auto obj_info = repo->GetObjectByPathFromTree(kRootId, "baz/bar_l"); + REQUIRE(obj_info); + CHECK(obj_info->id == kBarId); + CHECK(obj_info->type == ObjectType::Symlink); + CHECK(obj_info->symlink_content); + CHECK(*obj_info->symlink_content == "bar"); + } + } } TEST_CASE("Multi-threaded fake repository operations", "[git_repo]") { From 3d8c5f59656a32acd204855e2b712641bde46ac9 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Tue, 1 Aug 2023 11:57:34 +0200 Subject: [PATCH 24/97] just-mr: Add async map for resolving symlinks in Git trees --- src/other_tools/symlinks_map/TARGETS | 18 + .../symlinks_map/resolve_symlinks_map.cpp | 335 ++++++++++++++++++ .../symlinks_map/resolve_symlinks_map.hpp | 97 +++++ 3 files changed, 450 insertions(+) create mode 100644 src/other_tools/symlinks_map/TARGETS create mode 100644 src/other_tools/symlinks_map/resolve_symlinks_map.cpp create mode 100644 src/other_tools/symlinks_map/resolve_symlinks_map.hpp diff --git a/src/other_tools/symlinks_map/TARGETS b/src/other_tools/symlinks_map/TARGETS new file mode 100644 index 000000000..225000f5d --- /dev/null +++ b/src/other_tools/symlinks_map/TARGETS @@ -0,0 +1,18 @@ +{ "resolve_symlinks_map": + { "type": ["@", "rules", "CC", "library"] + , "name": ["resolve_symlinks_map"] + , "hdrs": ["resolve_symlinks_map.hpp"] + , "srcs": ["resolve_symlinks_map.cpp"] + , "deps": + [ ["src/buildtool/file_system", "git_repo"] + , ["src/buildtool/file_system", "object_type"] + , ["src/buildtool/multithreading", "async_map_consumer"] + , ["src/other_tools/just_mr", "utils"] + , ["src/utils/cpp", "path"] + , ["src/utils/cpp", "path_hash"] + ] + , "stage": ["src", "other_tools", "symlinks_map"] + , "private-deps": + [["@", "fmt", "", "fmt"], ["src/buildtool/storage", "config"]] + } +} diff --git a/src/other_tools/symlinks_map/resolve_symlinks_map.cpp b/src/other_tools/symlinks_map/resolve_symlinks_map.cpp new file mode 100644 index 000000000..88d09aecd --- /dev/null +++ b/src/other_tools/symlinks_map/resolve_symlinks_map.cpp @@ -0,0 +1,335 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/other_tools/symlinks_map/resolve_symlinks_map.hpp" + +#include "fmt/core.h" +#include "src/buildtool/file_system/git_repo.hpp" +#include "src/buildtool/storage/config.hpp" + +namespace { + +void ResolveKnownEntry(GitObjectToResolve const& obj, + GitRepo::TreeEntryInfo const& entry_info, + GitCASPtr const& just_git_cas, + ResolveSymlinksMap::SetterPtr const& setter, + ResolveSymlinksMap::LoggerPtr const& logger, + ResolveSymlinksMap::SubCallerPtr const& subcaller) { + // differentiated treatment based on object type + if (IsFileObject(entry_info.type)) { + // files are already resolved, so return the hash directly + (*setter)(ResolvedGitObject{.id = entry_info.id, + .type = entry_info.type, + .path = obj.rel_path}); + } + else if (IsTreeObject(entry_info.type)) { + // for tree types we resolve by rebuilding the tree from the + // resolved children + auto just_git_repo = GitRepo::Open(just_git_cas); + if (not just_git_repo) { + (*logger)("ResolveSymlinks: could not open Git cache repository!", + /*fatal=*/true); + return; + } + auto children = just_git_repo->ReadTree( + entry_info.id, + [](std::vector const& /*unused*/) { + return true; + }, + /*is_hex_id=*/true); + if (not children) { + (*logger)(fmt::format("ResolveSymlinks: failed to read entries of " + "subtree {} in root tree {}", + entry_info.id, + obj.root_tree_id), + /*fatal=*/true); + return; + } + // resolve children + std::vector children_info{}; + children_info.reserve(children->size()); + for (auto const& [raw_id, ev] : *children) { + for (auto const& e : ev) { + // must enforce ignore special at the tree level! + if (IsNonSpecialObject(e.type) or + obj.pragma_special != PragmaSpecial::Ignore) { + // children info is known, so pass this forward + if (IsSymlinkObject(e.type)) { + if (auto target = just_git_cas->ReadObject(raw_id)) { + children_info.emplace_back( + obj.root_tree_id, + obj.rel_path / e.name, + obj.pragma_special, + std::make_optional(GitRepo::TreeEntryInfo{ + .id = ToHexString(raw_id), + .type = e.type, + .symlink_content = *target})); + } + else { + (*logger)( + fmt::format("ResolveSymlinks: could not read " + "symlink {} in root tree {}", + (obj.rel_path / e.name).string(), + obj.root_tree_id), + /*fatal=*/true); + return; + } + } + else { + children_info.emplace_back( + obj.root_tree_id, + obj.rel_path / e.name, + obj.pragma_special, + GitRepo::TreeEntryInfo{ + .id = ToHexString(raw_id), + .type = e.type, + .symlink_content = std::nullopt}); + } + } + } + } + (*subcaller)( + children_info, + [children_info, parent = obj, just_git_cas, setter, logger]( + auto const& resolved_entries) { + // create the entries map of the children + GitRepo::tree_entries_t entries{}; + auto num = resolved_entries.size(); + entries.reserve(num); + for (auto i = 0; i < num; ++i) { + auto const& p = children_info[i].rel_path; + entries[*FromHexString(resolved_entries[i]->id)] + .emplace_back( + p.filename().string(), // we only need the name + resolved_entries[i]->type); + } + // create the tree inside our Git CAS, which is already + // existing by this point. Also, this operation is + // guarded internally, so no need for the + // critical_git_op map + auto just_git_repo = GitRepo::Open(just_git_cas); + if (not just_git_repo) { + (*logger)( + "ResolveSymlinks: could not open Git cache repository!", + /*fatal=*/true); + return; + } + auto tree_raw_id = just_git_repo->CreateTree(entries); + if (not tree_raw_id) { + (*logger)(fmt::format("ResolveSymlinks: failed to create " + "resolved tree {} in root tree {}", + parent.rel_path.string(), + parent.root_tree_id), + /*fatal=*/true); + return; + } + // set the resolved tree hash + (*setter)(ResolvedGitObject{.id = ToHexString(*tree_raw_id), + .type = ObjectType::Tree, + .path = parent.rel_path}); + }, + logger); + } + else { + // sanity check: cannot resolve a symlink called with ignore + // special, as that can only be handled by the parent tree + if (obj.pragma_special == PragmaSpecial::Ignore) { + (*logger)(fmt::format("ResolveSymlinks: asked to ignore symlink {} " + "in root tree {}", + obj.rel_path.string(), + obj.root_tree_id), + /*fatal=*/true); + return; + } + // target should have already been read + if (not entry_info.symlink_content) { + (*logger)(fmt::format("ResolveSymlinks: missing target of symlink " + "{} in root tree {}", + obj.rel_path.string(), + obj.root_tree_id), + /*fatal=*/true); + return; + } + // check if link target (unresolved) is confined to the tree + if (not PathIsConfined(*entry_info.symlink_content, obj.rel_path)) { + (*logger)(fmt::format("ResolveSymlinks: symlink {} is not confined " + "to tree {}", + obj.rel_path.string(), + obj.root_tree_id), + /*fatal=*/true); + return; + } + // if partially resolved, return non-upwards symlinks as-is + if (obj.pragma_special == PragmaSpecial::ResolvePartially and + PathIsNonUpwards(*entry_info.symlink_content)) { + // return as symlink object + (*setter)(ResolvedGitObject{.id = entry_info.id, + .type = ObjectType::Symlink, + .path = obj.rel_path}); + return; + } + // resolve the target + auto n_target = ToNormalPath(obj.rel_path.parent_path() / + *entry_info.symlink_content); + (*subcaller)( + {GitObjectToResolve(obj.root_tree_id, + n_target, + obj.pragma_special, + /*known_info=*/std::nullopt)}, + [setter](auto const& values) { + (*setter)(ResolvedGitObject{*values[0]}); + }, + logger); + } +} + +} // namespace + +auto CreateResolveSymlinksMap() -> ResolveSymlinksMap { + auto resolve_symlinks = [](auto /*unused*/, + auto setter, + auto logger, + auto subcaller, + auto const& key) { + // look up entry by its relative path + auto just_git_cas = GitCAS::Open(StorageConfig::GitRoot()); + if (not just_git_cas) { + (*logger)("ResolveSymlinks: could not open Git cache database!", + /*fatal=*/true); + return; + } + auto just_git_repo = GitRepo::Open(just_git_cas); + if (not just_git_repo) { + (*logger)("ResolveSymlinks: could not open Git cache repository!", + /*fatal=*/true); + return; + } + auto entry_info = key.known_info + ? key.known_info + : just_git_repo->GetObjectByPathFromTree( + key.root_tree_id, key.rel_path); + + // differentiate between existing path and non-existing + if (entry_info) { + ResolveKnownEntry( + key, *entry_info, just_git_cas, setter, logger, subcaller); + } + else { + // non-existing paths come from symlinks, so treat accordingly + // sanity check: pragma ignore special should not be set if here + if (key.pragma_special == PragmaSpecial::Ignore) { + (*logger)( + fmt::format("ResolveSymlinks: asked to ignore indirect " + "symlink path {} in root tree {}", + key.rel_path.string(), + key.root_tree_id), + /*fatal=*/true); + return; + } + auto parent_path = key.rel_path.parent_path(); + if (parent_path == key.rel_path) { + (*logger)(fmt::format("ResolveSymlinks: found unresolved path " + "{} in root tree {}", + key.rel_path.string(), + key.root_tree_id), + /*fatal=*/true); + return; + } + // resolve parent + (*subcaller)( + {GitObjectToResolve(key.root_tree_id, + parent_path, + key.pragma_special, + /*known_info=*/std::nullopt)}, + [key, + parent_path, + filename = key.rel_path.filename(), + just_git_cas, + setter, + logger, + subcaller](auto const& values) { + auto resolved_parent = *values[0]; + // parent must be a tree + if (not IsTreeObject(resolved_parent.type)) { + (*logger)( + fmt::format("ResolveSymlinks: path {} in root tree " + "{} failed to resolve to a tree", + parent_path.string(), + key.root_tree_id), + /*fatal=*/true); + return; + } + // check if filename exists in resolved parent tree + auto just_git_repo = GitRepo::Open(just_git_cas); + if (not just_git_repo) { + (*logger)( + "ResolveSymlinks: could not open Git cache " + "repository!", + /*fatal=*/true); + return; + } + auto entry_info = just_git_repo->GetObjectByPathFromTree( + resolved_parent.id, filename); + if (entry_info) { + ResolveKnownEntry( + GitObjectToResolve(key.root_tree_id, + resolved_parent.path / filename, + key.pragma_special, + /*known_info=*/std::nullopt), + std::move(*entry_info), + just_git_cas, + setter, + logger, + subcaller); + } + else { + // report unresolvable + (*logger)( + fmt::format( + "ResolveSymlinks: reached unresolvable " + "path {} in root tree {}", + (resolved_parent.path / filename).string(), + key.root_tree_id), + /*fatal=*/true); + } + }, + logger); + } + }; + return AsyncMapConsumer( + resolve_symlinks); +} + +auto DetectAndReportCycle(ResolveSymlinksMap const& map, + std::string const& root_tree_id) + -> std::optional { + using namespace std::string_literals; + auto cycle = map.DetectCycle(); + if (cycle) { + bool found{false}; + std::ostringstream oss{}; + oss << fmt::format("Cycle detected for Git tree {}:", root_tree_id) + << std::endl; + for (auto const& k : *cycle) { + auto match = (k == cycle->back()); + auto prefix{match ? found ? "`-- "s : ".-> "s + : found ? "| "s + : " "s}; + oss << prefix << k.rel_path << std::endl; + found = found or match; + } + return oss.str(); + } + return std::nullopt; +} diff --git a/src/other_tools/symlinks_map/resolve_symlinks_map.hpp b/src/other_tools/symlinks_map/resolve_symlinks_map.hpp new file mode 100644 index 000000000..8bae7583a --- /dev/null +++ b/src/other_tools/symlinks_map/resolve_symlinks_map.hpp @@ -0,0 +1,97 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_OTHER_TOOLS_SYMLINKS_MAP_RESOLVE_SYMLINKS_MAP_HPP +#define INCLUDED_SRC_OTHER_TOOLS_SYMLINKS_MAP_RESOLVE_SYMLINKS_MAP_HPP + +#include +#include +#include + +#include "src/buildtool/file_system/git_repo.hpp" +#include "src/buildtool/file_system/object_type.hpp" +#include "src/buildtool/multithreading/async_map_consumer.hpp" +#include "src/other_tools/just_mr/utils.hpp" +#include "src/utils/cpp/path.hpp" +#include "src/utils/cpp/path_hash.hpp" + +/// \brief Information needed to resolve an object (blob or tree) given its +/// path relative to the path of a root tree in a given CAS. +struct GitObjectToResolve { + // hash of the root tree + std::string root_tree_id{}; /* key */ + // path of this object relative to root tree, in normal form + std::filesystem::path rel_path{"."}; /* key */ + // how the tree should be resolved + PragmaSpecial pragma_special{}; /* key */ + // sometimes the info of the object at the required path is already known, + // so leverage this to avoid extra work + std::optional known_info{std::nullopt}; + + GitObjectToResolve() = default; // needed for cycle detection only! + + GitObjectToResolve(std::string root_tree_id_, + std::filesystem::path const& rel_path_, + PragmaSpecial const& pragma_special_, + std::optional known_info_) + : root_tree_id{std::move(root_tree_id_)}, + rel_path{ToNormalPath(rel_path_)}, + pragma_special{pragma_special_}, + known_info{std::move(known_info_)} {}; + + [[nodiscard]] auto operator==( + GitObjectToResolve const& other) const noexcept -> bool { + return root_tree_id == other.root_tree_id and + rel_path == other.rel_path and + pragma_special == other.pragma_special; + } +}; + +/// \brief For a possibly initially unresolved path by the end we should be able +/// to know its hash, its type, and its now resolved path. +struct ResolvedGitObject { + std::string id; + ObjectType type; + std::filesystem::path path; +}; + +/// \brief Maps information about a Git object to its Git ID, type, and path as +/// part of a Git tree where symlinks have been resolved according to the given +/// pragma value. +/// Returns a nullopt only if called on a symlink with pragma ignore special. +/// \note Call the map with type Tree and path "." to resolve a Git tree. +using ResolveSymlinksMap = + AsyncMapConsumer; + +[[nodiscard]] auto CreateResolveSymlinksMap() -> ResolveSymlinksMap; + +[[nodiscard]] auto DetectAndReportCycle(ResolveSymlinksMap const& map, + std::string const& root_tree_id) + -> std::optional; + +namespace std { +template <> +struct hash { + [[nodiscard]] auto operator()(const GitObjectToResolve& ct) const noexcept + -> std::size_t { + size_t seed{}; + hash_combine(&seed, ct.root_tree_id); + hash_combine(&seed, ct.rel_path); + hash_combine(&seed, ct.pragma_special); + return seed; + } +}; +} // namespace std + +#endif // INCLUDED_SRC_OTHER_TOOLS_SYMLINKS_MAP_RESOLVE_SYMLINKS_MAP_HPP From 643966865d6ff5292b59f876a78c7c67a90ddb16 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Wed, 19 Jul 2023 15:13:20 +0200 Subject: [PATCH 25/97] just-mr utils: Add resolved tree id file getter --- src/other_tools/just_mr/utils.cpp | 7 +++++++ src/other_tools/just_mr/utils.hpp | 11 +++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/other_tools/just_mr/utils.cpp b/src/other_tools/just_mr/utils.cpp index 925638241..57ef68306 100644 --- a/src/other_tools/just_mr/utils.cpp +++ b/src/other_tools/just_mr/utils.cpp @@ -55,6 +55,13 @@ auto GetDistdirTreeIDFile(std::string const& content) noexcept return StorageConfig::BuildRoot() / "distfiles-tree-map" / content; } +auto GetResolvedTreeIDFile(std::string const& tree_hash, + PragmaSpecial const& pragma_special) noexcept + -> std::filesystem::path { + return StorageConfig::BuildRoot() / "special-tree-map" / + kPragmaSpecialInverseMap.at(pragma_special) / tree_hash; +} + auto WriteTreeIDFile(std::filesystem::path const& tree_id_file, std::string const& tree_id) noexcept -> bool { // needs to be done safely, so use the rename trick diff --git a/src/other_tools/just_mr/utils.hpp b/src/other_tools/just_mr/utils.hpp index 678784452..92d679e20 100644 --- a/src/other_tools/just_mr/utils.hpp +++ b/src/other_tools/just_mr/utils.hpp @@ -102,6 +102,12 @@ std::unordered_map const kPragmaSpecialMap = { {"resolve-partially", PragmaSpecial::ResolvePartially}, {"resolve-completely", PragmaSpecial::ResolveCompletely}}; +/// \brief Pragma "special" value inverse map, from enum to string +std::unordered_map const kPragmaSpecialInverseMap = + {{PragmaSpecial::Ignore, "ignore"}, + {PragmaSpecial::ResolvePartially, "resolve-partially"}, + {PragmaSpecial::ResolveCompletely, "resolve-completely"}}; + namespace JustMR { struct Paths { @@ -169,6 +175,11 @@ namespace Utils { [[nodiscard]] auto GetDistdirTreeIDFile(std::string const& content) noexcept -> std::filesystem::path; +/// \brief Get the path to the file storing a resolved tree hash. +[[nodiscard]] auto GetResolvedTreeIDFile( + std::string const& tree_hash, + PragmaSpecial const& pragma_special) noexcept -> std::filesystem::path; + /// \brief Write a tree id to file. The parent folder of the file must exist! [[nodiscard]] auto WriteTreeIDFile(std::filesystem::path const& tree_id_file, std::string const& tree_id) noexcept -> bool; From c7cad41329bec5439196be119a2d31626be4e9fe Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Wed, 19 Jul 2023 15:14:29 +0200 Subject: [PATCH 26/97] just-mr maps: Add logic for resolving symlinks in file repositories --- src/other_tools/just_mr/TARGETS | 1 + src/other_tools/just_mr/main.cpp | 3 + src/other_tools/root_maps/TARGETS | 3 + src/other_tools/root_maps/fpath_git_map.cpp | 158 +++++++++++++++++--- src/other_tools/root_maps/fpath_git_map.hpp | 2 + 5 files changed, 144 insertions(+), 23 deletions(-) diff --git a/src/other_tools/just_mr/TARGETS b/src/other_tools/just_mr/TARGETS index 9bd706597..aac664879 100644 --- a/src/other_tools/just_mr/TARGETS +++ b/src/other_tools/just_mr/TARGETS @@ -20,6 +20,7 @@ , ["src/other_tools/just_mr/progress_reporting", "statistics"] , ["src/other_tools/just_mr/progress_reporting", "progress_reporter"] , ["src/buildtool/storage", "storage"] + , ["src/other_tools/symlinks_map", "resolve_symlinks_map"] ] , "stage": ["src", "other_tools", "just_mr"] , "private-ldflags": diff --git a/src/other_tools/just_mr/main.cpp b/src/other_tools/just_mr/main.cpp index c15970d23..3c8e5d572 100644 --- a/src/other_tools/just_mr/main.cpp +++ b/src/other_tools/just_mr/main.cpp @@ -33,6 +33,7 @@ #include "src/other_tools/ops_maps/git_update_map.hpp" #include "src/other_tools/ops_maps/repo_fetch_map.hpp" #include "src/other_tools/repo_map/repos_to_setup_map.hpp" +#include "src/other_tools/symlinks_map/resolve_symlinks_map.hpp" namespace { @@ -1144,6 +1145,7 @@ void DefaultReachableRepositories( arguments.common.git_path->string(), *arguments.common.local_launcher, arguments.common.jobs); + auto resolve_symlinks_map = CreateResolveSymlinksMap(); auto commit_git_map = CreateCommitGitMap(&critical_git_op_map, @@ -1158,6 +1160,7 @@ void DefaultReachableRepositories( auto fpath_git_map = CreateFilePathGitMap(arguments.just_cmd.subcmd_name, &critical_git_op_map, &import_to_git_map, + &resolve_symlinks_map, arguments.common.jobs); auto distdir_git_map = CreateDistdirGitMap(&content_cas_map, &import_to_git_map, diff --git a/src/other_tools/root_maps/TARGETS b/src/other_tools/root_maps/TARGETS index 469caf493..8ece55655 100644 --- a/src/other_tools/root_maps/TARGETS +++ b/src/other_tools/root_maps/TARGETS @@ -54,6 +54,7 @@ , ["src/utils/cpp", "path_hash"] , ["src/utils/cpp", "hash_combine"] , ["src/other_tools/just_mr", "utils"] + , ["src/other_tools/symlinks_map", "resolve_symlinks_map"] ] , "stage": ["src", "other_tools", "root_maps"] , "private-deps": @@ -61,6 +62,8 @@ , ["src/buildtool/storage", "config"] , ["src/utils/cpp", "tmp_dir"] , ["src/buildtool/file_system", "file_root"] + , ["src/buildtool/file_system", "git_repo"] + , ["src/other_tools/git_operations", "git_repo_remote"] ] } , "content_git_map": diff --git a/src/other_tools/root_maps/fpath_git_map.cpp b/src/other_tools/root_maps/fpath_git_map.cpp index e26b3a9d0..f0a12fd4a 100644 --- a/src/other_tools/root_maps/fpath_git_map.cpp +++ b/src/other_tools/root_maps/fpath_git_map.cpp @@ -16,20 +16,121 @@ #include "src/buildtool/execution_api/local/config.hpp" #include "src/buildtool/file_system/file_root.hpp" +#include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/storage/config.hpp" +#include "src/other_tools/git_operations/git_repo_remote.hpp" #include "src/utils/cpp/tmp_dir.hpp" +namespace { + +void ResolveFilePathTree( + + std::string const& repo_root, + std::string const& target_path, + std::string const& tree_hash, + std::optional const& pragma_special, + gsl::not_null const& resolve_symlinks_map, + gsl::not_null const& ts, + FilePathGitMap::SetterPtr const& ws_setter, + FilePathGitMap::LoggerPtr const& logger) { + if (pragma_special) { + // get the resolved tree + auto tree_id_file = + JustMR::Utils::GetResolvedTreeIDFile(tree_hash, *pragma_special); + if (FileSystemManager::Exists(tree_id_file)) { + // read resolved tree id + auto resolved_tree_id = FileSystemManager::ReadFile(tree_id_file); + if (not resolved_tree_id) { + (*logger)(fmt::format("Failed to read resolved " + "tree id from file {}", + tree_id_file.string()), + /*fatal=*/true); + return; + } + // set the workspace root + (*ws_setter)(nlohmann::json::array( + {FileRoot::kGitTreeMarker, *resolved_tree_id, repo_root})); + } + else { + // resolve tree + resolve_symlinks_map->ConsumeAfterKeysReady( + ts, + {GitObjectToResolve(tree_hash, + ".", + *pragma_special, + /*known_info=*/std::nullopt)}, + [resolve_symlinks_map, + tree_hash, + repo_root, + tree_id_file, + ws_setter, + logger](auto const& hashes) { + if (not hashes[0]) { + // check for cycles + auto error = DetectAndReportCycle(*resolve_symlinks_map, + tree_hash); + if (error) { + (*logger)(fmt::format("Failed to resolve symlinks " + "in tree {}:\n{}", + tree_hash, + *error), + /*fatal=*/true); + return; + } + (*logger)(fmt::format("Unknown error in resolving " + "symlinks in tree {}", + tree_hash), + /*fatal=*/true); + return; + } + auto const& resolved_tree = *hashes[0]; + // cache the resolved tree in the CAS map + if (not JustMR::Utils::WriteTreeIDFile(tree_id_file, + resolved_tree.id)) { + (*logger)(fmt::format("Failed to write resolved tree " + "id to file {}", + tree_id_file.string()), + /*fatal=*/true); + return; + } + // set the workspace root + (*ws_setter)( + nlohmann::json::array({FileRoot::kGitTreeMarker, + resolved_tree.id, + repo_root})); + }, + [logger, target_path](auto const& msg, bool fatal) { + (*logger)(fmt::format( + "While resolving symlinks for target {}:\n{}", + target_path, + msg), + fatal); + }); + } + } + else { + // set the workspace root as-is + (*ws_setter)(nlohmann::json::array( + {FileRoot::kGitTreeMarker, tree_hash, repo_root})); + } +} + +} // namespace + auto CreateFilePathGitMap( std::optional const& current_subcmd, gsl::not_null const& critical_git_op_map, gsl::not_null const& import_to_git_map, + gsl::not_null const& resolve_symlinks_map, std::size_t jobs) -> FilePathGitMap { - auto dir_to_git = [current_subcmd, critical_git_op_map, import_to_git_map]( - auto ts, - auto setter, - auto logger, - auto /*unused*/, - auto const& key) { + auto dir_to_git = [current_subcmd, + critical_git_op_map, + import_to_git_map, + resolve_symlinks_map](auto ts, + auto setter, + auto logger, + auto /*unused*/, + auto const& key) { // setup wrapped logger auto wrapped_logger = std::make_shared( [logger](auto const& msg, bool fatal) { @@ -75,6 +176,8 @@ auto CreateFilePathGitMap( pragma_special = key.pragma_special, git_cas = std::move(git_cas), repo_root = std::move(*repo_root), + resolve_symlinks_map, + ts, setter, logger](auto const& values) { GitOpValue op_result = *values[0]; @@ -102,19 +205,21 @@ auto CreateFilePathGitMap( msg), fatal); }); - // get tree id and return workspace root + // get tree id auto tree_hash = git_repo->GetSubtreeFromPath( fpath, *op_result.result, wrapped_logger); if (not tree_hash) { return; } - // set the workspace root - (*setter)(nlohmann::json::array( - {pragma_special == PragmaSpecial::Ignore - ? FileRoot::kGitTreeIgnoreSpecialMarker - : FileRoot::kGitTreeMarker, - *tree_hash, - repo_root})); + // resolve tree and set workspace root + ResolveFilePathTree(repo_root.string(), + fpath.string(), + *tree_hash, + pragma_special, + resolve_symlinks_map, + ts, + setter, + logger); }, [logger, target_path = *repo_root](auto const& msg, bool fatal) { @@ -159,8 +264,13 @@ auto CreateFilePathGitMap( {std::move(c_info)}, // tmp_dir passed, to ensure folder is not removed until import // to git is done - [tmp_dir, pragma_special = key.pragma_special, setter, logger]( - auto const& values) { + [tmp_dir, + fpath = key.fpath, + pragma_special = key.pragma_special, + resolve_symlinks_map, + ts, + setter, + logger](auto const& values) { // check for errors if (not values[0]->second) { (*logger)("Importing to git failed", @@ -169,13 +279,15 @@ auto CreateFilePathGitMap( } // we only need the tree std::string tree = values[0]->first; - // set the workspace root - (*setter)(nlohmann::json::array( - {pragma_special == PragmaSpecial::Ignore - ? FileRoot::kGitTreeIgnoreSpecialMarker - : FileRoot::kGitTreeMarker, - tree, - StorageConfig::GitRoot()})); + // resolve tree and set workspace root + ResolveFilePathTree(StorageConfig::GitRoot().string(), + fpath.string(), + tree, + pragma_special, + resolve_symlinks_map, + ts, + setter, + logger); }, [logger, target_path = key.fpath](auto const& msg, bool fatal) { (*logger)( diff --git a/src/other_tools/root_maps/fpath_git_map.hpp b/src/other_tools/root_maps/fpath_git_map.hpp index 25721c19f..39cae6538 100644 --- a/src/other_tools/root_maps/fpath_git_map.hpp +++ b/src/other_tools/root_maps/fpath_git_map.hpp @@ -18,6 +18,7 @@ #include "nlohmann/json.hpp" #include "src/other_tools/just_mr/utils.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" +#include "src/other_tools/symlinks_map/resolve_symlinks_map.hpp" #include "src/utils/cpp/hash_combine.hpp" #include "src/utils/cpp/path_hash.hpp" @@ -39,6 +40,7 @@ using FilePathGitMap = AsyncMapConsumer; std::optional const& current_subcmd, gsl::not_null const& critical_git_op_map, gsl::not_null const& import_to_git_map, + gsl::not_null const& resolve_symlinks_map, std::size_t jobs) -> FilePathGitMap; namespace std { From d95d2f9f79da0843593d68ad6d02abe520dcfbad Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Wed, 19 Jul 2023 16:23:23 +0200 Subject: [PATCH 27/97] just-mr maps: Add logic for resolving symlinks in archive repositories --- src/other_tools/just_mr/main.cpp | 1 + src/other_tools/root_maps/TARGETS | 3 + src/other_tools/root_maps/content_git_map.cpp | 151 +++++++++++++++--- src/other_tools/root_maps/content_git_map.hpp | 4 +- 4 files changed, 136 insertions(+), 23 deletions(-) diff --git a/src/other_tools/just_mr/main.cpp b/src/other_tools/just_mr/main.cpp index 3c8e5d572..b4a7b554f 100644 --- a/src/other_tools/just_mr/main.cpp +++ b/src/other_tools/just_mr/main.cpp @@ -1155,6 +1155,7 @@ void DefaultReachableRepositories( arguments.common.jobs); auto content_git_map = CreateContentGitMap(&content_cas_map, &import_to_git_map, + &resolve_symlinks_map, &critical_git_op_map, arguments.common.jobs); auto fpath_git_map = CreateFilePathGitMap(arguments.just_cmd.subcmd_name, diff --git a/src/other_tools/root_maps/TARGETS b/src/other_tools/root_maps/TARGETS index 8ece55655..06ff9ce2e 100644 --- a/src/other_tools/root_maps/TARGETS +++ b/src/other_tools/root_maps/TARGETS @@ -74,6 +74,7 @@ , "deps": [ ["src/other_tools/ops_maps", "content_cas_map"] , ["src/other_tools/ops_maps", "import_to_git_map"] + , ["src/other_tools/symlinks_map", "resolve_symlinks_map"] ] , "stage": ["src", "other_tools", "root_maps"] , "private-deps": @@ -85,6 +86,8 @@ , ["src/other_tools/just_mr/progress_reporting", "progress"] , ["src/other_tools/just_mr/progress_reporting", "statistics"] , ["src/buildtool/file_system", "file_root"] + , ["src/buildtool/file_system", "git_repo"] + , ["src/other_tools/git_operations", "git_repo_remote"] ] } , "tree_id_git_map": diff --git a/src/other_tools/root_maps/content_git_map.cpp b/src/other_tools/root_maps/content_git_map.cpp index df62fab67..0c991ef14 100644 --- a/src/other_tools/root_maps/content_git_map.cpp +++ b/src/other_tools/root_maps/content_git_map.cpp @@ -41,15 +41,117 @@ namespace { return "unrecognized repository type"; } +void ResolveContentTree( + std::string const& content, + std::string const& tree_hash, + bool is_cache_hit, + std::optional const& pragma_special, + gsl::not_null const& resolve_symlinks_map, + gsl::not_null const& ts, + ContentGitMap::SetterPtr const& ws_setter, + ContentGitMap::LoggerPtr const& logger) { + if (pragma_special) { + // get the resolved tree + auto tree_id_file = + JustMR::Utils::GetResolvedTreeIDFile(tree_hash, *pragma_special); + if (FileSystemManager::Exists(tree_id_file)) { + // read resolved tree id + auto resolved_tree_id = FileSystemManager::ReadFile(tree_id_file); + if (not resolved_tree_id) { + (*logger)(fmt::format("Failed to read resolved " + "tree id from file {}", + tree_id_file.string()), + /*fatal=*/true); + return; + } + // set the workspace root + (*ws_setter)(std::pair( + nlohmann::json::array({FileRoot::kGitTreeMarker, + *resolved_tree_id, + StorageConfig::GitRoot().string()}), + true // it is definitely a cache hit + )); + } + else { + // resolve tree + resolve_symlinks_map->ConsumeAfterKeysReady( + ts, + {GitObjectToResolve(tree_hash, + ".", + *pragma_special, + /*known_info=*/std::nullopt)}, + [resolve_symlinks_map, + tree_hash, + tree_id_file, + is_cache_hit, + ws_setter, + logger](auto const& hashes) { + if (not hashes[0]) { + // check for cycles + auto error = DetectAndReportCycle(*resolve_symlinks_map, + tree_hash); + if (error) { + (*logger)(fmt::format("Failed to resolve symlinks " + "in tree {}:\n{}", + tree_hash, + *error), + /*fatal=*/true); + return; + } + (*logger)(fmt::format("Unknown error in resolving " + "symlinks in tree {}", + tree_hash), + /*fatal=*/true); + return; + } + auto const& resolved_tree = *hashes[0]; + // cache the resolved tree in the CAS map + if (not JustMR::Utils::WriteTreeIDFile(tree_id_file, + resolved_tree.id)) { + (*logger)(fmt::format("Failed to write resolved tree " + "id to file {}", + tree_id_file.string()), + /*fatal=*/true); + return; + } + // set the workspace root + (*ws_setter)( + std::pair(nlohmann::json::array( + {FileRoot::kGitTreeMarker, + resolved_tree.id, + StorageConfig::GitRoot().string()}), + is_cache_hit)); + }, + [logger, content](auto const& msg, bool fatal) { + (*logger)(fmt::format("While resolving symlinks for " + "content {}:\n{}", + content, + msg), + fatal); + }); + } + } + else { + // set the workspace root as-is + (*ws_setter)(std::pair( + nlohmann::json::array({FileRoot::kGitTreeMarker, + tree_hash, + StorageConfig::GitRoot().string()}), + is_cache_hit)); + } +} + } // namespace auto CreateContentGitMap( gsl::not_null const& content_cas_map, gsl::not_null const& import_to_git_map, + gsl::not_null const& resolve_symlinks_map, gsl::not_null const& critical_git_op_map, std::size_t jobs) -> ContentGitMap { auto gitify_content = [content_cas_map, import_to_git_map, + resolve_symlinks_map, critical_git_op_map](auto ts, auto setter, auto logger, @@ -83,7 +185,10 @@ auto CreateContentGitMap( {std::move(op_key)}, [archive_tree_id = *archive_tree_id, subdir = key.subdir, + content = key.archive.content, pragma_special = key.pragma_special, + resolve_symlinks_map, + ts, setter, logger](auto const& values) { GitOpValue op_result = *values[0]; @@ -110,21 +215,21 @@ auto CreateContentGitMap( msg), fatal); }); - // get subtree id and return workspace root + // get subtree id auto subtree_hash = just_git_repo->GetSubtreeFromTree( archive_tree_id, subdir, wrapped_logger); if (not subtree_hash) { return; } - // set the workspace root - (*setter)(std::pair( - nlohmann::json::array( - {pragma_special == PragmaSpecial::Ignore - ? FileRoot::kGitTreeIgnoreSpecialMarker - : FileRoot::kGitTreeMarker, - *subtree_hash, - StorageConfig::GitRoot().string()}), - true)); + // resolve tree and set workspace root + ResolveContentTree(content, + *subtree_hash, + true, /*is_cache_hit*/ + pragma_special, + resolve_symlinks_map, + ts, + setter, + logger); }, [logger, target_path = StorageConfig::GitRoot()]( auto const& msg, bool fatal) { @@ -147,6 +252,7 @@ auto CreateContentGitMap( subdir = key.subdir, pragma_special = key.pragma_special, import_to_git_map, + resolve_symlinks_map, ts, setter, logger]([[maybe_unused]] auto const& values) { @@ -185,8 +291,11 @@ auto CreateContentGitMap( {std::move(c_info)}, [tmp_dir, // keep tmp_dir alive archive_tree_id_file, + content_id, subdir, pragma_special, + resolve_symlinks_map, + ts, setter, logger](auto const& values) { // check for errors @@ -216,14 +325,12 @@ auto CreateContentGitMap( /*fatal=*/true); return; } - // fetch all into Git cache auto just_git_repo = GitRepoRemote::Open(just_git_cas); if (not just_git_repo) { (*logger)( "Could not open Git cache repository!", /*fatal=*/true); - return; } // setup wrapped logger @@ -236,22 +343,22 @@ auto CreateContentGitMap( msg), fatal); }); - // get subtree id and return workspace root + // get subtree id auto subtree_hash = just_git_repo->GetSubtreeFromTree( archive_tree_id, subdir, wrapped_logger); if (not subtree_hash) { return; } - // set the workspace root - (*setter)(std::pair( - nlohmann::json::array( - {pragma_special == PragmaSpecial::Ignore - ? FileRoot::kGitTreeIgnoreSpecialMarker - : FileRoot::kGitTreeMarker, - *subtree_hash, - StorageConfig::GitRoot().string()}), - false)); + // resolve tree and set workspace root + ResolveContentTree(content_id, + *subtree_hash, + false, /*is_cache_hit*/ + pragma_special, + resolve_symlinks_map, + ts, + setter, + logger); }, [logger, target_path = tmp_dir->GetPath()]( auto const& msg, bool fatal) { diff --git a/src/other_tools/root_maps/content_git_map.hpp b/src/other_tools/root_maps/content_git_map.hpp index d2aeb22d2..6eed84483 100644 --- a/src/other_tools/root_maps/content_git_map.hpp +++ b/src/other_tools/root_maps/content_git_map.hpp @@ -19,15 +19,17 @@ #include "src/other_tools/ops_maps/content_cas_map.hpp" #include "src/other_tools/ops_maps/import_to_git_map.hpp" +#include "src/other_tools/symlinks_map/resolve_symlinks_map.hpp" /// \brief Maps the content of an archive to the resulting Git tree WS root, -/// togehter with the information whether it was a cache hit. +/// together with the information whether it was a cache hit. using ContentGitMap = AsyncMapConsumer>; [[nodiscard]] auto CreateContentGitMap( gsl::not_null const& content_cas_map, gsl::not_null const& import_to_git_map, + gsl::not_null const& resolve_symlinks_map, gsl::not_null const& critical_git_op_map, std::size_t jobs) -> ContentGitMap; From b6224603e89ddbf88493217c61ada85fb10e8bf3 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Tue, 25 Jul 2023 09:46:16 +0200 Subject: [PATCH 28/97] test: Add checks for 'special' pragma in just-mr Checks that the new 'special' pragma works for all supported repository types. In particular, checks that symlinks get resolved as expected and that resolved trees add their unresolved counterparts to the CAS for 'file' and 'archive' type repositories. Splits install-roots test into a basic check which works with the bootstrappable just-mr, and a new check focusing on the newly introduced 'special' pragma. --- test/end-to-end/just-mr/TARGETS | 14 +- .../just-mr/create_test_archives.cpp | 32 ++- .../just-mr/install-roots-symlinks.sh | 230 ++++++++++++++++++ test/end-to-end/just-mr/install-roots.sh | 35 +-- test/end-to-end/just-mr/just-mr.test.sh | 67 ++++- 5 files changed, 336 insertions(+), 42 deletions(-) create mode 100644 test/end-to-end/just-mr/install-roots-symlinks.sh diff --git a/test/end-to-end/just-mr/TARGETS b/test/end-to-end/just-mr/TARGETS index 2f44f9628..7cdc9ba53 100644 --- a/test/end-to-end/just-mr/TARGETS +++ b/test/end-to-end/just-mr/TARGETS @@ -60,6 +60,13 @@ , "deps": [["end-to-end", "mr-tool-under-test"], ["end-to-end", "tool-under-test"]] } +, "install-roots-symlinks": + { "type": ["@", "rules", "shell/test", "script"] + , "name": ["install-roots-symlinks"] + , "test": ["install-roots-symlinks.sh"] + , "deps": + [["end-to-end", "mr-tool-under-test"], ["end-to-end", "tool-under-test"]] + } , "TESTS": { "type": "install" , "tainted": ["test"] @@ -71,7 +78,12 @@ , { "type": "if" , "cond": {"type": "var", "name": "TEST_BOOTSTRAP_JUST_MR"} , "then": [] - , "else": ["just_mr_mp", "git-tree-verbosity", "defaults"] + , "else": + [ "install-roots-symlinks" + , "just_mr_mp" + , "git-tree-verbosity" + , "defaults" + ] } ] } diff --git a/test/end-to-end/just-mr/create_test_archives.cpp b/test/end-to-end/just-mr/create_test_archives.cpp index 2f4327b78..3fa6759a1 100644 --- a/test/end-to-end/just-mr/create_test_archives.cpp +++ b/test/end-to-end/just-mr/create_test_archives.cpp @@ -35,16 +35,27 @@ Structure of content tree: +--root +--bar +--foo + +--foo_l + +--baz_l +--baz +--bar +--foo + +--foo_l + +foo_l is symlink "baz/foo_l" [non-upwards, pointing to file] +baz_l is symlink "baz" [non-upwards, pointing to tree] +baz/foo_l is symlink "../foo_l" [upwards & confined, pointing to symlink] */ -auto const kExpected = filetree_t{{"root", {"", ObjectType::Tree}}, - {"root/foo", {"foo", ObjectType::File}}, - {"root/bar", {"bar", ObjectType::File}}, - {"root/baz", {"", ObjectType::Tree}}, - {"root/baz/foo", {"foo", ObjectType::File}}, - {"root/baz/bar", {"bar", ObjectType::File}}}; +auto const kExpected = + filetree_t{{"root", {"", ObjectType::Tree}}, + {"root/foo", {"foo", ObjectType::File}}, + {"root/bar", {"bar", ObjectType::File}}, + {"root/baz", {"", ObjectType::Tree}}, + {"root/baz_l", {"baz", ObjectType::Symlink}}, + {"root/foo_l", {"foo", ObjectType::Symlink}}, + {"root/baz/foo", {"foo", ObjectType::File}}, + {"root/baz/bar", {"bar", ObjectType::File}}, + {"root/baz/foo_l", {"../foo_l", ObjectType::Symlink}}}; void create_files(std::filesystem::path const& destDir = ".") { for (auto const& [path, file] : kExpected) { @@ -66,6 +77,15 @@ void create_files(std::filesystem::path const& destDir = ".") { std::exit(1); }; } break; + case ObjectType::Symlink: { + if (not FileSystemManager::CreateSymlink(content, + destDir / path)) { + Logger::Log(LogLevel::Error, + "Could not create test symlink at path {}", + (destDir / path).string()); + std::exit(1); + }; + } break; default: { Logger::Log(LogLevel::Error, "File system failure in creating test archive"); diff --git a/test/end-to-end/just-mr/install-roots-symlinks.sh b/test/end-to-end/just-mr/install-roots-symlinks.sh new file mode 100644 index 000000000..aa3ae8360 --- /dev/null +++ b/test/end-to-end/just-mr/install-roots-symlinks.sh @@ -0,0 +1,230 @@ +#!/bin/sh +# Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +### +# This test extends install-roots-basic with 'special' pragma use-cases +## + +set -eu + +readonly JUST="${PWD}/bin/tool-under-test" +readonly JUST_MR="${PWD}/bin/mr-tool-under-test" +readonly DISTDIR="${TEST_TMPDIR}/distfiles" +readonly LBR="${TEST_TMPDIR}/local-build-root" + +readonly INSTALL_DIR_RESOLVED="${TEST_TMPDIR}/installation-target-resolved" +readonly INSTALL_DIR_UNRESOLVED="${TEST_TMPDIR}/installation-target-unresolved" +readonly INSTALL_DIR_SPECIAL_IGNORE="${TEST_TMPDIR}/installation-target-special-ignore" +readonly INSTALL_DIR_SPECIAL_PARTIAL="${TEST_TMPDIR}/installation-target-special-partial" +readonly INSTALL_DIR_SPECIAL_COMPLETE="${TEST_TMPDIR}/installation-target-special-complete" + +readonly TEST_DATA="The content of the data file in foo" +readonly TEST_DIRS="bar/baz" +readonly DATA_PATH="bar/baz/data.txt" +readonly NON_UPWARDS_LINK_PATH="bar/baz/nonupwards" +readonly NON_UPWARDS_LINK_TARGET="data.txt" +readonly UPWARDS_LINK_PATH="bar/upwards" +readonly UPWARDS_LINK_TARGET="../bar/baz" +readonly INDIRECT_LINK_PATH="bar/indirect" +readonly INDIRECT_LINK_TARGET="upwards/data.txt" + +mkdir -p "${DISTDIR}" + +### +# Set up sample archives +## + +mkdir -p "foo/${TEST_DIRS}" +echo {} > foo/TARGETS +echo -n "${TEST_DATA}" > "foo/${DATA_PATH}" + +# add resolvable non-upwards symlink +ln -s "${NON_UPWARDS_LINK_TARGET}" "foo/${NON_UPWARDS_LINK_PATH}" +tar cf "${DISTDIR}/foo-1.2.3_v1.tar" foo \ + --sort=name --owner=root:0 --group=root:0 --mtime='UTC 1970-01-01' 2>&1 +foocontent_v1=$(git hash-object "${DISTDIR}/foo-1.2.3_v1.tar") +echo "Foo archive v1 has content ${foocontent_v1}" + +# get Git-tree of foo subdir with only non-upwards symlinks +cd foo +( + git init \ + && git config user.email "nobody@example.org" \ + && git config user.name "Nobody" \ + && git add . \ + && git commit -m "test" --date="1970-01-01T00:00Z" +) +UNRESOLVED_TREE=$(git log -n 1 --format='%T') +echo "Foo archive v1 has unresolved tree ${UNRESOLVED_TREE}" +rm -rf .git +cd .. + +# now add also a resolvable upwards symlink +ln -s "${UPWARDS_LINK_TARGET}" "foo/${UPWARDS_LINK_PATH}" +# ...and a resolvable non-upwards symlink pointing to it +ln -s "${INDIRECT_LINK_TARGET}" "foo/${INDIRECT_LINK_PATH}" +tar cf "${DISTDIR}/foo-1.2.3_v2.tar" foo \ + --sort=name --owner=root:0 --group=root:0 --mtime='UTC 1970-01-01' 2>&1 +foocontent_v2=$(git hash-object "${DISTDIR}/foo-1.2.3_v2.tar") +echo "Foo archive v2 has content ${foocontent_v2}" +rm -rf foo + +### +# Setup sample repository config +## + +touch ROOT +cat > repos.json <&1 +test "$(cat "${INSTALL_DIR_RESOLVED}/${DATA_PATH}")" = "${TEST_DATA}" +# non-upwards symlink is resolved +test "$(cat "${INSTALL_DIR_RESOLVED}/${NON_UPWARDS_LINK_PATH}")" = "${TEST_DATA}" + +# A resolved tree should add to CAS also its unresolved version; in our case, +# the unresolved tree has only non-upwards symlinks, so it can be installed +"${JUST}" install-cas --local-build-root "${LBR}" -o "${INSTALL_DIR_UNRESOLVED}" \ + "${UNRESOLVED_TREE}::t" 2>&1 +test "$(cat "${INSTALL_DIR_UNRESOLVED}/${DATA_PATH}")" = "${TEST_DATA}" +test "$(readlink "${INSTALL_DIR_UNRESOLVED}/${NON_UPWARDS_LINK_PATH}")" = "${NON_UPWARDS_LINK_TARGET}" + + +### +# Test archives with confined symlinks (upwards and non-upwards) +## + +echo === root with ignored special entries === + +# Read tree from repository configuration +TREE=$(jq -r '.repositories.foo_v2_ignore_special.workspace_root[1]' "${CONF}") +echo Tree is "${TREE}" + +# As the tree is known to just (in the git CAS), we should be able to install +# it with install-cas +"${JUST}" install-cas --local-build-root "${LBR}" -o "${INSTALL_DIR_SPECIAL_IGNORE}" \ + "${TREE}::t" 2>&1 +test "$(cat "${INSTALL_DIR_SPECIAL_IGNORE}/${DATA_PATH}")" = "${TEST_DATA}" +[ ! -e "${INSTALL_DIR_SPECIAL_IGNORE}/${NON_UPWARDS_LINK_PATH}" ] # symlink should be missing +[ ! -e "${INSTALL_DIR_SPECIAL_IGNORE}/${UPWARDS_LINK_PATH}" ] # symlink should be missing +[ ! -e "${INSTALL_DIR_SPECIAL_IGNORE}/${INDIRECT_LINK_PATH}" ] # symlink should be missing + + +echo === root with partially resolved symlinks === + +# Read tree from repository configuration +TREE=$(jq -r '.repositories.foo_v2_resolve_partially.workspace_root[1]' "${CONF}") +echo Tree is "${TREE}" + +# As the tree is known to just (in the git CAS), we should be able to install +# it with install-cas +"${JUST}" install-cas --local-build-root "${LBR}" -o "${INSTALL_DIR_SPECIAL_PARTIAL}" \ + "${TREE}::t" 2>&1 +test "$(cat "${INSTALL_DIR_SPECIAL_PARTIAL}/${DATA_PATH}")" = "${TEST_DATA}" +# non-upwards link remains as-is +test "$(readlink "${INSTALL_DIR_SPECIAL_PARTIAL}/${NON_UPWARDS_LINK_PATH}")" = "${NON_UPWARDS_LINK_TARGET}" +# upwards link is resolved, in this case to a tree +[ -d "${INSTALL_DIR_SPECIAL_PARTIAL}/${UPWARDS_LINK_PATH}" ] +test "$(cat "${INSTALL_DIR_SPECIAL_PARTIAL}/${UPWARDS_LINK_PATH}/data.txt")" = "${TEST_DATA}" +# indirect link is non-upwards, so it should remain as-is +test "$(readlink "${INSTALL_DIR_SPECIAL_PARTIAL}/${INDIRECT_LINK_PATH}")" = "${INDIRECT_LINK_TARGET}" + + +echo === root with fully resolved symlinks === + +# Read tree from repository configuration +TREE=$(jq -r '.repositories.foo_v2_resolve_completely.workspace_root[1]' "${CONF}") +echo Tree is "${TREE}" + +# As the tree is known to just (in the git CAS), we should be able to install +# it with install-cas +"${JUST}" install-cas --local-build-root "${LBR}" -o "${INSTALL_DIR_SPECIAL_COMPLETE}" \ + "${TREE}::t" 2>&1 +test "$(cat "${INSTALL_DIR_SPECIAL_COMPLETE}/${DATA_PATH}")" = "${TEST_DATA}" +# all links get resolved +test "$(cat "${INSTALL_DIR_SPECIAL_COMPLETE}/${NON_UPWARDS_LINK_PATH}")" = "${TEST_DATA}" +[ -d "${INSTALL_DIR_SPECIAL_COMPLETE}/${UPWARDS_LINK_PATH}" ] +test "$(cat "${INSTALL_DIR_SPECIAL_COMPLETE}/${UPWARDS_LINK_PATH}/data.txt")" = "${TEST_DATA}" +test "$(cat "${INSTALL_DIR_SPECIAL_COMPLETE}/${INDIRECT_LINK_PATH}")" = "${TEST_DATA}" + +echo OK diff --git a/test/end-to-end/just-mr/install-roots.sh b/test/end-to-end/just-mr/install-roots.sh index aab0c90b9..6ff664019 100644 --- a/test/end-to-end/just-mr/install-roots.sh +++ b/test/end-to-end/just-mr/install-roots.sh @@ -20,8 +20,7 @@ readonly JUST="${PWD}/bin/tool-under-test" readonly JUST_MR="${PWD}/bin/mr-tool-under-test" readonly DISTDIR="${TEST_TMPDIR}/distfiles" readonly LBR="${TEST_TMPDIR}/local-build-root" -readonly INSTALL_DIR_1="${TEST_TMPDIR}/installation-target-1" -readonly INSTALL_DIR_2="${TEST_TMPDIR}/installation-target-2" +readonly INSTALL_DIR="${TEST_TMPDIR}/installation-target" readonly TEST_DATA="The content of the data file in foo" readonly TEST_PATH="bar/baz" @@ -51,18 +50,9 @@ cat > repos.json <&1 -test "$(cat "${INSTALL_DIR_1}/${TEST_PATH}/data.txt")" = "${TEST_DATA}" -test "$(readlink "${INSTALL_DIR_1}/${TEST_PATH}/link")" = "${LINK_TARGET}" - -echo === ignore_special root === - -# Read tree from repository configuration -TREE=$(jq -r '.repositories.foo_ignore_special.workspace_root[1]' "${CONF}") -echo Tree is "${TREE}" - -# As the tree is known to just (in the git CAS), we should be able to install -# it with install-cas -"${JUST}" install-cas --local-build-root "${LBR}" -o "${INSTALL_DIR_2}" \ +"${JUST}" install-cas --local-build-root "${LBR}" -o "${INSTALL_DIR}" \ "${TREE}::t" 2>&1 -test "$(cat "${INSTALL_DIR_2}/${TEST_PATH}/data.txt")" = "${TEST_DATA}" -[ ! -e "${INSTALL_DIR_2}/${TEST_PATH}/link" ] # symlink should be missing +test "$(cat "${INSTALL_DIR}/${TEST_PATH}/data.txt")" = "${TEST_DATA}" +test "$(readlink "${INSTALL_DIR}/${TEST_PATH}/link")" = "${LINK_TARGET}" echo OK diff --git a/test/end-to-end/just-mr/just-mr.test.sh b/test/end-to-end/just-mr/just-mr.test.sh index cdd0ace41..d82e53090 100644 --- a/test/end-to-end/just-mr/just-mr.test.sh +++ b/test/end-to-end/just-mr/just-mr.test.sh @@ -53,10 +53,15 @@ readonly TGZ_REPO_SHA256=$(sha256sum tgz_repo.tar.gz | awk '{print $1}') readonly TGZ_REPO_SHA512=$(sha512sum tgz_repo.tar.gz | awk '{print $1}') echo "Set up file repos" +# add files mkdir -p "${FILE_ROOT}/test_dir1" echo test > "${FILE_ROOT}/test_dir1/test_file" mkdir -p "${FILE_ROOT}/test_dir2/test_dir3" echo test > "${FILE_ROOT}/test_dir2/test_dir3/test_file" +# add resolvable non-upwards symlink +ln -s test_file "${FILE_ROOT}/test_dir1/nonupwards" +# add resolvable upwards symlink +ln -s ../test_dir3/test_file "${FILE_ROOT}/test_dir2/test_dir3/upwards" echo "Set up local git repo" # NOTE: Libgit2 has no support for Git bundles yet @@ -67,6 +72,7 @@ mkdir -p foo/bar echo foo > foo.txt echo bar > foo/bar.txt echo baz > foo/bar/baz.txt +ln -s dummy foo/link EOF # set up git repo ( @@ -122,6 +128,20 @@ cat > test-repos.json < test-repos.json < test-repos.json < test-repos.json < Date: Tue, 11 Jul 2023 17:17:44 +0200 Subject: [PATCH 29/97] doc: Extended treatment of symlinks in roots and trees We now allow restrictive use of upwards symbolic links when importing filesystem directories and archives to git. --- doc/concepts/symlinks.md | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/doc/concepts/symlinks.md b/doc/concepts/symlinks.md index 34691ef32..2d33e3886 100644 --- a/doc/concepts/symlinks.md +++ b/doc/concepts/symlinks.md @@ -81,7 +81,7 @@ Our treatment of symbolic links ### "Ignore-special" roots -To allow working with source trees containing symbolic links, all the existing +For cases where we simply have no need for special entries, all the existing roots have "ignore-special" versions thereof. In such a root (regardless whether file based, or `git`-tree based), everything not a file or a directory is pretended to be absent. For any @@ -97,7 +97,7 @@ content fixed and hence eligible for target-level caching. ### Non-upwards relative symlinks as first-class objects -Finally, a restricted form of symlinks, more precisely *relative* +A restricted form of symlinks, more precisely *relative* *non-upwards symbolic links*, exist as first-class objects. That is, a new artifact type (besides blobs and trees) for relative non-upwards symbolic links has been introduced. Like any other artifact, @@ -117,3 +117,25 @@ non-upwards) symbolic links are, in many aspects, simple files with elevated permissions. As such, they locally use the existing file CAS. Remotely, the existing execution protocol already allows the handling of symbolic links via corresponding Protobuf messages, therefore no extensions are needed. + +### Import resolved `git`-trees + +Finally, to be as flexible as possible in handling external repositories with +(possibly) upwards symbolic links, we allow filesystem directories and archives +to be imported also as partially or completely resolved `git`-trees. + +In a *partially resolved tree*, all relative upwards symbolic links confined to +the tree get resolved, i.e., replaced by a copy of the entry they point to, if +existing, or removed otherwise. This of course leaves relative non-upwards +symbolic links in the `git`-tree, as they are supported objects. + +Alternatively, in a *completely resolved tree*, all relative symbolic links +confined to the tree (whether upwards or not) get resolved, resulting in a +`git`-tree free of all symbolic links. + +For reasons already described, absolute symbolic are never supported. + +As this process acts directly at the repository level, the resulting roots +remain cacheable and their trees accessible in constant time. Moreover, to +increase the chances of cache hits in `just-mr`, not only the resulting +resolved trees are stored, but also the original, unresolved ones. From 776347cd196ab16eec72e186dcf7f86dd620b42b Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Mon, 24 Jul 2023 15:05:33 +0200 Subject: [PATCH 30/97] man: Add 'special' pragma to just-mr repository config --- share/man/just-mr-repository-config.5.md | 39 ++++++++---------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/share/man/just-mr-repository-config.5.md b/share/man/just-mr-repository-config.5.md index 350de5d56..60a115ef0 100644 --- a/share/man/just-mr-repository-config.5.md +++ b/share/man/just-mr-repository-config.5.md @@ -40,11 +40,6 @@ The following fields are supported: - *`"path"`* provides the root directory containing the source files. This entry is mandatory. - - *`"ignore_special"`* is a boolean signaling that the resulting root - should ignore all special (i.e., neither file, executable, nor tree) - entries. Defaults to *`false`* if missing or not evaluating to - *`true`*. - ### *`"archive"`* / *`"zip"`* They define as workspace root a remote archive. The only difference @@ -76,11 +71,6 @@ The following fields are supported: This entry is optional. If missing, the root directory of the archive is used. - - *`"ignore_special"`* is a boolean signaling that the resulting root - should ignore all special (i.e., neither file, executable, nor tree) - entries. Defaults to *`false`* if missing or not evaluating to - *`true`*. - ### *`"git"`* It defines as workspace root a part of a Git repository. @@ -100,11 +90,6 @@ The following fields are supported: files. This entry is optional. If missing, the root directory of the Git repository is used. - - *`"ignore_special"`* is a boolean signaling that the resulting root - should ignore all special (i.e., neither file, executable, nor tree) - entries. Defaults to *`false`* if missing or not evaluating to - *`true`*. - ### *`"git tree"`* It defines as workspace root a known Git tree obtainable by a generic @@ -123,11 +108,6 @@ The following fields are supported: - *`"env"`* provides a map of envariables to be set for executing the command. - - *`"ignore_special"`* is a boolean signaling that the resulting root - should ignore all special (i.e., neither file, executable, nor tree) - entries. Defaults to *`false`* if missing or not evaluating to - *`true`*. - ### *`"distdir"`* It defines as workspace root a directory with the distribution archives @@ -139,11 +119,6 @@ The following fields are supported: - *`"repositories"`* provides a list of global names of repositories. This entry is mandatory. - - *`"ignore_special"`* is a boolean signaling that the resulting root - should ignore all special (i.e., neither file, executable, nor tree) - entries. Defaults to *`false`* if missing or not evaluating to - *`true`*. - ### Additional keys The key *`"pragma"`* is reserved for type-specific repository directives @@ -151,13 +126,23 @@ which alter the workspace root. It is given as a JSON object. The different workspace roots might support different keys for this object; unsupported keys are always ignored. -For a *`"file"`* workspace root, the pragma key *`"to_git"`* is -supported. If its value is *`true`*, it indicates that the workspace +For a *`"file"`* workspace root the pragma key *`"to_git"`* is +supported. If its value is *`true`* then it indicates that the workspace root should be returned as a Git tree. If the root directory is already part of a Git repository, its Git tree identifier is used; otherwise, the workspace root will be realized as a Git tree in the Git repository in **`just`**'s local build root. +For all workspace roots except *`"distdir"`* the pragma key *`"special"`* is +supported. If its value is *`"ignore"`* then it indicates that the workspace +root should ignore all special (i.e., neither file, executable, nor tree) +entries. For a *`"file"`* workspace root or for an *`"archive"`* workspace root +a value of *`"resolve-completely"`* indicates that the workspace root should +resolve all confined relative symbolic links, while a value of +*`"resolve-partially"`* indicates that the workspace root should resolve only +the confined relative upwards symbolic links; for a *`"file"`* workspace root +these two values imply *`"to_git"`* is *`true`*. + Repository description ---------------------- From 63ff75d2818a3613b01c5ad322802c305d2282f8 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Tue, 25 Jul 2023 15:37:53 +0200 Subject: [PATCH 31/97] just execute: Fix uncollected upwards symlinks Upwards symlinks should still be collected from actions, even if only the non-upwards symlinks are supported artifact types. The client side is thus the one responsible with enforcing the non-upwardness condition. --- .../execution_api/bazel_msg/bazel_blob.hpp | 2 +- .../bazel_msg/bazel_msg_factory.cpp | 11 +++--- .../execution_api/local/local_action.cpp | 36 +++++++------------ src/buildtool/file_system/file_root.hpp | 10 +++--- .../file_system/file_system_manager.hpp | 26 +++++++++----- 5 files changed, 43 insertions(+), 42 deletions(-) diff --git a/src/buildtool/execution_api/bazel_msg/bazel_blob.hpp b/src/buildtool/execution_api/bazel_msg/bazel_blob.hpp index ed82d92d5..944bb2d8a 100644 --- a/src/buildtool/execution_api/bazel_msg/bazel_blob.hpp +++ b/src/buildtool/execution_api/bazel_msg/bazel_blob.hpp @@ -39,7 +39,7 @@ struct BazelBlob { /// given path. [[nodiscard]] static inline auto CreateBlobFromPath( std::filesystem::path const& fpath) noexcept -> std::optional { - auto const type = FileSystemManager::Type(fpath); + auto const type = FileSystemManager::Type(fpath, /*allow_upwards=*/true); if (not type) { return std::nullopt; } diff --git a/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.cpp b/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.cpp index 9104b94db..145b51cd6 100644 --- a/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.cpp +++ b/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.cpp @@ -600,8 +600,7 @@ auto BazelMsgFactory::CreateDirectoryDigestFromLocalTree( if (IsSymlinkObject(type)) { // create and store symlink auto content = FileSystemManager::ReadSymlink(full_name); - if (content and PathIsNonUpwards(*content) and - store_symlink(*content)) { + if (content and store_symlink(*content)) { symlinks.emplace_back( CreateSymlinkNode(name.string(), *content, {})); return true; @@ -628,7 +627,8 @@ auto BazelMsgFactory::CreateDirectoryDigestFromLocalTree( return false; }; - if (FileSystemManager::ReadDirectory(root, dir_reader)) { + if (FileSystemManager::ReadDirectory( + root, dir_reader, /*allow_upwards=*/true)) { auto dir = CreateDirectory(files, dirs, symlinks, {}); if (auto bytes = SerializeMessage(dir)) { try { @@ -678,7 +678,7 @@ auto BazelMsgFactory::CreateGitTreeDigestFromLocalTree( try { if (IsSymlinkObject(type)) { auto content = FileSystemManager::ReadSymlink(full_name); - if (content and PathIsNonUpwards(*content)) { + if (content) { if (auto digest = store_symlink(*content)) { if (auto raw_id = FromHexString( NativeSupport::Unprefix(digest->hash()))) { @@ -716,7 +716,8 @@ auto BazelMsgFactory::CreateGitTreeDigestFromLocalTree( return false; }; - if (FileSystemManager::ReadDirectory(root, dir_reader)) { + if (FileSystemManager::ReadDirectory( + root, dir_reader, /*allow_upwards=*/true)) { if (auto tree = GitRepo::CreateShallowTree(entries)) { try { if (auto digest = store_tree(tree->second, entries)) { diff --git a/src/buildtool/execution_api/local/local_action.cpp b/src/buildtool/execution_api/local/local_action.cpp index d3ad7fd9c..3fcd8fe77 100644 --- a/src/buildtool/execution_api/local/local_action.cpp +++ b/src/buildtool/execution_api/local/local_action.cpp @@ -268,15 +268,14 @@ auto LocalAction::CollectOutputFileOrSymlink( std::string const& local_path) const noexcept -> std::optional { auto file_path = exec_path / local_path; - auto type = FileSystemManager::Type(file_path); + auto type = FileSystemManager::Type(file_path, /*allow_upwards=*/true); if (not type) { Logger::Log(LogLevel::Error, "expected known type at {}", local_path); return std::nullopt; } if (IsSymlinkObject(*type)) { auto content = FileSystemManager::ReadSymlink(file_path); - if (content and PathIsNonUpwards(*content) and - storage_->CAS().StoreBlob(*content)) { + if (content and storage_->CAS().StoreBlob(*content)) { auto out_symlink = bazel_re::OutputSymlink{}; out_symlink.set_path(local_path); out_symlink.set_target(*content); @@ -308,15 +307,14 @@ auto LocalAction::CollectOutputDirOrSymlink( std::string const& local_path) const noexcept -> std::optional { auto dir_path = exec_path / local_path; - auto type = FileSystemManager::Type(dir_path); + auto type = FileSystemManager::Type(dir_path, /*allow_upwards=*/true); if (not type) { Logger::Log(LogLevel::Error, "expected known type at {}", local_path); return std::nullopt; } if (IsSymlinkObject(*type)) { auto content = FileSystemManager::ReadSymlink(dir_path); - if (content and PathIsNonUpwards(*content) and - storage_->CAS().StoreBlob(*content)) { + if (content and storage_->CAS().StoreBlob(*content)) { auto out_symlink = bazel_re::OutputSymlink{}; out_symlink.set_path(local_path); out_symlink.set_target(*content); @@ -354,15 +352,10 @@ auto LocalAction::CollectAndStoreOutputs( } if (std::holds_alternative(*out)) { auto out_symlink = std::get(*out); - auto const& target = out_symlink.target(); - if (not PathIsNonUpwards(target)) { - logger_.Emit(LogLevel::Error, - "collected upwards output symlink {}", - path); - return false; - } - logger_.Emit( - LogLevel::Trace, " - symlink {}: {}", path, target); + logger_.Emit(LogLevel::Trace, + " - symlink {}: {}", + path, + out_symlink.target()); result->mutable_output_file_symlinks()->Add( std::move(out_symlink)); } @@ -383,15 +376,10 @@ auto LocalAction::CollectAndStoreOutputs( } if (std::holds_alternative(*out)) { auto out_symlink = std::get(*out); - auto const& target = out_symlink.target(); - if (not PathIsNonUpwards(target)) { - logger_.Emit(LogLevel::Error, - "collected upwards output symlink {}", - path); - return false; - } - logger_.Emit( - LogLevel::Trace, " - symlink {}: {}", path, target); + logger_.Emit(LogLevel::Trace, + " - symlink {}: {}", + path, + out_symlink.target()); result->mutable_output_file_symlinks()->Add( std::move(out_symlink)); } diff --git a/src/buildtool/file_system/file_root.hpp b/src/buildtool/file_system/file_root.hpp index b332f9cd6..feb5519af 100644 --- a/src/buildtool/file_system/file_root.hpp +++ b/src/buildtool/file_system/file_root.hpp @@ -362,7 +362,7 @@ class FileRoot { // std::holds_alternative(root_) == true auto root_path = std::get(root_) / path; auto exists = FileSystemManager::Exists(root_path); - auto type = FileSystemManager::Type(root_path); + auto type = FileSystemManager::Type(root_path, /*allow_upwards=*/true); return (ignore_special_ ? exists and type and IsNonSpecialObject(*type) : exists); } @@ -431,7 +431,8 @@ class FileRoot { return std::nullopt; } auto full_path = std::get(root_) / file_path; - if (auto type = FileSystemManager::Type(full_path)) { + if (auto type = + FileSystemManager::Type(full_path, /*allow_upwards=*/true)) { return IsSymlinkObject(*type) ? FileSystemManager::ReadSymlink(full_path) : FileSystemManager::ReadFile(full_path); @@ -461,6 +462,7 @@ class FileRoot { map.emplace(name.string(), type); return true; }, + /*allow_upwards=*/false, ignore_special_)) { return DirectoryEntries{std::move(map)}; } @@ -486,8 +488,8 @@ class FileRoot { } return std::nullopt; } - auto type = - FileSystemManager::Type(std::get(root_) / file_path); + auto type = FileSystemManager::Type( + std::get(root_) / file_path, /*allow_upwards=*/true); if (type and IsBlobObject(*type)) { return type; } diff --git a/src/buildtool/file_system/file_system_manager.hpp b/src/buildtool/file_system/file_system_manager.hpp index 04f84cf4a..6b6b675fb 100644 --- a/src/buildtool/file_system/file_system_manager.hpp +++ b/src/buildtool/file_system/file_system_manager.hpp @@ -588,7 +588,9 @@ class FileSystemManager { } /// \brief Gets type of object in path according to file system - [[nodiscard]] static auto Type(std::filesystem::path const& path) noexcept + /// \param allow_upwards Do not enforce non-upwardness in symlinks. + [[nodiscard]] static auto Type(std::filesystem::path const& path, + bool allow_upwards = false) noexcept -> std::optional { try { auto const status = std::filesystem::symlink_status(path); @@ -602,7 +604,7 @@ class FileSystemManager { return ObjectType::Tree; } if (std::filesystem::is_symlink(status) and - IsNonUpwardsSymlink(path)) { + (allow_upwards or IsNonUpwardsSymlink(path))) { return ObjectType::Symlink; } if (std::filesystem::exists(status)) { @@ -680,9 +682,11 @@ class FileSystemManager { /// \brief Read a filesystem directory tree. /// \param ignore_special If true, do not error out when encountering /// symlinks. + /// \param allow_upwards If true, do not enforce non-upwardness of symlinks. [[nodiscard]] static auto ReadDirectory( std::filesystem::path const& dir, ReadDirEntryFunc const& read_entry, + bool allow_upwards = false, bool ignore_special = false) noexcept -> bool { try { for (auto const& entry : std::filesystem::directory_iterator{dir}) { @@ -707,14 +711,20 @@ class FileSystemManager { // if not already ignored, check symlinks and only add the // non-upwards ones else if (std::filesystem::is_symlink(status)) { - if (IsNonUpwardsSymlink(entry)) { - type = ObjectType::Symlink; + if (not allow_upwards) { + if (IsNonUpwardsSymlink(entry)) { + type = ObjectType::Symlink; + } + else { + Logger::Log( + LogLevel::Error, + "unsupported upwards symlink dir entry {}", + entry.path().string()); + return false; + } } else { - Logger::Log(LogLevel::Error, - "unsupported upwards symlink dir entry {}", - entry.path().string()); - return false; + type = ObjectType::Symlink; } } else { From f75a44b00398a4ed403535380ead4d3e9e769a7c Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Thu, 27 Jul 2023 11:26:15 +0200 Subject: [PATCH 32/97] external: Fix absl export targets --- etc/import/absl/algorithm/TARGETS.absl | 4 +- etc/import/absl/base/TARGETS.absl | 20 +++++++++ etc/import/absl/cleanup/TARGETS.absl | 20 +++++++++ etc/import/absl/container/TARGETS.absl | 4 +- etc/import/absl/functional/TARGETS.absl | 60 +++++++++++++++++++++++++ etc/import/absl/memory/TARGETS.absl | 20 +++++++++ etc/import/absl/meta/TARGETS.absl | 20 +++++++++ etc/import/absl/strings/TARGETS.absl | 12 ++--- etc/import/absl/utility/TARGETS.absl | 20 +++++++++ 9 files changed, 170 insertions(+), 10 deletions(-) diff --git a/etc/import/absl/algorithm/TARGETS.absl b/etc/import/absl/algorithm/TARGETS.absl index 7114f67ac..e4af8e928 100644 --- a/etc/import/absl/algorithm/TARGETS.absl +++ b/etc/import/absl/algorithm/TARGETS.absl @@ -5,9 +5,9 @@ , "hdrs": ["algorithm.h"] , "deps": [["absl/base", "config"]] } -, "algorithm_container": +, "container": { "type": ["@", "rules", "CC", "library"] - , "name": ["algorithm_container"] + , "name": ["container"] , "stage": ["absl", "algorithm"] , "hdrs": ["container.h"] , "deps": diff --git a/etc/import/absl/base/TARGETS.absl b/etc/import/absl/base/TARGETS.absl index 664955e49..a433f6889 100644 --- a/etc/import/absl/base/TARGETS.absl +++ b/etc/import/absl/base/TARGETS.absl @@ -126,6 +126,26 @@ , "hdrs": ["config.h", "options.h", "policy_checks.h"] } , "core_headers": + { "type": "export" + , "target": "core_headers_internal" + , "flexible_config": + [ "ADD_CFLAGS" + , "ADD_CXXFLAGS" + , "AR" + , "ARCH" + , "CC" + , "CFLAGS" + , "COMPILER_FAMILY" + , "CXX" + , "CXXFLAGS" + , "DEBUG" + , "ENV" + , "HOST_ARCH" + , "OS" + , "TARGET_ARCH" + ] + } +, "core_headers_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["core_headers"] , "stage": ["absl", "base"] diff --git a/etc/import/absl/cleanup/TARGETS.absl b/etc/import/absl/cleanup/TARGETS.absl index 75e5ad21c..f6f9b1112 100644 --- a/etc/import/absl/cleanup/TARGETS.absl +++ b/etc/import/absl/cleanup/TARGETS.absl @@ -1,4 +1,24 @@ { "cleanup": + { "type": "export" + , "target": "cleanup_internal_" + , "flexible_config": + [ "ADD_CFLAGS" + , "ADD_CXXFLAGS" + , "AR" + , "ARCH" + , "CC" + , "CFLAGS" + , "COMPILER_FAMILY" + , "CXX" + , "CXXFLAGS" + , "DEBUG" + , "ENV" + , "HOST_ARCH" + , "OS" + , "TARGET_ARCH" + ] + } +, "cleanup_internal_": { "type": ["@", "rules", "CC", "library"] , "name": ["cleanup"] , "stage": ["absl", "cleanup"] diff --git a/etc/import/absl/container/TARGETS.absl b/etc/import/absl/container/TARGETS.absl index 10a0b7b62..ace89f596 100644 --- a/etc/import/absl/container/TARGETS.absl +++ b/etc/import/absl/container/TARGETS.absl @@ -122,7 +122,7 @@ [ "container_memory" , "hash_function_defaults" , "raw_hash_map" - , ["absl/algorithm", "algorithm_container"] + , ["absl/algorithm", "container"] , ["absl/base", "core_headers"] , ["absl/memory", "memory"] ] @@ -256,7 +256,7 @@ [ "container_memory" , "hash_function_defaults" , "raw_hash_set" - , ["absl/algorithm", "algorithm_container"] + , ["absl/algorithm", "container"] , ["absl/base", "core_headers"] , ["absl/memory", "memory"] ] diff --git a/etc/import/absl/functional/TARGETS.absl b/etc/import/absl/functional/TARGETS.absl index 3862316b1..cbf5adccd 100644 --- a/etc/import/absl/functional/TARGETS.absl +++ b/etc/import/absl/functional/TARGETS.absl @@ -1,4 +1,24 @@ { "function_ref": + { "type": "export" + , "target": "function_ref_internal" + , "flexible_config": + [ "ADD_CFLAGS" + , "ADD_CXXFLAGS" + , "AR" + , "ARCH" + , "CC" + , "CFLAGS" + , "COMPILER_FAMILY" + , "CXX" + , "CXXFLAGS" + , "DEBUG" + , "ENV" + , "HOST_ARCH" + , "OS" + , "TARGET_ARCH" + ] + } +, "function_ref_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["function_ref"] , "stage": ["absl", "functional"] @@ -10,6 +30,26 @@ ] } , "any_invocable": + { "type": "export" + , "target": "any_invocable_internal" + , "flexible_config": + [ "ADD_CFLAGS" + , "ADD_CXXFLAGS" + , "AR" + , "ARCH" + , "CC" + , "CFLAGS" + , "COMPILER_FAMILY" + , "CXX" + , "CXXFLAGS" + , "DEBUG" + , "ENV" + , "HOST_ARCH" + , "OS" + , "TARGET_ARCH" + ] + } +, "any_invocable_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["any_invocable"] , "stage": ["absl", "functional"] @@ -23,6 +63,26 @@ ] } , "bind_front": + { "type": "export" + , "target": "bind_front_internal" + , "flexible_config": + [ "ADD_CFLAGS" + , "ADD_CXXFLAGS" + , "AR" + , "ARCH" + , "CC" + , "CFLAGS" + , "COMPILER_FAMILY" + , "CXX" + , "CXXFLAGS" + , "DEBUG" + , "ENV" + , "HOST_ARCH" + , "OS" + , "TARGET_ARCH" + ] + } +, "bind_front_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["bind_front"] , "stage": ["absl", "functional"] diff --git a/etc/import/absl/memory/TARGETS.absl b/etc/import/absl/memory/TARGETS.absl index 2187f004f..c8ca115ce 100644 --- a/etc/import/absl/memory/TARGETS.absl +++ b/etc/import/absl/memory/TARGETS.absl @@ -1,4 +1,24 @@ { "memory": + { "type": "export" + , "target": "memory_internal" + , "flexible_config": + [ "ADD_CFLAGS" + , "ADD_CXXFLAGS" + , "AR" + , "ARCH" + , "CC" + , "CFLAGS" + , "COMPILER_FAMILY" + , "CXX" + , "CXXFLAGS" + , "DEBUG" + , "ENV" + , "HOST_ARCH" + , "OS" + , "TARGET_ARCH" + ] + } +, "memory_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["memory"] , "stage": ["absl", "memory"] diff --git a/etc/import/absl/meta/TARGETS.absl b/etc/import/absl/meta/TARGETS.absl index 56ed36017..4ccf97c9c 100644 --- a/etc/import/absl/meta/TARGETS.absl +++ b/etc/import/absl/meta/TARGETS.absl @@ -1,4 +1,24 @@ { "type_traits": + { "type": "export" + , "target": "type_traits_internal" + , "flexible_config": + [ "ADD_CFLAGS" + , "ADD_CXXFLAGS" + , "AR" + , "ARCH" + , "CC" + , "CFLAGS" + , "COMPILER_FAMILY" + , "CXX" + , "CXXFLAGS" + , "DEBUG" + , "ENV" + , "HOST_ARCH" + , "OS" + , "TARGET_ARCH" + ] + } +, "type_traits_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["type_traits"] , "stage": ["absl", "meta"] diff --git a/etc/import/absl/strings/TARGETS.absl b/etc/import/absl/strings/TARGETS.absl index 819914aed..6dfbcb94c 100644 --- a/etc/import/absl/strings/TARGETS.absl +++ b/etc/import/absl/strings/TARGETS.absl @@ -1,6 +1,6 @@ { "strings": { "type": "export" - , "target": "strings_internal_" + , "target": "strings_internal" , "flexible_config": [ "ADD_CFLAGS" , "ADD_CXXFLAGS" @@ -18,7 +18,7 @@ , "TARGET_ARCH" ] } -, "strings_internal_": +, "strings_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["strings"] , "stage": ["absl", "strings"] @@ -66,7 +66,7 @@ , "internal/memutil.h" ] , "deps": - [ "strings_internal" + [ "internal" , ["absl/base", "base"] , ["absl/base", "config"] , ["absl/base", "core_headers"] @@ -79,9 +79,9 @@ , ["absl/numeric", "int128"] ] } -, "strings_internal": +, "internal": { "type": ["@", "rules", "CC", "library"] - , "name": ["strings_internal"] + , "name": ["internal"] , "stage": ["absl", "strings"] , "hdrs": [ "internal/char_map.h" @@ -133,7 +133,7 @@ , "cordz_statistics" , "cordz_update_scope" , "cordz_update_tracker" - , "strings_internal" + , "internal" , "str_format" , "strings" , ["absl/base", "base"] diff --git a/etc/import/absl/utility/TARGETS.absl b/etc/import/absl/utility/TARGETS.absl index 65c5ac544..91197dcc9 100644 --- a/etc/import/absl/utility/TARGETS.absl +++ b/etc/import/absl/utility/TARGETS.absl @@ -1,4 +1,24 @@ { "utility": + { "type": "export" + , "target": "utility_internal" + , "flexible_config": + [ "ADD_CFLAGS" + , "ADD_CXXFLAGS" + , "AR" + , "ARCH" + , "CC" + , "CFLAGS" + , "COMPILER_FAMILY" + , "CXX" + , "CXXFLAGS" + , "DEBUG" + , "ENV" + , "HOST_ARCH" + , "OS" + , "TARGET_ARCH" + ] + } +, "utility_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["utility"] , "stage": ["absl", "utility"] From 02f3e87e60783acfcce3fbaca7c9a75b9fc6d9f7 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Wed, 26 Jul 2023 16:30:11 +0200 Subject: [PATCH 33/97] external: Update abseil to LTS 20230125.3 Also adds the extra targets needed by protobuf v23.4, which now has a dependency on abseil, and by grpc v1.56.2. --- etc/import/absl/algorithm/TARGETS.absl | 20 +++ etc/import/absl/base/TARGETS.absl | 40 +++++ etc/import/absl/container/TARGETS.absl | 47 +++++ etc/import/absl/debugging/TARGETS.absl | 14 ++ etc/import/absl/flags/TARGETS.absl | 164 +++++++++++++++++ etc/import/absl/log/TARGETS.absl | 178 +++++++++++++++++++ etc/import/absl/log/internal/TARGETS.absl | 204 ++++++++++++++++++++++ etc/import/absl/numeric/TARGETS.absl | 20 +++ etc/import/absl/strings/TARGETS.absl | 20 +++ etc/import/absl/types/TARGETS.absl | 7 + etc/repos.json | 10 +- 11 files changed, 719 insertions(+), 5 deletions(-) create mode 100644 etc/import/absl/flags/TARGETS.absl create mode 100644 etc/import/absl/log/TARGETS.absl create mode 100644 etc/import/absl/log/internal/TARGETS.absl diff --git a/etc/import/absl/algorithm/TARGETS.absl b/etc/import/absl/algorithm/TARGETS.absl index e4af8e928..810dfa02c 100644 --- a/etc/import/absl/algorithm/TARGETS.absl +++ b/etc/import/absl/algorithm/TARGETS.absl @@ -6,6 +6,26 @@ , "deps": [["absl/base", "config"]] } , "container": + { "type": "export" + , "target": "container_internal" + , "flexible_config": + [ "ADD_CFLAGS" + , "ADD_CXXFLAGS" + , "AR" + , "ARCH" + , "CC" + , "CFLAGS" + , "COMPILER_FAMILY" + , "CXX" + , "CXXFLAGS" + , "DEBUG" + , "ENV" + , "HOST_ARCH" + , "OS" + , "TARGET_ARCH" + ] + } +, "container_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["container"] , "stage": ["absl", "algorithm"] diff --git a/etc/import/absl/base/TARGETS.absl b/etc/import/absl/base/TARGETS.absl index a433f6889..a4d5b0555 100644 --- a/etc/import/absl/base/TARGETS.absl +++ b/etc/import/absl/base/TARGETS.absl @@ -169,6 +169,26 @@ , "deps": ["base_internal", "config"] } , "dynamic_annotations": + { "type": "export" + , "target": "dynamic_annotations_internal" + , "flexible_config": + [ "ADD_CFLAGS" + , "ADD_CXXFLAGS" + , "AR" + , "ARCH" + , "CC" + , "CFLAGS" + , "COMPILER_FAMILY" + , "CXX" + , "CXXFLAGS" + , "DEBUG" + , "ENV" + , "HOST_ARCH" + , "OS" + , "TARGET_ARCH" + ] + } +, "dynamic_annotations_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["dynamic_annotations"] , "stage": ["absl", "base"] @@ -176,6 +196,26 @@ , "deps": ["config", "core_headers"] } , "log_severity": + { "type": "export" + , "target": "log_severity_internal" + , "flexible_config": + [ "ADD_CFLAGS" + , "ADD_CXXFLAGS" + , "AR" + , "ARCH" + , "CC" + , "CFLAGS" + , "COMPILER_FAMILY" + , "CXX" + , "CXXFLAGS" + , "DEBUG" + , "ENV" + , "HOST_ARCH" + , "OS" + , "TARGET_ARCH" + ] + } +, "log_severity_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["log_severity"] , "stage": ["absl", "base"] diff --git a/etc/import/absl/container/TARGETS.absl b/etc/import/absl/container/TARGETS.absl index ace89f596..e74c658d1 100644 --- a/etc/import/absl/container/TARGETS.absl +++ b/etc/import/absl/container/TARGETS.absl @@ -261,4 +261,51 @@ , ["absl/memory", "memory"] ] } +, "btree": + { "type": "export" + , "target": "btree_internal" + , "flexible_config": + [ "ADD_CFLAGS" + , "ADD_CXXFLAGS" + , "AR" + , "ARCH" + , "CC" + , "CFLAGS" + , "COMPILER_FAMILY" + , "CXX" + , "CXXFLAGS" + , "DEBUG" + , "ENV" + , "HOST_ARCH" + , "OS" + , "TARGET_ARCH" + ] + } +, "btree_internal": + { "type": ["@", "rules", "CC", "library"] + , "name": ["btree"] + , "stage": ["absl", "container"] + , "hdrs": + [ "btree_map.h" + , "btree_set.h" + , "internal/btree.h" + , "internal/btree_container.h" + ] + , "deps": + [ "common" + , "common_policy_traits" + , "compressed_tuple" + , "container_memory" + , "layout" + , ["absl/base", "core_headers"] + , ["absl/base", "raw_logging_internal"] + , ["absl/base", "throw_delegate"] + , ["absl/memory", "memory"] + , ["absl/meta", "type_traits"] + , ["absl/strings", "strings"] + , ["absl/strings", "cord"] + , ["absl/types", "compare"] + , ["absl/utility", "utility"] + ] + } } diff --git a/etc/import/absl/debugging/TARGETS.absl b/etc/import/absl/debugging/TARGETS.absl index 1088c7388..658ec06d3 100644 --- a/etc/import/absl/debugging/TARGETS.absl +++ b/etc/import/absl/debugging/TARGETS.absl @@ -60,6 +60,20 @@ , ["absl/strings", "strings"] ] } +, "examine_stack": + { "type": ["@", "rules", "CC", "library"] + , "name": ["log_message"] + , "stage": ["absl", "debugging"] + , "srcs": ["internal/examine_stack.cc"] + , "hdrs": ["internal/examine_stack.h"] + , "deps": + [ "stacktrace" + , "symbolize" + , ["absl/base", "config"] + , ["absl/base", "core_headers"] + , ["absl/base", "raw_logging_internal"] + ] + } , "stacktrace": { "type": ["@", "rules", "CC", "library"] , "name": ["stacktrace"] diff --git a/etc/import/absl/flags/TARGETS.absl b/etc/import/absl/flags/TARGETS.absl new file mode 100644 index 000000000..d3f9d2c39 --- /dev/null +++ b/etc/import/absl/flags/TARGETS.absl @@ -0,0 +1,164 @@ +{ "path_util": + { "type": ["@", "rules", "CC", "library"] + , "name": ["path_util"] + , "stage": ["absl", "flags"] + , "hdrs": ["internal/path_util.h"] + , "deps": [["absl/base", "config"], ["absl/strings", "strings"]] + } +, "program_name": + { "type": ["@", "rules", "CC", "library"] + , "name": ["program_name"] + , "stage": ["absl", "flags"] + , "srcs": ["internal/program_name.cc"] + , "hdrs": ["internal/program_name.h"] + , "deps": + [ "path_util" + , ["absl/base", "config"] + , ["absl/base", "core_headers"] + , ["absl/strings", "strings"] + , ["absl/synchronization", "synchronization"] + ] + } +, "config": + { "type": ["@", "rules", "CC", "library"] + , "name": ["config"] + , "stage": ["absl", "flags"] + , "srcs": ["usage_config.cc"] + , "hdrs": ["config.h", "usage_config.h"] + , "deps": + [ "path_util" + , "program_name" + , ["absl/base", "config"] + , ["absl/base", "core_headers"] + , ["absl/strings", "strings"] + , ["absl/synchronization", "synchronization"] + ] + } +, "marshalling": + { "type": ["@", "rules", "CC", "library"] + , "name": ["marshalling"] + , "stage": ["absl", "flags"] + , "srcs": ["marshalling.cc"] + , "hdrs": ["marshalling.h"] + , "deps": + [ ["absl/base", "config"] + , ["absl/base", "core_headers"] + , ["absl/base", "log_severity"] + , ["absl/strings", "strings"] + , ["absl/strings", "str_format"] + , ["absl/types", "optional"] + ] + } +, "commandlineflag_internal": + { "type": ["@", "rules", "CC", "library"] + , "name": ["commandlineflag_internal"] + , "stage": ["absl", "flags"] + , "srcs": ["internal/commandlineflag.cc"] + , "hdrs": ["internal/commandlineflag.h"] + , "deps": [["absl/base", "config"], ["absl/base", "fast_type_id"]] + } +, "commandlineflag": + { "type": ["@", "rules", "CC", "library"] + , "name": ["commandlineflag"] + , "stage": ["absl", "flags"] + , "srcs": ["commandlineflag.cc"] + , "hdrs": ["commandlineflag.h"] + , "deps": + [ "commandlineflag_internal" + , ["absl/base", "config"] + , ["absl/base", "fast_type_id"] + , ["absl/strings", "strings"] + , ["absl/types", "optional"] + ] + } +, "private_handle_accessor": + { "type": ["@", "rules", "CC", "library"] + , "name": ["private_handle_accessor"] + , "stage": ["absl", "flags"] + , "srcs": ["internal/private_handle_accessor.cc"] + , "hdrs": ["internal/private_handle_accessor.h"] + , "deps": + [ "commandlineflag" + , "commandlineflag_internal" + , ["absl/base", "config"] + , ["absl/strings", "strings"] + ] + } +, "reflection": + { "type": ["@", "rules", "CC", "library"] + , "name": ["reflection"] + , "stage": ["absl", "flags"] + , "srcs": ["reflection.cc"] + , "hdrs": ["internal/registry.h", "reflection.h"] + , "deps": + [ "commandlineflag" + , "commandlineflag_internal" + , "config" + , "private_handle_accessor" + , ["absl/base", "config"] + , ["absl/base", "core_headers"] + , ["absl/container", "flat_hash_map"] + , ["absl/strings", "strings"] + , ["absl/synchronization", "synchronization"] + ] + } +, "flag_internal": + { "type": ["@", "rules", "CC", "library"] + , "name": ["flag_internal"] + , "stage": ["absl", "flags"] + , "srcs": ["internal/flag.cc"] + , "hdrs": ["internal/flag.h", "internal/sequence_lock.h"] + , "deps": + [ "commandlineflag" + , "commandlineflag_internal" + , "config" + , "marshalling" + , "reflection" + , ["absl/base", "base"] + , ["absl/base", "config"] + , ["absl/base", "core_headers"] + , ["absl/base", "dynamic_annotations"] + , ["absl/memory", "memory"] + , ["absl/meta", "type_traits"] + , ["absl/strings", "strings"] + , ["absl/synchronization", "synchronization"] + , ["absl/utility", "utility"] + ] + } +, "flag": + { "type": "export" + , "target": "flag_" + , "flexible_config": + [ "ADD_CFLAGS" + , "ADD_CXXFLAGS" + , "AR" + , "ARCH" + , "CC" + , "CFLAGS" + , "COMPILER_FAMILY" + , "CXX" + , "CXXFLAGS" + , "DEBUG" + , "ENV" + , "HOST_ARCH" + , "OS" + , "TARGET_ARCH" + ] + } +, "flag_": + { "type": ["@", "rules", "CC", "library"] + , "name": ["flag"] + , "stage": ["absl", "flags"] + , "srcs": ["flag.cc"] + , "hdrs": ["declare.h", "flag.h", "internal/flag_msvc.inc"] + , "deps": + [ "config" + , "flag_internal" + , "reflection" + , ["absl/base", "base"] + , ["absl/base", "config"] + , ["absl/base", "core_headers"] + , ["absl/strings", "strings"] + ] + } +} diff --git a/etc/import/absl/log/TARGETS.absl b/etc/import/absl/log/TARGETS.absl new file mode 100644 index 000000000..67f0f3dc6 --- /dev/null +++ b/etc/import/absl/log/TARGETS.absl @@ -0,0 +1,178 @@ +{ "absl_check": + { "type": "export" + , "target": "absl_check_internal" + , "flexible_config": + [ "ADD_CFLAGS" + , "ADD_CXXFLAGS" + , "AR" + , "ARCH" + , "CC" + , "CFLAGS" + , "COMPILER_FAMILY" + , "CXX" + , "CXXFLAGS" + , "DEBUG" + , "ENV" + , "HOST_ARCH" + , "OS" + , "TARGET_ARCH" + ] + } +, "absl_check_internal": + { "type": ["@", "rules", "CC", "library"] + , "name": ["absl_check"] + , "stage": ["absl", "log"] + , "hdrs": ["absl_check.h"] + , "deps": [["absl/log/internal", "check_impl"]] + } +, "absl_log": + { "type": "export" + , "target": "absl_log_internal" + , "flexible_config": + [ "ADD_CFLAGS" + , "ADD_CXXFLAGS" + , "AR" + , "ARCH" + , "CC" + , "CFLAGS" + , "COMPILER_FAMILY" + , "CXX" + , "CXXFLAGS" + , "DEBUG" + , "ENV" + , "HOST_ARCH" + , "OS" + , "TARGET_ARCH" + ] + } +, "absl_log_internal": + { "type": ["@", "rules", "CC", "library"] + , "name": ["absl_log"] + , "stage": ["absl", "log"] + , "hdrs": ["absl_log.h"] + , "deps": [["absl/log/internal", "log_impl"]] + } +, "die_if_null": + { "type": "export" + , "target": "die_if_null_internal" + , "flexible_config": + [ "ADD_CFLAGS" + , "ADD_CXXFLAGS" + , "AR" + , "ARCH" + , "CC" + , "CFLAGS" + , "COMPILER_FAMILY" + , "CXX" + , "CXXFLAGS" + , "DEBUG" + , "ENV" + , "HOST_ARCH" + , "OS" + , "TARGET_ARCH" + ] + } +, "die_if_null_internal": + { "type": ["@", "rules", "CC", "library"] + , "name": ["die_if_null"] + , "stage": ["absl", "log"] + , "srcs": ["die_if_null.cc"] + , "hdrs": ["die_if_null.h"] + , "deps": + [ "log" + , ["absl/base", "config"] + , ["absl/base", "core_headers"] + , ["absl/strings", "strings"] + ] + } +, "globals": + { "type": ["@", "rules", "CC", "library"] + , "name": ["globals"] + , "stage": ["absl", "log"] + , "srcs": ["globals.cc"] + , "hdrs": ["globals.h"] + , "deps": + [ ["absl/base", "atomic_hook"] + , ["absl/base", "config"] + , ["absl/base", "core_headers"] + , ["absl/base", "log_severity"] + , ["absl/hash", "hash"] + , ["absl/strings", "strings"] + ] + } +, "initialize": + { "type": "export" + , "target": "initialize_internal" + , "flexible_config": + [ "ADD_CFLAGS" + , "ADD_CXXFLAGS" + , "AR" + , "ARCH" + , "CC" + , "CFLAGS" + , "COMPILER_FAMILY" + , "CXX" + , "CXXFLAGS" + , "DEBUG" + , "ENV" + , "HOST_ARCH" + , "OS" + , "TARGET_ARCH" + ] + } +, "initialize_internal": + { "type": ["@", "rules", "CC", "library"] + , "name": ["initialize"] + , "stage": ["absl", "log"] + , "srcs": ["initialize.cc"] + , "hdrs": ["initialize.h"] + , "deps": + [ "globals" + , ["absl/base", "config"] + , ["absl/log/internal", "globals"] + , ["absl/time", "time"] + ] + } +, "log": + { "type": ["@", "rules", "CC", "library"] + , "name": ["log"] + , "stage": ["absl", "log"] + , "hdrs": ["log.h"] + , "deps": [["absl/log/internal", "log_impl"]] + } +, "log_entry": + { "type": ["@", "rules", "CC", "library"] + , "name": ["log_entry"] + , "stage": ["absl", "log"] + , "srcs": ["log_entry.cc"] + , "hdrs": ["log_entry.h"] + , "deps": + [ ["absl/base", "config"] + , ["absl/base", "core_headers"] + , ["absl/base", "log_severity"] + , ["absl/log/internal", "config"] + , ["absl/strings", "strings"] + , ["absl/time", "time"] + , ["absl/types", "span"] + ] + } +, "log_sink": + { "type": ["@", "rules", "CC", "library"] + , "name": ["log_sink"] + , "stage": ["absl", "log"] + , "srcs": ["log_sink.cc"] + , "hdrs": ["log_sink.h"] + , "deps": ["log_entry", ["absl/base", "config"]] + } +, "log_sink_registry": + { "type": ["@", "rules", "CC", "library"] + , "name": ["log_sink_registry"] + , "stage": ["absl", "log"] + , "hdrs": ["log_sink_registry.h"] + , "deps": + [ "log_sink" + , ["absl/base", "config"] + , ["absl/log/internal", "log_sink_set"] + ] + } +} diff --git a/etc/import/absl/log/internal/TARGETS.absl b/etc/import/absl/log/internal/TARGETS.absl new file mode 100644 index 000000000..dce2c666e --- /dev/null +++ b/etc/import/absl/log/internal/TARGETS.absl @@ -0,0 +1,204 @@ +{ "check_impl": + { "type": ["@", "rules", "CC", "library"] + , "name": ["check_impl"] + , "stage": ["absl", "log", "internal"] + , "hdrs": ["check_impl.h"] + , "deps": + [ "check_op" + , "conditions" + , "log_message" + , "strip" + , ["absl/base", "core_headers"] + ] + } +, "check_op": + { "type": ["@", "rules", "CC", "library"] + , "name": ["check_op"] + , "stage": ["absl", "log", "internal"] + , "srcs": ["check_op.cc"] + , "hdrs": ["check_op.h"] + , "deps": + [ "nullguard" + , "nullstream" + , "strip" + , ["absl/base", "config"] + , ["absl/base", "core_headers"] + , ["absl/strings", "strings"] + ] + } +, "conditions": + { "type": ["@", "rules", "CC", "library"] + , "name": ["conditions"] + , "stage": ["absl", "log", "internal"] + , "srcs": ["conditions.cc"] + , "hdrs": ["conditions.h"] + , "deps": + [ "voidify" + , ["absl/base", "base"] + , ["absl/base", "config"] + , ["absl/base", "core_headers"] + ] + } +, "config": + { "type": ["@", "rules", "CC", "library"] + , "name": ["config"] + , "stage": ["absl", "log", "internal"] + , "hdrs": ["config.h"] + , "deps": [["absl/base", "config"], ["absl/base", "core_headers"]] + } +, "format": + { "type": ["@", "rules", "CC", "library"] + , "name": ["format"] + , "stage": ["absl", "log", "internal"] + , "srcs": ["log_format.cc"] + , "hdrs": ["log_format.h"] + , "deps": + [ "append_truncated" + , "config" + , "globals" + , ["absl/base", "config"] + , ["absl/base", "core_headers"] + , ["absl/base", "log_severity"] + , ["absl/strings", "strings"] + , ["absl/strings", "str_format"] + , ["absl/time", "time"] + , ["absl/types", "span"] + ] + } +, "globals": + { "type": ["@", "rules", "CC", "library"] + , "name": ["globals"] + , "stage": ["absl", "log", "internal"] + , "srcs": ["globals.cc"] + , "hdrs": ["globals.h"] + , "deps": + [ ["absl/base", "config"] + , ["absl/base", "core_headers"] + , ["absl/base", "log_severity"] + , ["absl/base", "raw_logging_internal"] + , ["absl/strings", "strings"] + , ["absl/time", "time"] + ] + } +, "log_impl": + { "type": ["@", "rules", "CC", "library"] + , "name": ["log_impl"] + , "stage": ["absl", "log", "internal"] + , "hdrs": ["log_impl.h"] + , "deps": ["conditions", "log_message", "strip"] + } +, "log_message": + { "type": ["@", "rules", "CC", "library"] + , "name": ["log_message"] + , "stage": ["absl", "log", "internal"] + , "srcs": ["log_message.cc"] + , "hdrs": ["log_message.h"] + , "deps": + [ "append_truncated" + , "format" + , "globals" + , "log_sink_set" + , "nullguard" + , "proto" + , ["absl/base", "base"] + , ["absl/base", "config"] + , ["absl/base", "core_headers"] + , ["absl/base", "errno_saver"] + , ["absl/base", "log_severity"] + , ["absl/base", "raw_logging_internal"] + , ["absl/base", "strerror"] + , ["absl/container", "inlined_vector"] + , ["absl/debugging", "examine_stack"] + , ["absl/log", "globals"] + , ["absl/log", "log_entry"] + , ["absl/log", "log_sink"] + , ["absl/log", "log_sink_registry"] + , ["absl/memory", "memory"] + , ["absl/strings", "strings"] + , ["absl/time", "time"] + , ["absl/types", "span"] + ] + } +, "append_truncated": + { "type": ["@", "rules", "CC", "library"] + , "name": ["append_truncated"] + , "stage": ["absl", "log", "internal"] + , "hdrs": ["append_truncated.h"] + , "deps": + [ ["absl/base", "config"] + , ["absl/strings", "strings"] + , ["absl/types", "span"] + ] + } +, "log_sink_set": + { "type": ["@", "rules", "CC", "library"] + , "name": ["log_sink_set"] + , "stage": ["absl", "log", "internal"] + , "srcs": ["log_sink_set.cc"] + , "hdrs": ["log_sink_set.h"] + , "deps": + [ "config" + , "globals" + , ["absl/base", "base"] + , ["absl/base", "config"] + , ["absl/base", "core_headers"] + , ["absl/base", "log_severity"] + , ["absl/base", "raw_logging_internal"] + , ["absl/cleanup", "cleanup"] + , ["absl/log", "globals"] + , ["absl/log", "log_entry"] + , ["absl/log", "log_sink"] + , ["absl/strings", "strings"] + , ["absl/synchronization", "synchronization"] + , ["absl/types", "span"] + ] + } +, "nullguard": + { "type": ["@", "rules", "CC", "library"] + , "name": ["nullguard"] + , "stage": ["absl", "log", "internal"] + , "srcs": ["nullguard.cc"] + , "hdrs": ["nullguard.h"] + , "deps": [["absl/base", "config"], ["absl/base", "core_headers"]] + } +, "nullstream": + { "type": ["@", "rules", "CC", "library"] + , "name": ["nullstream"] + , "stage": ["absl", "log", "internal"] + , "hdrs": ["nullstream.h"] + , "deps": + [ ["absl/base", "config"] + , ["absl/base", "core_headers"] + , ["absl/base", "log_severity"] + , ["absl/strings", "strings"] + ] + } +, "strip": + { "type": ["@", "rules", "CC", "library"] + , "name": ["strip"] + , "stage": ["absl", "log", "internal"] + , "hdrs": ["strip.h"] + , "deps": ["log_message", "nullstream", ["absl/base", "log_severity"]] + } +, "voidify": + { "type": ["@", "rules", "CC", "library"] + , "name": ["voidify"] + , "stage": ["absl", "log", "internal"] + , "hdrs": ["voidify.h"] + , "deps": [["absl/base", "config"]] + } +, "proto": + { "type": ["@", "rules", "CC", "library"] + , "name": ["proto"] + , "stage": ["absl", "log", "internal"] + , "srcs": ["proto.cc"] + , "hdrs": ["proto.h"] + , "deps": + [ ["absl/base", "base"] + , ["absl/base", "config"] + , ["absl/base", "core_headers"] + , ["absl/strings", "strings"] + , ["absl/types", "span"] + ] + } +} diff --git a/etc/import/absl/numeric/TARGETS.absl b/etc/import/absl/numeric/TARGETS.absl index 3b4203a91..f38f11011 100644 --- a/etc/import/absl/numeric/TARGETS.absl +++ b/etc/import/absl/numeric/TARGETS.absl @@ -8,6 +8,26 @@ , "deps": ["bits", ["absl/base", "config"], ["absl/base", "core_headers"]] } , "bits": + { "type": "export" + , "target": "bits_internal" + , "flexible_config": + [ "ADD_CFLAGS" + , "ADD_CXXFLAGS" + , "AR" + , "ARCH" + , "CC" + , "CFLAGS" + , "COMPILER_FAMILY" + , "CXX" + , "CXXFLAGS" + , "DEBUG" + , "ENV" + , "HOST_ARCH" + , "OS" + , "TARGET_ARCH" + ] + } +, "bits_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["bits"] , "stage": ["absl", "numeric"] diff --git a/etc/import/absl/strings/TARGETS.absl b/etc/import/absl/strings/TARGETS.absl index 6dfbcb94c..81f5a8394 100644 --- a/etc/import/absl/strings/TARGETS.absl +++ b/etc/import/absl/strings/TARGETS.absl @@ -80,6 +80,26 @@ ] } , "internal": + { "type": "export" + , "target": "internal_" + , "flexible_config": + [ "ADD_CFLAGS" + , "ADD_CXXFLAGS" + , "AR" + , "ARCH" + , "CC" + , "CFLAGS" + , "COMPILER_FAMILY" + , "CXX" + , "CXXFLAGS" + , "DEBUG" + , "ENV" + , "HOST_ARCH" + , "OS" + , "TARGET_ARCH" + ] + } +, "internal_": { "type": ["@", "rules", "CC", "library"] , "name": ["internal"] , "stage": ["absl", "strings"] diff --git a/etc/import/absl/types/TARGETS.absl b/etc/import/absl/types/TARGETS.absl index 8bb1094b5..a6de1d33b 100644 --- a/etc/import/absl/types/TARGETS.absl +++ b/etc/import/absl/types/TARGETS.absl @@ -115,4 +115,11 @@ , ["absl/meta", "type_traits"] ] } +, "compare": + { "type": ["@", "rules", "CC", "library"] + , "name": ["compare"] + , "stage": ["absl", "types"] + , "hdrs": ["compare.h"] + , "deps": [["absl/base", "core_headers"], ["absl/meta", "type_traits"]] + } } diff --git a/etc/repos.json b/etc/repos.json index 1b00ea63a..58b6c8e71 100644 --- a/etc/repos.json +++ b/etc/repos.json @@ -296,11 +296,11 @@ , "com_google_absl": { "repository": { "type": "archive" - , "content": "8cc2a99cfd2426ba6fbdc4499a07bccfd19be8a6" - , "fetch": "https://github.com/abseil/abseil-cpp/archive/refs/tags/20230125.0.tar.gz" - , "sha256": "3ea49a7d97421b88a8c48a0de16c16048e17725c7ec0f1d3ea2683a2a75adc21" - , "sha512": "b3d334215c78b31a2eb10bd9d4a978cd48367866d6daa2065c6c727282bafe19ef7ff5bd7cd4ed5c319d3b04e0711222e08ddbe7621ef1e079fed93a7307112f" - , "subdir": "abseil-cpp-20230125.0" + , "content": "08a0c4a58d13f234bd3f1de6c73395ec5ab7d5f6" + , "fetch": "https://github.com/abseil/abseil-cpp/archive/refs/tags/20230125.3.tar.gz" + , "sha256": "5366d7e7fa7ba0d915014d387b66d0d002c03236448e1ba9ef98122c13b35c36" + , "sha512": "50509acfc4128fd31435631f71ac8cd0350acd9e290f78502723149016e7f07c9d84182ba99e0938b1873fecda09393d3fd7af8dabfb0d89cdcdd8a69a917e70" + , "subdir": "abseil-cpp-20230125.3" } , "target_root": "import targets" , "target_file_name": "TARGETS.absl" From 5969b46f5aeb51f864070af37ec57fb3870d8e22 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Wed, 26 Jul 2023 16:35:42 +0200 Subject: [PATCH 34/97] external: Update protobuf to v23.4 This corresponds to the highest current version found in popular distros (in this case, the one in Arch Linux). The abseil library is now a dependency of protobuf (for logging). --- etc/defaults/CC/TARGETS.protobuf | 3 + .../src/google/protobuf/TARGETS.protobuf | 589 ++++++++++-------- .../third_party/utf8_range/TARGETS.protobuf | 29 + etc/repos.json | 15 +- 4 files changed, 365 insertions(+), 271 deletions(-) create mode 100644 etc/import/third_party/utf8_range/TARGETS.protobuf diff --git a/etc/defaults/CC/TARGETS.protobuf b/etc/defaults/CC/TARGETS.protobuf index bc78b33ad..2bde69ea6 100644 --- a/etc/defaults/CC/TARGETS.protobuf +++ b/etc/defaults/CC/TARGETS.protobuf @@ -19,6 +19,7 @@ , "case": { "msvc": [ "/wd4065" + , "/wd4146" , "/wd4244" , "/wd4251" , "/wd4267" @@ -38,6 +39,7 @@ , "-Wno-sign-conversion" , "-Wno-unused-function" , "-Wno-deprecated-declarations" + , "-Wno-nonnull" ] } , "default": @@ -48,6 +50,7 @@ , "-Wno-unused-function" , "-Wno-deprecated-declarations" , "-Wno-deprecated-enum-enum-conversion" + , "-Wno-nonnull" ] } ] diff --git a/etc/import/src/google/protobuf/TARGETS.protobuf b/etc/import/src/google/protobuf/TARGETS.protobuf index 80707adf8..0589f9387 100644 --- a/etc/import/src/google/protobuf/TARGETS.protobuf +++ b/etc/import/src/google/protobuf/TARGETS.protobuf @@ -22,249 +22,245 @@ { "type": ["@", "rules", "data", "staged"] , "srcs": [ "well_known_protos" - , "type.pb.h" - , "repeated_field.h" - , "reflection_internal.h" - , "struct.pb.h" - , "generated_message_tctable_decl.h" - , "wire_format.h" - , "descriptor.pb.h" - , "generated_message_tctable_impl.h" - , "reflection.h" , "any.h" - , "map.h" - , "package_info.h" - , "generated_message_bases.h" - , "service.h" - , "test_util_lite.h" - , "port.h" - , "metadata_lite.h" - , "map_test_util.h" - , "arenaz_sampler.h" - , "map_field.h" - , "testing/file.h" - , "testing/googletest.h" , "any.pb.h" - , "map_lite_test_util.h" - , "field_access_listener.h" - , "descriptor_database.h" - , "extension_set_inl.h" + , "api.pb.h" , "arena.h" - , "timestamp.pb.h" - , "io/package_info.h" - , "io/tokenizer.h" - , "io/zero_copy_stream.h" - , "io/zero_copy_stream_impl.h" - , "io/strtod.h" - , "io/printer.h" - , "io/zero_copy_stream_impl_lite.h" - , "io/io_win32.h" - , "io/coded_stream.h" - , "io/gzip_stream.h" - , "has_bits.h" - , "source_context.pb.h" + , "arenastring.h" + , "arenaz_sampler.h" + , "arena_align.h" + , "arena_allocation_policy.h" + , "arena_cleanup.h" + , "arena_config.h" , "arena_test_util.h" + , "compiler/allowlists/allowlist.h" + , "compiler/allowlists/allowlists.h" + , "compiler/annotation_test_util.h" , "compiler/code_generator.h" - , "compiler/parser.h" - , "compiler/plugin.h" - , "compiler/scc.h" + , "compiler/command_line_interface.h" + , "compiler/cpp/cpp_generator.h" + , "compiler/cpp/enum.h" + , "compiler/cpp/extension.h" + , "compiler/cpp/field.h" + , "compiler/cpp/field_generators/generators.h" + , "compiler/cpp/file.h" + , "compiler/cpp/generator.h" + , "compiler/cpp/helpers.h" + , "compiler/cpp/message.h" + , "compiler/cpp/message_layout_helper.h" + , "compiler/cpp/names.h" + , "compiler/cpp/options.h" + , "compiler/cpp/padding_optimizer.h" + , "compiler/cpp/parse_function_generator.h" + , "compiler/cpp/service.h" + , "compiler/cpp/tracker.h" + , "compiler/cpp/unittest.h" + , "compiler/csharp/csharp_doc_comment.h" , "compiler/csharp/csharp_enum.h" - , "compiler/csharp/csharp_repeated_message_field.h" , "compiler/csharp/csharp_enum_field.h" - , "compiler/csharp/csharp_options.h" - , "compiler/csharp/csharp_message_field.h" , "compiler/csharp/csharp_field_base.h" - , "compiler/csharp/csharp_source_generator_base.h" - , "compiler/csharp/csharp_reflection_class.h" + , "compiler/csharp/csharp_generator.h" + , "compiler/csharp/csharp_helpers.h" + , "compiler/csharp/csharp_map_field.h" + , "compiler/csharp/csharp_message.h" + , "compiler/csharp/csharp_message_field.h" + , "compiler/csharp/csharp_options.h" , "compiler/csharp/csharp_primitive_field.h" - , "compiler/csharp/csharp_doc_comment.h" + , "compiler/csharp/csharp_reflection_class.h" , "compiler/csharp/csharp_repeated_enum_field.h" + , "compiler/csharp/csharp_repeated_message_field.h" , "compiler/csharp/csharp_repeated_primitive_field.h" - , "compiler/csharp/csharp_generator.h" + , "compiler/csharp/csharp_source_generator_base.h" , "compiler/csharp/csharp_wrapper_field.h" - , "compiler/csharp/csharp_helpers.h" - , "compiler/csharp/csharp_message.h" - , "compiler/csharp/csharp_names.h" - , "compiler/csharp/csharp_map_field.h" - , "compiler/package_info.h" - , "compiler/python/python_generator.h" - , "compiler/python/generator.h" - , "compiler/python/helpers.h" - , "compiler/python/pyi_generator.h" - , "compiler/cpp/extension.h" - , "compiler/cpp/enum.h" - , "compiler/cpp/generator.h" - , "compiler/cpp/helpers.h" - , "compiler/cpp/service.h" - , "compiler/cpp/map_field.h" - , "compiler/cpp/cpp_generator.h" - , "compiler/cpp/names.h" - , "compiler/cpp/string_field.h" - , "compiler/cpp/unittest.h" - , "compiler/cpp/message_layout_helper.h" - , "compiler/cpp/parse_function_generator.h" - , "compiler/cpp/primitive_field.h" - , "compiler/cpp/enum_field.h" - , "compiler/cpp/field.h" - , "compiler/cpp/options.h" - , "compiler/cpp/file.h" - , "compiler/cpp/message.h" - , "compiler/cpp/message_field.h" - , "compiler/cpp/padding_optimizer.h" - , "compiler/php/php_generator.h" - , "compiler/subprocess.h" - , "compiler/plugin.pb.h" - , "compiler/java/extension_lite.h" - , "compiler/java/extension.h" - , "compiler/java/shared_code_generator.h" - , "compiler/java/enum.h" - , "compiler/java/generator.h" - , "compiler/java/string_field_lite.h" - , "compiler/java/helpers.h" - , "compiler/java/service.h" - , "compiler/java/map_field.h" - , "compiler/java/names.h" - , "compiler/java/string_field.h" - , "compiler/java/kotlin_generator.h" + , "compiler/csharp/names.h" + , "compiler/importer.h" , "compiler/java/context.h" - , "compiler/java/generator_factory.h" , "compiler/java/doc_comment.h" - , "compiler/java/primitive_field.h" - , "compiler/java/enum_field_lite.h" + , "compiler/java/enum.h" , "compiler/java/enum_field.h" - , "compiler/java/message_lite.h" - , "compiler/java/message_builder_lite.h" + , "compiler/java/enum_field_lite.h" + , "compiler/java/enum_lite.h" + , "compiler/java/extension.h" + , "compiler/java/extension_lite.h" , "compiler/java/field.h" - , "compiler/java/options.h" - , "compiler/java/java_generator.h" , "compiler/java/file.h" - , "compiler/java/primitive_field_lite.h" - , "compiler/java/message.h" - , "compiler/java/name_resolver.h" - , "compiler/java/message_field.h" + , "compiler/java/generator.h" + , "compiler/java/generator_factory.h" + , "compiler/java/helpers.h" + , "compiler/java/java_generator.h" + , "compiler/java/kotlin_generator.h" + , "compiler/java/map_field.h" , "compiler/java/map_field_lite.h" - , "compiler/java/enum_lite.h" + , "compiler/java/message.h" , "compiler/java/message_builder.h" + , "compiler/java/message_builder_lite.h" + , "compiler/java/message_field.h" , "compiler/java/message_field_lite.h" + , "compiler/java/message_lite.h" + , "compiler/java/message_serialization.h" + , "compiler/java/names.h" + , "compiler/java/name_resolver.h" + , "compiler/java/options.h" + , "compiler/java/primitive_field.h" + , "compiler/java/primitive_field_lite.h" + , "compiler/java/service.h" + , "compiler/java/shared_code_generator.h" + , "compiler/java/string_field.h" + , "compiler/java/string_field_lite.h" + , "compiler/mock_code_generator.h" + , "compiler/objectivec/enum.h" + , "compiler/objectivec/enum_field.h" + , "compiler/objectivec/extension.h" + , "compiler/objectivec/field.h" + , "compiler/objectivec/file.h" + , "compiler/objectivec/generator.h" + , "compiler/objectivec/helpers.h" + , "compiler/objectivec/import_writer.h" + , "compiler/objectivec/line_consumer.h" + , "compiler/objectivec/map_field.h" + , "compiler/objectivec/message.h" + , "compiler/objectivec/message_field.h" + , "compiler/objectivec/names.h" + , "compiler/objectivec/nsobject_methods.h" + , "compiler/objectivec/oneof.h" + , "compiler/objectivec/options.h" + , "compiler/objectivec/primitive_field.h" + , "compiler/objectivec/text_format_decode_data.h" + , "compiler/package_info.h" + , "compiler/parser.h" + , "compiler/php/names.h" + , "compiler/php/php_generator.h" + , "compiler/plugin.h" + , "compiler/plugin.pb.h" + , "compiler/python/generator.h" + , "compiler/python/helpers.h" + , "compiler/python/pyi_generator.h" + , "compiler/python/python_generator.h" + , "compiler/retention.h" , "compiler/ruby/ruby_generator.h" - , "compiler/objectivec/objectivec_nsobject_methods.h" - , "compiler/objectivec/objectivec_enum.h" - , "compiler/objectivec/objectivec_primitive_field.h" - , "compiler/objectivec/objectivec_message_field.h" - , "compiler/objectivec/objectivec_message.h" - , "compiler/objectivec/objectivec_field.h" - , "compiler/objectivec/objectivec_enum_field.h" - , "compiler/objectivec/objectivec_map_field.h" - , "compiler/objectivec/objectivec_file.h" - , "compiler/objectivec/objectivec_generator.h" - , "compiler/objectivec/objectivec_helpers.h" - , "compiler/objectivec/objectivec_extension.h" - , "compiler/objectivec/objectivec_oneof.h" + , "compiler/rust/accessors/accessors.h" + , "compiler/rust/context.h" + , "compiler/rust/generator.h" + , "compiler/rust/message.h" + , "compiler/rust/naming.h" + , "compiler/scc.h" + , "compiler/subprocess.h" , "compiler/zip_writer.h" - , "compiler/command_line_interface.h" - , "compiler/mock_code_generator.h" - , "compiler/annotation_test_util.h" - , "compiler/importer.h" - , "dynamic_message.h" - , "parse_context.h" - , "map_entry_lite.h" + , "descriptor.h" + , "descriptor.pb.h" + , "descriptor_database.h" + , "descriptor_legacy.h" , "duration.pb.h" - , "util/time_util.h" - , "util/package_info.h" - , "util/delimited_message_util.h" - , "util/field_comparator.h" - , "util/field_mask_util.h" - , "util/internal/object_location_tracker.h" - , "util/internal/expecting_objectwriter.h" - , "util/internal/json_objectwriter.h" - , "util/internal/json_escaping.h" - , "util/internal/mock_error_listener.h" - , "util/internal/constants.h" - , "util/internal/field_mask_utility.h" - , "util/internal/json_stream_parser.h" - , "util/internal/protostream_objectsource.h" - , "util/internal/error_listener.h" - , "util/internal/utility.h" - , "util/internal/type_info.h" - , "util/internal/structured_objectwriter.h" - , "util/internal/object_source.h" - , "util/internal/protostream_objectwriter.h" - , "util/internal/type_info_test_helper.h" - , "util/internal/object_writer.h" - , "util/internal/proto_writer.h" - , "util/internal/default_value_objectwriter.h" - , "util/internal/location_tracker.h" - , "util/internal/datapiece.h" - , "util/json_util.h" - , "util/type_resolver.h" - , "util/type_resolver_util.h" - , "util/message_differencer.h" - , "repeated_ptr_field.h" - , "unknown_field_set.h" + , "dynamic_message.h" + , "empty.pb.h" + , "endian.h" + , "explicitly_constructed.h" + , "extension_set.h" + , "extension_set_inl.h" + , "field_access_listener.h" , "field_mask.pb.h" - , "test_util.h" - , "map_type_handler.h" - , "map_test_util_impl.h" - , "arena_impl.h" - , "test_util2.h" + , "generated_enum_reflection.h" , "generated_enum_util.h" - , "message_lite.h" - , "string_member_robber.h" - , "descriptor.h" - , "map_entry.h" - , "metadata.h" - , "inlined_string_field.h" - , "message.h" + , "generated_message_bases.h" + , "generated_message_reflection.h" + , "generated_message_tctable_decl.h" + , "generated_message_tctable_gen.h" + , "generated_message_tctable_impl.h" + , "generated_message_util.h" + , "has_bits.h" , "implicit_weak_message.h" - , "reflection_tester.h" - , "wrappers.pb.h" + , "inlined_string_field.h" + , "io/coded_stream.h" + , "io/gzip_stream.h" + , "io/io_win32.h" + , "io/package_info.h" + , "io/printer.h" + , "io/strtod.h" + , "io/test_zero_copy_stream.h" + , "io/tokenizer.h" + , "io/zero_copy_sink.h" + , "io/zero_copy_stream.h" + , "io/zero_copy_stream_impl.h" + , "io/zero_copy_stream_impl_lite.h" + , "json/internal/descriptor_traits.h" + , "json/internal/lexer.h" + , "json/internal/message_path.h" + , "json/internal/parser.h" + , "json/internal/parser_traits.h" + , "json/internal/test_input_stream.h" + , "json/internal/unparser.h" + , "json/internal/unparser_traits.h" + , "json/internal/untyped_message.h" + , "json/internal/writer.h" + , "json/internal/zero_copy_buffered_stream.h" + , "json/json.h" + , "map.h" + , "map_entry.h" + , "map_entry_lite.h" + , "map_field.h" , "map_field_inl.h" , "map_field_lite.h" - , "text_format.h" - , "extension_set.h" - , "empty.pb.h" - , "endian.h" - , "wire_format_lite.h" - , "generated_enum_reflection.h" + , "map_lite_test_util.h" + , "map_test_util.h" + , "map_test_util_impl.h" + , "map_type_handler.h" + , "message.h" + , "message_lite.h" + , "metadata.h" + , "metadata_lite.h" + , "package_info.h" + , "parse_context.h" + , "port.h" + , "reflection.h" + , "reflection_internal.h" + , "reflection_mode.h" , "reflection_ops.h" - , "arenastring.h" - , "explicitly_constructed.h" - , "api.pb.h" - , "stubs/substitute.h" - , "stubs/stl_util.h" - , "stubs/bytestream.h" - , "stubs/macros.h" - , "stubs/int128.h" - , "stubs/platform_macros.h" - , "stubs/port.h" + , "reflection_tester.h" + , "repeated_field.h" + , "repeated_ptr_field.h" + , "serial_arena.h" + , "service.h" + , "source_context.pb.h" + , "string_block.h" + , "string_member_robber.h" + , "struct.pb.h" , "stubs/callback.h" - , "stubs/logging.h" - , "stubs/template_util.h" - , "stubs/time.h" - , "stubs/casts.h" , "stubs/common.h" - , "stubs/stringpiece.h" - , "stubs/mathutil.h" - , "stubs/strutil.h" - , "stubs/status.h" - , "stubs/map_util.h" - , "stubs/once.h" - , "stubs/stringprintf.h" - , "stubs/statusor.h" + , "stubs/platform_macros.h" + , "stubs/port.h" , "stubs/status_macros.h" - , "stubs/hash.h" - , "stubs/mutex.h" - , "generated_message_reflection.h" - , "generated_message_util.h" - , "proto3_lite_unittest.inc" - , "message_unittest.inc" + , "testing/file.h" + , "testing/googletest.h" + , "test_util.h" + , "test_util2.h" + , "test_util_lite.h" + , "text_format.h" + , "thread_safe_arena.h" + , "timestamp.pb.h" + , "type.pb.h" + , "unknown_field_set.h" + , "util/delimited_message_util.h" + , "util/field_comparator.h" + , "util/field_mask_util.h" + , "util/json_util.h" + , "util/message_differencer.h" + , "util/package_info.h" + , "util/time_util.h" + , "util/type_resolver.h" + , "util/type_resolver_util.h" + , "varint_shuffle.h" + , "wire_format.h" + , "wire_format_lite.h" + , "wrappers.pb.h" , "compiler/cpp/unittest.inc" - , "port_undef.inc" + , "map_test.inc" , "map_test_util.inc" + , "message_unittest.inc" + , "port_def.inc" + , "port_undef.inc" + , "proto3_lite_unittest.inc" , "test_util.inc" , "wire_format_unittest.inc" - , "port_def.inc" - , "map_test.inc" ] , "stage": ["google", "protobuf"] } @@ -296,6 +292,8 @@ , "srcs": [ "any_lite.cc" , "arena.cc" + , "arena_align.cc" + , "arena_config.cc" , "arenastring.cc" , "arenaz_sampler.cc" , "extension_set.cc" @@ -306,109 +304,158 @@ , "inlined_string_field.cc" , "io/coded_stream.cc" , "io/io_win32.cc" - , "io/strtod.cc" , "io/zero_copy_stream.cc" , "io/zero_copy_stream_impl.cc" , "io/zero_copy_stream_impl_lite.cc" , "map.cc" , "message_lite.cc" , "parse_context.cc" + , "port.cc" , "repeated_field.cc" , "repeated_ptr_field.cc" - , "stubs/bytestream.cc" , "stubs/common.cc" - , "stubs/int128.cc" - , "stubs/status.cc" - , "stubs/statusor.cc" - , "stubs/stringpiece.cc" - , "stubs/stringprintf.cc" - , "stubs/structurally_valid.cc" - , "stubs/strutil.cc" - , "stubs/time.cc" , "wire_format_lite.cc" ] + , "deps": + [ ["@", "absl", "absl/container", "btree"] + , ["@", "absl", "absl/container", "flat_hash_set"] + , ["@", "absl", "absl/hash", "hash"] + , ["@", "absl", "absl/log", "absl_check"] + , ["@", "absl", "absl/log", "absl_log"] + , ["@", "absl", "absl/meta", "type_traits"] + , ["@", "absl", "absl/numeric", "bits"] + , ["@", "absl", "absl/status", "status"] + , ["@", "absl", "absl/strings", "internal"] + , ["@", "absl", "absl/synchronization", "synchronization"] + , ["@", "absl", "absl/time", "time"] + , ["third_party/utf8_range", "utf8_validity"] + ] } , "libprotobuf": { "type": ["@", "rules", "CC", "library"] , "name": ["protobuf"] , "hdrs": ["protobuf_headers"] , "srcs": - [ "any.cc" - , "any.pb.cc" + [ "any.pb.cc" , "api.pb.cc" + , "duration.pb.cc" + , "empty.pb.cc" + , "field_mask.pb.cc" + , "source_context.pb.cc" + , "struct.pb.cc" + , "timestamp.pb.cc" + , "type.pb.cc" + , "wrappers.pb.cc" + , "any.cc" + , "any_lite.cc" + , "arena.cc" + , "arena_align.cc" + , "arena_config.cc" + , "arenastring.cc" + , "arenaz_sampler.cc" , "compiler/importer.cc" , "compiler/parser.cc" , "descriptor.cc" , "descriptor.pb.cc" , "descriptor_database.cc" - , "duration.pb.cc" , "dynamic_message.cc" - , "empty.pb.cc" + , "extension_set.cc" , "extension_set_heavy.cc" - , "field_mask.pb.cc" + , "generated_enum_util.cc" , "generated_message_bases.cc" , "generated_message_reflection.cc" , "generated_message_tctable_full.cc" + , "generated_message_tctable_gen.cc" + , "generated_message_tctable_lite.cc" + , "generated_message_util.cc" + , "implicit_weak_message.cc" + , "inlined_string_field.cc" + , "io/coded_stream.cc" , "io/gzip_stream.cc" + , "io/io_win32.cc" , "io/printer.cc" + , "io/strtod.cc" , "io/tokenizer.cc" + , "io/zero_copy_sink.cc" + , "io/zero_copy_stream.cc" + , "io/zero_copy_stream_impl.cc" + , "io/zero_copy_stream_impl_lite.cc" + , "json/internal/lexer.cc" + , "json/internal/message_path.cc" + , "json/internal/parser.cc" + , "json/internal/unparser.cc" + , "json/internal/untyped_message.cc" + , "json/internal/writer.cc" + , "json/internal/zero_copy_buffered_stream.cc" + , "json/json.cc" + , "map.cc" , "map_field.cc" , "message.cc" + , "message_lite.cc" + , "parse_context.cc" + , "port.cc" + , "reflection_mode.cc" , "reflection_ops.cc" + , "repeated_field.cc" + , "repeated_ptr_field.cc" , "service.cc" - , "source_context.pb.cc" - , "struct.pb.cc" - , "stubs/substitute.cc" + , "stubs/common.cc" , "text_format.cc" - , "timestamp.pb.cc" - , "type.pb.cc" , "unknown_field_set.cc" , "util/delimited_message_util.cc" , "util/field_comparator.cc" , "util/field_mask_util.cc" - , "util/internal/datapiece.cc" - , "util/internal/default_value_objectwriter.cc" - , "util/internal/error_listener.cc" - , "util/internal/field_mask_utility.cc" - , "util/internal/json_escaping.cc" - , "util/internal/json_objectwriter.cc" - , "util/internal/json_stream_parser.cc" - , "util/internal/object_writer.cc" - , "util/internal/proto_writer.cc" - , "util/internal/protostream_objectsource.cc" - , "util/internal/protostream_objectwriter.cc" - , "util/internal/type_info.cc" - , "util/internal/utility.cc" - , "util/json_util.cc" , "util/message_differencer.cc" , "util/time_util.cc" , "util/type_resolver_util.cc" , "wire_format.cc" - , "wrappers.pb.cc" + , "wire_format_lite.cc" + ] + , "deps": + [ ["", "libprotobuf_lite"] + , ["@", "absl", "absl/base", "base"] + , ["@", "absl", "absl/base", "dynamic_annotations"] + , ["@", "absl", "absl/container", "btree"] + , ["@", "absl", "absl/container", "flat_hash_map"] + , ["@", "absl", "absl/container", "flat_hash_set"] + , ["@", "absl", "absl/hash", "hash"] + , ["@", "absl", "absl/log", "absl_check"] + , ["@", "absl", "absl/log", "absl_log"] + , ["@", "absl", "absl/log", "die_if_null"] + , ["@", "absl", "absl/status", "statusor"] + , ["@", "absl", "absl/strings", "internal"] + , ["@", "absl", "absl/synchronization", "synchronization"] + , ["@", "absl", "absl/time", "time"] + , ["third_party/utf8_range", "utf8_validity"] + , ["@", "zlib", "", "zlib"] ] - , "deps": [["", "libprotobuf_lite"], ["@", "zlib", "", "zlib"]] } , "libprotoc": { "type": ["@", "rules", "CC", "library"] , "name": ["libprotoc"] , "srcs": - [ "compiler/code_generator.cc" + [ "compiler/main.cc" + , "compiler/allowlists/empty_package.cc" + , "compiler/allowlists/weak_imports.cc" + , "compiler/code_generator.cc" , "compiler/command_line_interface.cc" , "compiler/cpp/enum.cc" - , "compiler/cpp/enum_field.cc" , "compiler/cpp/extension.cc" , "compiler/cpp/field.cc" + , "compiler/cpp/field_generators/cord_field.cc" + , "compiler/cpp/field_generators/enum_field.cc" + , "compiler/cpp/field_generators/map_field.cc" + , "compiler/cpp/field_generators/message_field.cc" + , "compiler/cpp/field_generators/primitive_field.cc" + , "compiler/cpp/field_generators/string_field.cc" , "compiler/cpp/file.cc" , "compiler/cpp/generator.cc" , "compiler/cpp/helpers.cc" - , "compiler/cpp/map_field.cc" , "compiler/cpp/message.cc" - , "compiler/cpp/message_field.cc" , "compiler/cpp/padding_optimizer.cc" , "compiler/cpp/parse_function_generator.cc" - , "compiler/cpp/primitive_field.cc" , "compiler/cpp/service.cc" - , "compiler/cpp/string_field.cc" + , "compiler/cpp/tracker.cc" , "compiler/csharp/csharp_doc_comment.cc" , "compiler/csharp/csharp_enum.cc" , "compiler/csharp/csharp_enum_field.cc" @@ -425,6 +472,7 @@ , "compiler/csharp/csharp_repeated_primitive_field.cc" , "compiler/csharp/csharp_source_generator_base.cc" , "compiler/csharp/csharp_wrapper_field.cc" + , "compiler/csharp/names.cc" , "compiler/java/context.cc" , "compiler/java/doc_comment.cc" , "compiler/java/enum.cc" @@ -447,42 +495,55 @@ , "compiler/java/message_field.cc" , "compiler/java/message_field_lite.cc" , "compiler/java/message_lite.cc" + , "compiler/java/message_serialization.cc" , "compiler/java/name_resolver.cc" + , "compiler/java/names.cc" , "compiler/java/primitive_field.cc" , "compiler/java/primitive_field_lite.cc" , "compiler/java/service.cc" , "compiler/java/shared_code_generator.cc" , "compiler/java/string_field.cc" , "compiler/java/string_field_lite.cc" - , "compiler/objectivec/objectivec_enum.cc" - , "compiler/objectivec/objectivec_enum_field.cc" - , "compiler/objectivec/objectivec_extension.cc" - , "compiler/objectivec/objectivec_field.cc" - , "compiler/objectivec/objectivec_file.cc" - , "compiler/objectivec/objectivec_generator.cc" - , "compiler/objectivec/objectivec_helpers.cc" - , "compiler/objectivec/objectivec_map_field.cc" - , "compiler/objectivec/objectivec_message.cc" - , "compiler/objectivec/objectivec_message_field.cc" - , "compiler/objectivec/objectivec_oneof.cc" - , "compiler/objectivec/objectivec_primitive_field.cc" + , "compiler/objectivec/enum.cc" + , "compiler/objectivec/enum_field.cc" + , "compiler/objectivec/extension.cc" + , "compiler/objectivec/field.cc" + , "compiler/objectivec/file.cc" + , "compiler/objectivec/generator.cc" + , "compiler/objectivec/helpers.cc" + , "compiler/objectivec/import_writer.cc" + , "compiler/objectivec/line_consumer.cc" + , "compiler/objectivec/map_field.cc" + , "compiler/objectivec/message.cc" + , "compiler/objectivec/message_field.cc" + , "compiler/objectivec/names.cc" + , "compiler/objectivec/oneof.cc" + , "compiler/objectivec/primitive_field.cc" + , "compiler/objectivec/text_format_decode_data.cc" + , "compiler/php/names.cc" , "compiler/php/php_generator.cc" , "compiler/plugin.cc" , "compiler/plugin.pb.cc" , "compiler/python/generator.cc" , "compiler/python/helpers.cc" , "compiler/python/pyi_generator.cc" + , "compiler/retention.cc" , "compiler/ruby/ruby_generator.cc" + , "compiler/rust/accessors/accessors.cc" + , "compiler/rust/accessors/singular_bytes.cc" + , "compiler/rust/accessors/singular_scalar.cc" + , "compiler/rust/context.cc" + , "compiler/rust/generator.cc" + , "compiler/rust/message.cc" + , "compiler/rust/naming.cc" , "compiler/subprocess.cc" , "compiler/zip_writer.cc" ] - , "deps": [["", "libprotobuf"]] + , "deps": [["", "libprotobuf"], ["@", "absl", "absl/log", "initialize"]] } , "protoc": { "type": ["@", "rules", "CC", "binary"] , "name": ["protoc"] - , "srcs": ["compiler/main.cc"] , "private-deps": [["", "libprotoc"]] - , "private-ldflags": ["-pthread"] } } diff --git a/etc/import/third_party/utf8_range/TARGETS.protobuf b/etc/import/third_party/utf8_range/TARGETS.protobuf new file mode 100644 index 000000000..44c5340a1 --- /dev/null +++ b/etc/import/third_party/utf8_range/TARGETS.protobuf @@ -0,0 +1,29 @@ +{ "utf8_validity": + { "type": "export" + , "target": "utf8_validity_internal" + , "flexible_config": + [ "OS" + , "ARCH" + , "HOST_ARCH" + , "TARGET_ARCH" + , "COMPILER_FAMILY" + , "DEBUG" + , "CXX" + , "CXXFLAGS" + , "ADD_CXXFLAGS" + , "AR" + , "ENV" + , "CC" + , "CFLAGS" + , "ADD_CFLAGS" + , "PKG_CONFIG_ARGS" + ] + } +, "utf8_validity_internal": + { "type": ["@", "rules", "CC", "library"] + , "name": ["utf8_validity"] + , "hdrs": ["utf8_validity.h"] + , "srcs": ["utf8_validity.cc"] + , "deps": [["@", "absl", "absl/strings", "strings"]] + } +} diff --git a/etc/repos.json b/etc/repos.json index 58b6c8e71..73001b941 100644 --- a/etc/repos.json +++ b/etc/repos.json @@ -250,16 +250,17 @@ } , "protobuf": { "repository": - { "type": "zip" - , "content": "5bc92c917a0714d0b37cbb9dcbbf434d132edc00" - , "fetch": "https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.zip" - , "sha256": "f7042d540c969b00db92e8e1066a9b8099c8379c33f40f360eb9e1d98a36ca26" - , "sha512": "7f1c9a4297a4e47705d64932483f31ed25da928d24e351d94899038f852015d4a16c88c2b962ac4a0c62e6a3f6689f87fb4cef7165d327aae5381a7b975bdf2e" - , "subdir": "protobuf-3.21.12" + { "type": "archive" + , "content": "3e7ee8fe7138998e31bd367fdd6feaeb1d98ab39" + , "fetch": "https://github.com/protocolbuffers/protobuf/releases/download/v23.4/protobuf-23.4.tar.gz" + , "sha256": "a700a49470d301f1190a487a923b5095bf60f08f4ae4cac9f5f7c36883d17971" + , "sha512": "b93a4e0339ecbe085796de5b6c61feae35229d40db9019c043090bcaa483a1cce78a99487c5638482c68832fcede5579a0e5ec731221a88359b80db9bb6dc566" + , "subdir": "protobuf-23.4" } , "target_root": "import targets" , "target_file_name": "TARGETS.protobuf" - , "bindings": {"rules": "rules-protobuf", "zlib": "zlib"} + , "bindings": + {"rules": "rules-protobuf", "zlib": "zlib", "absl": "com_google_absl"} , "pkg_bootstrap": { "copy": ["bin/protoc", "include/google/protobuf", "proto/google/protobuf"] From 2180d357614e4a8bccffec9d8501f7e9659ce24e Mon Sep 17 00:00:00 2001 From: Alberto Sartori Date: Fri, 4 Aug 2023 12:17:22 +0200 Subject: [PATCH 35/97] update grpc to version 1.56.2 --- etc/import/TARGETS.grpc | 975 +++++++++--- etc/import/include/grpc/TARGETS.grpc | 5 +- etc/import/include/grpcpp/TARGETS.grpc | 1 + etc/import/src/core/TARGETS.grpc | 1366 ++++++++++++----- .../src/core/ext/upb-generated/TARGETS.grpc | 588 +++---- .../core/ext/upbdefs-generated/TARGETS.grpc | 3 +- .../third_party/address_sorting/TARGETS.grpc | 15 + etc/import/third_party/upb/TARGETS.grpc | 613 ++++++-- .../third_party/utf8_range/TARGETS.grpc | 8 + etc/repos.json | 12 +- 10 files changed, 2616 insertions(+), 970 deletions(-) create mode 100644 etc/import/third_party/address_sorting/TARGETS.grpc create mode 100644 etc/import/third_party/utf8_range/TARGETS.grpc diff --git a/etc/import/TARGETS.grpc b/etc/import/TARGETS.grpc index 627f8fd45..b6ef81bbf 100644 --- a/etc/import/TARGETS.grpc +++ b/etc/import/TARGETS.grpc @@ -5,6 +5,7 @@ , "src/cpp/client/client_callback.cc" , "src/cpp/client/client_context.cc" , "src/cpp/client/client_interceptor.cc" + , "src/cpp/client/client_stats_interceptor.cc" , "src/cpp/client/create_channel.cc" , "src/cpp/client/create_channel_internal.cc" , "src/cpp/client/create_channel_posix.cc" @@ -38,6 +39,7 @@ { "type": ["@", "rules", "data", "staged"] , "srcs": [ "src/cpp/client/create_channel_internal.h" + , "src/cpp/client/client_stats_interceptor.h" , "src/cpp/common/channel_filter.h" , "src/cpp/server/dynamic_thread_pool.h" , "src/cpp/server/external_connection_acceptor_impl.h" @@ -46,11 +48,40 @@ , "src/cpp/thread_manager/thread_manager.h" ] } -, "grpcxx_public_headers": - { "type": ["@", "rules", "data", "staged"] +, "grpc_unsecure": + { "type": ["@", "rules", "CC", "library"] + , "name": ["grpc_unsecure"] , "srcs": - [ ["./", "include/grpc++", "grpc++_public_headers"] - , ["./", "include/grpcpp", "grpcpp_public_headers"] + [ "src/core/lib/surface/init.cc" + , "src/core/plugin_registry/grpc_plugin_registry.cc" + , "src/core/plugin_registry/grpc_plugin_registry_noextra.cc" + ] + , "hdrs": [["include/grpc", "grpc_public_headers"]] + , "deps": + [ "channel_stack_builder" + , "config" + , "exec_ctx" + , "gpr" + , "grpc_base" + , "grpc_client_channel" + , "grpc_common" + , "grpc_http_filters" + , "grpc_security_base" + , "grpc_trace" + , "http_connect_handshaker" + , "iomgr_timer" + , ["src/core", "channel_args"] + , ["src/core", "channel_init"] + , ["src/core", "channel_stack_type"] + , ["src/core", "default_event_engine"] + , ["src/core", "experiments"] + , ["src/core", "forkable"] + , ["src/core", "grpc_authorization_base"] + , ["src/core", "init_internally"] + , ["src/core", "posix_event_engine_timer_manager"] + , ["src/core", "slice"] + , ["src/core", "tcp_connect_handshaker"] + , ["@", "absl", "absl/base", "core_headers"] ] } , "grpc": @@ -82,7 +113,7 @@ , "src/core/plugin_registry/grpc_plugin_registry.cc" , "src/core/plugin_registry/grpc_plugin_registry_extra.cc" ] - , "hdrs": [["./", "include/grpc", "grpc_public_headers"]] + , "hdrs": [["include/grpc", "grpc_public_headers"]] , "deps": [ "channel_stack_builder" , "config" @@ -108,6 +139,7 @@ , "uri_parser" , ["src/core", "channel_args"] , ["src/core", "channel_init"] + , ["src/core", "channel_creds_registry_init"] , ["src/core", "channel_stack_type"] , ["src/core", "default_event_engine"] , ["src/core", "experiments"] @@ -120,7 +152,21 @@ , ["src/core", "grpc_insecure_credentials"] , ["src/core", "grpc_local_credentials"] , ["src/core", "grpc_oauth2_credentials"] + , ["src/core", "grpc_lb_policy_rls"] + , ["src/core", "grpc_lb_policy_xds_cluster_resolver"] + , ["src/core", "grpc_lb_policy_xds_cluster_manager"] + , ["src/core", "grpc_lb_policy_xds_cluster_impl"] + , ["src/core", "grpc_lb_policy_xds_override_host"] + , ["src/core", "grpc_lb_policy_cds"] + , ["src/core", "grpc_lb_policy_ring_hash"] + , ["src/core", "grpc_lb_policy_xds_wrr_locality"] + , ["src/core", "grpc_rbac_filter"] + , ["src/core", "grpc_resolver_c2p"] + , ["src/core", "grpc_resolver_xds"] , ["src/core", "grpc_ssl_credentials"] + , ["src/core", "grpc_stateful_session_filter"] + , ["src/core", "grpc_xds_channel_stack_modifier"] + , ["src/core", "grpc_xds_client"] , ["src/core", "grpc_tls_credentials"] , ["src/core", "grpc_transport_chttp2_alpn"] , ["src/core", "httpcli_ssl_credentials"] @@ -132,17 +178,6 @@ , ["src/core", "slice_refcount"] , ["src/core", "tcp_connect_handshaker"] , ["src/core", "useful"] - , ["src/core", "grpc_lb_policy_cds"] - , ["src/core", "grpc_lb_policy_xds_cluster_impl"] - , ["src/core", "grpc_lb_policy_xds_cluster_manager"] - , ["src/core", "grpc_lb_policy_xds_cluster_resolver"] - , ["src/core", "grpc_lb_policy_xds_override_host"] - , ["src/core", "grpc_lb_policy_xds_wrr_locality"] - , ["src/core", "grpc_resolver_xds"] - , ["src/core", "grpc_resolver_c2p"] - , ["src/core", "grpc_xds_server_config_fetcher"] - , ["src/core", "grpc_stateful_session_filter"] - , ["src/core", "channel_creds_registry_init"] , ["@", "absl", "absl/base", "core_headers"] ] } @@ -172,54 +207,48 @@ , "name": ["gpr"] , "srcs": [ "src/core/lib/gpr/alloc.cc" - , "src/core/lib/gpr/cpu_iphone.cc" - , "src/core/lib/gpr/cpu_linux.cc" - , "src/core/lib/gpr/cpu_posix.cc" - , "src/core/lib/gpr/cpu_windows.cc" + , "src/core/lib/gpr/android/log.cc" + , "src/core/lib/gpr/iphone/cpu.cc" + , "src/core/lib/gpr/linux/cpu.cc" + , "src/core/lib/gpr/linux/log.cc" , "src/core/lib/gpr/log.cc" - , "src/core/lib/gpr/log_android.cc" - , "src/core/lib/gpr/log_linux.cc" - , "src/core/lib/gpr/log_posix.cc" - , "src/core/lib/gpr/log_windows.cc" + , "src/core/lib/gpr/msys/tmpfile.cc" + , "src/core/lib/gpr/posix/cpu.cc" + , "src/core/lib/gpr/posix/log.cc" + , "src/core/lib/gpr/posix/string.cc" + , "src/core/lib/gpr/posix/sync.cc" + , "src/core/lib/gpr/posix/time.cc" + , "src/core/lib/gpr/posix/tmpfile.cc" , "src/core/lib/gpr/string.cc" - , "src/core/lib/gpr/string_posix.cc" - , "src/core/lib/gpr/string_util_windows.cc" - , "src/core/lib/gpr/string_windows.cc" , "src/core/lib/gpr/sync.cc" , "src/core/lib/gpr/sync_abseil.cc" - , "src/core/lib/gpr/sync_posix.cc" - , "src/core/lib/gpr/sync_windows.cc" , "src/core/lib/gpr/time.cc" - , "src/core/lib/gpr/time_posix.cc" , "src/core/lib/gpr/time_precise.cc" - , "src/core/lib/gpr/time_windows.cc" - , "src/core/lib/gpr/tmpfile_msys.cc" - , "src/core/lib/gpr/tmpfile_posix.cc" - , "src/core/lib/gpr/tmpfile_windows.cc" + , "src/core/lib/gpr/windows/cpu.cc" + , "src/core/lib/gpr/windows/log.cc" + , "src/core/lib/gpr/windows/string.cc" + , "src/core/lib/gpr/windows/string_util.cc" + , "src/core/lib/gpr/windows/sync.cc" + , "src/core/lib/gpr/windows/time.cc" + , "src/core/lib/gpr/windows/tmpfile.cc" , "src/core/lib/gpr/wrap_memcpy.cc" , "src/core/lib/gprpp/crash.cc" , "src/core/lib/gprpp/fork.cc" - , "src/core/lib/gprpp/global_config_env.cc" , "src/core/lib/gprpp/host_port.cc" , "src/core/lib/gprpp/mpscq.cc" - , "src/core/lib/gprpp/stat_posix.cc" - , "src/core/lib/gprpp/stat_windows.cc" - , "src/core/lib/gprpp/thd_posix.cc" - , "src/core/lib/gprpp/thd_windows.cc" + , "src/core/lib/gprpp/posix/stat.cc" + , "src/core/lib/gprpp/posix/thd.cc" , "src/core/lib/gprpp/time_util.cc" + , "src/core/lib/gprpp/windows/stat.cc" + , "src/core/lib/gprpp/windows/thd.cc" ] , "hdrs": - [ ["./", "include/grpc", "gpr_public_headers"] - , "src/core/lib/gpr/alloc.h" + [ "src/core/lib/gpr/alloc.h" , "src/core/lib/gpr/string.h" , "src/core/lib/gpr/time_precise.h" , "src/core/lib/gpr/tmpfile.h" , "src/core/lib/gprpp/crash.h" , "src/core/lib/gprpp/fork.h" - , "src/core/lib/gprpp/global_config.h" - , "src/core/lib/gprpp/global_config_custom.h" - , "src/core/lib/gprpp/global_config_env.h" - , "src/core/lib/gprpp/global_config_generic.h" , "src/core/lib/gprpp/host_port.h" , "src/core/lib/gprpp/memory.h" , "src/core/lib/gprpp/mpscq.h" @@ -227,9 +256,11 @@ , "src/core/lib/gprpp/sync.h" , "src/core/lib/gprpp/thd.h" , "src/core/lib/gprpp/time_util.h" + , ["include/grpc", "gpr_public_headers"] ] , "deps": - [ "debug_location" + [ "config_vars" + , "debug_location" , ["src/core", "construct_destruct"] , ["src/core", "env"] , ["src/core", "event_engine_thread_local"] @@ -241,17 +272,27 @@ , ["src/core", "useful"] , ["@", "absl", "absl/base", "base"] , ["@", "absl", "absl/base", "core_headers"] + , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/memory", "memory"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "cord"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/synchronization", "synchronization"] , ["@", "absl", "absl/time", "time"] , ["@", "absl", "absl/types", "optional"] ] } +, "gpr_public_hdrs": + { "type": ["@", "rules", "CC", "library"] + , "name": ["gpr_public_hdrs"] + , "hdrs": [["include/grpc", "gpr_public_headers"]] + , "deps": + [ ["@", "absl", "absl/strings", "strings"] + , ["@", "absl", "absl/types", "variant"] + ] + } , "cpp_impl_of": { "type": ["@", "rules", "CC", "library"] , "name": ["cpp_impl_of"] @@ -261,55 +302,60 @@ { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_common"] , "deps": - [ "grpc_base" - , "census" + [ "census" + , "grpc_base" , "grpc_resolver_dns_ares" , "grpc_resolver_fake" , ["src/core", "grpc_backend_metric_filter"] - , ["src/core", "grpc_deadline_filter"] + , ["src/core", "grpc_channel_idle_filter"] , ["src/core", "grpc_client_authority_filter"] + , ["src/core", "grpc_deadline_filter"] + , ["src/core", "grpc_fault_injection_filter"] , ["src/core", "grpc_lb_policy_grpclb"] , ["src/core", "grpc_lb_policy_outlier_detection"] , ["src/core", "grpc_lb_policy_pick_first"] , ["src/core", "grpc_lb_policy_priority"] - , ["src/core", "grpc_lb_policy_ring_hash"] , ["src/core", "grpc_lb_policy_round_robin"] , ["src/core", "grpc_lb_policy_weighted_round_robin"] , ["src/core", "grpc_lb_policy_weighted_target"] - , ["src/core", "grpc_lb_policy_rls"] - , ["src/core", "grpc_channel_idle_filter"] , ["src/core", "grpc_message_size_filter"] , ["src/core", "grpc_resolver_binder"] , ["src/core", "grpc_resolver_dns_native"] + , ["src/core", "grpc_resolver_dns_plugin"] , ["src/core", "grpc_resolver_sockaddr"] , ["src/core", "grpc_transport_chttp2_client_connector"] , ["src/core", "grpc_transport_chttp2_server"] , ["src/core", "grpc_transport_inproc"] - , ["src/core", "grpc_fault_injection_filter"] ] } , "grpc_public_hdrs": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_public_hdrs"] - , "hdrs": - [ ["./", "include/grpc", "grpc_public_headers"] - , ["./", "include/grpc", "gpr_public_headers"] + , "hdrs": [["include/grpc", "grpc_public_headers"]] + , "deps": + [ "gpr_public_hdrs" + , ["@", "absl", "absl/status", "statusor"] + , ["@", "absl", "absl/strings", "strings"] ] } , "grpc++_public_hdrs": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc++_public_hdrs"] - , "hdrs": ["grpcxx_public_headers"] + , "hdrs": + [ ["include/grpc++", "grpc++_public_headers"] + , ["include/grpcpp", "grpcpp_public_headers"] + ] , "deps": [ "grpc_public_hdrs" , ["src/core", "gpr_atm"] + , ["@", "absl", "absl/strings", "cord"] , ["@", "absl", "absl/synchronization", "synchronization"] , ["@", "protobuf", "", "libprotobuf"] ] } , "grpc++": { "type": "export" - , "target": "grpc++internal" + , "target": "grpc++_internal" , "flexible_config": [ "OS" , "ARCH" @@ -328,22 +374,101 @@ , "PKG_CONFIG_ARGS" ] } -, "grpc++internal": +, "grpc++_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc++"] , "hdrs": - [ "grpcxx_public_headers" - , "src/cpp/client/secure_credentials.h" + [ "src/cpp/client/secure_credentials.h" , "src/cpp/common/secure_auth_context.h" , "src/cpp/server/secure_server_credentials.h" + , ["include/grpc++", "grpc++_public_headers"] + , ["include/grpcpp", "grpcpp_public_headers"] ] , "deps": [ "grpc++_base" , ["src/core", "gpr_atm"] , ["src/core", "slice"] - , "grpc++_xds_client" - , "grpc++_xds_server" - , "grpc++_binder" + , ["@", "absl", "absl/strings", "cord"] + ] + } +, "grpc_cronet_hdrs": + { "type": ["@", "rules", "CC", "library"] + , "name": ["grpc_cronet_hdrs"] + , "hdrs": ["include/grpc/grpc_cronet.h"] + , "deps": ["gpr_public_hdrs", "grpc_base"] + } +, "grpc++_cronet_credentials": + { "type": ["@", "rules", "CC", "library"] + , "name": ["grpc++_cronet_credentials"] + , "srcs": ["src/cpp/client/cronet_credentials.cc"] + , "hdrs": ["include/grpcpp/security/cronet_credentials.h"] + , "deps": ["grpc++_base", "grpc_cronet_hdrs", "grpc_public_hdrs"] + } +, "grpc_authorization_provider": + { "type": ["@", "rules", "CC", "library"] + , "name": ["grpc_authorization_provider"] + , "srcs": + [ "src/core/lib/security/authorization/grpc_authorization_policy_provider.cc" + , "src/core/lib/security/authorization/rbac_translator.cc" + ] + , "hdrs": + [ "src/core/lib/security/authorization/grpc_authorization_policy_provider.h" + , "src/core/lib/security/authorization/rbac_translator.h" + ] + , "deps": + [ "gpr" + , "grpc_base" + , "grpc_public_hdrs" + , "grpc_trace" + , "ref_counted_ptr" + , ["src/core", "error"] + , ["src/core", "grpc_audit_logging"] + , ["src/core", "grpc_authorization_base"] + , ["src/core", "grpc_matchers"] + , ["src/core", "grpc_rbac_engine"] + , ["src/core", "json"] + , ["src/core", "json_reader"] + , ["src/core", "slice"] + , ["src/core", "slice_refcount"] + , ["src/core", "status_helper"] + , ["src/core", "useful"] + , ["@", "absl", "absl/base", "core_headers"] + , ["@", "absl", "absl/status", "status"] + , ["@", "absl", "absl/status", "statusor"] + , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] + , ["@", "absl", "absl/types", "optional"] + ] + } +, "grpc++_authorization_provider": + { "type": ["@", "rules", "CC", "library"] + , "name": ["grpc++_authorization_provider"] + , "srcs": ["src/cpp/server/authorization_policy_provider.cc"] + , "hdrs": ["include/grpcpp/security/authorization_policy_provider.h"] + , "deps": + [ "gpr" + , "grpc++" + , "grpc++_public_hdrs" + , "grpc_authorization_provider" + , "grpc_public_hdrs" + ] + } +, "grpc_cel_engine": + { "type": ["@", "rules", "CC", "library"] + , "name": ["grpc_cel_engine"] + , "srcs": ["src/core/lib/security/authorization/cel_authorization_engine.cc"] + , "hdrs": ["src/core/lib/security/authorization/cel_authorization_engine.h"] + , "deps": + [ "gpr" + , "grpc_mock_cel" + , ["src/core", "grpc_authorization_base"] + , ["src/core/ext/upb-generated", "upb-generated-lib"] + , ["@", "absl", "absl/container", "flat_hash_set"] + , ["@", "absl", "absl/strings", "strings"] + , ["@", "absl", "absl/types", "optional"] + , ["@", "absl", "absl/types", "span"] + , ["third_party/upb", "collections"] + , ["third_party/upb", "upb"] ] } , "grpc++_binder": @@ -370,8 +495,7 @@ , "src/core/ext/transport/binder/wire_format/wire_writer.cc" ] , "hdrs": - [ ["./", "include/grpcpp", "grpc++_binder_grpcpp_headers"] - , "src/core/ext/transport/binder/client/binder_connector.h" + [ "src/core/ext/transport/binder/client/binder_connector.h" , "src/core/ext/transport/binder/client/channel_create_impl.h" , "src/core/ext/transport/binder/client/connection_id_generator.h" , "src/core/ext/transport/binder/client/endpoint_binder_pool.h" @@ -391,6 +515,9 @@ , "src/core/ext/transport/binder/wire_format/wire_reader.h" , "src/core/ext/transport/binder/wire_format/wire_reader_impl.h" , "src/core/ext/transport/binder/wire_format/wire_writer.h" + , "include/grpcpp/create_channel_binder.h" + , "include/grpcpp/security/binder_credentials.h" + , "include/grpcpp/security/binder_security_policy.h" ] , "deps": [ "config" @@ -419,6 +546,7 @@ , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/cleanup", "cleanup"] , ["@", "absl", "absl/container", "flat_hash_map"] + , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/hash", "hash"] , ["@", "absl", "absl/memory", "memory"] , ["@", "absl", "absl/meta", "type_traits"] @@ -456,15 +584,62 @@ ] , "deps": ["gpr", "grpc", "grpc++_base"] } +, "grpc++_unsecure": + { "type": ["@", "rules", "CC", "library"] + , "name": ["grpc++_unsecure"] + , "srcs": + [ "src/cpp/client/insecure_credentials.cc" + , "src/cpp/common/insecure_create_auth_context.cc" + , "src/cpp/server/insecure_server_credentials.cc" + ] + , "deps": + [ "gpr" + , "grpc++_base_unsecure" + , "grpc++_codegen_proto" + , "grpc_public_hdrs" + , "grpc_unsecure" + , ["src/core", "grpc_insecure_credentials"] + ] + } +, "grpc++_error_details": + { "type": ["@", "rules", "CC", "library"] + , "name": ["grpc++_error_details"] + , "srcs": ["src/cpp/util/error_details.cc"] + , "hdrs": + [ "include/grpc++/support/error_details.h" + , "include/grpcpp/support/error_details.h" + ] + , "deps": ["grpc++"] + } +, "grpc++_alts": + { "type": ["@", "rules", "CC", "library"] + , "name": ["grpc++_alts"] + , "srcs": ["src/cpp/common/alts_context.cc", "src/cpp/common/alts_util.cc"] + , "hdrs": + [ "include/grpcpp/security/alts_context.h" + , "include/grpcpp/security/alts_util.h" + ] + , "deps": + [ "alts_upb" + , "gpr" + , "grpc++" + , "grpc_base" + , "tsi_alts_credentials" + , ["third_party/upb", "collections"] + , ["third_party/upb", "upb"] + ] + } , "census": { "type": ["@", "rules", "CC", "library"] , "name": ["census"] , "srcs": ["src/core/ext/filters/census/grpc_context.cc"] - , "hdrs": [["./", "include/grpc", "census_headers"]] - , "deps": ["grpc_base"] + , "hdrs": [["include/grpc", "census_headers"]] + , "deps": + ["gpr", "grpc_base", "grpc_public_hdrs", "grpc_trace", "legacy_context"] } , "gpr_platform": { "type": ["@", "rules", "CC", "library"] + , "name": ["gpr_platform"] , "hdrs": [ "include/grpc/impl/codegen/port_platform.h" , "include/grpc/support/port_platform.h" @@ -475,35 +650,35 @@ { "type": ["@", "rules", "CC", "library"] , "name": ["event_engine_base_hdrs"] , "hdrs": - [ ["./", "include/grpc", "grpc_public_headers"] - , ["./", "include/grpc", "grpc_public_event_engine_headers"] + [ ["include/grpc", "grpc_public_event_engine_headers"] + , ["include/grpc", "grpc_public_headers"] ] , "deps": [ "gpr" + , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/time", "time"] , ["@", "absl", "absl/types", "optional"] - , ["@", "absl", "absl/functional", "any_invocable"] ] } , "grpc_base": { "type": ["@", "rules", "CC", "library"] - , "arguments_config": ["ARCH", "TARGET_ARCH"] , "name": ["grpc_base"] , "srcs": - [ "src/core/lib/channel/channel_stack.cc" + [ "src/core/lib/channel/call_tracer.cc" + , "src/core/lib/channel/channel_stack.cc" , "src/core/lib/channel/channel_stack_builder_impl.cc" , "src/core/lib/channel/channel_trace.cc" , "src/core/lib/channel/channelz.cc" , "src/core/lib/channel/channelz_registry.cc" , "src/core/lib/channel/connected_channel.cc" , "src/core/lib/channel/promise_based_filter.cc" + , "src/core/lib/channel/server_call_tracer_filter.cc" , "src/core/lib/channel/status_util.cc" , "src/core/lib/compression/compression.cc" , "src/core/lib/compression/compression_internal.cc" , "src/core/lib/compression/message_compress.cc" - , "src/core/lib/iomgr/buffer_list.cc" , "src/core/lib/iomgr/call_combiner.cc" , "src/core/lib/iomgr/cfstream_handle.cc" , "src/core/lib/iomgr/dualstack_socket_posix.cc" @@ -517,12 +692,14 @@ , "src/core/lib/iomgr/ev_poll_posix.cc" , "src/core/lib/iomgr/ev_posix.cc" , "src/core/lib/iomgr/ev_windows.cc" + , "src/core/lib/iomgr/event_engine_shims/closure.cc" + , "src/core/lib/iomgr/event_engine_shims/endpoint.cc" + , "src/core/lib/iomgr/event_engine_shims/tcp_client.cc" , "src/core/lib/iomgr/fork_posix.cc" , "src/core/lib/iomgr/fork_windows.cc" , "src/core/lib/iomgr/gethostname_fallback.cc" , "src/core/lib/iomgr/gethostname_host_name_max.cc" , "src/core/lib/iomgr/gethostname_sysconf.cc" - , "src/core/lib/iomgr/internal_errqueue.cc" , "src/core/lib/iomgr/iocp_windows.cc" , "src/core/lib/iomgr/iomgr.cc" , "src/core/lib/iomgr/iomgr_posix.cc" @@ -581,6 +758,7 @@ , "src/core/lib/surface/server.cc" , "src/core/lib/surface/validate_metadata.cc" , "src/core/lib/surface/version.cc" + , "src/core/lib/transport/batch_builder.cc" , "src/core/lib/transport/connectivity_state.cc" , "src/core/lib/transport/error_utils.cc" , "src/core/lib/transport/metadata_batch.cc" @@ -589,14 +767,9 @@ , "src/core/lib/transport/timeout_encoding.cc" , "src/core/lib/transport/transport.cc" , "src/core/lib/transport/transport_op_string.cc" - , "src/core/lib/iomgr/event_engine_shims/closure.cc" - , "src/core/lib/iomgr/event_engine_shims/endpoint.cc" - , "src/core/lib/iomgr/event_engine_shims/tcp_client.cc" ] , "hdrs": - [ ["./", "include/grpc", "grpc_public_headers"] - , ["./", "include/grpc", "grpc_public_event_engine_headers"] - , "src/core/lib/channel/call_finalization.h" + [ "src/core/lib/channel/call_finalization.h" , "src/core/lib/channel/call_tracer.h" , "src/core/lib/channel/channel_stack.h" , "src/core/lib/channel/channel_stack_builder_impl.h" @@ -604,13 +777,11 @@ , "src/core/lib/channel/channelz.h" , "src/core/lib/channel/channelz_registry.h" , "src/core/lib/channel/connected_channel.h" - , "src/core/lib/channel/context.h" , "src/core/lib/channel/promise_based_filter.h" , "src/core/lib/channel/status_util.h" , "src/core/lib/compression/compression_internal.h" , "src/core/lib/compression/message_compress.h" , "src/core/lib/iomgr/block_annotate.h" - , "src/core/lib/iomgr/buffer_list.h" , "src/core/lib/iomgr/call_combiner.h" , "src/core/lib/iomgr/cfstream_handle.h" , "src/core/lib/iomgr/dynamic_annotations.h" @@ -622,8 +793,10 @@ , "src/core/lib/iomgr/ev_epoll1_linux.h" , "src/core/lib/iomgr/ev_poll_posix.h" , "src/core/lib/iomgr/ev_posix.h" + , "src/core/lib/iomgr/event_engine_shims/closure.h" + , "src/core/lib/iomgr/event_engine_shims/endpoint.h" + , "src/core/lib/iomgr/event_engine_shims/tcp_client.h" , "src/core/lib/iomgr/gethostname.h" - , "src/core/lib/iomgr/internal_errqueue.h" , "src/core/lib/iomgr/iocp_windows.h" , "src/core/lib/iomgr/iomgr.h" , "src/core/lib/iomgr/load_file.h" @@ -669,42 +842,34 @@ , "src/core/lib/surface/lame_client.h" , "src/core/lib/surface/server.h" , "src/core/lib/surface/validate_metadata.h" + , "src/core/lib/transport/batch_builder.h" , "src/core/lib/transport/connectivity_state.h" + , "src/core/lib/transport/custom_metadata.h" , "src/core/lib/transport/error_utils.h" , "src/core/lib/transport/metadata_batch.h" , "src/core/lib/transport/parsed_metadata.h" + , "src/core/lib/transport/simple_slice_based_metadata.h" , "src/core/lib/transport/status_conversion.h" , "src/core/lib/transport/timeout_encoding.h" , "src/core/lib/transport/transport.h" , "src/core/lib/transport/transport_impl.h" - , "src/core/lib/iomgr/event_engine_shims/closure.h" - , "src/core/lib/iomgr/event_engine_shims/endpoint.h" - , "src/core/lib/iomgr/event_engine_shims/tcp_client.h" + , ["include/grpc", "grpc_public_event_engine_headers"] + , ["include/grpc", "grpc_public_headers"] ] - , "private-ldflags": - { "type": "case" - , "expr": - { "type": "var" - , "name": "TARGET_ARCH" - , "default": - { "type": "var" - , "name": "ARCH" - , "default": - {"type": "fail", "msg": "Required variable 'ARCH' is not set."} - } - } - , "case": {"arm": ["-latomic"]} - } , "deps": [ "channel_stack_builder" , "config" + , "config_vars" , "cpp_impl_of" , "debug_location" , "exec_ctx" , "gpr" , "grpc_public_hdrs" , "grpc_trace" + , "iomgr_buffer_list" + , "iomgr_internal_errqueue" , "iomgr_timer" + , "legacy_context" , "orphanable" , "parse_address" , "promise" @@ -713,6 +878,7 @@ , "stats" , "uri_parser" , "work_serializer" + , ["src/core", "1999"] , ["src/core", "activity"] , ["src/core", "arena"] , ["src/core", "arena_promise"] @@ -720,6 +886,7 @@ , ["src/core", "basic_join"] , ["src/core", "basic_seq"] , ["src/core", "bitset"] + , ["src/core", "cancel_callback"] , ["src/core", "channel_args"] , ["src/core", "channel_args_endpoint_config"] , ["src/core", "channel_args_preconditioning"] @@ -728,40 +895,51 @@ , ["src/core", "channel_stack_type"] , ["src/core", "chunked_vector"] , ["src/core", "closure"] + , ["src/core", "construct_destruct"] , ["src/core", "context"] , ["src/core", "default_event_engine"] , ["src/core", "dual_ref_counted"] , ["src/core", "error"] , ["src/core", "event_engine_common"] + , ["src/core", "event_engine_memory_allocator_factory"] , ["src/core", "event_engine_shim"] , ["src/core", "event_engine_tcp_socket_utils"] , ["src/core", "event_engine_trace"] , ["src/core", "event_log"] , ["src/core", "experiments"] + , ["src/core", "for_each"] , ["src/core", "gpr_atm"] , ["src/core", "gpr_manual_constructor"] , ["src/core", "gpr_spinlock"] , ["src/core", "grpc_sockaddr"] , ["src/core", "http2_errors"] + , ["src/core", "if"] , ["src/core", "init_internally"] , ["src/core", "iomgr_fwd"] , ["src/core", "iomgr_port"] , ["src/core", "json"] + , ["src/core", "json_writer"] + , ["src/core", "latch"] + , ["src/core", "loop"] , ["src/core", "map"] , ["src/core", "match"] , ["src/core", "memory_quota"] + , ["src/core", "metadata_compression_traits"] , ["src/core", "no_destruct"] , ["src/core", "notification"] , ["src/core", "packed_table"] + , ["src/core", "per_cpu"] , ["src/core", "pipe"] , ["src/core", "poll"] , ["src/core", "pollset_set"] , ["src/core", "posix_event_engine_base_hdrs"] , ["src/core", "promise_status"] + , ["src/core", "race"] , ["src/core", "ref_counted"] , ["src/core", "resolved_address"] , ["src/core", "resource_quota"] , ["src/core", "resource_quota_trace"] + , ["src/core", "seq"] , ["src/core", "slice"] , ["src/core", "slice_buffer"] , ["src/core", "slice_cast"] @@ -776,6 +954,8 @@ , ["src/core", "try_join"] , ["src/core", "try_seq"] , ["src/core", "useful"] + , ["src/core", "windows_event_engine"] + , ["src/core", "windows_event_engine_listener"] , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/cleanup", "cleanup"] , ["@", "absl", "absl/container", "flat_hash_map"] @@ -785,8 +965,8 @@ , ["@", "absl", "absl/meta", "type_traits"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/time", "time"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] @@ -794,6 +974,96 @@ , ["@", "zlib", "", "zlib"] ] } +, "legacy_context": + { "type": ["@", "rules", "CC", "library"] + , "name": ["legacy_context"] + , "hdrs": ["src/core/lib/channel/context.h"] + , "deps": ["gpr_platform", ["src/core", "context"]] + } +, "lb_load_data_store": + { "type": ["@", "rules", "CC", "library"] + , "name": ["lb_load_data_store"] + , "srcs": ["src/cpp/server/load_reporter/load_data_store.cc"] + , "hdrs": + [ "src/cpp/server/load_reporter/constants.h" + , "src/cpp/server/load_reporter/load_data_store.h" + ] + , "deps": ["gpr", "gpr_platform", "grpc++", ["src/core", "grpc_sockaddr"]] + } +, "lb_server_load_reporting_service_server_builder_plugin": + { "type": ["@", "rules", "CC", "library"] + , "name": ["lb_server_load_reporting_service_server_builder_plugin"] + , "srcs": + [ "src/cpp/server/load_reporter/load_reporting_service_server_builder_plugin.cc" + ] + , "hdrs": + [ "src/cpp/server/load_reporter/load_reporting_service_server_builder_plugin.h" + ] + , "deps": ["gpr_platform", "grpc++", "lb_load_reporter_service"] + } +, "grpcpp_server_load_reporting": + { "type": ["@", "rules", "CC", "library"] + , "name": ["grpcpp_server_load_reporting"] + , "srcs": + [ "src/cpp/server/load_reporter/load_reporting_service_server_builder_option.cc" + , "src/cpp/server/load_reporter/util.cc" + ] + , "hdrs": ["include/grpcpp/ext/server_load_reporting.h"] + , "deps": + [ "gpr" + , "gpr_platform" + , "grpc" + , "grpc++" + , "grpc++_public_hdrs" + , "grpc_public_hdrs" + , "lb_server_load_reporting_service_server_builder_plugin" + , ["src/core", "lb_server_load_reporting_filter"] + ] + } +, "lb_load_reporter_service": + { "type": ["@", "rules", "CC", "library"] + , "name": ["lb_load_reporter_service"] + , "srcs": + ["src/cpp/server/load_reporter/load_reporter_async_service_impl.cc"] + , "hdrs": ["src/cpp/server/load_reporter/load_reporter_async_service_impl.h"] + , "deps": + [ "gpr" + , "grpc++" + , "lb_load_reporter" + , ["@", "absl", "absl/memory", "memory"] + , ["@", "protobuf", "", "libprotobuf"] + ] + } +, "lb_get_cpu_stats": + { "type": ["@", "rules", "CC", "library"] + , "name": ["lb_get_cpu_stats"] + , "srcs": + [ "src/cpp/server/load_reporter/get_cpu_stats_linux.cc" + , "src/cpp/server/load_reporter/get_cpu_stats_macos.cc" + , "src/cpp/server/load_reporter/get_cpu_stats_unsupported.cc" + , "src/cpp/server/load_reporter/get_cpu_stats_windows.cc" + ] + , "hdrs": ["src/cpp/server/load_reporter/get_cpu_stats.h"] + , "deps": ["gpr", "gpr_platform"] + } +, "lb_load_reporter": + { "type": ["@", "rules", "CC", "library"] + , "name": ["lb_load_reporter"] + , "srcs": ["src/cpp/server/load_reporter/load_reporter.cc"] + , "hdrs": + [ "src/cpp/server/load_reporter/constants.h" + , "src/cpp/server/load_reporter/load_reporter.h" + ] + , "deps": + [ "gpr" + , "lb_get_cpu_stats" + , "lb_load_data_store" + , ["src/proto/grpc/lb/v1", "load_reporter_proto"] + , ["@", "opencensus-stats", "", ""] + , ["@", "opencensus-tags", "", ""] + , ["@", "protobuf", "", "libprotobuf"] + ] + } , "grpc_security_base": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_security_base"] @@ -811,8 +1081,7 @@ , "src/core/lib/security/transport/tsi_error.cc" ] , "hdrs": - [ ["./", "include/grpc", "grpc_public_headers"] - , "src/core/lib/security/context/security_context.h" + [ "src/core/lib/security/context/security_context.h" , "src/core/lib/security/credentials/call_creds_util.h" , "src/core/lib/security/credentials/composite/composite_credentials.h" , "src/core/lib/security/credentials/credentials.h" @@ -822,6 +1091,7 @@ , "src/core/lib/security/transport/secure_endpoint.h" , "src/core/lib/security/transport/security_handshaker.h" , "src/core/lib/security/transport/tsi_error.h" + , ["include/grpc", "grpc_public_headers"] ] , "deps": [ "config" @@ -832,6 +1102,7 @@ , "grpc_public_hdrs" , "grpc_trace" , "handshaker" + , "legacy_context" , "promise" , "ref_counted_ptr" , "stats" @@ -899,7 +1170,7 @@ , "src/core/tsi/alts/handshaker/transport_security_common_api.cc" ] , "hdrs": - [ ["./", "include/grpc", "grpc_secure_public_headers"] + [ ["include/grpc", "grpc_secure_public_headers"] , "src/core/lib/security/credentials/alts/check_gcp_environment.h" , "src/core/lib/security/credentials/alts/grpc_alts_credentials_options.h" , "src/core/tsi/alts/handshaker/transport_security_common_api.h" @@ -907,6 +1178,7 @@ , "deps": [ "gpr" , "grpc_public_hdrs" + , ["third_party/upb", "upb"] , ["src/core/ext/upb-generated", "upb-generated-lib"] ] } @@ -920,18 +1192,17 @@ , "tsi_fake_credentials" , ["src/core", "tsi_local_credentials"] , ["src/core", "useful"] - , ["@", "libssl", "", "ssl"] - , ["@", "libssl", "", "crypto"] - , ["third_party/upb", "upb"] , ["@", "absl", "absl/strings", "strings"] + , ["@", "ssl", "", "crypto"] + , ["@", "ssl", "", "ssl"] + , ["third_party/upb", "upb"] ] } , "grpc++_base": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc++_base"] , "srcs": - [ "grpcxx_sources" - , "src/cpp/client/insecure_credentials.cc" + [ "src/cpp/client/insecure_credentials.cc" , "src/cpp/client/secure_credentials.cc" , "src/cpp/common/auth_property_iterator.cc" , "src/cpp/common/secure_auth_context.cc" @@ -942,13 +1213,15 @@ , "src/cpp/common/tls_credentials_options.cc" , "src/cpp/server/insecure_server_credentials.cc" , "src/cpp/server/secure_server_credentials.cc" + , ["", "grpcxx_sources"] ] , "hdrs": - [ "grpcxx_public_headers" - , "grpcxx_headers" - , "src/cpp/client/secure_credentials.h" + [ "src/cpp/client/secure_credentials.h" , "src/cpp/common/secure_auth_context.h" , "src/cpp/server/secure_server_credentials.h" + , ["", "grpcxx_headers"] + , ["include/grpc++", "grpc++_public_headers"] + , ["include/grpcpp", "grpcpp_public_headers"] ] , "deps": [ "channel_stack_builder" @@ -967,6 +1240,7 @@ , "grpcpp_call_metric_recorder" , "grpcpp_status" , "iomgr_timer" + , "legacy_context" , "ref_counted_ptr" , ["src/core", "arena"] , ["src/core", "channel_args"] @@ -979,10 +1253,12 @@ , ["src/core", "error"] , ["src/core", "gpr_atm"] , ["src/core", "gpr_manual_constructor"] + , ["src/core", "grpc_audit_logging"] , ["src/core", "grpc_backend_metric_provider"] , ["src/core", "grpc_service_config"] , ["src/core", "grpc_transport_inproc"] , ["src/core", "json"] + , ["src/core", "json_reader"] , ["src/core", "ref_counted"] , ["src/core", "resource_quota"] , ["src/core", "slice"] @@ -994,17 +1270,72 @@ , ["src/core", "time"] , ["src/core", "useful"] , ["@", "absl", "absl/base", "core_headers"] + , ["@", "absl", "absl/container", "inlined_vector"] + , ["@", "absl", "absl/memory", "memory"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/synchronization", "synchronization"] - , ["@", "absl", "absl/memory", "memory"] , ["@", "absl", "absl/types", "optional"] - , ["@", "absl", "absl/container", "inlined_vector"] + , ["@", "protobuf", "", "libprotobuf"] , ["third_party/upb", "upb"] , ["src/core/ext/upb-generated", "upb-generated-lib"] + ] + } +, "grpc++_base_unsecure": + { "type": ["@", "rules", "CC", "library"] + , "name": ["grpc++_base_unsecure"] + , "srcs": [["", "grpcxx_sources"]] + , "hdrs": + [ ["", "grpcxx_headers"] + , ["include/grpc++", "grpc++_public_headers"] + , ["include/grpcpp", "grpcpp_public_headers"] + ] + , "deps": + [ "channel_stack_builder" + , "config" + , "exec_ctx" + , "gpr" + , "grpc_base" + , "grpc_health_upb" + , "grpc_public_hdrs" + , "grpc_service_config_impl" + , "grpc_trace" + , "grpc_unsecure" + , "grpcpp_backend_metric_recorder" + , "grpcpp_call_metric_recorder" + , "grpcpp_status" + , "iomgr_timer" + , "legacy_context" + , "ref_counted_ptr" + , ["src/core", "arena"] + , ["src/core", "channel_args"] + , ["src/core", "channel_init"] + , ["src/core", "closure"] + , ["src/core", "error"] + , ["src/core", "gpr_atm"] + , ["src/core", "gpr_manual_constructor"] + , ["src/core", "grpc_backend_metric_provider"] + , ["src/core", "grpc_insecure_credentials"] + , ["src/core", "grpc_service_config"] + , ["src/core", "grpc_transport_inproc"] + , ["src/core", "ref_counted"] + , ["src/core", "resource_quota"] + , ["src/core", "slice"] + , ["src/core", "socket_mutator"] + , ["src/core", "time"] + , ["src/core", "useful"] + , ["@", "absl", "absl/base", "core_headers"] + , ["@", "absl", "absl/memory", "memory"] + , ["@", "absl", "absl/status", "status"] + , ["@", "absl", "absl/status", "statusor"] + , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] + , ["@", "absl", "absl/synchronization", "synchronization"] + , ["@", "absl", "absl/types", "optional"] , ["@", "protobuf", "", "libprotobuf"] + , ["third_party/upb", "upb"] ] } , "grpc++_codegen_proto": @@ -1025,15 +1356,34 @@ { "type": ["@", "rules", "CC", "library"] , "name": ["grpc++_config_proto"] , "hdrs": - [ ["./", "include/grpc++", "grpc++_config_proto_headers"] - , ["./", "include/grpcpp", "grpcpp_config_proto_headers"] + [ ["include/grpc++", "grpc++_config_proto_headers"] + , ["include/grpcpp", "grpcpp_config_proto_headers"] ] , "deps": [["@", "protobuf", "", "libprotobuf"]] } +, "grpc++_reflection": + { "type": ["@", "rules", "CC", "library"] + , "name": ["grpc++_reflection"] + , "srcs": + [ "src/cpp/ext/proto_server_reflection.cc" + , "src/cpp/ext/proto_server_reflection_plugin.cc" + ] + , "hdrs": + [ "src/cpp/ext/proto_server_reflection.h" + , "include/grpc++/ext/proto_server_reflection_plugin.h" + , "include/grpcpp/ext/proto_server_reflection_plugin.h" + ] + , "deps": + [ "grpc++" + , "grpc++_config_proto" + , ["src/proto/grpc/reflection/v1alpha", "reflection_proto"] + , ["@", "protobuf", "", "libprotobuf"] + ] + } , "grpcpp_call_metric_recorder": { "type": ["@", "rules", "CC", "library"] , "name": ["grpcpp_call_metric_recorder"] - , "hdrs": [["./", "include/grpcpp", "grpcpp_call_metric_recorder_headers"]] + , "hdrs": [["include/grpcpp", "grpcpp_call_metric_recorder_headers"]] , "deps": [ "grpc++_public_hdrs" , ["@", "absl", "absl/strings", "strings"] @@ -1045,8 +1395,8 @@ , "name": ["grpcpp_backend_metric_recorder"] , "srcs": ["src/cpp/server/backend_metric_recorder.cc"] , "hdrs": - [ ["./", "include/grpcpp", "grpcpp_backend_metric_recorder_headers"] - , "src/cpp/server/backend_metric_recorder.h" + [ "src/cpp/server/backend_metric_recorder.h" + , ["include/grpcpp", "grpcpp_backend_metric_recorder_headers"] ] , "deps": [ "gpr" @@ -1059,6 +1409,158 @@ , ["@", "absl", "absl/strings", "strings"] ] } +, "grpcpp_orca_service": + { "type": ["@", "rules", "CC", "library"] + , "name": ["grpcpp_orca_service"] + , "srcs": ["src/cpp/server/orca/orca_service.cc"] + , "hdrs": ["include/grpcpp/ext/orca_service.h"] + , "deps": + [ "debug_location" + , "exec_ctx" + , "gpr" + , "grpc++" + , "grpc_base" + , "grpcpp_backend_metric_recorder" + , "ref_counted_ptr" + , ["@", "protobuf", "", "libprotobuf"] + , ["src/core", "default_event_engine"] + , ["src/core", "grpc_backend_metric_data"] + , ["src/core", "ref_counted"] + , ["src/core", "time"] + , ["src/core/ext/upb-generated", "upb-generated-lib"] + , ["@", "absl", "absl/base", "core_headers"] + , ["@", "absl", "absl/strings", "strings"] + , ["@", "absl", "absl/time", "time"] + , ["@", "absl", "absl/types", "optional"] + , ["third_party/upb", "upb"] + ] + } +, "grpcpp_channelz": + { "type": ["@", "rules", "CC", "library"] + , "name": ["grpcpp_channelz"] + , "srcs": + [ "src/cpp/server/channelz/channelz_service.cc" + , "src/cpp/server/channelz/channelz_service_plugin.cc" + ] + , "hdrs": + [ "src/cpp/server/channelz/channelz_service.h" + , "include/grpcpp/ext/channelz_service_plugin.h" + ] + , "deps": + [ "gpr" + , "grpc" + , "grpc++" + , "grpc++_config_proto" + , ["src/proto/grpc/channelz", "channelz_proto"] + , ["@", "protobuf", "", "libprotobuf"] + ] + } +, "grpcpp_csds": + { "type": ["@", "rules", "CC", "library"] + , "name": ["grpcpp_csds"] + , "srcs": ["src/cpp/server/csds/csds.cc"] + , "hdrs": ["src/cpp/server/csds/csds.h"] + , "deps": + [ "gpr" + , "grpc" + , "grpc++_base" + , ["src/proto/grpc/testing/xds/v3", "csds_proto"] + , ["@", "absl", "absl/status", "status"] + , ["@", "absl", "absl/status", "statusor"] + ] + } +, "grpcpp_admin": + { "type": ["@", "rules", "CC", "library"] + , "name": ["grpcpp_admin"] + , "srcs": ["src/cpp/server/admin/admin_services.cc"] + , "hdrs": ["include/grpcpp/ext/admin_services.h"] + , "deps": + [ "gpr" + , "grpc++" + , "grpcpp_channelz" + , ["@", "absl", "absl/memory", "memory"] + ] + } +, "grpc++_test": + { "type": ["@", "rules", "CC", "library"] + , "name": ["grpc++_test"] + , "srcs": ["src/cpp/client/channel_test_peer.cc"] + , "hdrs": + [ "include/grpc++/test/mock_stream.h" + , "include/grpc++/test/server_context_test_spouse.h" + , "include/grpcpp/test/channel_test_peer.h" + , "include/grpcpp/test/client_context_test_peer.h" + , "include/grpcpp/test/default_reactor_test_peer.h" + , "include/grpcpp/test/mock_stream.h" + , "include/grpcpp/test/server_context_test_spouse.h" + ] + , "deps": ["grpc++", "grpc_base", ["@", "gtest", "", ""]] + } +, "grpc_opencensus_plugin": + { "type": ["@", "rules", "CC", "library"] + , "name": ["grpc_opencensus_plugin"] + , "srcs": + [ "src/cpp/ext/filters/census/client_filter.cc" + , "src/cpp/ext/filters/census/context.cc" + , "src/cpp/ext/filters/census/grpc_plugin.cc" + , "src/cpp/ext/filters/census/measures.cc" + , "src/cpp/ext/filters/census/rpc_encoding.cc" + , "src/cpp/ext/filters/census/server_call_tracer.cc" + , "src/cpp/ext/filters/census/views.cc" + ] + , "hdrs": + [ "include/grpcpp/opencensus.h" + , "src/cpp/ext/filters/census/client_filter.h" + , "src/cpp/ext/filters/census/context.h" + , "src/cpp/ext/filters/census/grpc_plugin.h" + , "src/cpp/ext/filters/census/measures.h" + , "src/cpp/ext/filters/census/open_census_call_tracer.h" + , "src/cpp/ext/filters/census/rpc_encoding.h" + , "src/cpp/ext/filters/census/server_call_tracer.h" + ] + , "deps": + [ "channel_stack_builder" + , "config" + , "gpr" + , "grpc++_base" + , "grpc_base" + , "grpc_public_hdrs" + , "legacy_context" + , ["src/core", "arena"] + , ["src/core", "arena_promise"] + , ["src/core", "channel_args"] + , ["src/core", "channel_fwd"] + , ["src/core", "channel_stack_type"] + , ["src/core", "context"] + , ["src/core", "error"] + , ["src/core", "experiments"] + , ["src/core", "slice"] + , ["src/core", "slice_buffer"] + , ["src/core", "slice_refcount"] + , ["@", "absl", "absl/base", "base"] + , ["@", "absl", "absl/base", "core_headers"] + , ["@", "absl", "absl/meta", "type_traits"] + , ["@", "absl", "absl/status", "status"] + , ["@", "absl", "absl/status", "statusor"] + , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] + , ["@", "absl", "absl/time", "time"] + , ["@", "absl", "absl/types", "optional"] + , ["@", "opencensus-stats", "", ""] + , ["@", "opencensus-tags", "", ""] + , ["@", "opencensus-tags-context_util", "", ""] + , ["@", "opencensus-trace", "", ""] + , ["@", "opencensus-trace-context_util", "", ""] + , ["@", "opencensus-trace-propagation", "", ""] + , ["@", "opencensus-trace-span_context", "", ""] + ] + } +, "grpcpp_gcp_observability": + { "type": ["@", "rules", "CC", "library"] + , "name": ["grpcpp_gcp_observability"] + , "hdrs": ["include/grpcpp/ext/gcp_observability.h"] + , "deps": [["src/cpp/ext/gcp", "observability"]] + } , "work_serializer": { "type": ["@", "rules", "CC", "library"] , "name": ["work_serializer"] @@ -1077,7 +1579,43 @@ , "name": ["grpc_trace"] , "srcs": ["src/core/lib/debug/trace.cc"] , "hdrs": ["src/core/lib/debug/trace.h"] - , "deps": ["gpr", "grpc_public_hdrs"] + , "deps": + [ "config_vars" + , "gpr" + , "grpc_public_hdrs" + , ["@", "absl", "absl/strings", "strings"] + ] + } +, "load_config": + { "type": ["@", "rules", "CC", "library"] + , "name": ["load_config"] + , "srcs": ["src/core/lib/config/load_config.cc"] + , "hdrs": ["src/core/lib/config/load_config.h"] + , "deps": + [ "gpr_platform" + , ["src/core", "env"] + , ["src/core", "gpr_log_internal"] + , ["@", "absl", "absl/flags", "flag"] + , ["@", "absl", "absl/flags", "marshalling"] + , ["@", "absl", "absl/strings", "strings"] + , ["@", "absl", "absl/types", "optional"] + ] + } +, "config_vars": + { "type": ["@", "rules", "CC", "library"] + , "name": ["config_vars"] + , "srcs": + [ "src/core/lib/config/config_vars.cc" + , "src/core/lib/config/config_vars_non_generated.cc" + ] + , "hdrs": ["src/core/lib/config/config_vars.h"] + , "deps": + [ "gpr_platform" + , "load_config" + , ["@", "absl", "absl/flags", "flag"] + , ["@", "absl", "absl/strings", "strings"] + , ["@", "absl", "absl/types", "optional"] + ] } , "config": { "type": ["@", "rules", "CC", "library"] @@ -1121,6 +1659,7 @@ [ "gpr_platform" , ["src/core", "poll"] , ["src/core", "promise_like"] + , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/types", "optional"] ] @@ -1229,8 +1768,8 @@ , ["src/core", "resolved_address"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] ] } , "iomgr_timer": @@ -1243,11 +1782,11 @@ , "src/core/lib/iomgr/timer_manager.cc" ] , "hdrs": - [ "src/core/lib/iomgr/timer.h" + [ "src/core/lib/iomgr/iomgr.h" + , "src/core/lib/iomgr/timer.h" , "src/core/lib/iomgr/timer_generic.h" , "src/core/lib/iomgr/timer_heap.h" , "src/core/lib/iomgr/timer_manager.h" - , "src/core/lib/iomgr/iomgr.h" ] , "deps": [ "event_engine_base_hdrs" @@ -1262,8 +1801,29 @@ , ["src/core", "time"] , ["src/core", "time_averaged_stats"] , ["src/core", "useful"] + , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/strings", "strings"] + ] + } +, "iomgr_internal_errqueue": + { "type": ["@", "rules", "CC", "library"] + , "name": ["iomgr_internal_errqueue"] + , "srcs": ["src/core/lib/iomgr/internal_errqueue.cc"] + , "hdrs": ["src/core/lib/iomgr/internal_errqueue.h"] + , "deps": ["gpr", ["src/core", "iomgr_port"], ["src/core", "strerror"]] + } +, "iomgr_buffer_list": + { "type": ["@", "rules", "CC", "library"] + , "name": ["iomgr_buffer_list"] + , "srcs": ["src/core/lib/iomgr/buffer_list.cc"] + , "hdrs": ["src/core/lib/iomgr/buffer_list.h"] + , "deps": + [ "gpr" + , "iomgr_internal_errqueue" + , ["src/core", "error"] + , ["src/core", "iomgr_port"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] ] } , "uri_parser": @@ -1275,8 +1835,8 @@ [ "gpr" , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] ] } , "parse_address": @@ -1359,10 +1919,13 @@ , ["src/core", "json"] , ["src/core", "json_args"] , ["src/core", "json_object_loader"] + , ["src/core", "json_reader"] + , ["src/core", "json_writer"] , ["src/core", "service_config_parser"] , ["src/core", "slice"] , ["src/core", "slice_refcount"] , ["src/core", "validation_errors"] + , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] @@ -1381,8 +1944,8 @@ , ["src/core", "useful"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] ] } , "grpc_resolver": @@ -1409,8 +1972,8 @@ , ["src/core", "iomgr_fwd"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] ] } , "grpc_client_channel": @@ -1428,7 +1991,6 @@ , "src/core/ext/filters/client_channel/config_selector.cc" , "src/core/ext/filters/client_channel/dynamic_filters.cc" , "src/core/ext/filters/client_channel/global_subchannel_pool.cc" - , "src/core/ext/filters/client_channel/health/health_check_client.cc" , "src/core/ext/filters/client_channel/http_proxy.cc" , "src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc" , "src/core/ext/filters/client_channel/lb_policy/oob_backend_metric.cc" @@ -1447,14 +2009,13 @@ , "src/core/ext/filters/client_channel/client_channel.h" , "src/core/ext/filters/client_channel/client_channel_channelz.h" , "src/core/ext/filters/client_channel/client_channel_factory.h" + , "src/core/ext/filters/client_channel/client_channel_internal.h" , "src/core/ext/filters/client_channel/client_channel_service_config.h" , "src/core/ext/filters/client_channel/config_selector.h" , "src/core/ext/filters/client_channel/connector.h" , "src/core/ext/filters/client_channel/dynamic_filters.h" , "src/core/ext/filters/client_channel/global_subchannel_pool.h" - , "src/core/ext/filters/client_channel/health/health_check_client.h" , "src/core/ext/filters/client_channel/http_proxy.h" - , "src/core/ext/filters/client_channel/lb_call_state_internal.h" , "src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h" , "src/core/ext/filters/client_channel/lb_policy/oob_backend_metric.h" , "src/core/ext/filters/client_channel/lb_policy/oob_backend_metric_internal.h" @@ -1471,6 +2032,7 @@ [ "backoff" , "channel_stack_builder" , "config" + , "config_vars" , "debug_location" , "exec_ctx" , "gpr" @@ -1481,14 +2043,16 @@ , "grpc_trace" , "http_connect_handshaker" , "iomgr_timer" + , "legacy_context" , "orphanable" - , ["@", "protobuf", "", "libprotobuf"] + , "parse_address" , "ref_counted_ptr" , "server_address" , "sockaddr_utils" , "stats" , "uri_parser" , "work_serializer" + , ["@", "protobuf", "", "libprotobuf"] , ["src/core", "arena"] , ["src/core", "channel_args"] , ["src/core", "channel_fwd"] @@ -1530,17 +2094,20 @@ , ["src/core", "unique_type_name"] , ["src/core", "useful"] , ["src/core", "validation_errors"] + , ["src/core/ext/upb-generated", "upb-generated-lib"] , ["@", "absl", "absl/base", "core_headers"] + , ["@", "absl", "absl/cleanup", "cleanup"] + , ["@", "absl", "absl/container", "flat_hash_set"] , ["@", "absl", "absl/container", "inlined_vector"] + , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "cord"] - , ["@", "absl", "absl/synchronization", "synchronization"] + , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] + , ["third_party/upb", "collections"] , ["third_party/upb", "upb"] - , ["src/core/ext/upb-generated", "upb-generated-lib"] ] } , "grpc_resolver_dns_ares": @@ -1555,12 +2122,14 @@ , "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc" ] , "hdrs": - [ "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h" + [ "src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.h" + , "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h" , "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h" ] , "deps": [ "backoff" , "config" + , "config_vars" , "debug_location" , "exec_ctx" , "gpr" @@ -1580,15 +2149,14 @@ , ["src/core", "closure"] , ["src/core", "error"] , ["src/core", "event_engine_common"] - , ["src/core", "grpc_resolver_dns_selection"] , ["src/core", "grpc_service_config"] , ["src/core", "grpc_sockaddr"] , ["src/core", "iomgr_fwd"] , ["src/core", "iomgr_port"] - , ["src/core", "json"] , ["src/core", "polling_resolver"] , ["src/core", "pollset_set"] , ["src/core", "resolved_address"] + , ["src/core", "service_config_helper"] , ["src/core", "slice"] , ["src/core", "status_helper"] , ["src/core", "time"] @@ -1596,11 +2164,11 @@ , ["@", "absl", "absl/container", "flat_hash_set"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "cares", "", "ares"] - , ["third_party", "address_sorting"] + , ["third_party/address_sorting", "address_sorting"] ] } , "httpcli": @@ -1648,8 +2216,8 @@ , ["@", "absl", "absl/functional", "bind_front"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] ] } @@ -1730,6 +2298,8 @@ , ["src/core", "httpcli_ssl_credentials"] , ["src/core", "iomgr_fwd"] , ["src/core", "json"] + , ["src/core", "json_reader"] + , ["src/core", "json_writer"] , ["src/core", "slice"] , ["src/core", "slice_refcount"] , ["src/core", "time"] @@ -1738,12 +2308,12 @@ , ["src/core", "useful"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/time", "time"] , ["@", "absl", "absl/types", "optional"] - , ["@", "libssl", "", "crypto"] - , ["@", "libssl", "", "ssl"] + , ["@", "ssl", "", "crypto"] + , ["@", "ssl", "", "ssl"] ] } , "grpc_credentials_util": @@ -1762,7 +2332,8 @@ , "src/core/lib/security/util/json_util.h" ] , "deps": - [ "gpr" + [ "config_vars" + , "gpr" , "grpc_base" , "grpc_security_base" , ["src/core", "error"] @@ -1796,8 +2367,10 @@ , "tsi_base" , ["src/core", "channel_args"] , ["src/core", "closure"] + , ["src/core", "env"] , ["src/core", "pollset_set"] , ["src/core", "slice"] + , ["@", "absl", "absl/strings", "strings"] , ["third_party/upb", "upb"] , ["src/core/ext/upb-generated", "upb-generated-lib"] ] @@ -1844,8 +2417,8 @@ , ["src/core", "slice"] , ["src/core", "slice_buffer"] , ["src/core", "useful"] - , ["@", "libssl", "", "crypto"] - , ["@", "libssl", "", "ssl"] + , ["@", "ssl", "", "crypto"] + , ["@", "ssl", "", "ssl"] ] } , "tsi_ssl_session_cache": @@ -1867,7 +2440,7 @@ , ["src/core", "ref_counted"] , ["src/core", "slice"] , ["@", "absl", "absl/memory", "memory"] - , ["@", "libssl", "", "ssl"] + , ["@", "ssl", "", "ssl"] ] } , "tsi_ssl_credentials": @@ -1875,20 +2448,19 @@ , "name": ["tsi_ssl_credentials"] , "srcs": [ "src/core/lib/security/security_connector/ssl_utils.cc" - , "src/core/lib/security/security_connector/ssl_utils_config.cc" , "src/core/tsi/ssl/key_logging/ssl_key_logging.cc" , "src/core/tsi/ssl_transport_security.cc" , "src/core/tsi/ssl_transport_security_utils.cc" ] , "hdrs": [ "src/core/lib/security/security_connector/ssl_utils.h" - , "src/core/lib/security/security_connector/ssl_utils_config.h" , "src/core/tsi/ssl/key_logging/ssl_key_logging.h" , "src/core/tsi/ssl_transport_security.h" , "src/core/tsi/ssl_transport_security_utils.h" ] , "deps": - [ "gpr" + [ "config_vars" + , "gpr" , "grpc_base" , "grpc_credentials_util" , "grpc_public_hdrs" @@ -1906,19 +2478,13 @@ , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "strings"] - , ["@", "libssl", "", "crypto"] - , ["@", "libssl", "", "ssl"] + , ["@", "ssl", "", "crypto"] + , ["@", "ssl", "", "ssl"] ] } , "grpc_http_filters": { "type": ["@", "rules", "CC", "library"] - , "arguments_config": ["COMPILER_FAMILY"] , "name": ["grpc_http_filters"] - , "private-cflags": - { "type": "case" - , "expr": {"type": "var", "name": "COMPILER_FAMILY", "default": "unknown"} - , "case": {"gnu": ["-Wno-array-bounds"]} - } , "srcs": [ "src/core/ext/filters/http/client/http_client_filter.cc" , "src/core/ext/filters/http/http_filters_plugin.cc" @@ -1937,6 +2503,7 @@ , "grpc_base" , "grpc_public_hdrs" , "grpc_trace" + , "legacy_context" , "promise" , ["src/core", "activity"] , ["src/core", "arena"] @@ -1952,6 +2519,7 @@ , ["src/core", "percent_encoding"] , ["src/core", "pipe"] , ["src/core", "poll"] + , ["src/core", "prioritized_race"] , ["src/core", "race"] , ["src/core", "slice"] , ["src/core", "slice_buffer"] @@ -1960,8 +2528,8 @@ , ["@", "absl", "absl/meta", "type_traits"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] ] } @@ -2001,7 +2569,6 @@ , "src/core/ext/xds/xds_resource_type_impl.h" , "src/core/ext/xds/xds_transport.h" ] - , "proto": [["@", "google_apis", "", "google_rpc_status_proto"]] , "deps": [ "backoff" , "debug_location" @@ -2010,31 +2577,52 @@ , "gpr" , "grpc_trace" , "orphanable" - , ["@", "protobuf", "", "libprotobuf"] , "ref_counted_ptr" , "uri_parser" , "work_serializer" + , ["@", "protobuf", "", "libprotobuf"] , ["src/core", "default_event_engine"] , ["src/core", "dual_ref_counted"] , ["src/core", "env"] , ["src/core", "json"] + , ["src/core", "per_cpu"] , ["src/core", "ref_counted"] , ["src/core", "time"] , ["src/core", "upb_utils"] , ["src/core", "useful"] + , ["src/core/ext/upb-generated", "upb-generated-lib"] + , ["src/core/ext/upbdefs-generated", "upbdefs-generated-lib"] , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/memory", "memory"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] + , ["third_party/upb", "json"] + , ["third_party/upb", "reflection"] + , ["third_party/upb", "textformat"] , ["third_party/upb", "upb"] - , ["third_party/upb", "upb_textformat"] - , ["third_party/upb", "upb_json"] - , ["third_party/upb", "upb_reflection"] + ] + } +, "grpc_mock_cel": + { "type": ["@", "rules", "CC", "library"] + , "name": ["grpc_mock_cel"] + , "hdrs": + [ "src/core/lib/security/authorization/mock_cel/activation.h" + , "src/core/lib/security/authorization/mock_cel/cel_expr_builder_factory.h" + , "src/core/lib/security/authorization/mock_cel/cel_expression.h" + , "src/core/lib/security/authorization/mock_cel/cel_value.h" + , "src/core/lib/security/authorization/mock_cel/evaluator_core.h" + , "src/core/lib/security/authorization/mock_cel/flat_expr_builder.h" + ] + , "deps": + [ "gpr_public_hdrs" , ["src/core/ext/upb-generated", "upb-generated-lib"] - , ["src/core/ext/upbdefs-generated", "upbdefs-generated-lib"] + , ["@", "absl", "absl/status", "status"] + , ["@", "absl", "absl/status", "statusor"] + , ["@", "absl", "absl/strings", "strings"] + , ["@", "absl", "absl/types", "span"] ] } , "grpc_resolver_fake": @@ -2088,14 +2676,32 @@ , "gpr_platform" , "grpc_base" , "grpc_trace" + , "hpack_parse_result" , "http_trace" - , ["src/core", "error"] , ["src/core", "hpack_constants"] , ["src/core", "no_destruct"] , ["src/core", "slice"] + , ["@", "absl", "absl/functional", "function_ref"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "strings"] + ] + } +, "hpack_parse_result": + { "type": ["@", "rules", "CC", "library"] + , "name": ["hpack_parse_result"] + , "srcs": ["src/core/ext/transport/chttp2/transport/hpack_parse_result.cc"] + , "hdrs": ["src/core/ext/transport/chttp2/transport/hpack_parse_result.h"] + , "deps": + [ "gpr" + , "grpc_base" + , ["src/core", "error"] + , ["src/core", "hpack_constants"] + , ["src/core", "slice"] + , ["src/core", "status_helper"] + , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] + , ["@", "absl", "absl/types", "optional"] ] } , "hpack_parser": @@ -2110,19 +2716,20 @@ , "grpc_base" , "grpc_public_hdrs" , "grpc_trace" + , "hpack_parse_result" , "hpack_parser_table" , "stats" , ["src/core", "decode_huff"] , ["src/core", "error"] , ["src/core", "hpack_constants"] + , ["src/core", "match"] + , ["src/core", "random_early_detection"] , ["src/core", "slice"] , ["src/core", "slice_refcount"] , ["src/core", "stats_data"] - , ["src/core", "status_helper"] , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "strings"] - , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "span"] , ["@", "absl", "absl/types", "variant"] @@ -2145,6 +2752,7 @@ , "http_trace" , ["src/core", "hpack_constants"] , ["src/core", "hpack_encoder_table"] + , ["src/core", "metadata_compression_traits"] , ["src/core", "slice"] , ["src/core", "slice_buffer"] , ["src/core", "time"] @@ -2166,13 +2774,18 @@ , "hdrs": ["src/core/ext/transport/chttp2/transport/varint.h"] , "deps": ["gpr", ["@", "absl", "absl/base", "core_headers"]] } +, "chttp2_context_list_entry": + { "type": ["@", "rules", "CC", "library"] + , "name": ["chttp2_context_list_entry"] + , "hdrs": ["src/core/ext/transport/chttp2/transport/context_list_entry.h"] + , "deps": ["gpr"] + } , "grpc_transport_chttp2": { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_transport_chttp2"] , "srcs": [ "src/core/ext/transport/chttp2/transport/bin_decoder.cc" , "src/core/ext/transport/chttp2/transport/chttp2_transport.cc" - , "src/core/ext/transport/chttp2/transport/context_list.cc" , "src/core/ext/transport/chttp2/transport/frame_data.cc" , "src/core/ext/transport/chttp2/transport/frame_goaway.cc" , "src/core/ext/transport/chttp2/transport/frame_ping.cc" @@ -2187,7 +2800,6 @@ , "hdrs": [ "src/core/ext/transport/chttp2/transport/bin_decoder.h" , "src/core/ext/transport/chttp2/transport/chttp2_transport.h" - , "src/core/ext/transport/chttp2/transport/context_list.h" , "src/core/ext/transport/chttp2/transport/frame_data.h" , "src/core/ext/transport/chttp2/transport/frame_goaway.h" , "src/core/ext/transport/chttp2/transport/frame_ping.h" @@ -2198,7 +2810,8 @@ , "src/core/ext/transport/chttp2/transport/stream_map.h" ] , "deps": - [ "chttp2_frame" + [ "chttp2_context_list_entry" + , "chttp2_frame" , "chttp2_varint" , "debug_location" , "exec_ctx" @@ -2211,7 +2824,8 @@ , "hpack_parser_table" , "http_trace" , "httpcli" - , "iomgr_timer" + , "iomgr_buffer_list" + , "legacy_context" , "ref_counted_ptr" , "stats" , ["src/core", "arena"] @@ -2221,11 +2835,12 @@ , ["src/core", "chttp2_flow_control"] , ["src/core", "closure"] , ["src/core", "error"] - , ["src/core", "gpr_atm"] + , ["src/core", "experiments"] , ["src/core", "http2_errors"] , ["src/core", "http2_settings"] , ["src/core", "init_internally"] , ["src/core", "iomgr_fwd"] + , ["src/core", "iomgr_port"] , ["src/core", "memory_quota"] , ["src/core", "poll"] , ["src/core", "ref_counted"] @@ -2241,9 +2856,9 @@ , ["src/core", "useful"] , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/status", "status"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "cord"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] ] } @@ -2252,11 +2867,11 @@ , "name": ["grpcpp_status"] , "srcs": ["src/cpp/util/status.cc"] , "hdrs": - [ "include/grpcpp/impl/status.h" + [ "include/grpc++/impl/codegen/status.h" + , "include/grpc++/support/status.h" , "include/grpcpp/impl/codegen/status.h" + , "include/grpcpp/impl/status.h" , "include/grpcpp/support/status.h" - , "include/grpc++/impl/codegen/status.h" - , "include/grpc++/support/status.h" ] , "deps": ["gpr_platform", "grpc++_public_hdrs", "grpc_public_hdrs"] } diff --git a/etc/import/include/grpc/TARGETS.grpc b/etc/import/include/grpc/TARGETS.grpc index 2f1e64e29..892aa6393 100644 --- a/etc/import/include/grpc/TARGETS.grpc +++ b/etc/import/include/grpc/TARGETS.grpc @@ -5,7 +5,9 @@ , "support/atm_gcc_atomic.h" , "support/atm_gcc_sync.h" , "support/atm_windows.h" + , "support/atm.h" , "support/cpu.h" + , "support/json.h" , "support/log.h" , "support/log_windows.h" , "support/port_platform.h" @@ -38,7 +40,8 @@ , "grpc_public_headers": { "type": ["@", "rules", "data", "staged"] , "srcs": - [ "byte_buffer.h" + [ "grpc_audit_logging.h" + , "byte_buffer.h" , "byte_buffer_reader.h" , "compression.h" , "fork.h" diff --git a/etc/import/include/grpcpp/TARGETS.grpc b/etc/import/include/grpcpp/TARGETS.grpc index 112311617..0781bca8a 100644 --- a/etc/import/include/grpcpp/TARGETS.grpc +++ b/etc/import/include/grpcpp/TARGETS.grpc @@ -38,6 +38,7 @@ , "impl/status.h" , "impl/sync.h" , "resource_quota.h" + , "security/audit_logging.h" , "security/auth_context.h" , "security/auth_metadata_processor.h" , "security/credentials.h" diff --git a/etc/import/src/core/TARGETS.grpc b/etc/import/src/core/TARGETS.grpc index a0be6d472..9d1529595 100644 --- a/etc/import/src/core/TARGETS.grpc +++ b/etc/import/src/core/TARGETS.grpc @@ -1,19 +1,19 @@ { "channel_fwd": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["channel_fwd"] + , "stage": ["src", "core"] , "hdrs": ["lib/channel/channel_fwd.h"] } , "slice_cast": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["slice_cast"] + , "stage": ["src", "core"] , "hdrs": [["include/grpc", "slice_cast_headers"]] } , "event_engine_common": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["event_engine_common"] + , "stage": ["src", "core"] , "srcs": [ "lib/event_engine/event_engine.cc" , "lib/event_engine/resolved_address.cc" @@ -42,39 +42,50 @@ } , "transport_fwd": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["transport_fwd"] + , "stage": ["src", "core"] , "hdrs": ["lib/transport/transport_fwd.h"] } , "atomic_utils": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["atomic_utils"] + , "stage": ["src", "core"] , "hdrs": ["lib/gprpp/atomic_utils.h"] , "deps": [["", "gpr"]] } -, "experiments": +, "metadata_compression_traits": { "type": ["@", "rules", "CC", "library"] + , "name": ["metadata_compression_traits"] , "stage": ["src", "core"] + , "hdrs": ["lib/transport/metadata_compression_traits.h"] + , "deps": [["", "gpr_platform"]] + } +, "experiments": + { "type": ["@", "rules", "CC", "library"] , "name": ["experiments"] + , "stage": ["src", "core"] , "srcs": ["lib/experiments/config.cc", "lib/experiments/experiments.cc"] , "hdrs": ["lib/experiments/config.h", "lib/experiments/experiments.h"] - , "private-hdrs": [] , "deps": - ["no_destruct", ["", "gpr"], ["@", "absl", "absl/strings", "strings"]] + [ "no_destruct" + , ["", "config_vars"] + , ["", "gpr"] + , ["@", "absl", "absl/functional", "any_invocable"] + , ["@", "absl", "absl/strings", "strings"] + ] } , "init_internally": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["init_internally"] + , "stage": ["src", "core"] , "srcs": ["lib/surface/init_internally.cc"] , "hdrs": ["lib/surface/init_internally.h"] , "deps": [["", "gpr_platform"]] } , "useful": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["useful"] + , "stage": ["src", "core"] , "hdrs": ["lib/gpr/useful.h"] , "deps": [ ["", "gpr_platform"] @@ -84,56 +95,49 @@ } , "examine_stack": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["examine_stack"] + , "stage": ["src", "core"] , "srcs": ["lib/gprpp/examine_stack.cc"] , "hdrs": ["lib/gprpp/examine_stack.h"] , "deps": [["", "gpr_platform"], ["@", "absl", "absl/types", "optional"]] } , "gpr_atm": { "type": ["@", "rules", "CC", "library"] - , "arguments_config": ["ARCH", "TARGET_ARCH"] , "name": ["gpr_atm"] + , "stage": ["src", "core"] , "srcs": ["lib/gpr/atm.cc"] , "hdrs": [["include/grpc", "gpr_atm_headers"]] - , "private-ldflags": - { "type": "case" - , "expr": - { "type": "var" - , "name": "TARGET_ARCH" - , "default": - { "type": "var" - , "name": "ARCH" - , "default": - {"type": "fail", "msg": "Required variable 'ARCH' is not set."} - } - } - , "case": {"arm": ["-latomic"]} - } , "deps": ["useful", ["", "gpr_platform"]] } , "gpr_manual_constructor": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["gpr_manual_constructor"] + , "stage": ["src", "core"] , "hdrs": ["lib/gprpp/manual_constructor.h"] , "deps": ["construct_destruct", ["", "gpr_platform"]] } , "gpr_spinlock": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["gpr_spinlock"] + , "stage": ["src", "core"] , "hdrs": ["lib/gpr/spinlock.h"] , "deps": ["gpr_atm", ["", "gpr_platform"]] } -, "env": +, "gpr_log_internal": { "type": ["@", "rules", "CC", "library"] + , "name": ["gpr_log_internal"] , "stage": ["src", "core"] + , "hdrs": ["lib/gpr/log_internal.h"] + , "deps": [["", "gpr_platform"]] + } +, "env": + { "type": ["@", "rules", "CC", "library"] , "name": ["env"] + , "stage": ["src", "core"] , "srcs": - [ "lib/gprpp/env_linux.cc" - , "lib/gprpp/env_posix.cc" - , "lib/gprpp/env_windows.cc" + [ "lib/gprpp/linux/env.cc" + , "lib/gprpp/posix/env.cc" + , "lib/gprpp/windows/env.cc" ] , "hdrs": ["lib/gprpp/env.h"] , "deps": @@ -141,22 +145,22 @@ } , "chunked_vector": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["chunked_vector"] + , "stage": ["src", "core"] , "hdrs": ["lib/gprpp/chunked_vector.h"] , "deps": ["arena", "gpr_manual_constructor", ["", "gpr"]] } , "construct_destruct": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["construct_destruct"] + , "stage": ["src", "core"] , "hdrs": ["lib/gprpp/construct_destruct.h"] , "deps": [["", "gpr_platform"]] } , "status_helper": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["status_helper"] + , "stage": ["src", "core"] , "srcs": ["lib/gprpp/status_helper.cc"] , "hdrs": ["lib/gprpp/status_helper.h"] , "proto": [["@", "google_apis", "", "google_rpc_status_proto"]] @@ -166,8 +170,8 @@ , ["", "debug_location"] , ["", "gpr"] , ["@", "absl", "absl/status", "status"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "cord"] + , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/time", "time"] , ["@", "absl", "absl/types", "optional"] , ["third_party/upb", "upb"] @@ -176,17 +180,16 @@ } , "unique_type_name": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["unique_type_name"] + , "stage": ["src", "core"] , "hdrs": ["lib/gprpp/unique_type_name.h"] - , "private-hdrs": [] , "deps": ["useful", ["", "gpr_platform"], ["@", "absl", "absl/strings", "strings"]] } , "validation_errors": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["validation_errors"] + , "stage": ["src", "core"] , "srcs": ["lib/gprpp/validation_errors.cc"] , "hdrs": ["lib/gprpp/validation_errors.h"] , "deps": @@ -197,23 +200,23 @@ } , "overload": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["overload"] + , "stage": ["src", "core"] , "hdrs": ["lib/gprpp/overload.h"] , "deps": [["", "gpr_platform"]] } , "match": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["match"] + , "stage": ["src", "core"] , "hdrs": ["lib/gprpp/match.h"] , "deps": ["overload", ["", "gpr_platform"], ["@", "absl", "absl/types", "variant"]] } , "table": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["table"] + , "stage": ["src", "core"] , "hdrs": ["lib/gprpp/table.h"] , "deps": [ "bitset" @@ -224,58 +227,101 @@ } , "packed_table": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["packed_table"] + , "stage": ["src", "core"] , "hdrs": ["lib/gprpp/packed_table.h"] , "deps": ["sorted_pack", "table", ["", "gpr_platform"]] } , "bitset": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["bitset"] + , "stage": ["src", "core"] , "hdrs": ["lib/gprpp/bitset.h"] , "deps": ["useful", ["", "gpr_platform"]] } , "no_destruct": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["no_destruct"] + , "stage": ["src", "core"] , "hdrs": ["lib/gprpp/no_destruct.h"] , "deps": ["construct_destruct", ["", "gpr_platform"]] } , "tchar": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["tchar"] + , "stage": ["src", "core"] , "srcs": ["lib/gprpp/tchar.cc"] , "hdrs": ["lib/gprpp/tchar.h"] , "deps": [["", "gpr_platform"]] } , "poll": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["poll"] + , "stage": ["src", "core"] , "hdrs": ["lib/promise/poll.h"] , "deps": ["construct_destruct", ["", "gpr"], ["", "gpr_platform"]] } -, "context": +, "map_pipe": + { "type": ["@", "rules", "CC", "library"] + , "name": ["map_pipe"] + , "stage": ["src", "core"] + , "hdrs": ["lib/promise/map_pipe.h"] + , "deps": + [ "for_each" + , "map" + , "pipe" + , "poll" + , "promise_factory" + , "promise_trace" + , "try_seq" + , ["", "gpr"] + , ["", "gpr_platform"] + , ["", "grpc_trace"] + , ["@", "absl", "absl/status", "status"] + ] + } +, "1999": { "type": ["@", "rules", "CC", "library"] + , "name": ["1999"] , "stage": ["src", "core"] + , "srcs": ["lib/promise/party.cc"] + , "hdrs": ["lib/promise/party.h"] + , "deps": + [ "activity" + , "arena" + , "construct_destruct" + , "context" + , "promise_factory" + , "promise_trace" + , "ref_counted" + , ["", "event_engine_base_hdrs"] + , ["", "exec_ctx"] + , ["", "gpr"] + , ["", "grpc_trace"] + , ["", "ref_counted_ptr"] + , ["@", "absl", "absl/base", "core_headers"] + , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] + ] + } +, "context": + { "type": ["@", "rules", "CC", "library"] , "name": ["context"] + , "stage": ["src", "core"] , "hdrs": ["lib/promise/context.h"] , "deps": [["", "gpr"]] } , "map": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["map"] + , "stage": ["src", "core"] , "hdrs": ["lib/promise/map.h"] , "deps": ["poll", "promise_like", ["", "gpr_platform"]] } , "sleep": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["sleep"] + , "stage": ["src", "core"] , "srcs": ["lib/promise/sleep.cc"] , "hdrs": ["lib/promise/sleep.h"] , "deps": @@ -292,8 +338,8 @@ } , "arena_promise": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["arena_promise"] + , "stage": ["src", "core"] , "hdrs": ["lib/promise/arena_promise.h"] , "deps": [ "arena" @@ -306,23 +352,34 @@ } , "promise_like": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["promise_like"] + , "stage": ["src", "core"] , "hdrs": ["lib/promise/detail/promise_like.h"] , "deps": ["poll", ["", "gpr_platform"], ["@", "absl", "absl/meta", "type_traits"]] } -, "promise_factory": +, "cancel_callback": { "type": ["@", "rules", "CC", "library"] + , "name": ["cancel_callback"] , "stage": ["src", "core"] + , "hdrs": ["lib/promise/cancel_callback.h"] + , "deps": ["promise_like", ["", "gpr_platform"]] + } +, "promise_factory": + { "type": ["@", "rules", "CC", "library"] , "name": ["promise_factory"] + , "stage": ["src", "core"] , "hdrs": ["lib/promise/detail/promise_factory.h"] - , "deps": ["promise_like", ["", "gpr_platform"]] + , "deps": + [ "promise_like" + , ["", "gpr_platform"] + , ["@", "absl", "absl/meta", "type_traits"] + ] } , "if": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["if"] + , "stage": ["src", "core"] , "hdrs": ["lib/promise/if.h"] , "deps": [ "construct_destruct" @@ -336,8 +393,8 @@ } , "promise_status": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["promise_status"] + , "stage": ["src", "core"] , "hdrs": ["lib/promise/detail/status.h"] , "deps": [ ["", "gpr_platform"] @@ -347,18 +404,26 @@ } , "race": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["race"] + , "stage": ["src", "core"] , "hdrs": ["lib/promise/race.h"] , "deps": [["", "gpr_platform"]] } -, "loop": +, "prioritized_race": { "type": ["@", "rules", "CC", "library"] + , "name": ["prioritized_race"] , "stage": ["src", "core"] + , "hdrs": ["lib/promise/prioritized_race.h"] + , "deps": [["", "gpr_platform"]] + } +, "loop": + { "type": ["@", "rules", "CC", "library"] , "name": ["loop"] + , "stage": ["src", "core"] , "hdrs": ["lib/promise/loop.h"] , "deps": - [ "poll" + [ "construct_destruct" + , "poll" , "promise_factory" , ["", "gpr_platform"] , ["@", "absl", "absl/status", "status"] @@ -368,8 +433,8 @@ } , "basic_join": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["basic_join"] + , "stage": ["src", "core"] , "hdrs": ["lib/promise/detail/basic_join.h"] , "deps": [ "bitset" @@ -380,10 +445,21 @@ , ["@", "absl", "absl/utility", "utility"] ] } -, "try_join": +, "join": { "type": ["@", "rules", "CC", "library"] + , "name": ["join"] , "stage": ["src", "core"] + , "hdrs": ["lib/promise/join.h"] + , "deps": + [ "basic_join" + , ["", "gpr_platform"] + , ["@", "absl", "absl/meta", "type_traits"] + ] + } +, "try_join": + { "type": ["@", "rules", "CC", "library"] , "name": ["try_join"] + , "stage": ["src", "core"] , "hdrs": ["lib/promise/try_join.h"] , "deps": [ "basic_join" @@ -397,15 +473,15 @@ } , "switch": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["switch"] + , "stage": ["src", "core"] , "hdrs": ["lib/promise/detail/switch.h"] , "deps": [["", "gpr_platform"]] } , "basic_seq": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["basic_seq"] + , "stage": ["src", "core"] , "hdrs": ["lib/promise/detail/basic_seq.h"] , "deps": [ "construct_destruct" @@ -420,15 +496,15 @@ } , "seq": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["seq"] + , "stage": ["src", "core"] , "hdrs": ["lib/promise/seq.h"] , "deps": ["basic_seq", "poll", "promise_like", ["", "gpr_platform"]] } , "try_seq": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["try_seq"] + , "stage": ["src", "core"] , "hdrs": ["lib/promise/try_seq.h"] , "deps": [ "basic_seq" @@ -443,8 +519,8 @@ } , "activity": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["activity"] + , "stage": ["src", "core"] , "srcs": ["lib/promise/activity.cc"] , "hdrs": ["lib/promise/activity.h"] , "deps": @@ -452,6 +528,7 @@ , "construct_destruct" , "context" , "no_destruct" + , "poll" , "promise_factory" , "promise_status" , ["", "gpr"] @@ -459,13 +536,14 @@ , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] ] } , "exec_ctx_wakeup_scheduler": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["exec_ctx_wakeup_scheduler"] + , "stage": ["src", "core"] , "hdrs": ["lib/promise/exec_ctx_wakeup_scheduler.h"] , "deps": [ "closure" @@ -476,21 +554,34 @@ , ["@", "absl", "absl/status", "status"] ] } -, "intra_activity_waiter": +, "event_engine_wakeup_scheduler": { "type": ["@", "rules", "CC", "library"] + , "name": ["event_engine_wakeup_scheduler"] , "stage": ["src", "core"] - , "name": ["intra_activity_waiter"] - , "hdrs": ["lib/promise/intra_activity_waiter.h"] - , "deps": ["activity", "poll", ["", "gpr_platform"]] + , "hdrs": ["lib/promise/event_engine_wakeup_scheduler.h"] + , "deps": + [["", "event_engine_base_hdrs"], ["", "exec_ctx"], ["", "gpr_platform"]] } -, "latch": +, "wait_set": { "type": ["@", "rules", "CC", "library"] + , "name": ["wait_set"] , "stage": ["src", "core"] + , "hdrs": ["lib/promise/wait_set.h"] + , "deps": + [ "activity" + , "poll" + , ["", "gpr_platform"] + , ["@", "absl", "absl/container", "flat_hash_set"] + , ["@", "absl", "absl/hash", "hash"] + ] + } +, "latch": + { "type": ["@", "rules", "CC", "library"] , "name": ["latch"] + , "stage": ["src", "core"] , "hdrs": ["lib/promise/latch.h"] , "deps": [ "activity" - , "intra_activity_waiter" , "poll" , "promise_trace" , ["", "gpr"] @@ -500,8 +591,8 @@ } , "interceptor_list": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["interceptor_list"] + , "stage": ["src", "core"] , "hdrs": ["lib/promise/interceptor_list.h"] , "deps": [ "arena" @@ -513,15 +604,15 @@ , ["", "debug_location"] , ["", "gpr"] , ["", "grpc_trace"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] ] } , "pipe": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["pipe"] + , "stage": ["src", "core"] , "hdrs": ["lib/promise/pipe.h"] , "deps": [ "activity" @@ -529,7 +620,6 @@ , "context" , "if" , "interceptor_list" - , "intra_activity_waiter" , "map" , "poll" , "promise_trace" @@ -538,7 +628,6 @@ , ["", "gpr"] , ["", "grpc_trace"] , ["", "ref_counted_ptr"] - , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] @@ -546,16 +635,49 @@ } , "promise_trace": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["promise_trace"] + , "stage": ["src", "core"] , "srcs": ["lib/promise/trace.cc"] , "hdrs": ["lib/promise/trace.h"] , "deps": [["", "gpr_platform"], ["", "grpc_trace"]] } -, "ref_counted": +, "mpsc": + { "type": ["@", "rules", "CC", "library"] + , "name": ["mpsc"] + , "stage": ["src", "core"] + , "hdrs": ["lib/promise/mpsc.h"] + , "deps": + [ "activity" + , "poll" + , "ref_counted" + , "wait_set" + , ["", "gpr"] + , ["", "ref_counted_ptr"] + , ["@", "absl", "absl/base", "core_headers"] + ] + } +, "for_each": { "type": ["@", "rules", "CC", "library"] + , "name": ["for_each"] , "stage": ["src", "core"] + , "hdrs": ["lib/promise/for_each.h"] + , "deps": + [ "activity" + , "construct_destruct" + , "poll" + , "promise_factory" + , "promise_trace" + , ["", "gpr"] + , ["", "gpr_platform"] + , ["", "grpc_trace"] + , ["@", "absl", "absl/status", "status"] + , ["@", "absl", "absl/strings", "strings"] + ] + } +, "ref_counted": + { "type": ["@", "rules", "CC", "library"] , "name": ["ref_counted"] + , "stage": ["src", "core"] , "hdrs": ["lib/gprpp/ref_counted.h"] , "deps": [ "atomic_utils" @@ -566,8 +688,8 @@ } , "dual_ref_counted": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["dual_ref_counted"] + , "stage": ["src", "core"] , "hdrs": ["lib/gprpp/dual_ref_counted.h"] , "deps": [ ["", "debug_location"] @@ -576,17 +698,25 @@ , ["", "ref_counted_ptr"] ] } -, "handshaker_factory": +, "uuid_v4": { "type": ["@", "rules", "CC", "library"] + , "name": ["uuid_v4"] , "stage": ["src", "core"] + , "srcs": ["lib/gprpp/uuid_v4.cc"] + , "hdrs": ["lib/gprpp/uuid_v4.h"] + , "deps": [["", "gpr"], ["@", "absl", "absl/strings", "str_format"]] + } +, "handshaker_factory": + { "type": ["@", "rules", "CC", "library"] , "name": ["handshaker_factory"] + , "stage": ["src", "core"] , "hdrs": ["lib/transport/handshaker_factory.h"] , "deps": ["channel_args", "iomgr_fwd", ["", "gpr_platform"]] } , "handshaker_registry": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["handshaker_registry"] + , "stage": ["src", "core"] , "srcs": ["lib/transport/handshaker_registry.cc"] , "hdrs": ["lib/transport/handshaker_registry.h"] , "deps": @@ -594,8 +724,8 @@ } , "tcp_connect_handshaker": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["tcp_connect_handshaker"] + , "stage": ["src", "core"] , "srcs": ["lib/transport/tcp_connect_handshaker.cc"] , "hdrs": ["lib/transport/tcp_connect_handshaker.h"] , "deps": @@ -627,8 +757,8 @@ } , "channel_creds_registry": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["channel_creds_registry"] + , "stage": ["src", "core"] , "hdrs": ["lib/security/credentials/channel_creds_registry.h"] , "deps": [ "json" @@ -639,8 +769,8 @@ } , "event_engine_memory_allocator": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["event_engine_memory_allocator"] + , "stage": ["src", "core"] , "srcs": ["lib/event_engine/memory_allocator.cc"] , "hdrs": [["include/grpc", "event_engine_memory_allocator_headers"]] , "deps": @@ -650,13 +780,24 @@ , ["@", "absl", "absl/strings", "strings"] ] } -, "memory_quota": +, "event_engine_memory_allocator_factory": { "type": ["@", "rules", "CC", "library"] + , "name": ["event_engine_memory_allocator_factory"] , "stage": ["src", "core"] + , "hdrs": ["lib/event_engine/memory_allocator_factory.h"] + , "deps": + [ "event_engine_memory_allocator" + , "memory_quota" + , ["", "gpr_platform"] + , ["@", "absl", "absl/strings", "strings"] + ] + } +, "memory_quota": + { "type": ["@", "rules", "CC", "library"] , "name": ["memory_quota"] + , "stage": ["src", "core"] , "srcs": ["lib/resource_quota/memory_quota.cc"] , "hdrs": ["lib/resource_quota/memory_quota.h"] - , "private-hdrs": [] , "deps": [ "activity" , "basic_seq" @@ -680,14 +821,13 @@ , ["@", "absl", "absl/container", "flat_hash_set"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/strings", "strings"] - , ["@", "absl", "absl/synchronization", "synchronization"] , ["@", "absl", "absl/types", "optional"] ] } , "periodic_update": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["periodic_update"] + , "stage": ["src", "core"] , "srcs": ["lib/resource_quota/periodic_update.cc"] , "hdrs": ["lib/resource_quota/periodic_update.h"] , "deps": @@ -699,8 +839,8 @@ } , "arena": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["arena"] + , "stage": ["src", "core"] , "srcs": ["lib/resource_quota/arena.cc"] , "hdrs": ["lib/resource_quota/arena.h"] , "deps": @@ -715,8 +855,8 @@ } , "thread_quota": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["thread_quota"] + , "stage": ["src", "core"] , "srcs": ["lib/resource_quota/thread_quota.cc"] , "hdrs": ["lib/resource_quota/thread_quota.h"] , "deps": @@ -728,16 +868,16 @@ } , "resource_quota_trace": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["resource_quota_trace"] + , "stage": ["src", "core"] , "srcs": ["lib/resource_quota/trace.cc"] , "hdrs": ["lib/resource_quota/trace.h"] , "deps": [["", "gpr_platform"], ["", "grpc_trace"]] } , "resource_quota": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["resource_quota"] + , "stage": ["src", "core"] , "srcs": ["lib/resource_quota/resource_quota.cc"] , "hdrs": ["lib/resource_quota/resource_quota.h"] , "deps": @@ -754,10 +894,10 @@ } , "slice_refcount": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["slice_refcount"] + , "stage": ["src", "core"] , "srcs": ["lib/slice/slice_refcount.cc"] - , "hdrs": ["lib/slice/slice_refcount.h", ["include/grpc", "slice_headers"]] + , "hdrs": ["lib/slice/slice_refcount.h", ["include/grpc", "slice.h"]] , "deps": [ ["", "debug_location"] , ["", "event_engine_base_hdrs"] @@ -767,14 +907,14 @@ } , "slice": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["slice"] + , "stage": ["src", "core"] , "srcs": ["lib/slice/slice.cc", "lib/slice/slice_string_helpers.cc"] , "hdrs": [ "lib/slice/slice.h" , "lib/slice/slice_internal.h" , "lib/slice/slice_string_helpers.h" - , ["include/grpc", "slice_headers"] + , ["include/grpc", "slice.h"] ] , "deps": [ "slice_cast" @@ -788,17 +928,16 @@ } , "slice_buffer": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["slice_buffer"] + , "stage": ["src", "core"] , "srcs": ["lib/slice/slice_buffer.cc"] - , "hdrs": - ["lib/slice/slice_buffer.h", ["include/grpc", "slice_buffer_headers"]] + , "hdrs": ["lib/slice/slice_buffer.h", ["include/grpc", "slice_buffer.h"]] , "deps": ["slice", "slice_refcount", ["", "gpr"]] } , "error": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["error"] + , "stage": ["src", "core"] , "srcs": ["lib/iomgr/error.cc"] , "hdrs": ["lib/iomgr/error.h"] , "deps": @@ -817,8 +956,8 @@ } , "closure": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["closure"] + , "stage": ["src", "core"] , "srcs": ["lib/iomgr/closure.cc"] , "hdrs": ["lib/iomgr/closure.h"] , "deps": @@ -831,8 +970,8 @@ } , "time": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["time"] + , "stage": ["src", "core"] , "srcs": ["lib/gprpp/time.cc"] , "hdrs": ["lib/gprpp/time.h"] , "deps": @@ -846,22 +985,22 @@ } , "iomgr_port": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["iomgr_port"] + , "stage": ["src", "core"] , "hdrs": ["lib/iomgr/port.h"] , "deps": [["", "gpr_platform"]] } , "iomgr_fwd": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["iomgr_fwd"] + , "stage": ["src", "core"] , "hdrs": ["lib/iomgr/iomgr_fwd.h"] , "deps": [["", "gpr_platform"]] } , "grpc_sockaddr": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_sockaddr"] + , "stage": ["src", "core"] , "srcs": ["lib/iomgr/sockaddr_utils_posix.cc", "lib/iomgr/socket_utils_windows.cc"] , "hdrs": @@ -874,23 +1013,23 @@ } , "avl": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["avl"] + , "stage": ["src", "core"] , "hdrs": ["lib/avl/avl.h"] , "deps": ["useful", ["", "gpr_platform"]] } , "time_averaged_stats": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["time_averaged_stats"] + , "stage": ["src", "core"] , "srcs": ["lib/gprpp/time_averaged_stats.cc"] , "hdrs": ["lib/gprpp/time_averaged_stats.h"] , "deps": [["", "gpr"]] } , "forkable": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["forkable"] + , "stage": ["src", "core"] , "srcs": ["lib/event_engine/forkable.cc"] , "hdrs": ["lib/event_engine/forkable.h"] , "deps": @@ -902,8 +1041,8 @@ } , "event_engine_poller": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["event_engine_poller"] + , "stage": ["src", "core"] , "hdrs": ["lib/event_engine/poller.h"] , "deps": [ ["", "event_engine_base_hdrs"] @@ -911,29 +1050,44 @@ , ["@", "absl", "absl/functional", "function_ref"] ] } -, "event_engine_executor": +, "event_engine_time_util": + { "type": ["@", "rules", "CC", "library"] + , "name": ["event_engine_time_util"] + , "stage": ["src", "core"] + , "srcs": ["lib/event_engine/time_util.cc"] + , "hdrs": ["lib/event_engine/time_util.h"] + , "deps": [["", "event_engine_base_hdrs"], ["", "gpr_platform"]] + } +, "event_engine_work_queue": { "type": ["@", "rules", "CC", "library"] + , "name": ["event_engine_work_queue"] , "stage": ["src", "core"] - , "name": ["event_engine_executor"] - , "hdrs": ["lib/event_engine/executor/executor.h"] + , "hdrs": ["lib/event_engine/work_queue/work_queue.h"] , "deps": [ ["", "event_engine_base_hdrs"] - , ["", "gpr_platform"] + , ["", "gpr"] , ["@", "absl", "absl/functional", "any_invocable"] ] } -, "event_engine_time_util": +, "event_engine_basic_work_queue": { "type": ["@", "rules", "CC", "library"] + , "name": ["event_engine_basic_work_queue"] , "stage": ["src", "core"] - , "name": ["event_engine_time_util"] - , "srcs": ["lib/event_engine/time_util.cc"] - , "hdrs": ["lib/event_engine/time_util.h"] - , "deps": [["", "event_engine_base_hdrs"], ["", "gpr_platform"]] + , "srcs": ["lib/event_engine/work_queue/basic_work_queue.cc"] + , "hdrs": ["lib/event_engine/work_queue/basic_work_queue.h"] + , "deps": + [ "common_event_engine_closures" + , "event_engine_work_queue" + , ["", "event_engine_base_hdrs"] + , ["", "gpr"] + , ["@", "absl", "absl/base", "core_headers"] + , ["@", "absl", "absl/functional", "any_invocable"] + ] } , "common_event_engine_closures": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["common_event_engine_closures"] + , "stage": ["src", "core"] , "hdrs": ["lib/event_engine/common_closures.h"] , "deps": [ ["", "event_engine_base_hdrs"] @@ -943,8 +1097,8 @@ } , "posix_event_engine_timer": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["posix_event_engine_timer"] + , "stage": ["src", "core"] , "srcs": [ "lib/event_engine/posix_engine/timer.cc" , "lib/event_engine/posix_engine/timer_heap.cc" @@ -965,35 +1119,50 @@ } , "event_engine_thread_local": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["event_engine_thread_local"] + , "stage": ["src", "core"] , "srcs": ["lib/event_engine/thread_local.cc"] , "hdrs": ["lib/event_engine/thread_local.h"] , "deps": [["", "gpr_platform"]] } , "event_engine_thread_pool": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["event_engine_thread_pool"] - , "srcs": ["lib/event_engine/thread_pool.cc"] - , "hdrs": ["lib/event_engine/thread_pool.h"] + , "stage": ["src", "core"] + , "srcs": + [ "lib/event_engine/thread_pool/original_thread_pool.cc" + , "lib/event_engine/thread_pool/thread_pool_factory.cc" + , "lib/event_engine/thread_pool/work_stealing_thread_pool.cc" + ] + , "hdrs": + [ "lib/event_engine/thread_pool/original_thread_pool.h" + , "lib/event_engine/thread_pool/thread_pool.h" + , "lib/event_engine/thread_pool/work_stealing_thread_pool.h" + ] , "deps": - [ "event_engine_executor" + [ "common_event_engine_closures" + , "event_engine_basic_work_queue" , "event_engine_thread_local" + , "event_engine_trace" + , "event_engine_work_queue" + , "experiments" , "forkable" , "time" , "useful" + , ["", "backoff"] , ["", "event_engine_base_hdrs"] , ["", "gpr"] + , ["", "grpc_trace"] , ["@", "absl", "absl/base", "core_headers"] + , ["@", "absl", "absl/container", "flat_hash_set"] , ["@", "absl", "absl/functional", "any_invocable"] , ["@", "absl", "absl/time", "time"] ] } , "posix_event_engine_base_hdrs": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["posix_event_engine_base_hdrs"] + , "stage": ["src", "core"] , "hdrs": ["lib/event_engine/posix.h"] , "deps": [ ["", "event_engine_base_hdrs"] @@ -1005,8 +1174,8 @@ } , "posix_event_engine_timer_manager": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["posix_event_engine_timer_manager"] + , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/timer_manager.cc"] , "hdrs": ["lib/event_engine/posix_engine/timer_manager.h"] , "deps": @@ -1025,8 +1194,8 @@ } , "posix_event_engine_event_poller": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["posix_event_engine_event_poller"] + , "stage": ["src", "core"] , "hdrs": ["lib/event_engine/posix_engine/event_poller.h"] , "deps": [ "event_engine_poller" @@ -1040,8 +1209,8 @@ } , "posix_event_engine_closure": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["posix_event_engine_closure"] + , "stage": ["src", "core"] , "hdrs": ["lib/event_engine/posix_engine/posix_engine_closure.h"] , "deps": [ ["", "event_engine_base_hdrs"] @@ -1052,8 +1221,8 @@ } , "posix_event_engine_lockfree_event": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["posix_event_engine_lockfree_event"] + , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/lockfree_event.cc"] , "hdrs": ["lib/event_engine/posix_engine/lockfree_event.h"] , "deps": @@ -1067,15 +1236,15 @@ } , "posix_event_engine_wakeup_fd_posix": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["posix_event_engine_wakeup_fd_posix"] + , "stage": ["src", "core"] , "hdrs": ["lib/event_engine/posix_engine/wakeup_fd_posix.h"] , "deps": [["", "gpr_platform"], ["@", "absl", "absl/status", "status"]] } , "posix_event_engine_wakeup_fd_posix_pipe": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["posix_event_engine_wakeup_fd_posix_pipe"] + , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/wakeup_fd_pipe.cc"] , "hdrs": ["lib/event_engine/posix_engine/wakeup_fd_pipe.h"] , "deps": @@ -1090,8 +1259,8 @@ } , "posix_event_engine_wakeup_fd_posix_eventfd": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["posix_event_engine_wakeup_fd_posix_eventfd"] + , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/wakeup_fd_eventfd.cc"] , "hdrs": ["lib/event_engine/posix_engine/wakeup_fd_eventfd.h"] , "deps": @@ -1106,8 +1275,8 @@ } , "posix_event_engine_wakeup_fd_posix_default": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["posix_event_engine_wakeup_fd_posix_default"] + , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/wakeup_fd_posix_default.cc"] , "hdrs": ["lib/event_engine/posix_engine/wakeup_fd_posix_default.h"] , "deps": @@ -1122,8 +1291,8 @@ } , "posix_event_engine_poller_posix_epoll1": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["posix_event_engine_poller_posix_epoll1"] + , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/ev_epoll1_linux.cc"] , "hdrs": ["lib/event_engine/posix_engine/ev_epoll1_linux.h"] , "deps": @@ -1147,14 +1316,14 @@ , ["@", "absl", "absl/functional", "function_ref"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] ] } , "posix_event_engine_poller_posix_poll": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["posix_event_engine_poller_posix_poll"] + , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/ev_poll_posix.cc"] , "hdrs": ["lib/event_engine/posix_engine/ev_poll_posix.h"] , "deps": @@ -1178,14 +1347,14 @@ , ["@", "absl", "absl/functional", "function_ref"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] ] } , "posix_event_engine_poller_posix_default": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["posix_event_engine_poller_posix_default"] + , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/event_poller_posix_default.cc"] , "hdrs": ["lib/event_engine/posix_engine/event_poller_posix_default.h"] , "deps": @@ -1193,22 +1362,23 @@ , "posix_event_engine_event_poller" , "posix_event_engine_poller_posix_epoll1" , "posix_event_engine_poller_posix_poll" + , ["", "config_vars"] , ["", "gpr"] , ["@", "absl", "absl/strings", "strings"] ] } , "posix_event_engine_internal_errqueue": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["posix_event_engine_internal_errqueue"] + , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/internal_errqueue.cc"] , "hdrs": ["lib/event_engine/posix_engine/internal_errqueue.h"] , "deps": ["iomgr_port", "strerror", ["", "gpr"]] } , "posix_event_engine_traced_buffer_list": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["posix_event_engine_traced_buffer_list"] + , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/traced_buffer_list.cc"] , "hdrs": ["lib/event_engine/posix_engine/traced_buffer_list.h"] , "deps": @@ -1222,8 +1392,8 @@ } , "posix_event_engine_endpoint": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["posix_event_engine_endpoint"] + , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/posix_endpoint.cc"] , "hdrs": ["lib/event_engine/posix_engine/posix_endpoint.h"] , "deps": @@ -1263,16 +1433,21 @@ } , "event_engine_utils": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["event_engine_utils"] + , "stage": ["src", "core"] , "srcs": ["lib/event_engine/utils.cc"] , "hdrs": ["lib/event_engine/utils.h"] - , "deps": ["time", ["", "event_engine_base_hdrs"], ["", "gpr_platform"]] + , "deps": + [ "time" + , ["", "event_engine_base_hdrs"] + , ["", "gpr_platform"] + , ["@", "absl", "absl/strings", "strings"] + ] } , "posix_event_engine_tcp_socket_utils": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["posix_event_engine_tcp_socket_utils"] + , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/tcp_socket_utils.cc"] , "hdrs": ["lib/event_engine/posix_engine/tcp_socket_utils.h"] , "deps": @@ -1282,6 +1457,7 @@ , "socket_mutator" , "status_helper" , "strerror" + , "time" , "useful" , ["", "event_engine_base_hdrs"] , ["", "gpr"] @@ -1295,8 +1471,8 @@ } , "posix_event_engine_listener_utils": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["posix_event_engine_listener_utils"] + , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/posix_engine_listener_utils.cc"] , "hdrs": ["lib/event_engine/posix_engine/posix_engine_listener_utils.h"] , "deps": @@ -1315,8 +1491,8 @@ } , "posix_event_engine_listener": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["posix_event_engine_listener"] + , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/posix_engine_listener.cc"] , "hdrs": ["lib/event_engine/posix_engine/posix_engine_listener.h"] , "deps": @@ -1330,6 +1506,7 @@ , "posix_event_engine_tcp_socket_utils" , "socket_mutator" , "status_helper" + , "time" , ["", "event_engine_base_hdrs"] , ["", "gpr"] , ["@", "absl", "absl/base", "core_headers"] @@ -1337,20 +1514,18 @@ , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] - , ["@", "absl", "absl/synchronization", "synchronization"] , ["@", "absl", "absl/types", "optional"] ] } , "posix_event_engine": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["posix_event_engine"] + , "stage": ["src", "core"] , "srcs": ["lib/event_engine/posix_engine/posix_engine.cc"] , "hdrs": ["lib/event_engine/posix_engine/posix_engine.h"] , "deps": [ "event_engine_common" , "event_engine_poller" - , "event_engine_shim" , "event_engine_tcp_socket_utils" , "event_engine_thread_pool" , "event_engine_trace" @@ -1366,6 +1541,7 @@ , "posix_event_engine_tcp_socket_utils" , "posix_event_engine_timer" , "posix_event_engine_timer_manager" + , "useful" , ["", "event_engine_base_hdrs"] , ["", "gpr"] , ["", "grpc_trace"] @@ -1382,8 +1558,8 @@ } , "windows_event_engine": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["windows_event_engine"] + , "stage": ["src", "core"] , "srcs": ["lib/event_engine/windows/windows_engine.cc"] , "hdrs": ["lib/event_engine/windows/windows_engine.h"] , "deps": @@ -1391,7 +1567,6 @@ , "common_event_engine_closures" , "error" , "event_engine_common" - , "event_engine_executor" , "event_engine_tcp_socket_utils" , "event_engine_thread_pool" , "event_engine_trace" @@ -1400,6 +1575,7 @@ , "posix_event_engine_timer_manager" , "time" , "windows_endpoint" + , "windows_event_engine_listener" , "windows_iocp" , ["", "event_engine_base_hdrs"] , ["", "gpr"] @@ -1408,17 +1584,124 @@ , ["@", "absl", "absl/strings", "strings"] ] } -, "event_engine_tcp_socket_utils": +, "windows_iocp": + { "type": ["@", "rules", "CC", "library"] + , "name": ["windows_iocp"] + , "stage": ["src", "core"] + , "srcs": + [ "lib/event_engine/windows/iocp.cc" + , "lib/event_engine/windows/win_socket.cc" + ] + , "hdrs": + [ "lib/event_engine/windows/iocp.h" + , "lib/event_engine/windows/win_socket.h" + ] + , "deps": + [ "error" + , "event_engine_poller" + , "event_engine_tcp_socket_utils" + , "event_engine_thread_pool" + , "event_engine_time_util" + , "event_engine_trace" + , ["", "debug_location"] + , ["", "event_engine_base_hdrs"] + , ["", "gpr"] + , ["", "gpr_platform"] + , ["@", "absl", "absl/base", "core_headers"] + , ["@", "absl", "absl/functional", "any_invocable"] + , ["@", "absl", "absl/status", "status"] + , ["@", "absl", "absl/strings", "str_format"] + ] + } +, "windows_endpoint": + { "type": ["@", "rules", "CC", "library"] + , "name": ["windows_endpoint"] + , "stage": ["src", "core"] + , "srcs": ["lib/event_engine/windows/windows_endpoint.cc"] + , "hdrs": ["lib/event_engine/windows/windows_endpoint.h"] + , "deps": + [ "error" + , "event_engine_tcp_socket_utils" + , "event_engine_thread_pool" + , "event_engine_trace" + , "status_helper" + , "windows_iocp" + , ["", "debug_location"] + , ["", "event_engine_base_hdrs"] + , ["", "gpr"] + , ["", "gpr_platform"] + , ["@", "absl", "absl/cleanup", "cleanup"] + , ["@", "absl", "absl/functional", "any_invocable"] + , ["@", "absl", "absl/status", "status"] + , ["@", "absl", "absl/strings", "str_format"] + ] + } +, "windows_event_engine_listener": + { "type": ["@", "rules", "CC", "library"] + , "name": ["windows_event_engine_listener"] + , "stage": ["src", "core"] + , "srcs": ["lib/event_engine/windows/windows_listener.cc"] + , "hdrs": ["lib/event_engine/windows/windows_listener.h"] + , "deps": + [ "common_event_engine_closures" + , "error" + , "event_engine_tcp_socket_utils" + , "event_engine_thread_pool" + , "event_engine_trace" + , "windows_endpoint" + , "windows_iocp" + , ["", "event_engine_base_hdrs"] + , ["", "gpr"] + , ["", "gpr_platform"] + , ["@", "absl", "absl/base", "core_headers"] + , ["@", "absl", "absl/status", "status"] + , ["@", "absl", "absl/status", "statusor"] + , ["@", "absl", "absl/strings", "str_format"] + ] + } +, "cf_event_engine": { "type": ["@", "rules", "CC", "library"] + , "name": ["cf_event_engine"] , "stage": ["src", "core"] + , "srcs": + [ "lib/event_engine/cf_engine/cf_engine.cc" + , "lib/event_engine/cf_engine/cfstream_endpoint.cc" + ] + , "hdrs": + [ "lib/event_engine/cf_engine/cf_engine.h" + , "lib/event_engine/cf_engine/cfstream_endpoint.h" + , "lib/event_engine/cf_engine/cftype_unique_ref.h" + ] + , "deps": + [ "event_engine_common" + , "event_engine_tcp_socket_utils" + , "event_engine_thread_pool" + , "event_engine_trace" + , "event_engine_utils" + , "init_internally" + , "posix_event_engine_closure" + , "posix_event_engine_event_poller" + , "posix_event_engine_lockfree_event" + , "posix_event_engine_timer_manager" + , "ref_counted" + , "strerror" + , ["", "event_engine_base_hdrs"] + , ["", "gpr"] + , ["", "ref_counted_ptr"] + , ["", "sockaddr_utils"] + , ["@", "absl", "absl/strings", "str_format"] + ] + } +, "event_engine_tcp_socket_utils": + { "type": ["@", "rules", "CC", "library"] , "name": ["event_engine_tcp_socket_utils"] + , "stage": ["src", "core"] , "srcs": ["lib/event_engine/tcp_socket_utils.cc"] , "hdrs": ["lib/event_engine/tcp_socket_utils.h"] , "deps": [ "iomgr_port" , "resolved_address" , "status_helper" - , "event_engine_common" , ["", "event_engine_base_hdrs"] , ["", "gpr"] , ["", "gpr_platform"] @@ -1426,53 +1709,44 @@ , ["", "uri_parser"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] ] } , "event_engine_trace": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["event_engine_trace"] + , "stage": ["src", "core"] , "srcs": ["lib/event_engine/trace.cc"] , "hdrs": ["lib/event_engine/trace.h"] , "deps": [["", "gpr"], ["", "gpr_platform"], ["", "grpc_trace"]] } , "event_engine_shim": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["event_engine_shim"] + , "stage": ["src", "core"] , "srcs": ["lib/event_engine/shim.cc"] , "hdrs": ["lib/event_engine/shim.h"] , "deps": ["experiments", "iomgr_port", ["", "gpr_platform"]] } , "default_event_engine_factory": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] - , "arguments_config": ["OS"] , "name": ["default_event_engine_factory"] + , "stage": ["src", "core"] , "srcs": ["lib/event_engine/default_event_engine_factory.cc"] , "hdrs": ["lib/event_engine/default_event_engine_factory.h"] , "deps": - { "type": "++" - , "$1": - [ [ ["", "event_engine_base_hdrs"] - , ["", "gpr_platform"] - , ["@", "absl", "absl/memory", "memory"] - ] - , { "type": "case*" - , "expr": {"type": "var", "name": "OS"} - , "case": [["windows", ["windows_event_engine"]]] - , "default": ["posix_event_engine"] - } - ] - } + [ ["", "event_engine_base_hdrs"] + , ["", "gpr_platform"] + , ["@", "absl", "absl/memory", "memory"] + , "posix_event_engine" + ] } , "channel_args_endpoint_config": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["channel_args_endpoint_config"] + , "stage": ["src", "core"] , "srcs": ["lib/event_engine/channel_args_endpoint_config.cc"] , "hdrs": ["lib/event_engine/channel_args_endpoint_config.h"] , "deps": @@ -1483,10 +1757,25 @@ , ["@", "absl", "absl/types", "optional"] ] } -, "default_event_engine": +, "thready_event_engine": { "type": ["@", "rules", "CC", "library"] + , "name": ["thready_event_engine"] , "stage": ["src", "core"] + , "srcs": ["lib/event_engine/thready_event_engine/thready_event_engine.cc"] + , "hdrs": ["lib/event_engine/thready_event_engine/thready_event_engine.h"] + , "deps": + [ ["", "event_engine_base_hdrs"] + , ["", "gpr"] + , ["@", "absl", "absl/functional", "any_invocable"] + , ["@", "absl", "absl/status", "status"] + , ["@", "absl", "absl/status", "statusor"] + , ["@", "absl", "absl/strings", "strings"] + ] + } +, "default_event_engine": + { "type": ["@", "rules", "CC", "library"] , "name": ["default_event_engine"] + , "stage": ["src", "core"] , "srcs": ["lib/event_engine/default_event_engine.cc"] , "hdrs": ["lib/event_engine/default_event_engine.h"] , "deps": @@ -1495,6 +1784,7 @@ , "default_event_engine_factory" , "event_engine_trace" , "no_destruct" + , "thready_event_engine" , ["", "config"] , ["", "debug_location"] , ["", "event_engine_base_hdrs"] @@ -1505,8 +1795,8 @@ } , "channel_args_preconditioning": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["channel_args_preconditioning"] + , "stage": ["src", "core"] , "srcs": ["lib/channel/channel_args_preconditioning.cc"] , "hdrs": ["lib/channel/channel_args_preconditioning.h"] , "deps": @@ -1514,32 +1804,37 @@ } , "pid_controller": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["pid_controller"] + , "stage": ["src", "core"] , "srcs": ["lib/transport/pid_controller.cc"] , "hdrs": ["lib/transport/pid_controller.h"] , "deps": ["useful", ["", "gpr_platform"]] } , "bdp_estimator": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["bdp_estimator"] + , "stage": ["src", "core"] , "srcs": ["lib/transport/bdp_estimator.cc"] , "hdrs": ["lib/transport/bdp_estimator.h"] - , "deps": ["time", ["", "gpr"], ["", "grpc_trace"]] + , "deps": + [ "time" + , ["", "gpr"] + , ["", "grpc_trace"] + , ["@", "absl", "absl/strings", "strings"] + ] } , "percent_encoding": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["percent_encoding"] + , "stage": ["src", "core"] , "srcs": ["lib/slice/percent_encoding.cc"] , "hdrs": ["lib/slice/percent_encoding.h"] , "deps": ["bitset", "slice", ["", "gpr"]] } , "socket_mutator": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["socket_mutator"] + , "stage": ["src", "core"] , "srcs": ["lib/iomgr/socket_mutator.cc"] , "hdrs": ["lib/iomgr/socket_mutator.h"] , "deps": @@ -1547,39 +1842,45 @@ } , "pollset_set": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["pollset_set"] + , "stage": ["src", "core"] , "srcs": ["lib/iomgr/pollset_set.cc"] , "hdrs": ["lib/iomgr/pollset_set.h"] , "deps": ["iomgr_fwd", ["", "gpr"]] } , "histogram_view": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["histogram_view"] + , "stage": ["src", "core"] , "srcs": ["lib/debug/histogram_view.cc"] , "hdrs": ["lib/debug/histogram_view.h"] , "deps": [["", "gpr"]] } , "stats_data": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["stats_data"] + , "stage": ["src", "core"] , "srcs": ["lib/debug/stats_data.cc"] , "hdrs": ["lib/debug/stats_data.h"] - , "deps": ["histogram_view", "per_cpu", ["", "gpr_platform"]] - } -, "per_cpu": + , "deps": + [ "histogram_view" + , "per_cpu" + , ["", "gpr_platform"] + , ["@", "absl", "absl/strings", "strings"] + ] + } +, "per_cpu": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["per_cpu"] + , "stage": ["src", "core"] + , "srcs": ["lib/gprpp/per_cpu.cc"] , "hdrs": ["lib/gprpp/per_cpu.h"] - , "deps": [["", "exec_ctx"], ["", "gpr"]] + , "deps": ["useful", ["", "exec_ctx"], ["", "gpr"]] } , "event_log": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["event_log"] + , "stage": ["src", "core"] , "srcs": ["lib/debug/event_log.cc"] , "hdrs": ["lib/debug/event_log.h"] , "deps": @@ -1592,8 +1893,8 @@ } , "load_file": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["load_file"] + , "stage": ["src", "core"] , "srcs": ["lib/gprpp/load_file.cc"] , "hdrs": ["lib/gprpp/load_file.h"] , "deps": @@ -1607,22 +1908,22 @@ } , "http2_errors": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["http2_errors"] + , "stage": ["src", "core"] , "hdrs": ["lib/transport/http2_errors.h"] } , "channel_stack_type": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["channel_stack_type"] + , "stage": ["src", "core"] , "srcs": ["lib/surface/channel_stack_type.cc"] , "hdrs": ["lib/surface/channel_stack_type.h"] , "deps": [["", "gpr_platform"]] } , "channel_init": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["channel_init"] + , "stage": ["src", "core"] , "srcs": ["lib/surface/channel_init.cc"] , "hdrs": ["lib/surface/channel_init.h"] , "deps": @@ -1630,34 +1931,37 @@ } , "single_set_ptr": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["single_set_ptr"] + , "stage": ["src", "core"] , "hdrs": ["lib/gprpp/single_set_ptr.h"] , "deps": [["", "gpr"]] } , "grpc_service_config": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_service_config"] + , "stage": ["src", "core"] , "hdrs": [ "lib/service_config/service_config.h" , "lib/service_config/service_config_call_data.h" ] , "deps": - [ "ref_counted" + [ "arena" + , "chunked_vector" + , "ref_counted" , "service_config_parser" , "slice_refcount" , "unique_type_name" , "useful" , ["", "gpr_platform"] + , ["", "legacy_context"] , ["", "ref_counted_ptr"] , ["@", "absl", "absl/strings", "strings"] ] } , "service_config_parser": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["service_config_parser"] + , "stage": ["src", "core"] , "srcs": ["lib/service_config/service_config_parser.cc"] , "hdrs": ["lib/service_config/service_config_parser.h"] , "deps": @@ -1670,15 +1974,15 @@ } , "notification": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["notification"] + , "stage": ["src", "core"] , "hdrs": ["lib/gprpp/notification.h"] , "deps": [["", "gpr"], ["@", "absl", "absl/time", "time"]] } , "channel_args": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["channel_args"] + , "stage": ["src", "core"] , "srcs": ["lib/channel/channel_args.cc"] , "hdrs": ["lib/channel/channel_args.h"] , "deps": @@ -1694,23 +1998,23 @@ , ["", "gpr"] , ["", "ref_counted_ptr"] , ["@", "absl", "absl/meta", "type_traits"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] ] } , "resolved_address": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["resolved_address"] + , "stage": ["src", "core"] , "hdrs": ["lib/iomgr/resolved_address.h"] , "deps": ["iomgr_port", ["", "gpr_platform"]] } , "lb_policy": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["lb_policy"] + , "stage": ["src", "core"] , "srcs": ["lib/load_balancing/lb_policy.cc"] , "hdrs": ["lib/load_balancing/lb_policy.h"] , "deps": @@ -1726,12 +2030,14 @@ , ["", "debug_location"] , ["", "event_engine_base_hdrs"] , ["", "exec_ctx"] + , ["", "gpr"] , ["", "gpr_platform"] , ["", "grpc_trace"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "server_address"] , ["", "work_serializer"] + , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] @@ -1741,8 +2047,8 @@ } , "lb_policy_factory": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["lb_policy_factory"] + , "stage": ["src", "core"] , "hdrs": ["lib/load_balancing/lb_policy_factory.h"] , "deps": [ "json" @@ -1756,8 +2062,8 @@ } , "lb_policy_registry": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["lb_policy_registry"] + , "stage": ["src", "core"] , "srcs": ["lib/load_balancing/lb_policy_registry.cc"] , "hdrs": ["lib/load_balancing/lb_policy_registry.h"] , "deps": @@ -1769,14 +2075,14 @@ , ["", "ref_counted_ptr"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] ] } , "subchannel_interface": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["subchannel_interface"] + , "stage": ["src", "core"] , "hdrs": ["lib/load_balancing/subchannel_interface.h"] , "deps": [ "dual_ref_counted" @@ -1789,8 +2095,8 @@ } , "proxy_mapper": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["proxy_mapper"] + , "stage": ["src", "core"] , "hdrs": ["lib/handshaker/proxy_mapper.h"] , "deps": [ "channel_args" @@ -1802,8 +2108,8 @@ } , "proxy_mapper_registry": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["proxy_mapper_registry"] + , "stage": ["src", "core"] , "srcs": ["lib/handshaker/proxy_mapper_registry.cc"] , "hdrs": ["lib/handshaker/proxy_mapper_registry.h"] , "deps": @@ -1817,8 +2123,8 @@ } , "grpc_server_config_selector": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_server_config_selector"] + , "stage": ["src", "core"] , "hdrs": ["ext/filters/server_config_selector/server_config_selector.h"] , "deps": [ "dual_ref_counted" @@ -1835,8 +2141,8 @@ } , "grpc_server_config_selector_filter": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_server_config_selector_filter"] + , "stage": ["src", "core"] , "srcs": ["ext/filters/server_config_selector/server_config_selector_filter.cc"] , "hdrs": @@ -1852,6 +2158,7 @@ , "status_helper" , ["", "gpr"] , ["", "grpc_base"] + , ["", "legacy_context"] , ["", "promise"] , ["", "ref_counted_ptr"] , ["@", "absl", "absl/base", "core_headers"] @@ -1862,30 +2169,32 @@ } , "sorted_pack": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["sorted_pack"] + , "stage": ["src", "core"] , "hdrs": ["lib/gprpp/sorted_pack.h"] , "deps": [["", "gpr_platform"]] } , "certificate_provider_factory": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["certificate_provider_factory"] + , "stage": ["src", "core"] , "hdrs": ["lib/security/certificate_provider/certificate_provider_factory.h"] , "deps": - [ "error" - , "json" + [ "json" + , "json_args" , "ref_counted" + , "validation_errors" , ["", "alts_util"] , ["", "gpr"] , ["", "ref_counted_ptr"] + , ["@", "absl", "absl/strings", "strings"] ] } , "certificate_provider_registry": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["certificate_provider_registry"] + , "stage": ["src", "core"] , "srcs": ["lib/security/certificate_provider/certificate_provider_registry.cc"] , "hdrs": @@ -1896,10 +2205,33 @@ , ["@", "absl", "absl/strings", "strings"] ] } -, "grpc_authorization_base": +, "grpc_audit_logging": { "type": ["@", "rules", "CC", "library"] + , "name": ["grpc_audit_logging"] , "stage": ["src", "core"] + , "srcs": + [ "lib/security/authorization/audit_logging.cc" + , "lib/security/authorization/stdout_logger.cc" + ] + , "hdrs": + [ "lib/security/authorization/audit_logging.h" + , "lib/security/authorization/stdout_logger.h" + ] + , "deps": + [ ["", "gpr"] + , ["", "grpc_base"] + , ["@", "absl", "absl/base", "core_headers"] + , ["@", "absl", "absl/status", "status"] + , ["@", "absl", "absl/status", "statusor"] + , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] + , ["@", "absl", "absl/time", "time"] + ] + } +, "grpc_authorization_base": + { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_authorization_base"] + , "stage": ["src", "core"] , "srcs": [ "lib/security/authorization/authorization_policy_provider_vtable.cc" , "lib/security/authorization/evaluate_args.cc" @@ -1937,8 +2269,8 @@ } , "grpc_fake_credentials": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_fake_credentials"] + , "stage": ["src", "core"] , "srcs": [ "lib/security/credentials/fake/fake_credentials.cc" , "lib/security/security_connector/fake/fake_security_connector.cc" @@ -1969,15 +2301,15 @@ , ["", "tsi_fake_credentials"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] ] } , "grpc_insecure_credentials": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_insecure_credentials"] + , "stage": ["src", "core"] , "srcs": [ "lib/security/credentials/insecure/insecure_credentials.cc" , "lib/security/security_connector/insecure/insecure_security_connector.cc" @@ -2009,8 +2341,8 @@ } , "tsi_local_credentials": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["tsi_local_credentials"] + , "stage": ["src", "core"] , "srcs": ["tsi/local_transport_security.cc"] , "hdrs": ["tsi/local_transport_security.h"] , "deps": @@ -2022,8 +2354,8 @@ } , "grpc_local_credentials": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_local_credentials"] + , "stage": ["src", "core"] , "srcs": [ "lib/security/credentials/local/local_credentials.cc" , "lib/security/security_connector/local/local_security_connector.cc" @@ -2064,8 +2396,8 @@ } , "grpc_ssl_credentials": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_ssl_credentials"] + , "stage": ["src", "core"] , "srcs": [ "lib/security/credentials/ssl/ssl_credentials.cc" , "lib/security/security_connector/ssl/ssl_security_connector.cc" @@ -2095,15 +2427,15 @@ , ["", "tsi_ssl_credentials"] , ["", "tsi_ssl_session_cache"] , ["@", "absl", "absl/status", "status"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] ] } , "grpc_google_default_credentials": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_google_default_credentials"] + , "stage": ["src", "core"] , "srcs": [ "lib/security/credentials/google_default/credentials_generic.cc" , "lib/security/credentials/google_default/google_default_credentials.cc" @@ -2123,6 +2455,7 @@ , "grpc_ssl_credentials" , "iomgr_fwd" , "json" + , "json_reader" , "slice" , "slice_refcount" , "status_helper" @@ -2149,16 +2482,16 @@ } , "strerror": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["strerror"] + , "stage": ["src", "core"] , "srcs": ["lib/gprpp/strerror.cc"] , "hdrs": ["lib/gprpp/strerror.h"] , "deps": [["", "gpr_platform"], ["@", "absl", "absl/strings", "str_format"]] } , "grpc_tls_credentials": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_tls_credentials"] + , "stage": ["src", "core"] , "srcs": [ "lib/security/credentials/tls/grpc_tls_certificate_distributor.cc" , "lib/security/credentials/tls/grpc_tls_certificate_provider.cc" @@ -2204,18 +2537,18 @@ , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/container", "inlined_vector"] , ["@", "absl", "absl/functional", "bind_front"] - , ["@", "absl", "absl/status", "statusor"] + , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] - , ["@", "libssl", "", "crypto"] - , ["@", "libssl", "", "ssl"] + , ["@", "ssl", "", "crypto"] + , ["@", "ssl", "", "ssl"] ] } , "grpc_iam_credentials": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_iam_credentials"] + , "stage": ["src", "core"] , "srcs": ["lib/security/credentials/iam/iam_credentials.cc"] , "hdrs": ["lib/security/credentials/iam/iam_credentials.h"] , "deps": @@ -2231,15 +2564,15 @@ , ["", "promise"] , ["", "ref_counted_ptr"] , ["@", "absl", "absl/status", "statusor"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] ] } , "grpc_oauth2_credentials": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_oauth2_credentials"] + , "stage": ["src", "core"] , "srcs": ["lib/security/credentials/oauth2/oauth2_credentials.cc"] , "hdrs": ["lib/security/credentials/oauth2/oauth2_credentials.h"] , "deps": @@ -2250,6 +2583,7 @@ , "error" , "httpcli_ssl_credentials" , "json" + , "json_reader" , "poll" , "pollset_set" , "ref_counted" @@ -2271,15 +2605,15 @@ , ["", "uri_parser"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] ] } , "grpc_external_account_credentials": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_external_account_credentials"] + , "stage": ["src", "core"] , "srcs": [ "lib/security/credentials/external/aws_external_account_credentials.cc" , "lib/security/credentials/external/aws_request_signer.cc" @@ -2301,11 +2635,12 @@ , "grpc_oauth2_credentials" , "httpcli_ssl_credentials" , "json" + , "json_reader" + , "json_writer" , "slice" , "slice_refcount" , "status_helper" , "time" - , "grpc_insecure_credentials" , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_credentials_util"] @@ -2316,17 +2651,17 @@ , ["", "uri_parser"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/time", "time"] , ["@", "absl", "absl/types", "optional"] - , ["@", "libssl", "", "crypto"] + , ["@", "ssl", "", "crypto"] ] } , "httpcli_ssl_credentials": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["httpcli_ssl_credentials"] + , "stage": ["src", "core"] , "srcs": ["lib/http/httpcli_security_connector.cc"] , "hdrs": ["lib/http/httpcli_ssl_credentials.h"] , "deps": @@ -2353,31 +2688,31 @@ } , "tsi_ssl_types": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["tsi_ssl_types"] + , "stage": ["src", "core"] , "hdrs": ["tsi/ssl_types.h"] - , "deps": [["", "gpr_platform"], ["@", "libssl", "", "ssl"]] + , "deps": [["", "gpr_platform"], ["@", "ssl", "", "ssl"]] } , "grpc_matchers": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_matchers"] + , "stage": ["src", "core"] , "srcs": ["lib/matchers/matchers.cc"] , "hdrs": ["lib/matchers/matchers.h"] , "deps": [ ["", "gpr"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "re2", "", "re2"] ] } , "grpc_rbac_engine": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_rbac_engine"] + , "stage": ["src", "core"] , "srcs": [ "lib/security/authorization/grpc_authorization_engine.cc" , "lib/security/authorization/matchers.cc" @@ -2389,7 +2724,8 @@ , "lib/security/authorization/rbac_policy.h" ] , "deps": - [ "grpc_authorization_base" + [ "grpc_audit_logging" + , "grpc_authorization_base" , "grpc_matchers" , "resolved_address" , ["", "gpr"] @@ -2398,30 +2734,48 @@ , ["", "sockaddr_utils"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] ] } , "json": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["json"] - , "srcs": ["lib/json/json_reader.cc", "lib/json/json_writer.cc"] + , "stage": ["src", "core"] , "hdrs": ["lib/json/json.h"] + , "deps": [["", "gpr"]] + } +, "json_reader": + { "type": ["@", "rules", "CC", "library"] + , "name": ["json_reader"] + , "stage": ["src", "core"] + , "srcs": ["lib/json/json_reader.cc"] + , "hdrs": ["lib/json/json_reader.h"] , "deps": - [ ["", "gpr"] + [ "json" + , "match" + , ["", "gpr"] , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] + , ["@", "absl", "absl/types", "variant"] ] } -, "json_util": +, "json_writer": { "type": ["@", "rules", "CC", "library"] + , "name": ["json_writer"] , "stage": ["src", "core"] + , "srcs": ["lib/json/json_writer.cc"] + , "hdrs": ["lib/json/json_writer.h"] + , "deps": ["json", ["", "gpr"], ["@", "absl", "absl/strings", "strings"]] + } +, "json_util": + { "type": ["@", "rules", "CC", "library"] , "name": ["json_util"] + , "stage": ["src", "core"] , "srcs": ["lib/json/json_util.cc"] , "hdrs": ["lib/json/json_util.h"] , "deps": @@ -2438,15 +2792,15 @@ } , "json_args": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["json_args"] + , "stage": ["src", "core"] , "hdrs": ["lib/json/json_args.h"] , "deps": [["", "gpr"], ["@", "absl", "absl/strings", "strings"]] } , "json_object_loader": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["json_object_loader"] + , "stage": ["src", "core"] , "srcs": ["lib/json/json_object_loader.cc"] , "hdrs": ["lib/json/json_object_loader.h"] , "deps": @@ -2458,6 +2812,7 @@ , ["", "gpr"] , ["", "ref_counted_ptr"] , ["@", "absl", "absl/meta", "type_traits"] + , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] @@ -2465,8 +2820,8 @@ } , "json_channel_args": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["json_channel_args"] + , "stage": ["src", "core"] , "hdrs": ["lib/json/json_channel_args.h"] , "deps": [ "channel_args" @@ -2478,16 +2833,16 @@ } , "idle_filter_state": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["idle_filter_state"] + , "stage": ["src", "core"] , "srcs": ["ext/filters/channel_idle/idle_filter_state.cc"] , "hdrs": ["ext/filters/channel_idle/idle_filter_state.h"] , "deps": [["", "gpr_platform"]] } , "grpc_channel_idle_filter": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_channel_idle_filter"] + , "stage": ["src", "core"] , "srcs": ["ext/filters/channel_idle/channel_idle_filter.cc"] , "hdrs": ["ext/filters/channel_idle/channel_idle_filter.h"] , "deps": @@ -2526,8 +2881,8 @@ } , "grpc_deadline_filter": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_deadline_filter"] + , "stage": ["src", "core"] , "srcs": ["ext/filters/deadline/deadline_filter.cc"] , "hdrs": ["ext/filters/deadline/deadline_filter.h"] , "deps": @@ -2556,8 +2911,8 @@ } , "grpc_client_authority_filter": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_client_authority_filter"] + , "stage": ["src", "core"] , "srcs": ["ext/filters/http/client_authority_filter.cc"] , "hdrs": ["ext/filters/http/client_authority_filter.h"] , "deps": @@ -2578,41 +2933,47 @@ } , "grpc_message_size_filter": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_message_size_filter"] + , "stage": ["src", "core"] , "srcs": ["ext/filters/message_size/message_size_filter.cc"] , "hdrs": ["ext/filters/message_size/message_size_filter.h"] , "deps": - [ "channel_args" + [ "activity" + , "arena" + , "arena_promise" + , "channel_args" , "channel_fwd" , "channel_init" , "channel_stack_type" - , "closure" - , "error" + , "context" , "grpc_service_config" , "json" , "json_args" , "json_object_loader" + , "latch" + , "poll" + , "race" , "service_config_parser" + , "slice" , "slice_buffer" - , "status_helper" , "validation_errors" , ["", "channel_stack_builder"] , ["", "config"] - , ["", "debug_location"] , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_public_hdrs"] - , ["@", "absl", "absl/status", "status"] - , ["@", "absl", "absl/strings", "strings"] + , ["", "grpc_trace"] + , ["", "legacy_context"] + , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] ] } , "grpc_fault_injection_filter": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_fault_injection_filter"] + , "stage": ["src", "core"] , "srcs": [ "ext/filters/fault_injection/fault_injection_filter.cc" , "ext/filters/fault_injection/fault_injection_service_config_parser.cc" @@ -2640,18 +3001,19 @@ , ["", "grpc_base"] , ["", "grpc_public_hdrs"] , ["", "grpc_trace"] + , ["", "legacy_context"] , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] + , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] - , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/types", "optional"] ] } , "grpc_rbac_filter": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_rbac_filter"] + , "stage": ["src", "core"] , "srcs": [ "ext/filters/rbac/rbac_filter.cc" , "ext/filters/rbac/rbac_service_config_parser.cc" @@ -2665,6 +3027,7 @@ , "channel_fwd" , "closure" , "error" + , "grpc_audit_logging" , "grpc_authorization_base" , "grpc_matchers" , "grpc_rbac_engine" @@ -2682,6 +3045,7 @@ , ["", "grpc_base"] , ["", "grpc_public_hdrs"] , ["", "grpc_security_base"] + , ["", "legacy_context"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] @@ -2690,8 +3054,8 @@ } , "grpc_stateful_session_filter": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_stateful_session_filter"] + , "stage": ["src", "core"] , "srcs": [ "ext/filters/stateful_session/stateful_session_filter.cc" , "ext/filters/stateful_session/stateful_session_service_config_parser.cc" @@ -2722,6 +3086,7 @@ , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_trace"] + , ["", "legacy_context"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] @@ -2729,8 +3094,8 @@ } , "grpc_lb_policy_grpclb": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_lb_policy_grpclb"] + , "stage": ["src", "core"] , "srcs": [ "ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc" , "ext/filters/client_channel/lb_policy/grpclb/grpclb.cc" @@ -2799,31 +3164,39 @@ , ["@", "absl", "absl/meta", "type_traits"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] , ["third_party/upb", "upb"] , ["src/core/ext/upb-generated", "upb-generated-lib"] ] } -, "grpc_backend_metric_data": +, "random_early_detection": { "type": ["@", "rules", "CC", "library"] + , "name": ["random_early_detection"] , "stage": ["src", "core"] + , "srcs": ["lib/backoff/random_early_detection.cc"] + , "hdrs": ["lib/backoff/random_early_detection.h"] + , "deps": [["", "gpr_platform"], ["@", "absl", "absl/random", "random"]] + } +, "grpc_backend_metric_data": + { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_backend_metric_data"] + , "stage": ["src", "core"] , "hdrs": ["ext/filters/client_channel/lb_policy/backend_metric_data.h"] , "deps": [["", "gpr_platform"], ["@", "absl", "absl/strings", "strings"]] } , "grpc_backend_metric_provider": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_backend_metric_provider"] + , "stage": ["src", "core"] , "hdrs": ["ext/filters/backend_metrics/backend_metric_provider.h"] } , "grpc_lb_policy_rls": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_lb_policy_rls"] + , "stage": ["src", "core"] , "srcs": ["ext/filters/client_channel/lb_policy/rls/rls.cc"] , "deps": [ "channel_args" @@ -2834,6 +3207,7 @@ , "json" , "json_args" , "json_object_loader" + , "json_writer" , "lb_policy" , "lb_policy_factory" , "lb_policy_registry" @@ -2865,8 +3239,8 @@ , ["@", "absl", "absl/hash", "hash"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["third_party/upb", "upb"] , ["src/core/ext/upb-generated", "upb-generated-lib"] @@ -2874,8 +3248,8 @@ } , "upb_utils": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["upb_utils"] + , "stage": ["src", "core"] , "hdrs": ["ext/xds/upb_utils.h"] , "deps": [ ["", "gpr_platform"] @@ -2885,11 +3259,12 @@ } , "grpc_xds_client": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_xds_client"] + , "stage": ["src", "core"] , "srcs": [ "ext/xds/certificate_provider_store.cc" , "ext/xds/file_watcher_certificate_provider_factory.cc" + , "ext/xds/xds_audit_logger_registry.cc" , "ext/xds/xds_bootstrap_grpc.cc" , "ext/xds/xds_certificate_provider.cc" , "ext/xds/xds_client_grpc.cc" @@ -2912,6 +3287,7 @@ , "hdrs": [ "ext/xds/certificate_provider_store.h" , "ext/xds/file_watcher_certificate_provider_factory.h" + , "ext/xds/xds_audit_logger_registry.h" , "ext/xds/xds_bootstrap_grpc.h" , "ext/xds/xds_certificate_provider.h" , "ext/xds/xds_client_grpc.h" @@ -2931,7 +3307,6 @@ , "ext/xds/xds_transport_grpc.h" , "lib/security/credentials/xds/xds_credentials.h" ] - , "proto": [["@", "google_apis", "", "google_rpc_status_proto"]] , "deps": [ "certificate_provider_factory" , "certificate_provider_registry" @@ -2942,6 +3317,7 @@ , "default_event_engine" , "env" , "error" + , "grpc_audit_logging" , "grpc_fake_credentials" , "grpc_fault_injection_filter" , "grpc_lb_xds_channel_args" @@ -2957,7 +3333,9 @@ , "json" , "json_args" , "json_object_loader" + , "json_reader" , "json_util" + , "json_writer" , "lb_policy_registry" , "match" , "pollset_set" @@ -2991,29 +3369,35 @@ , ["", "uri_parser"] , ["", "work_serializer"] , ["", "xds_client"] + , ["@", "protobuf", "", "libprotobuf"] + , ["src/core/ext/upb-generated", "upb-generated-lib"] + , ["src/core/ext/upbdefs-generated", "upbdefs-generated-lib"] , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/functional", "bind_front"] , ["@", "absl", "absl/memory", "memory"] + , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] + , ["@", "absl", "absl/synchronization", "synchronization"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "span"] , ["@", "absl", "absl/types", "variant"] - , ["third_party/upb", "upb"] - , ["third_party/upb", "upb_textformat"] - , ["third_party/upb", "upb_json"] - , ["third_party/upb", "upb_reflection"] , ["@", "re2", "", "re2"] + , ["third_party/upb", "collections"] + , ["third_party/upb", "json"] + , ["third_party/upb", "reflection"] + , ["third_party/upb", "textformat"] + , ["third_party/upb", "upb"] , ["src/core/ext/upb-generated", "upb-generated-lib"] , ["src/core/ext/upbdefs-generated", "upbdefs-generated-lib"] ] } , "grpc_xds_channel_stack_modifier": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_xds_channel_stack_modifier"] + , "stage": ["src", "core"] , "srcs": ["ext/xds/xds_channel_stack_modifier.cc"] , "hdrs": ["ext/xds/xds_channel_stack_modifier.h"] , "deps": @@ -3032,8 +3416,8 @@ } , "grpc_xds_server_config_fetcher": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_xds_server_config_fetcher"] + , "stage": ["src", "core"] , "srcs": ["ext/xds/xds_server_config_fetcher.cc"] , "deps": [ "channel_args" @@ -3074,8 +3458,8 @@ } , "channel_creds_registry_init": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["channel_creds_registry_init"] + , "stage": ["src", "core"] , "srcs": ["lib/security/credentials/channel_creds_registry_init.cc"] , "deps": [ "channel_creds_registry" @@ -3091,8 +3475,8 @@ } , "grpc_lb_policy_cds": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_lb_policy_cds"] + , "stage": ["src", "core"] , "srcs": ["ext/filters/client_channel/lb_policy/xds/cds.cc"] , "deps": [ "channel_args" @@ -3103,6 +3487,7 @@ , "json" , "json_args" , "json_object_loader" + , "json_writer" , "lb_policy" , "lb_policy_factory" , "lb_policy_registry" @@ -3130,14 +3515,14 @@ } , "grpc_lb_xds_channel_args": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_lb_xds_channel_args"] + , "stage": ["src", "core"] , "hdrs": ["ext/filters/client_channel/lb_policy/xds/xds_channel_args.h"] } , "grpc_lb_xds_attributes": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_lb_xds_attributes"] + , "stage": ["src", "core"] , "srcs": ["ext/filters/client_channel/lb_policy/xds/xds_attributes.cc"] , "hdrs": ["ext/filters/client_channel/lb_policy/xds/xds_attributes.h"] , "deps": @@ -3151,8 +3536,8 @@ } , "grpc_lb_policy_xds_cluster_resolver": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_lb_policy_xds_cluster_resolver"] + , "stage": ["src", "core"] , "srcs": ["ext/filters/client_channel/lb_policy/xds/xds_cluster_resolver.cc"] , "deps": @@ -3164,6 +3549,7 @@ , "json" , "json_args" , "json_object_loader" + , "json_writer" , "lb_policy" , "lb_policy_factory" , "lb_policy_registry" @@ -3191,11 +3577,12 @@ } , "grpc_lb_policy_xds_cluster_impl": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_lb_policy_xds_cluster_impl"] + , "stage": ["src", "core"] , "srcs": ["ext/filters/client_channel/lb_policy/xds/xds_cluster_impl.cc"] , "deps": [ "channel_args" + , "grpc_backend_metric_data" , "grpc_lb_xds_attributes" , "grpc_lb_xds_channel_args" , "grpc_xds_client" @@ -3229,8 +3616,8 @@ } , "grpc_lb_policy_xds_cluster_manager": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_lb_policy_xds_cluster_manager"] + , "stage": ["src", "core"] , "srcs": ["ext/filters/client_channel/lb_policy/xds/xds_cluster_manager.cc"] , "deps": [ "channel_args" @@ -3265,8 +3652,8 @@ } , "grpc_lb_policy_xds_wrr_locality": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_lb_policy_xds_wrr_locality"] + , "stage": ["src", "core"] , "srcs": ["ext/filters/client_channel/lb_policy/xds/xds_wrr_locality.cc"] , "deps": [ "channel_args" @@ -3274,6 +3661,7 @@ , "json" , "json_args" , "json_object_loader" + , "json_writer" , "lb_policy" , "lb_policy_factory" , "lb_policy_registry" @@ -3296,8 +3684,8 @@ } , "grpc_lb_address_filtering": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_lb_address_filtering"] + , "stage": ["src", "core"] , "srcs": ["ext/filters/client_channel/lb_policy/address_filtering.cc"] , "hdrs": ["ext/filters/client_channel/lb_policy/address_filtering.h"] , "deps": @@ -3307,35 +3695,74 @@ , ["@", "absl", "absl/strings", "strings"] ] } -, "grpc_lb_subchannel_list": +, "health_check_client": { "type": ["@", "rules", "CC", "library"] + , "name": ["health_check_client"] , "stage": ["src", "core"] - , "name": ["grpc_lb_subchannel_list"] - , "hdrs": ["ext/filters/client_channel/lb_policy/subchannel_list.h"] + , "srcs": ["ext/filters/client_channel/lb_policy/health_check_client.cc"] + , "hdrs": + [ "ext/filters/client_channel/lb_policy/health_check_client.h" + , "ext/filters/client_channel/lb_policy/health_check_client_internal.h" + ] , "deps": - [ "channel_args" - , "dual_ref_counted" - , "gpr_manual_constructor" + [ "closure" + , "error" + , "iomgr_fwd" + , "pollset_set" + , "slice" + , "subchannel_interface" + , "unique_type_name" + , ["", "debug_location"] + , ["", "exec_ctx"] + , ["", "gpr"] + , ["", "grpc_base"] + , ["", "grpc_client_channel"] + , ["", "grpc_public_hdrs"] + , ["", "grpc_trace"] + , ["", "orphanable"] + , ["", "ref_counted_ptr"] + , ["", "work_serializer"] + , ["@", "absl", "absl/base", "core_headers"] + , ["@", "absl", "absl/status", "status"] + , ["@", "absl", "absl/status", "statusor"] + , ["@", "absl", "absl/strings", "strings"] + , ["third_party/upb", "upb"] + , ["src/core/ext/upb-generated", "upb-generated-lib"] + ] + } +, "grpc_lb_subchannel_list": + { "type": ["@", "rules", "CC", "library"] + , "name": ["grpc_lb_subchannel_list"] + , "stage": ["src", "core"] + , "hdrs": ["ext/filters/client_channel/lb_policy/subchannel_list.h"] + , "deps": + [ "channel_args" + , "dual_ref_counted" + , "gpr_manual_constructor" + , "health_check_client" , "iomgr_fwd" , "lb_policy" , "subchannel_interface" , ["", "debug_location"] , ["", "gpr"] , ["", "grpc_base"] + , ["", "grpc_client_channel"] , ["", "ref_counted_ptr"] , ["", "server_address"] + , ["", "work_serializer"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/types", "optional"] ] } , "grpc_lb_policy_pick_first": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_lb_policy_pick_first"] + , "stage": ["src", "core"] , "srcs": ["ext/filters/client_channel/lb_policy/pick_first/pick_first.cc"] , "deps": [ "channel_args" , "grpc_lb_subchannel_list" + , "grpc_outlier_detection_header" , "json" , "lb_policy" , "lb_policy_factory" @@ -3348,6 +3775,7 @@ , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "server_address"] + , ["", "work_serializer"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] @@ -3356,8 +3784,8 @@ } , "grpc_lb_policy_ring_hash": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_lb_policy_ring_hash"] + , "stage": ["src", "core"] , "srcs": ["ext/filters/client_channel/lb_policy/ring_hash/ring_hash.cc"] , "hdrs": ["ext/filters/client_channel/lb_policy/ring_hash/ring_hash.h"] , "deps": @@ -3365,11 +3793,13 @@ , "closure" , "error" , "grpc_lb_subchannel_list" + , "grpc_service_config" , "json" , "json_args" , "json_object_loader" , "lb_policy" , "lb_policy_factory" + , "ref_counted" , "subchannel_interface" , "unique_type_name" , "validation_errors" @@ -3396,8 +3826,8 @@ } , "grpc_lb_policy_round_robin": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_lb_policy_round_robin"] + , "stage": ["src", "core"] , "srcs": ["ext/filters/client_channel/lb_policy/round_robin/round_robin.cc"] , "deps": [ "channel_args" @@ -3414,6 +3844,8 @@ , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "server_address"] + , ["", "work_serializer"] + , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] @@ -3422,8 +3854,8 @@ } , "static_stride_scheduler": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["static_stride_scheduler"] + , "stage": ["src", "core"] , "srcs": [ "ext/filters/client_channel/lb_policy/weighted_round_robin/static_stride_scheduler.cc" ] @@ -3439,8 +3871,8 @@ } , "grpc_lb_policy_weighted_round_robin": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_lb_policy_weighted_round_robin"] + , "stage": ["src", "core"] , "srcs": [ "ext/filters/client_channel/lb_policy/weighted_round_robin/weighted_round_robin.cc" ] @@ -3470,6 +3902,7 @@ , ["", "ref_counted_ptr"] , ["", "server_address"] , ["", "sockaddr_utils"] + , ["", "work_serializer"] , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] @@ -3480,8 +3913,8 @@ } , "grpc_outlier_detection_header": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_outlier_detection_header"] + , "stage": ["src", "core"] , "hdrs": [ "ext/filters/client_channel/lb_policy/outlier_detection/outlier_detection.h" ] @@ -3492,13 +3925,14 @@ , "time" , "validation_errors" , ["", "gpr_platform"] + , ["", "server_address"] , ["@", "absl", "absl/types", "optional"] ] } , "grpc_lb_policy_outlier_detection": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_lb_policy_outlier_detection"] + , "stage": ["src", "core"] , "srcs": [ "ext/filters/client_channel/lb_policy/outlier_detection/outlier_detection.cc" ] @@ -3535,8 +3969,8 @@ } , "grpc_lb_policy_priority": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_lb_policy_priority"] + , "stage": ["src", "core"] , "srcs": ["ext/filters/client_channel/lb_policy/priority/priority.cc"] , "deps": [ "channel_args" @@ -3570,8 +4004,8 @@ } , "grpc_lb_policy_weighted_target": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_lb_policy_weighted_target"] + , "stage": ["src", "core"] , "srcs": ["ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc"] , "deps": @@ -3598,6 +4032,7 @@ , ["", "ref_counted_ptr"] , ["", "server_address"] , ["", "work_serializer"] + , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] @@ -3607,8 +4042,8 @@ } , "grpc_lb_policy_xds_override_host": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_lb_policy_xds_override_host"] + , "stage": ["src", "core"] , "srcs": ["ext/filters/client_channel/lb_policy/xds/xds_override_host.cc"] , "hdrs": ["ext/filters/client_channel/lb_policy/xds/xds_override_host.h"] , "deps": @@ -3642,16 +4077,56 @@ , ["", "work_serializer"] , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/status", "status"] + , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] - , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] ] } -, "grpc_backend_metric_filter": +, "lb_server_load_reporting_filter": { "type": ["@", "rules", "CC", "library"] + , "name": ["lb_server_load_reporting_filter"] , "stage": ["src", "core"] + , "srcs": ["ext/filters/load_reporting/server_load_reporting_filter.cc"] + , "hdrs": + [ "ext/filters/load_reporting/registered_opencensus_objects.h" + , "ext/filters/load_reporting/server_load_reporting_filter.h" + , ["src/cpp/server/load_reporter", "constants.h"] + ] + , "deps": + [ "arena_promise" + , "channel_args" + , "channel_fwd" + , "channel_stack_type" + , "context" + , "grpc_sockaddr" + , "resolved_address" + , "seq" + , "slice" + , ["", "channel_stack_builder"] + , ["", "config"] + , ["", "gpr"] + , ["", "gpr_platform"] + , ["", "grpc_base"] + , ["", "grpc_public_hdrs"] + , ["", "grpc_security_base"] + , ["", "parse_address"] + , ["", "promise"] + , ["", "uri_parser"] + , ["@", "absl", "absl/meta", "type_traits"] + , ["@", "absl", "absl/status", "status"] + , ["@", "absl", "absl/status", "statusor"] + , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] + , ["@", "absl", "absl/types", "optional"] + , ["@", "opencensus-stats", "", ""] + , ["@", "opencensus-tags", "", ""] + ] + } +, "grpc_backend_metric_filter": + { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_backend_metric_filter"] + , "stage": ["src", "core"] , "srcs": ["ext/filters/backend_metrics/backend_metric_filter.cc"] , "hdrs": ["ext/filters/backend_metrics/backend_metric_filter.h"] , "deps": @@ -3670,6 +4145,7 @@ , ["", "gpr_platform"] , ["", "grpc_base"] , ["", "grpc_trace"] + , ["", "legacy_context"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] @@ -3679,8 +4155,8 @@ } , "polling_resolver": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["polling_resolver"] + , "stage": ["src", "core"] , "srcs": ["ext/filters/client_channel/resolver/polling_resolver.cc"] , "hdrs": ["ext/filters/client_channel/resolver/polling_resolver.h"] , "deps": @@ -3705,24 +4181,96 @@ , ["@", "absl", "absl/types", "optional"] ] } -, "grpc_resolver_dns_selection": +, "service_config_helper": { "type": ["@", "rules", "CC", "library"] + , "name": ["service_config_helper"] , "stage": ["src", "core"] - , "name": ["grpc_resolver_dns_selection"] , "srcs": - ["ext/filters/client_channel/resolver/dns/dns_resolver_selection.cc"] + [ "ext/filters/client_channel/resolver/dns/event_engine/service_config_helper.cc" + ] , "hdrs": - ["ext/filters/client_channel/resolver/dns/dns_resolver_selection.h"] - , "deps": [["", "gpr"]] + [ "ext/filters/client_channel/resolver/dns/event_engine/service_config_helper.h" + ] + , "deps": + [ "json" + , "json_args" + , "json_object_loader" + , "json_reader" + , "json_writer" + , "status_helper" + , ["", "gpr_platform"] + , ["", "grpc_base"] + , ["@", "absl", "absl/status", "statusor"] + , ["@", "absl", "absl/strings", "strings"] + ] } -, "grpc_resolver_dns_native": +, "grpc_resolver_dns_event_engine": + { "type": ["@", "rules", "CC", "library"] + , "name": ["grpc_resolver_dns_event_engine"] + , "stage": ["src", "core"] + , "srcs": + [ "ext/filters/client_channel/resolver/dns/event_engine/event_engine_client_channel_resolver.cc" + ] + , "hdrs": + [ "ext/filters/client_channel/resolver/dns/event_engine/event_engine_client_channel_resolver.h" + ] + , "deps": + [ "channel_args" + , "event_engine_common" + , "event_engine_utils" + , "grpc_service_config" + , "polling_resolver" + , "service_config_helper" + , "time" + , "validation_errors" + , ["", "backoff"] + , ["", "debug_location"] + , ["", "gpr"] + , ["", "gpr_platform"] + , ["", "grpc_base"] + , ["", "grpc_grpclb_balancer_addresses"] + , ["", "grpc_resolver"] + , ["", "grpc_service_config_impl"] + , ["", "grpc_trace"] + , ["", "orphanable"] + , ["", "ref_counted_ptr"] + , ["", "server_address"] + , ["", "uri_parser"] + , ["@", "absl", "absl/base", "core_headers"] + , ["@", "absl", "absl/cleanup", "cleanup"] + , ["@", "absl", "absl/container", "flat_hash_set"] + , ["@", "absl", "absl/status", "status"] + , ["@", "absl", "absl/status", "statusor"] + , ["@", "absl", "absl/strings", "strings"] + , ["@", "absl", "absl/types", "optional"] + ] + } +, "grpc_resolver_dns_plugin": { "type": ["@", "rules", "CC", "library"] + , "name": ["grpc_resolver_dns_plugin"] , "stage": ["src", "core"] + , "srcs": ["ext/filters/client_channel/resolver/dns/dns_resolver_plugin.cc"] + , "hdrs": ["ext/filters/client_channel/resolver/dns/dns_resolver_plugin.h"] + , "deps": + [ "experiments" + , "grpc_resolver_dns_event_engine" + , "grpc_resolver_dns_native" + , ["", "config"] + , ["", "config_vars"] + , ["", "gpr"] + , ["", "grpc_resolver"] + , ["", "grpc_resolver_dns_ares"] + , ["@", "absl", "absl/strings", "strings"] + ] + } +, "grpc_resolver_dns_native": + { "type": ["@", "rules", "CC", "library"] , "name": ["grpc_resolver_dns_native"] + , "stage": ["src", "core"] , "srcs": ["ext/filters/client_channel/resolver/dns/native/dns_resolver.cc"] + , "hdrs": ["ext/filters/client_channel/resolver/dns/native/dns_resolver.h"] , "deps": [ "channel_args" - , "grpc_resolver_dns_selection" , "polling_resolver" , "resolved_address" , "time" @@ -3746,8 +4294,8 @@ } , "grpc_resolver_sockaddr": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_resolver_sockaddr"] + , "stage": ["src", "core"] , "srcs": ["ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc"] , "deps": @@ -3767,8 +4315,8 @@ } , "grpc_resolver_binder": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_resolver_binder"] + , "stage": ["src", "core"] , "srcs": ["ext/filters/client_channel/resolver/binder/binder_resolver.cc"] , "deps": [ "channel_args" @@ -3789,28 +4337,36 @@ } , "grpc_resolver_xds_header": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_resolver_xds_header"] + , "stage": ["src", "core"] , "hdrs": ["ext/filters/client_channel/resolver/xds/xds_resolver.h"] - , "deps": ["unique_type_name", ["", "gpr_platform"]] + , "deps": + [ "grpc_service_config" + , "unique_type_name" + , ["", "gpr_platform"] + , ["@", "absl", "absl/strings", "strings"] + ] } , "grpc_resolver_xds": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_resolver_xds"] + , "stage": ["src", "core"] , "srcs": ["ext/filters/client_channel/resolver/xds/xds_resolver.cc"] - , "hdrs": [] , "deps": [ "arena" + , "arena_promise" , "channel_args" , "channel_fwd" + , "context" , "dual_ref_counted" , "grpc_lb_policy_ring_hash" + , "grpc_resolver_xds_header" , "grpc_service_config" , "grpc_xds_client" , "iomgr_fwd" , "match" , "pollset_set" + , "ref_counted" , "slice" , "time" , "unique_type_name" @@ -3823,6 +4379,7 @@ , ["", "grpc_resolver"] , ["", "grpc_service_config_impl"] , ["", "grpc_trace"] + , ["", "legacy_context"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "server_address"] @@ -3833,8 +4390,8 @@ , ["@", "absl", "absl/random", "random"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] , ["@", "absl", "absl/types", "variant"] , ["@", "re2", "", "re2"] @@ -3843,19 +4400,18 @@ } , "grpc_resolver_c2p": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_resolver_c2p"] + , "stage": ["src", "core"] , "srcs": ["ext/filters/client_channel/resolver/google_c2p/google_c2p_resolver.cc"] , "deps": [ "channel_args" - , "closure" , "env" - , "error" + , "gcp_metadata_query" , "grpc_xds_client" , "json" + , "json_writer" , "resource_quota" - , "status_helper" , "time" , ["", "alts_util"] , ["", "config"] @@ -3863,31 +4419,27 @@ , ["", "gpr"] , ["", "grpc_base"] , ["", "grpc_resolver"] - , ["", "grpc_security_base"] - , ["", "httpcli"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "uri_parser"] , ["", "work_serializer"] , ["", "xds_client"] - , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] , ["@", "absl", "absl/strings", "strings"] - , ["@", "absl", "absl/strings", "str_format"] , ["@", "absl", "absl/types", "optional"] ] } , "hpack_constants": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["hpack_constants"] + , "stage": ["src", "core"] , "hdrs": ["ext/transport/chttp2/transport/hpack_constants.h"] , "deps": [["", "gpr_platform"]] } , "hpack_encoder_table": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["hpack_encoder_table"] + , "stage": ["src", "core"] , "srcs": ["ext/transport/chttp2/transport/hpack_encoder_table.cc"] , "hdrs": ["ext/transport/chttp2/transport/hpack_encoder_table.h"] , "deps": @@ -3898,8 +4450,8 @@ } , "chttp2_flow_control": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["chttp2_flow_control"] + , "stage": ["src", "core"] , "srcs": ["ext/transport/chttp2/transport/flow_control.cc"] , "hdrs": ["ext/transport/chttp2/transport/flow_control.h"] , "deps": @@ -3914,47 +4466,47 @@ , ["", "grpc_trace"] , ["@", "absl", "absl/functional", "function_ref"] , ["@", "absl", "absl/status", "status"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] ] } , "huffsyms": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["huffsyms"] + , "stage": ["src", "core"] , "srcs": ["ext/transport/chttp2/transport/huffsyms.cc"] , "hdrs": ["ext/transport/chttp2/transport/huffsyms.h"] , "deps": [["", "gpr_platform"]] } , "decode_huff": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["decode_huff"] + , "stage": ["src", "core"] , "srcs": ["ext/transport/chttp2/transport/decode_huff.cc"] , "hdrs": ["ext/transport/chttp2/transport/decode_huff.h"] , "deps": [["", "gpr_platform"]] } , "http2_settings": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["http2_settings"] + , "stage": ["src", "core"] , "srcs": ["ext/transport/chttp2/transport/http2_settings.cc"] , "hdrs": ["ext/transport/chttp2/transport/http2_settings.h"] , "deps": ["http2_errors", "useful", ["", "gpr_platform"]] } , "grpc_transport_chttp2_alpn": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_transport_chttp2_alpn"] + , "stage": ["src", "core"] , "srcs": ["ext/transport/chttp2/alpn/alpn.cc"] , "hdrs": ["ext/transport/chttp2/alpn/alpn.h"] , "deps": ["useful", ["", "gpr"]] } , "grpc_transport_chttp2_client_connector": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_transport_chttp2_client_connector"] + , "stage": ["src", "core"] , "srcs": ["ext/transport/chttp2/client/chttp2_connector.cc"] , "hdrs": ["ext/transport/chttp2/client/chttp2_connector.h"] , "deps": @@ -3996,8 +4548,8 @@ } , "grpc_transport_chttp2_server": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_transport_chttp2_server"] + , "stage": ["src", "core"] , "srcs": ["ext/transport/chttp2/server/chttp2_server.cc"] , "hdrs": ["ext/transport/chttp2/server/chttp2_server.h"] , "deps": @@ -4026,7 +4578,6 @@ , ["", "grpc_trace"] , ["", "grpc_transport_chttp2"] , ["", "handshaker"] - , ["", "iomgr_timer"] , ["", "orphanable"] , ["", "ref_counted_ptr"] , ["", "sockaddr_utils"] @@ -4034,15 +4585,15 @@ , ["@", "absl", "absl/base", "core_headers"] , ["@", "absl", "absl/status", "status"] , ["@", "absl", "absl/status", "statusor"] - , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] , ["@", "absl", "absl/types", "optional"] ] } , "grpc_transport_inproc": { "type": ["@", "rules", "CC", "library"] - , "stage": ["src", "core"] , "name": ["grpc_transport_inproc"] + , "stage": ["src", "core"] , "srcs": [ "ext/transport/inproc/inproc_plugin.cc" , "ext/transport/inproc/inproc_transport.cc" @@ -4075,4 +4626,119 @@ , ["@", "absl", "absl/types", "optional"] ] } +, "chaotic_good_frame": + { "type": ["@", "rules", "CC", "library"] + , "name": ["chaotic_good_frame"] + , "stage": ["src", "core"] + , "srcs": ["ext/transport/chaotic_good/frame.cc"] + , "hdrs": ["ext/transport/chaotic_good/frame.h"] + , "deps": + [ "arena" + , "bitset" + , "chaotic_good_frame_header" + , "no_destruct" + , "slice" + , "slice_buffer" + , "status_helper" + , ["", "gpr"] + , ["", "gpr_platform"] + , ["", "grpc_base"] + , ["", "hpack_encoder"] + , ["", "hpack_parser"] + , ["@", "absl", "absl/status", "status"] + , ["@", "absl", "absl/status", "statusor"] + , ["@", "absl", "absl/types", "variant"] + ] + } +, "chaotic_good_frame_header": + { "type": ["@", "rules", "CC", "library"] + , "name": ["chaotic_good_frame_header"] + , "stage": ["src", "core"] + , "srcs": ["ext/transport/chaotic_good/frame_header.cc"] + , "hdrs": ["ext/transport/chaotic_good/frame_header.h"] + , "deps": + [ "bitset" + , ["", "gpr_platform"] + , ["@", "absl", "absl/status", "status"] + , ["@", "absl", "absl/status", "statusor"] + ] + } +, "gcp_metadata_query": + { "type": ["@", "rules", "CC", "library"] + , "name": ["gcp_metadata_query"] + , "stage": ["src", "core"] + , "srcs": ["ext/gcp/metadata_query.cc"] + , "hdrs": ["ext/gcp/metadata_query.h"] + , "deps": + [ "closure" + , "error" + , "status_helper" + , "time" + , ["", "gpr"] + , ["", "gpr_platform"] + , ["", "grpc_base"] + , ["", "grpc_security_base"] + , ["", "grpc_trace"] + , ["", "httpcli"] + , ["", "orphanable"] + , ["", "ref_counted_ptr"] + , ["", "uri_parser"] + , ["@", "absl", "absl/functional", "any_invocable"] + , ["@", "absl", "absl/status", "status"] + , ["@", "absl", "absl/status", "statusor"] + , ["@", "absl", "absl/strings", "str_format"] + , ["@", "absl", "absl/strings", "strings"] + ] + } +, "logging_sink": + { "type": ["@", "rules", "CC", "library"] + , "name": ["logging_sink"] + , "stage": ["src", "core"] + , "hdrs": ["ext/filters/logging/logging_sink.h"] + , "deps": + [ "time" + , ["", "gpr_platform"] + , ["@", "absl", "absl/numeric", "int128"] + , ["@", "absl", "absl/strings", "strings"] + ] + } +, "logging_filter": + { "type": ["@", "rules", "CC", "library"] + , "name": ["logging_filter"] + , "stage": ["src", "core"] + , "srcs": ["ext/filters/logging/logging_filter.cc"] + , "hdrs": ["ext/filters/logging/logging_filter.h"] + , "deps": + [ "arena" + , "arena_promise" + , "cancel_callback" + , "channel_args" + , "channel_fwd" + , "channel_stack_type" + , "context" + , "logging_sink" + , "map" + , "pipe" + , "poll" + , "slice" + , "slice_buffer" + , "time" + , ["", "channel_stack_builder"] + , ["", "config"] + , ["", "gpr"] + , ["", "gpr_platform"] + , ["", "grpc_base"] + , ["", "grpc_client_channel"] + , ["", "grpc_public_hdrs"] + , ["", "grpc_resolver"] + , ["", "legacy_context"] + , ["", "uri_parser"] + , ["@", "absl", "absl/numeric", "int128"] + , ["@", "absl", "absl/random", "distributions"] + , ["@", "absl", "absl/random", "random"] + , ["@", "absl", "absl/status", "statusor"] + , ["@", "absl", "absl/strings", "strings"] + , ["@", "absl", "absl/types", "optional"] + ] + } } diff --git a/etc/import/src/core/ext/upb-generated/TARGETS.grpc b/etc/import/src/core/ext/upb-generated/TARGETS.grpc index df9f81d6e..e7ed8b7b5 100644 --- a/etc/import/src/core/ext/upb-generated/TARGETS.grpc +++ b/etc/import/src/core/ext/upb-generated/TARGETS.grpc @@ -2,334 +2,334 @@ { "type": ["@", "rules", "CC", "library"] , "name": ["upb-generated-lib"] , "pure C": ["YES"] + , "srcs": + [ "google/protobuf/struct.upb.c" + , "google/protobuf/descriptor.upb.c" + , "google/protobuf/any.upb.c" + , "google/protobuf/empty.upb.c" + , "google/protobuf/duration.upb.c" + , "google/protobuf/timestamp.upb.c" + , "google/protobuf/wrappers.upb.c" + , "google/api/httpbody.upb.c" + , "google/api/annotations.upb.c" + , "google/api/expr/v1alpha1/syntax.upb.c" + , "google/api/expr/v1alpha1/checked.upb.c" + , "google/api/http.upb.c" + , "google/type/expr.upb.c" + , "google/rpc/status.upb.c" + , "validate/validate.upb.c" + , "envoy/admin/v3/config_dump.upb.c" + , "envoy/admin/v3/tap.upb.c" + , "envoy/admin/v3/certs.upb.c" + , "envoy/admin/v3/mutex_stats.upb.c" + , "envoy/admin/v3/metrics.upb.c" + , "envoy/admin/v3/memory.upb.c" + , "envoy/admin/v3/clusters.upb.c" + , "envoy/admin/v3/listeners.upb.c" + , "envoy/admin/v3/config_dump_shared.upb.c" + , "envoy/admin/v3/init_dump.upb.c" + , "envoy/admin/v3/server_info.upb.c" + , "envoy/config/tap/v3/common.upb.c" + , "envoy/config/metrics/v3/metrics_service.upb.c" + , "envoy/config/metrics/v3/stats.upb.c" + , "envoy/config/bootstrap/v3/bootstrap.upb.c" + , "envoy/config/accesslog/v3/accesslog.upb.c" + , "envoy/config/common/matcher/v3/matcher.upb.c" + , "envoy/config/cluster/v3/cluster.upb.c" + , "envoy/config/cluster/v3/circuit_breaker.upb.c" + , "envoy/config/cluster/v3/outlier_detection.upb.c" + , "envoy/config/cluster/v3/filter.upb.c" + , "envoy/config/overload/v3/overload.upb.c" + , "envoy/config/trace/v3/lightstep.upb.c" + , "envoy/config/trace/v3/service.upb.c" + , "envoy/config/trace/v3/trace.upb.c" + , "envoy/config/trace/v3/zipkin.upb.c" + , "envoy/config/trace/v3/xray.upb.c" + , "envoy/config/trace/v3/http_tracer.upb.c" + , "envoy/config/trace/v3/opencensus.upb.c" + , "envoy/config/trace/v3/skywalking.upb.c" + , "envoy/config/trace/v3/opentelemetry.upb.c" + , "envoy/config/trace/v3/datadog.upb.c" + , "envoy/config/trace/v3/dynamic_ot.upb.c" + , "envoy/config/listener/v3/api_listener.upb.c" + , "envoy/config/listener/v3/listener_components.upb.c" + , "envoy/config/listener/v3/listener.upb.c" + , "envoy/config/listener/v3/udp_listener_config.upb.c" + , "envoy/config/listener/v3/quic_config.upb.c" + , "envoy/config/endpoint/v3/endpoint.upb.c" + , "envoy/config/endpoint/v3/endpoint_components.upb.c" + , "envoy/config/endpoint/v3/load_report.upb.c" + , "envoy/config/rbac/v3/rbac.upb.c" + , "envoy/config/core/v3/udp_socket_config.upb.c" + , "envoy/config/core/v3/http_uri.upb.c" + , "envoy/config/core/v3/resolver.upb.c" + , "envoy/config/core/v3/base.upb.c" + , "envoy/config/core/v3/socket_option.upb.c" + , "envoy/config/core/v3/proxy_protocol.upb.c" + , "envoy/config/core/v3/substitution_format_string.upb.c" + , "envoy/config/core/v3/extension.upb.c" + , "envoy/config/core/v3/grpc_method_list.upb.c" + , "envoy/config/core/v3/grpc_service.upb.c" + , "envoy/config/core/v3/event_service_config.upb.c" + , "envoy/config/core/v3/backoff.upb.c" + , "envoy/config/core/v3/config_source.upb.c" + , "envoy/config/core/v3/address.upb.c" + , "envoy/config/core/v3/protocol.upb.c" + , "envoy/config/core/v3/health_check.upb.c" + , "envoy/config/route/v3/route_components.upb.c" + , "envoy/config/route/v3/route.upb.c" + , "envoy/config/route/v3/scoped_route.upb.c" + , "envoy/extensions/transport_sockets/tls/v3/cert.upb.c" + , "envoy/extensions/transport_sockets/tls/v3/common.upb.c" + , "envoy/extensions/transport_sockets/tls/v3/secret.upb.c" + , "envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upb.c" + , "envoy/extensions/transport_sockets/tls/v3/tls.upb.c" + , "envoy/extensions/http/stateful_session/cookie/v3/cookie.upb.c" + , "envoy/extensions/load_balancing_policies/common/v3/common.upb.c" + , "envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.upb.c" + , "envoy/extensions/load_balancing_policies/wrr_locality/v3/wrr_locality.upb.c" + , "envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.upb.c" + , "envoy/extensions/clusters/aggregate/v3/cluster.upb.c" + , "envoy/extensions/filters/http/stateful_session/v3/stateful_session.upb.c" + , "envoy/extensions/filters/http/fault/v3/fault.upb.c" + , "envoy/extensions/filters/http/rbac/v3/rbac.upb.c" + , "envoy/extensions/filters/http/router/v3/router.upb.c" + , "envoy/extensions/filters/common/fault/v3/fault.upb.c" + , "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb.c" + , "envoy/type/tracing/v3/custom_tag.upb.c" + , "envoy/type/http/v3/cookie.upb.c" + , "envoy/type/http/v3/path_transformation.upb.c" + , "envoy/type/v3/ratelimit_unit.upb.c" + , "envoy/type/v3/ratelimit_strategy.upb.c" + , "envoy/type/v3/hash_policy.upb.c" + , "envoy/type/v3/percent.upb.c" + , "envoy/type/v3/http.upb.c" + , "envoy/type/v3/semantic_version.upb.c" + , "envoy/type/v3/range.upb.c" + , "envoy/type/v3/http_status.upb.c" + , "envoy/type/v3/token_bucket.upb.c" + , "envoy/type/matcher/v3/struct.upb.c" + , "envoy/type/matcher/v3/metadata.upb.c" + , "envoy/type/matcher/v3/filter_state.upb.c" + , "envoy/type/matcher/v3/path.upb.c" + , "envoy/type/matcher/v3/value.upb.c" + , "envoy/type/matcher/v3/number.upb.c" + , "envoy/type/matcher/v3/status_code_input.upb.c" + , "envoy/type/matcher/v3/node.upb.c" + , "envoy/type/matcher/v3/regex.upb.c" + , "envoy/type/matcher/v3/string.upb.c" + , "envoy/type/matcher/v3/http_inputs.upb.c" + , "envoy/type/metadata/v3/metadata.upb.c" + , "envoy/service/load_stats/v3/lrs.upb.c" + , "envoy/service/status/v3/csds.upb.c" + , "envoy/service/discovery/v3/ads.upb.c" + , "envoy/service/discovery/v3/discovery.upb.c" + , "envoy/annotations/deprecation.upb.c" + , "envoy/annotations/resource.upb.c" + , "xds/data/orca/v3/orca_load_report.upb.c" + , "xds/type/v3/cel.upb.c" + , "xds/type/v3/typed_struct.upb.c" + , "xds/type/v3/range.upb.c" + , "xds/type/matcher/v3/domain.upb.c" + , "xds/type/matcher/v3/cel.upb.c" + , "xds/type/matcher/v3/range.upb.c" + , "xds/type/matcher/v3/ip.upb.c" + , "xds/type/matcher/v3/regex.upb.c" + , "xds/type/matcher/v3/string.upb.c" + , "xds/type/matcher/v3/matcher.upb.c" + , "xds/type/matcher/v3/http_inputs.upb.c" + , "xds/core/v3/collection_entry.upb.c" + , "xds/core/v3/cidr.upb.c" + , "xds/core/v3/authority.upb.c" + , "xds/core/v3/resource.upb.c" + , "xds/core/v3/extension.upb.c" + , "xds/core/v3/resource_locator.upb.c" + , "xds/core/v3/context_params.upb.c" + , "xds/core/v3/resource_name.upb.c" + , "xds/service/orca/v3/orca.upb.c" + , "xds/annotations/v3/sensitive.upb.c" + , "xds/annotations/v3/security.upb.c" + , "xds/annotations/v3/versioning.upb.c" + , "xds/annotations/v3/migrate.upb.c" + , "xds/annotations/v3/status.upb.c" + , "opencensus/proto/trace/v1/trace_config.upb.c" + , "udpa/annotations/sensitive.upb.c" + , "udpa/annotations/security.upb.c" + , "udpa/annotations/versioning.upb.c" + , "udpa/annotations/migrate.upb.c" + , "udpa/annotations/status.upb.c" + , "src/proto/grpc/lookup/v1/rls_config.upb.c" + , "src/proto/grpc/lookup/v1/rls.upb.c" + , "src/proto/grpc/gcp/transport_security_common.upb.c" + , "src/proto/grpc/gcp/altscontext.upb.c" + , "src/proto/grpc/gcp/handshaker.upb.c" + , "src/proto/grpc/lb/v1/load_balancer.upb.c" + , "src/proto/grpc/health/v1/health.upb.c" + ] , "hdrs": - [ "envoy/admin/v3/certs.upb.h" + [ "google/protobuf/any.upb.h" + , "google/protobuf/duration.upb.h" + , "google/protobuf/wrappers.upb.h" + , "google/protobuf/struct.upb.h" + , "google/protobuf/empty.upb.h" + , "google/protobuf/descriptor.upb.h" + , "google/protobuf/timestamp.upb.h" + , "google/api/http.upb.h" + , "google/api/annotations.upb.h" + , "google/api/expr/v1alpha1/syntax.upb.h" + , "google/api/expr/v1alpha1/checked.upb.h" + , "google/api/httpbody.upb.h" + , "google/type/expr.upb.h" + , "google/rpc/status.upb.h" + , "validate/validate.upb.h" + , "envoy/admin/v3/mutex_stats.upb.h" + , "envoy/admin/v3/tap.upb.h" + , "envoy/admin/v3/server_info.upb.h" , "envoy/admin/v3/clusters.upb.h" - , "envoy/admin/v3/config_dump.upb.h" - , "envoy/admin/v3/config_dump_shared.upb.h" + , "envoy/admin/v3/certs.upb.h" + , "envoy/admin/v3/metrics.upb.h" , "envoy/admin/v3/init_dump.upb.h" , "envoy/admin/v3/listeners.upb.h" + , "envoy/admin/v3/config_dump_shared.upb.h" , "envoy/admin/v3/memory.upb.h" - , "envoy/admin/v3/metrics.upb.h" - , "envoy/admin/v3/mutex_stats.upb.h" - , "envoy/admin/v3/server_info.upb.h" - , "envoy/admin/v3/tap.upb.h" - , "envoy/annotations/deprecation.upb.h" - , "envoy/annotations/resource.upb.h" - , "envoy/config/accesslog/v3/accesslog.upb.h" + , "envoy/admin/v3/config_dump.upb.h" + , "envoy/config/tap/v3/common.upb.h" + , "envoy/config/metrics/v3/stats.upb.h" + , "envoy/config/metrics/v3/metrics_service.upb.h" , "envoy/config/bootstrap/v3/bootstrap.upb.h" - , "envoy/config/cluster/v3/circuit_breaker.upb.h" + , "envoy/config/accesslog/v3/accesslog.upb.h" + , "envoy/config/common/matcher/v3/matcher.upb.h" , "envoy/config/cluster/v3/cluster.upb.h" , "envoy/config/cluster/v3/filter.upb.h" + , "envoy/config/cluster/v3/circuit_breaker.upb.h" , "envoy/config/cluster/v3/outlier_detection.upb.h" - , "envoy/config/common/matcher/v3/matcher.upb.h" - , "envoy/config/core/v3/address.upb.h" - , "envoy/config/core/v3/backoff.upb.h" + , "envoy/config/overload/v3/overload.upb.h" + , "envoy/config/trace/v3/http_tracer.upb.h" + , "envoy/config/trace/v3/dynamic_ot.upb.h" + , "envoy/config/trace/v3/skywalking.upb.h" + , "envoy/config/trace/v3/trace.upb.h" + , "envoy/config/trace/v3/xray.upb.h" + , "envoy/config/trace/v3/opencensus.upb.h" + , "envoy/config/trace/v3/service.upb.h" + , "envoy/config/trace/v3/opentelemetry.upb.h" + , "envoy/config/trace/v3/zipkin.upb.h" + , "envoy/config/trace/v3/lightstep.upb.h" + , "envoy/config/trace/v3/datadog.upb.h" + , "envoy/config/listener/v3/api_listener.upb.h" + , "envoy/config/listener/v3/udp_listener_config.upb.h" + , "envoy/config/listener/v3/listener.upb.h" + , "envoy/config/listener/v3/listener_components.upb.h" + , "envoy/config/listener/v3/quic_config.upb.h" + , "envoy/config/endpoint/v3/load_report.upb.h" + , "envoy/config/endpoint/v3/endpoint_components.upb.h" + , "envoy/config/endpoint/v3/endpoint.upb.h" + , "envoy/config/rbac/v3/rbac.upb.h" + , "envoy/config/core/v3/extension.upb.h" , "envoy/config/core/v3/base.upb.h" , "envoy/config/core/v3/config_source.upb.h" - , "envoy/config/core/v3/event_service_config.upb.h" - , "envoy/config/core/v3/extension.upb.h" , "envoy/config/core/v3/grpc_method_list.upb.h" - , "envoy/config/core/v3/grpc_service.upb.h" - , "envoy/config/core/v3/health_check.upb.h" - , "envoy/config/core/v3/http_uri.upb.h" + , "envoy/config/core/v3/address.upb.h" + , "envoy/config/core/v3/resolver.upb.h" , "envoy/config/core/v3/protocol.upb.h" + , "envoy/config/core/v3/udp_socket_config.upb.h" + , "envoy/config/core/v3/event_service_config.upb.h" + , "envoy/config/core/v3/health_check.upb.h" , "envoy/config/core/v3/proxy_protocol.upb.h" - , "envoy/config/core/v3/resolver.upb.h" - , "envoy/config/core/v3/socket_option.upb.h" + , "envoy/config/core/v3/http_uri.upb.h" + , "envoy/config/core/v3/grpc_service.upb.h" + , "envoy/config/core/v3/backoff.upb.h" , "envoy/config/core/v3/substitution_format_string.upb.h" - , "envoy/config/core/v3/udp_socket_config.upb.h" - , "envoy/config/endpoint/v3/endpoint.upb.h" - , "envoy/config/endpoint/v3/endpoint_components.upb.h" - , "envoy/config/endpoint/v3/load_report.upb.h" - , "envoy/config/listener/v3/api_listener.upb.h" - , "envoy/config/listener/v3/listener.upb.h" - , "envoy/config/listener/v3/listener_components.upb.h" - , "envoy/config/listener/v3/quic_config.upb.h" - , "envoy/config/listener/v3/udp_listener_config.upb.h" - , "envoy/config/metrics/v3/metrics_service.upb.h" - , "envoy/config/metrics/v3/stats.upb.h" - , "envoy/config/overload/v3/overload.upb.h" - , "envoy/config/rbac/v3/rbac.upb.h" + , "envoy/config/core/v3/socket_option.upb.h" + , "envoy/config/route/v3/scoped_route.upb.h" , "envoy/config/route/v3/route.upb.h" , "envoy/config/route/v3/route_components.upb.h" - , "envoy/config/route/v3/scoped_route.upb.h" - , "envoy/config/tap/v3/common.upb.h" - , "envoy/config/trace/v3/datadog.upb.h" - , "envoy/config/trace/v3/dynamic_ot.upb.h" - , "envoy/config/trace/v3/http_tracer.upb.h" - , "envoy/config/trace/v3/lightstep.upb.h" - , "envoy/config/trace/v3/opencensus.upb.h" - , "envoy/config/trace/v3/opentelemetry.upb.h" - , "envoy/config/trace/v3/service.upb.h" - , "envoy/config/trace/v3/skywalking.upb.h" - , "envoy/config/trace/v3/trace.upb.h" - , "envoy/config/trace/v3/xray.upb.h" - , "envoy/config/trace/v3/zipkin.upb.h" + , "envoy/extensions/transport_sockets/tls/v3/tls.upb.h" + , "envoy/extensions/transport_sockets/tls/v3/common.upb.h" + , "envoy/extensions/transport_sockets/tls/v3/cert.upb.h" + , "envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upb.h" + , "envoy/extensions/transport_sockets/tls/v3/secret.upb.h" + , "envoy/extensions/http/stateful_session/cookie/v3/cookie.upb.h" + , "envoy/extensions/load_balancing_policies/common/v3/common.upb.h" + , "envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.upb.h" + , "envoy/extensions/load_balancing_policies/wrr_locality/v3/wrr_locality.upb.h" + , "envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.upb.h" , "envoy/extensions/clusters/aggregate/v3/cluster.upb.h" - , "envoy/extensions/filters/common/fault/v3/fault.upb.h" + , "envoy/extensions/filters/http/stateful_session/v3/stateful_session.upb.h" , "envoy/extensions/filters/http/fault/v3/fault.upb.h" , "envoy/extensions/filters/http/rbac/v3/rbac.upb.h" , "envoy/extensions/filters/http/router/v3/router.upb.h" - , "envoy/extensions/filters/http/stateful_session/v3/stateful_session.upb.h" + , "envoy/extensions/filters/common/fault/v3/fault.upb.h" , "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb.h" - , "envoy/extensions/http/stateful_session/cookie/v3/cookie.upb.h" - , "envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.upb.h" - , "envoy/extensions/load_balancing_policies/common/v3/common.upb.h" - , "envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.upb.h" - , "envoy/extensions/load_balancing_policies/wrr_locality/v3/wrr_locality.upb.h" - , "envoy/extensions/transport_sockets/tls/v3/cert.upb.h" - , "envoy/extensions/transport_sockets/tls/v3/common.upb.h" - , "envoy/extensions/transport_sockets/tls/v3/secret.upb.h" - , "envoy/extensions/transport_sockets/tls/v3/tls.upb.h" - , "envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upb.h" - , "envoy/service/discovery/v3/ads.upb.h" - , "envoy/service/discovery/v3/discovery.upb.h" - , "envoy/service/load_stats/v3/lrs.upb.h" - , "envoy/service/status/v3/csds.upb.h" + , "envoy/type/tracing/v3/custom_tag.upb.h" , "envoy/type/http/v3/cookie.upb.h" , "envoy/type/http/v3/path_transformation.upb.h" - , "envoy/type/matcher/v3/filter_state.upb.h" + , "envoy/type/v3/semantic_version.upb.h" + , "envoy/type/v3/http_status.upb.h" + , "envoy/type/v3/http.upb.h" + , "envoy/type/v3/range.upb.h" + , "envoy/type/v3/ratelimit_unit.upb.h" + , "envoy/type/v3/ratelimit_strategy.upb.h" + , "envoy/type/v3/hash_policy.upb.h" + , "envoy/type/v3/percent.upb.h" + , "envoy/type/v3/token_bucket.upb.h" + , "envoy/type/matcher/v3/regex.upb.h" + , "envoy/type/matcher/v3/path.upb.h" + , "envoy/type/matcher/v3/node.upb.h" , "envoy/type/matcher/v3/http_inputs.upb.h" + , "envoy/type/matcher/v3/value.upb.h" , "envoy/type/matcher/v3/metadata.upb.h" - , "envoy/type/matcher/v3/node.upb.h" - , "envoy/type/matcher/v3/number.upb.h" - , "envoy/type/matcher/v3/path.upb.h" - , "envoy/type/matcher/v3/regex.upb.h" + , "envoy/type/matcher/v3/struct.upb.h" , "envoy/type/matcher/v3/status_code_input.upb.h" + , "envoy/type/matcher/v3/number.upb.h" , "envoy/type/matcher/v3/string.upb.h" - , "envoy/type/matcher/v3/struct.upb.h" - , "envoy/type/matcher/v3/value.upb.h" + , "envoy/type/matcher/v3/filter_state.upb.h" , "envoy/type/metadata/v3/metadata.upb.h" - , "envoy/type/tracing/v3/custom_tag.upb.h" - , "envoy/type/v3/hash_policy.upb.h" - , "envoy/type/v3/http.upb.h" - , "envoy/type/v3/http_status.upb.h" - , "envoy/type/v3/percent.upb.h" - , "envoy/type/v3/range.upb.h" - , "envoy/type/v3/ratelimit_strategy.upb.h" - , "envoy/type/v3/ratelimit_unit.upb.h" - , "envoy/type/v3/semantic_version.upb.h" - , "envoy/type/v3/token_bucket.upb.h" - , "google/api/annotations.upb.h" - , "google/api/expr/v1alpha1/checked.upb.h" - , "google/api/expr/v1alpha1/syntax.upb.h" - , "google/api/http.upb.h" - , "google/api/httpbody.upb.h" - , "google/protobuf/any.upb.h" - , "google/protobuf/descriptor.upb.h" - , "google/protobuf/duration.upb.h" - , "google/protobuf/empty.upb.h" - , "google/protobuf/struct.upb.h" - , "google/protobuf/timestamp.upb.h" - , "google/protobuf/wrappers.upb.h" - , "google/rpc/status.upb.h" - , "google/type/expr.upb.h" + , "envoy/service/load_stats/v3/lrs.upb.h" + , "envoy/service/status/v3/csds.upb.h" + , "envoy/service/discovery/v3/discovery.upb.h" + , "envoy/service/discovery/v3/ads.upb.h" + , "envoy/annotations/deprecation.upb.h" + , "envoy/annotations/resource.upb.h" + , "xds/data/orca/v3/orca_load_report.upb.h" + , "xds/type/v3/range.upb.h" + , "xds/type/v3/typed_struct.upb.h" + , "xds/type/v3/cel.upb.h" + , "xds/type/matcher/v3/matcher.upb.h" + , "xds/type/matcher/v3/regex.upb.h" + , "xds/type/matcher/v3/ip.upb.h" + , "xds/type/matcher/v3/range.upb.h" + , "xds/type/matcher/v3/http_inputs.upb.h" + , "xds/type/matcher/v3/string.upb.h" + , "xds/type/matcher/v3/cel.upb.h" + , "xds/type/matcher/v3/domain.upb.h" + , "xds/core/v3/resource_locator.upb.h" + , "xds/core/v3/extension.upb.h" + , "xds/core/v3/resource_name.upb.h" + , "xds/core/v3/cidr.upb.h" + , "xds/core/v3/authority.upb.h" + , "xds/core/v3/context_params.upb.h" + , "xds/core/v3/resource.upb.h" + , "xds/core/v3/collection_entry.upb.h" + , "xds/service/orca/v3/orca.upb.h" + , "xds/annotations/v3/security.upb.h" + , "xds/annotations/v3/status.upb.h" + , "xds/annotations/v3/migrate.upb.h" + , "xds/annotations/v3/versioning.upb.h" + , "xds/annotations/v3/sensitive.upb.h" , "opencensus/proto/trace/v1/trace_config.upb.h" + , "udpa/annotations/security.upb.h" + , "udpa/annotations/status.upb.h" + , "udpa/annotations/migrate.upb.h" + , "udpa/annotations/versioning.upb.h" + , "udpa/annotations/sensitive.upb.h" + , "src/proto/grpc/lookup/v1/rls_config.upb.h" + , "src/proto/grpc/lookup/v1/rls.upb.h" , "src/proto/grpc/gcp/altscontext.upb.h" , "src/proto/grpc/gcp/handshaker.upb.h" , "src/proto/grpc/gcp/transport_security_common.upb.h" - , "src/proto/grpc/health/v1/health.upb.h" , "src/proto/grpc/lb/v1/load_balancer.upb.h" - , "src/proto/grpc/lookup/v1/rls.upb.h" - , "src/proto/grpc/lookup/v1/rls_config.upb.h" - , "udpa/annotations/migrate.upb.h" - , "udpa/annotations/security.upb.h" - , "udpa/annotations/sensitive.upb.h" - , "udpa/annotations/status.upb.h" - , "udpa/annotations/versioning.upb.h" - , "validate/validate.upb.h" - , "xds/annotations/v3/migrate.upb.h" - , "xds/annotations/v3/security.upb.h" - , "xds/annotations/v3/sensitive.upb.h" - , "xds/annotations/v3/status.upb.h" - , "xds/annotations/v3/versioning.upb.h" - , "xds/core/v3/authority.upb.h" - , "xds/core/v3/cidr.upb.h" - , "xds/core/v3/collection_entry.upb.h" - , "xds/core/v3/context_params.upb.h" - , "xds/core/v3/extension.upb.h" - , "xds/core/v3/resource.upb.h" - , "xds/core/v3/resource_locator.upb.h" - , "xds/core/v3/resource_name.upb.h" - , "xds/data/orca/v3/orca_load_report.upb.h" - , "xds/service/orca/v3/orca.upb.h" - , "xds/type/matcher/v3/cel.upb.h" - , "xds/type/matcher/v3/domain.upb.h" - , "xds/type/matcher/v3/http_inputs.upb.h" - , "xds/type/matcher/v3/ip.upb.h" - , "xds/type/matcher/v3/matcher.upb.h" - , "xds/type/matcher/v3/range.upb.h" - , "xds/type/matcher/v3/regex.upb.h" - , "xds/type/matcher/v3/string.upb.h" - , "xds/type/v3/cel.upb.h" - , "xds/type/v3/range.upb.h" - , "xds/type/v3/typed_struct.upb.h" - ] - , "srcs": - [ "envoy/admin/v3/certs.upb.c" - , "envoy/admin/v3/clusters.upb.c" - , "envoy/admin/v3/config_dump.upb.c" - , "envoy/admin/v3/config_dump_shared.upb.c" - , "envoy/admin/v3/init_dump.upb.c" - , "envoy/admin/v3/listeners.upb.c" - , "envoy/admin/v3/memory.upb.c" - , "envoy/admin/v3/metrics.upb.c" - , "envoy/admin/v3/mutex_stats.upb.c" - , "envoy/admin/v3/server_info.upb.c" - , "envoy/admin/v3/tap.upb.c" - , "envoy/annotations/deprecation.upb.c" - , "envoy/annotations/resource.upb.c" - , "envoy/config/accesslog/v3/accesslog.upb.c" - , "envoy/config/bootstrap/v3/bootstrap.upb.c" - , "envoy/config/cluster/v3/circuit_breaker.upb.c" - , "envoy/config/cluster/v3/cluster.upb.c" - , "envoy/config/cluster/v3/filter.upb.c" - , "envoy/config/cluster/v3/outlier_detection.upb.c" - , "envoy/config/common/matcher/v3/matcher.upb.c" - , "envoy/config/core/v3/address.upb.c" - , "envoy/config/core/v3/backoff.upb.c" - , "envoy/config/core/v3/base.upb.c" - , "envoy/config/core/v3/config_source.upb.c" - , "envoy/config/core/v3/event_service_config.upb.c" - , "envoy/config/core/v3/extension.upb.c" - , "envoy/config/core/v3/grpc_method_list.upb.c" - , "envoy/config/core/v3/grpc_service.upb.c" - , "envoy/config/core/v3/health_check.upb.c" - , "envoy/config/core/v3/http_uri.upb.c" - , "envoy/config/core/v3/protocol.upb.c" - , "envoy/config/core/v3/proxy_protocol.upb.c" - , "envoy/config/core/v3/resolver.upb.c" - , "envoy/config/core/v3/socket_option.upb.c" - , "envoy/config/core/v3/substitution_format_string.upb.c" - , "envoy/config/core/v3/udp_socket_config.upb.c" - , "envoy/config/endpoint/v3/endpoint.upb.c" - , "envoy/config/endpoint/v3/endpoint_components.upb.c" - , "envoy/config/endpoint/v3/load_report.upb.c" - , "envoy/config/listener/v3/api_listener.upb.c" - , "envoy/config/listener/v3/listener.upb.c" - , "envoy/config/listener/v3/listener_components.upb.c" - , "envoy/config/listener/v3/quic_config.upb.c" - , "envoy/config/listener/v3/udp_listener_config.upb.c" - , "envoy/config/metrics/v3/metrics_service.upb.c" - , "envoy/config/metrics/v3/stats.upb.c" - , "envoy/config/overload/v3/overload.upb.c" - , "envoy/config/rbac/v3/rbac.upb.c" - , "envoy/config/route/v3/route.upb.c" - , "envoy/config/route/v3/route_components.upb.c" - , "envoy/config/route/v3/scoped_route.upb.c" - , "envoy/config/tap/v3/common.upb.c" - , "envoy/config/trace/v3/datadog.upb.c" - , "envoy/config/trace/v3/dynamic_ot.upb.c" - , "envoy/config/trace/v3/http_tracer.upb.c" - , "envoy/config/trace/v3/lightstep.upb.c" - , "envoy/config/trace/v3/opencensus.upb.c" - , "envoy/config/trace/v3/opentelemetry.upb.c" - , "envoy/config/trace/v3/service.upb.c" - , "envoy/config/trace/v3/skywalking.upb.c" - , "envoy/config/trace/v3/trace.upb.c" - , "envoy/config/trace/v3/xray.upb.c" - , "envoy/config/trace/v3/zipkin.upb.c" - , "envoy/extensions/clusters/aggregate/v3/cluster.upb.c" - , "envoy/extensions/filters/common/fault/v3/fault.upb.c" - , "envoy/extensions/filters/http/fault/v3/fault.upb.c" - , "envoy/extensions/filters/http/rbac/v3/rbac.upb.c" - , "envoy/extensions/filters/http/router/v3/router.upb.c" - , "envoy/extensions/filters/http/stateful_session/v3/stateful_session.upb.c" - , "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb.c" - , "envoy/extensions/http/stateful_session/cookie/v3/cookie.upb.c" - , "envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.upb.c" - , "envoy/extensions/load_balancing_policies/common/v3/common.upb.c" - , "envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.upb.c" - , "envoy/extensions/load_balancing_policies/wrr_locality/v3/wrr_locality.upb.c" - , "envoy/extensions/transport_sockets/tls/v3/cert.upb.c" - , "envoy/extensions/transport_sockets/tls/v3/common.upb.c" - , "envoy/extensions/transport_sockets/tls/v3/secret.upb.c" - , "envoy/extensions/transport_sockets/tls/v3/tls.upb.c" - , "envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upb.c" - , "envoy/service/discovery/v3/ads.upb.c" - , "envoy/service/discovery/v3/discovery.upb.c" - , "envoy/service/load_stats/v3/lrs.upb.c" - , "envoy/service/status/v3/csds.upb.c" - , "envoy/type/http/v3/cookie.upb.c" - , "envoy/type/http/v3/path_transformation.upb.c" - , "envoy/type/matcher/v3/filter_state.upb.c" - , "envoy/type/matcher/v3/http_inputs.upb.c" - , "envoy/type/matcher/v3/metadata.upb.c" - , "envoy/type/matcher/v3/node.upb.c" - , "envoy/type/matcher/v3/number.upb.c" - , "envoy/type/matcher/v3/path.upb.c" - , "envoy/type/matcher/v3/regex.upb.c" - , "envoy/type/matcher/v3/status_code_input.upb.c" - , "envoy/type/matcher/v3/string.upb.c" - , "envoy/type/matcher/v3/struct.upb.c" - , "envoy/type/matcher/v3/value.upb.c" - , "envoy/type/metadata/v3/metadata.upb.c" - , "envoy/type/tracing/v3/custom_tag.upb.c" - , "envoy/type/v3/hash_policy.upb.c" - , "envoy/type/v3/http.upb.c" - , "envoy/type/v3/http_status.upb.c" - , "envoy/type/v3/percent.upb.c" - , "envoy/type/v3/range.upb.c" - , "envoy/type/v3/ratelimit_strategy.upb.c" - , "envoy/type/v3/ratelimit_unit.upb.c" - , "envoy/type/v3/semantic_version.upb.c" - , "envoy/type/v3/token_bucket.upb.c" - , "google/api/annotations.upb.c" - , "google/api/expr/v1alpha1/checked.upb.c" - , "google/api/expr/v1alpha1/syntax.upb.c" - , "google/api/http.upb.c" - , "google/api/httpbody.upb.c" - , "google/protobuf/any.upb.c" - , "google/protobuf/descriptor.upb.c" - , "google/protobuf/duration.upb.c" - , "google/protobuf/empty.upb.c" - , "google/protobuf/struct.upb.c" - , "google/protobuf/timestamp.upb.c" - , "google/protobuf/wrappers.upb.c" - , "google/rpc/status.upb.c" - , "google/type/expr.upb.c" - , "opencensus/proto/trace/v1/trace_config.upb.c" - , "src/proto/grpc/gcp/altscontext.upb.c" - , "src/proto/grpc/gcp/handshaker.upb.c" - , "src/proto/grpc/gcp/transport_security_common.upb.c" - , "src/proto/grpc/health/v1/health.upb.c" - , "src/proto/grpc/lb/v1/load_balancer.upb.c" - , "src/proto/grpc/lookup/v1/rls.upb.c" - , "src/proto/grpc/lookup/v1/rls_config.upb.c" - , "udpa/annotations/migrate.upb.c" - , "udpa/annotations/security.upb.c" - , "udpa/annotations/sensitive.upb.c" - , "udpa/annotations/status.upb.c" - , "udpa/annotations/versioning.upb.c" - , "validate/validate.upb.c" - , "xds/annotations/v3/migrate.upb.c" - , "xds/annotations/v3/security.upb.c" - , "xds/annotations/v3/sensitive.upb.c" - , "xds/annotations/v3/status.upb.c" - , "xds/annotations/v3/versioning.upb.c" - , "xds/core/v3/authority.upb.c" - , "xds/core/v3/cidr.upb.c" - , "xds/core/v3/collection_entry.upb.c" - , "xds/core/v3/context_params.upb.c" - , "xds/core/v3/extension.upb.c" - , "xds/core/v3/resource.upb.c" - , "xds/core/v3/resource_locator.upb.c" - , "xds/core/v3/resource_name.upb.c" - , "xds/data/orca/v3/orca_load_report.upb.c" - , "xds/service/orca/v3/orca.upb.c" - , "xds/type/matcher/v3/cel.upb.c" - , "xds/type/matcher/v3/domain.upb.c" - , "xds/type/matcher/v3/http_inputs.upb.c" - , "xds/type/matcher/v3/ip.upb.c" - , "xds/type/matcher/v3/matcher.upb.c" - , "xds/type/matcher/v3/range.upb.c" - , "xds/type/matcher/v3/regex.upb.c" - , "xds/type/matcher/v3/string.upb.c" - , "xds/type/v3/cel.upb.c" - , "xds/type/v3/range.upb.c" - , "xds/type/v3/typed_struct.upb.c" + , "src/proto/grpc/health/v1/health.upb.h" ] - , "deps": [["third_party/upb", "upb"]] + , "deps": [["third_party/upb", "message_accessors"]] } } diff --git a/etc/import/src/core/ext/upbdefs-generated/TARGETS.grpc b/etc/import/src/core/ext/upbdefs-generated/TARGETS.grpc index b561236af..040ef56cd 100644 --- a/etc/import/src/core/ext/upbdefs-generated/TARGETS.grpc +++ b/etc/import/src/core/ext/upbdefs-generated/TARGETS.grpc @@ -306,7 +306,8 @@ ] , "deps": [ ["src/core/ext/upb-generated", "upb-generated-lib"] - , ["third_party/upb", "upb_reflection"] + , ["third_party/upb", "reflection"] + , ["third_party/upb", "reflection_internal"] ] } } diff --git a/etc/import/third_party/address_sorting/TARGETS.grpc b/etc/import/third_party/address_sorting/TARGETS.grpc new file mode 100644 index 000000000..94af3d454 --- /dev/null +++ b/etc/import/third_party/address_sorting/TARGETS.grpc @@ -0,0 +1,15 @@ +{ "address_sorting": + { "type": ["@", "rules", "CC", "library"] + , "name": ["address_sorting"] + , "srcs": + [ "address_sorting.c" + , "address_sorting_posix.c" + , "address_sorting_windows.c" + ] + , "hdrs": + [ "address_sorting_internal.h" + , ["./", "include/address_sorting", "address_sorting_headers"] + ] + , "pure C": ["YES"] + } +} diff --git a/etc/import/third_party/upb/TARGETS.grpc b/etc/import/third_party/upb/TARGETS.grpc index a73566219..a34c9c5ce 100644 --- a/etc/import/third_party/upb/TARGETS.grpc +++ b/etc/import/third_party/upb/TARGETS.grpc @@ -1,201 +1,538 @@ -{ "upb_port": +{ "port": { "type": ["@", "rules", "CC", "library"] - , "name": ["upb_port"] - , "pure C": ["YES"] + , "name": ["port"] , "hdrs": - [ "upb/internal/vsnprintf_compat.h" - , "upb/port_def.inc" - , "upb/port_undef.inc" + [ "upb/port/atomic.h" + , "upb/port/vsnprintf_compat.h" + , "upb/port/def.inc" + , "upb/port/undef.inc" ] + , "pure C": ["YES"] } , "upb": { "type": ["@", "rules", "CC", "library"] , "name": ["upb"] - , "pure C": ["YES"] - , "srcs": - [ "upb/arena.c" - , "upb/decode.c" - , "upb/encode.c" - , "upb/msg.c" - , "upb/status.c" - , "upb/table.c" - , "upb/upb.c" - ] , "hdrs": - [ "upb/arena.h" + [ "upb/alloc.h" + , "upb/arena.h" + , "upb/array.h" + , "upb/base/descriptor_constants.h" + , "upb/base/status.h" + , "upb/base/string_view.h" + , "upb/collections/array.h" , "upb/decode.h" , "upb/encode.h" , "upb/extension_registry.h" + , "upb/map.h" + , "upb/mem/alloc.h" + , "upb/mem/arena.h" + , "upb/message/extension_internal.h" + , "upb/message/message.h" + , "upb/mini_table/extension_registry.h" , "upb/msg.h" , "upb/status.h" + , "upb/string_view.h" , "upb/upb.h" , "upb/upb.hpp" - , "upb/msg_internal.h" + , "upb/wire/common.h" + , "upb/wire/decode.h" + , "upb/wire/encode.h" ] - , "private-hdrs": - ["upb/internal/decode.h", "upb/internal/table.h", "upb/internal/upb.h"] , "deps": - ["upb_extension_registry", "upb_fastdecode", "upb_port", "upb_utf8_range"] + [ "base" + , "collections_internal" + , "fastdecode" + , "hash" + , "lex" + , "mem" + , "message_internal" + , "mini_table_internal" + , "port" + , "wire" + ] + , "pure C": ["YES"] + } +, "base": + { "type": ["@", "rules", "CC", "library"] + , "name": ["base"] + , "srcs": ["upb/base/status.c"] + , "hdrs": + [ "upb/base/descriptor_constants.h" + , "upb/base/log2.h" + , "upb/base/status.h" + , "upb/base/string_view.h" + ] + , "deps": ["port"] + , "pure C": ["YES"] + } +, "mini_table": + { "type": ["@", "rules", "CC", "library"] + , "name": ["mini_table"] + , "hdrs": + [ "upb/mini_table.h" + , "upb/mini_table/decode.h" + , "upb/mini_table/extension_registry.h" + , "upb/mini_table/types.h" + ] + , "deps": ["base", "mem", "mini_table_internal", "port"] + , "pure C": ["YES"] + } +, "mini_table_internal": + { "type": ["@", "rules", "CC", "library"] + , "name": ["mini_table_internal"] + , "srcs": + [ "upb/mini_table/common.c" + , "upb/mini_table/decode.c" + , "upb/mini_table/encode.c" + , "upb/mini_table/extension_registry.c" + ] + , "hdrs": + [ "upb/mini_table/common.h" + , "upb/mini_table/common_internal.h" + , "upb/mini_table/decode.h" + , "upb/mini_table/encode_internal.h" + , "upb/mini_table/encode_internal.hpp" + , "upb/mini_table/enum_internal.h" + , "upb/mini_table/extension_internal.h" + , "upb/mini_table/extension_registry.h" + , "upb/mini_table/field_internal.h" + , "upb/mini_table/file_internal.h" + , "upb/mini_table/message_internal.h" + , "upb/mini_table/sub_internal.h" + , "upb/mini_table/types.h" + ] + , "deps": ["base", "hash", "mem", "port"] + , "pure C": ["YES"] + } +, "message": + { "type": ["@", "rules", "CC", "library"] + , "name": ["message"] + , "hdrs": ["upb/message/message.h"] + , "deps": ["mem", "message_internal", "mini_table", "port"] + , "pure C": ["YES"] + } +, "message_internal": + { "type": ["@", "rules", "CC", "library"] + , "name": ["message_internal"] + , "srcs": ["upb/message/message.c"] + , "hdrs": + [ "upb/message/extension_internal.h" + , "upb/message/internal.h" + , "upb/message/message.h" + ] + , "deps": ["base", "hash", "mem", "mini_table_internal", "port"] + , "pure C": ["YES"] } -, "upb_extension_registry": +, "message_accessors": { "type": ["@", "rules", "CC", "library"] - , "name": ["upb_extension_registry"] + , "name": ["message_accessors"] + , "srcs": ["upb/message/accessors.c"] + , "hdrs": ["upb/message/accessors.h", "upb/message/accessors_internal.h"] + , "deps": + [ "collections_internal" + , "eps_copy_input_stream" + , "hash" + , "message_internal" + , "mini_table_internal" + , "port" + , "upb" + , "wire" + , "wire_reader" + ] , "pure C": ["YES"] - , "hdrs": ["upb/extension_registry.h"] - , "srcs": ["upb/extension_registry.c"] - , "private-hdrs": ["upb/msg.h", "upb/msg_internal.h", "upb/upb.h"] - , "deps": ["upb_port", "upb_table"] } -, "upb_mini_table_internal": +, "message_promote": { "type": ["@", "rules", "CC", "library"] - , "name": ["upb_mini_table_internal"] + , "name": ["message_promote"] + , "srcs": ["upb/message/promote.c"] + , "hdrs": ["upb/message/promote.h"] + , "deps": + [ "collections_internal" + , "eps_copy_input_stream" + , "hash" + , "message_accessors" + , "message_internal" + , "mini_table_internal" + , "port" + , "upb" + , "wire" + , "wire_reader" + ] , "pure C": ["YES"] - , "hdrs": ["upb/msg_internal.h"] - , "deps": ["upb_extension_registry", "upb_port", "upb_table", "upb"] } -, "upb_mini_table": +, "message_copy": { "type": ["@", "rules", "CC", "library"] - , "name": ["upb_mini_table"] + , "name": ["message_copy"] + , "srcs": ["upb/message/copy.c"] + , "hdrs": ["upb/message/copy.h"] + , "deps": + [ "collections_internal" + , "message_accessors" + , "message_internal" + , "mini_table_internal" + , "port" + , "upb" + ] , "pure C": ["YES"] - , "srcs": ["upb/mini_table.c"] - , "hdrs": ["upb/mini_table.h", "upb/mini_table.hpp"] + } +, "fastdecode": + { "type": ["@", "rules", "CC", "library"] + , "name": ["fastdecode"] , "deps": - ["upb_extension_registry", "upb_mini_table_internal", "upb_port", "upb"] + [ "base" + , "collections_internal" + , "hash" + , "mem_internal" + , "message_internal" + , "mini_table_internal" + , "port" + , "wire" + ] + , "pure C": ["YES"] } -, "upb_fastdecode": +, "generated_code_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me": { "type": ["@", "rules", "CC", "library"] - , "name": ["upb_fastdecode"] + , "name": + [ "generated_code_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me" + ] + , "hdrs": + [ "upb/collections/array.h" + , "upb/collections/array_internal.h" + , "upb/collections/map_gencode_util.h" + , "upb/collections/message_value.h" + , "upb/extension_registry.h" + , "upb/message/accessors.h" + , "upb/message/accessors_internal.h" + , "upb/message/extension_internal.h" + , "upb/message/internal.h" + , "upb/message/message.h" + , "upb/mini_table/common.h" + , "upb/mini_table/enum_internal.h" + , "upb/mini_table/extension_internal.h" + , "upb/mini_table/field_internal.h" + , "upb/mini_table/file_internal.h" + , "upb/mini_table/message_internal.h" + , "upb/mini_table/sub_internal.h" + , "upb/mini_table/types.h" + , "upb/port/def.inc" + , "upb/port/undef.inc" + , "upb/wire/decode.h" + , "upb/wire/decode_fast.h" + , "upb/wire/encode.h" + ] + , "deps": ["base", "collections_internal", "hash", "upb"] , "pure C": ["YES"] - , "srcs": ["upb/decode_fast.c"] + } +, "generated_cpp_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me": + { "type": ["@", "rules", "CC", "library"] + , "name": + [ "generated_cpp_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me" + ] , "hdrs": - [ "upb/decode.h" - , "upb/decode_fast.h" - , "upb/internal/decode.h" - , "upb/internal/upb.h" - , "upb/msg.h" - , "upb/msg_internal.h" + [ "upb/message/copy.h" + , "upb/message/extension_internal.h" + , "upb/message/internal.h" + , "upb/message/message.h" + , "upb/mini_table/common.h" + , "upb/mini_table/enum_internal.h" + , "upb/mini_table/extension_internal.h" + , "upb/mini_table/field_internal.h" + , "upb/mini_table/file_internal.h" + , "upb/mini_table/message_internal.h" + , "upb/mini_table/sub_internal.h" + , "upb/mini_table/types.h" + , "upb/port/def.inc" + , "upb/port/undef.inc" + , "upb/upb.hpp" + , "upb/wire/decode.h" + , "upb/wire/decode_fast.h" + , "upb/wire/encode.h" + ] + , "deps": + [ "base" + , "collections_internal" + , "hash" + , "message_copy" + , "mini_table" + , "upb" + ] + , "pure C": ["YES"] + } +, "generated_reflection_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me": + { "type": ["@", "rules", "CC", "library"] + , "name": + [ "generated_reflection_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me" + ] + , "hdrs": + [ "upb/port/def.inc" + , "upb/port/undef.inc" + , "upb/reflection/def.h" + , "upb/reflection/def_pool_internal.h" + ] + , "deps": + [ "base" + , "descriptor_upb_proto" + , "hash" + , "mini_table_internal" + , "reflection_internal" + ] + , "pure C": ["YES"] + } +, "collections": + { "type": ["@", "rules", "CC", "library"] + , "name": ["collections"] + , "hdrs": ["upb/collections/array.h", "upb/collections/map.h"] + , "deps": ["base", "collections_internal", "mem", "port"] + , "pure C": ["YES"] + } +, "collections_internal": + { "type": ["@", "rules", "CC", "library"] + , "name": ["collections_internal"] + , "srcs": + [ "upb/collections/array.c" + , "upb/collections/map.c" + , "upb/collections/map_sorter.c" + ] + , "hdrs": + [ "upb/collections/array.h" + , "upb/collections/array_internal.h" + , "upb/collections/map.h" + , "upb/collections/map_gencode_util.h" + , "upb/collections/map_internal.h" + , "upb/collections/map_sorter_internal.h" + , "upb/collections/message_value.h" ] , "deps": - ["upb_extension_registry", "upb_port", "upb_table", "upb_utf8_range"] + ["base", "hash", "mem", "message_internal", "mini_table_internal", "port"] + , "pure C": ["YES"] } -, "upb_collections": +, "reflection": { "type": ["@", "rules", "CC", "library"] - , "name": ["upb_collections"] + , "name": ["reflection"] + , "hdrs": + [ "upb/def.h" + , "upb/def.hpp" + , "upb/reflection.h" + , "upb/reflection.hpp" + , "upb/reflection/def.h" + , "upb/reflection/def.hpp" + , "upb/reflection/message.h" + , "upb/reflection/message.hpp" + ] + , "deps": ["collections", "port", "upb"] , "pure C": ["YES"] + } +, "reflection_internal": + { "type": ["@", "rules", "CC", "library"] + , "name": ["reflection_internal"] + , "srcs": + [ "upb/reflection/def_builder.c" + , "upb/reflection/def_pool.c" + , "upb/reflection/def_type.c" + , "upb/reflection/desc_state.c" + , "upb/reflection/enum_def.c" + , "upb/reflection/enum_reserved_range.c" + , "upb/reflection/enum_value_def.c" + , "upb/reflection/extension_range.c" + , "upb/reflection/field_def.c" + , "upb/reflection/file_def.c" + , "upb/reflection/message.c" + , "upb/reflection/message_def.c" + , "upb/reflection/message_reserved_range.c" + , "upb/reflection/method_def.c" + , "upb/reflection/oneof_def.c" + , "upb/reflection/service_def.c" + ] , "hdrs": - ["upb/array.h", "upb/collections.h", "upb/map.h", "upb/message_value.h"] - , "srcs": ["upb/array.c", "upb/map.c"] + [ "upb/reflection/common.h" + , "upb/reflection/def.h" + , "upb/reflection/def.hpp" + , "upb/reflection/def_builder_internal.h" + , "upb/reflection/def_pool.h" + , "upb/reflection/def_pool_internal.h" + , "upb/reflection/def_type.h" + , "upb/reflection/desc_state_internal.h" + , "upb/reflection/enum_def.h" + , "upb/reflection/enum_def_internal.h" + , "upb/reflection/enum_reserved_range.h" + , "upb/reflection/enum_reserved_range_internal.h" + , "upb/reflection/enum_value_def.h" + , "upb/reflection/enum_value_def_internal.h" + , "upb/reflection/extension_range.h" + , "upb/reflection/extension_range_internal.h" + , "upb/reflection/field_def.h" + , "upb/reflection/field_def_internal.h" + , "upb/reflection/file_def.h" + , "upb/reflection/file_def_internal.h" + , "upb/reflection/message.h" + , "upb/reflection/message.hpp" + , "upb/reflection/message_def.h" + , "upb/reflection/message_def_internal.h" + , "upb/reflection/message_reserved_range.h" + , "upb/reflection/message_reserved_range_internal.h" + , "upb/reflection/method_def.h" + , "upb/reflection/method_def_internal.h" + , "upb/reflection/oneof_def.h" + , "upb/reflection/oneof_def_internal.h" + , "upb/reflection/service_def.h" + , "upb/reflection/service_def_internal.h" + ] , "deps": - [ "upb_mini_table" - , "upb_port" - , "upb_table" + [ "collections" + , "hash" + , "message_accessors" + , "mini_table_internal" + , "port" , "upb" , ["src/core/ext/upb-generated", "upb-generated-lib"] ] + , "pure C": ["YES"] } -, "upb_reflection": - { "type": ["@", "rules", "CC", "library"] - , "arguments_config": ["ARCH", "TARGET_ARCH"] - , "name": ["upb_reflection"] - , "pure C": ["YES"] - , "hdrs": - ["upb/def.h", "upb/def.hpp", "upb/reflection.h", "upb/reflection.hpp"] - , "srcs": ["upb/def.c", "upb/reflection.c"] - , "private-hdrs": ["upb/msg.h"] - , "private-cflags": - { "type": "case" - , "expr": - { "type": "var" - , "name": "TARGET_ARCH" - , "default": - { "type": "var" - , "name": "ARCH" - , "default": - {"type": "fail", "msg": "Required variable 'ARCH' is not set."} - } - } - , "case": {"arm": ["-Wno-type-limits"], "arm64": ["-Wno-type-limits"]} - } +, "textformat": + { "type": ["@", "rules", "CC", "library"] + , "name": ["textformat"] + , "srcs": ["upb/text/encode.c"] + , "hdrs": ["upb/text/encode.h", "upb/text_encode.h"] + , "deps": + [ "collections_internal" + , "eps_copy_input_stream" + , "lex" + , "port" + , "reflection" + , "reflection_internal" + , "wire" + , "wire_reader" + , "wire_types" + ] + , "pure C": ["YES"] + } +, "json": + { "type": ["@", "rules", "CC", "library"] + , "name": ["json"] + , "srcs": ["upb/json/decode.c", "upb/json/encode.c"] + , "hdrs": + [ "upb/json/decode.h" + , "upb/json/encode.h" + , "upb/json_decode.h" + , "upb/json_encode.h" + ] , "deps": - [ "upb_collections" - , "upb_mini_table" - , "upb_port" - , "upb_table" + [ "collections" + , "lex" + , "port" + , "reflection" + , "reflection_internal" , "upb" , ["src/core/ext/upb-generated", "upb-generated-lib"] ] + , "pure C": ["YES"] } -, "upb_textformat": +, "mem": { "type": ["@", "rules", "CC", "library"] - , "name": ["upb_textformat"] + , "name": ["mem"] + , "hdrs": ["upb/mem/alloc.h", "upb/mem/arena.h"] + , "deps": ["mem_internal", "port"] , "pure C": ["YES"] - , "hdrs": ["upb/text_encode.h"] - , "srcs": ["upb/text_encode.c"] - , "private-hdrs": ["upb/internal/upb.h"] - , "deps": ["upb_port", "upb_reflection", "upb_table"] } -, "upb_json": +, "mem_internal": { "type": ["@", "rules", "CC", "library"] - , "name": ["upb_json"] + , "name": ["mem_internal"] + , "srcs": ["upb/mem/alloc.c", "upb/mem/arena.c"] + , "hdrs": ["upb/mem/alloc.h", "upb/mem/arena.h", "upb/mem/arena_internal.h"] + , "deps": ["port"] , "pure C": ["YES"] - , "hdrs": ["upb/json_decode.h", "upb/json_encode.h"] - , "srcs": ["upb/json_decode.c", "upb/json_encode.c"] - , "private-hdrs": ["upb/internal/upb.h"] - , "deps": ["upb_port", "upb_reflection", "upb"] } -, "upb_table": +, "wire": { "type": ["@", "rules", "CC", "library"] - , "name": ["upb_table"] + , "name": ["wire"] + , "hdrs": ["upb/wire/decode.h", "upb/wire/encode.h"] + , "deps": + ["mem", "message_internal", "mini_table_internal", "port", "wire_internal"] , "pure C": ["YES"] + } +, "wire_internal": + { "type": ["@", "rules", "CC", "library"] + , "name": ["wire_internal"] + , "srcs": + ["upb/wire/decode.c", "upb/wire/decode_fast.c", "upb/wire/encode.c"] , "hdrs": - [ "upb/arena.h" - , "upb/internal/table.h" - , "upb/status.h" - , "upb/table_internal.h" - , "upb/upb.h" + [ "upb/wire/common.h" + , "upb/wire/common_internal.h" + , "upb/wire/decode.h" + , "upb/wire/decode_fast.h" + , "upb/wire/decode_internal.h" + , "upb/wire/encode.h" + , "upb/wire/swap_internal.h" + ] + , "deps": + [ "base" + , "collections_internal" + , "eps_copy_input_stream" + , "hash" + , "mem_internal" + , "message_internal" + , "mini_table_internal" + , "port" + , "wire_reader" + , "wire_types" + , ["third_party/utf8_range", ""] ] - , "deps": ["upb_port"] + , "pure C": ["YES"] + } +, "wire_types": + { "type": ["@", "rules", "CC", "library"] + , "name": ["wire_types"] + , "hdrs": ["upb/wire/types.h"] + , "pure C": ["YES"] } -, "upb_utf8_range": +, "eps_copy_input_stream": { "type": ["@", "rules", "CC", "library"] - , "arguments_config": ["TARGET_ARCH", "COMPILER_FAMILY"] - , "name": ["upb_utf8_range"] + , "name": ["eps_copy_input_stream"] + , "srcs": ["upb/wire/eps_copy_input_stream.c"] + , "hdrs": ["upb/wire/eps_copy_input_stream.h"] + , "deps": ["mem", "port"] , "pure C": ["YES"] - , "hdrs": ["third_party/utf8_range/utf8_range.h"] + } +, "upb/wire/swap_internal.h-lib": + { "type": ["@", "rules", "CC", "library"] + , "name": ["upb/wire/swap_internal.h-lib"] + , "stage": [] + , "hdrs": ["upb/wire/swap_internal.h"] + } +, "wire_reader": + { "type": ["@", "rules", "CC", "library"] + , "name": ["wire_reader"] + , "srcs": ["upb/wire/reader.c", "upb/wire/swap_internal.h"] + , "hdrs": ["upb/wire/reader.h"] + , "deps": ["eps_copy_input_stream", "port", "wire_types"] + , "pure C": ["YES"] + , "private-deps": ["upb/wire/swap_internal.h-lib"] + } +, "hash": + { "type": ["@", "rules", "CC", "library"] + , "name": ["hash"] + , "srcs": ["upb/hash/common.c"] + , "hdrs": + ["upb/hash/common.h", "upb/hash/int_table.h", "upb/hash/str_table.h"] + , "deps": ["base", "mem", "port"] + , "pure C": ["YES"] + } +, "lex": + { "type": ["@", "rules", "CC", "library"] + , "name": ["lex"] , "srcs": - { "type": "++" - , "$1": - [ ["third_party/utf8_range/naive.c"] - , { "type": "case*" - , "expr": {"type": "var", "name": "TARGET_ARCH"} - , "case": - [ ["arm", ["third_party/utf8_range/range2-neon.c"]] - , ["arm64", ["third_party/utf8_range/range2-neon.c"]] - , [ "x86_64" - , { "type": "case*" - , "expr": {"type": "var", "name": "COMPILER_FAMILY"} - , "case": [["msvc", []]] - , "default": ["third_party/utf8_range/range2-sse.c"] - } - ] - ] - } - ] - } - , "private-cflags": - { "type": "case*" - , "expr": {"type": "var", "name": "TARGET_ARCH"} - , "case": - [ ["arm", ["-mfpu=neon"]] - , ["arm64", ["-march=armv8-a+crypto"]] - , [ "x86_64" - , { "type": "case*" - , "expr": {"type": "var", "name": "COMPILER_FAMILY"} - , "case": [["msvc", []]] - , "default": ["-msse4.1"] - } - ] - ] - } + [ "upb/lex/atoi.c" + , "upb/lex/round_trip.c" + , "upb/lex/strtod.c" + , "upb/lex/unicode.c" + ] + , "hdrs": + [ "upb/lex/atoi.h" + , "upb/lex/round_trip.h" + , "upb/lex/strtod.h" + , "upb/lex/unicode.h" + ] + , "deps": ["port"] + , "pure C": ["YES"] } } diff --git a/etc/import/third_party/utf8_range/TARGETS.grpc b/etc/import/third_party/utf8_range/TARGETS.grpc new file mode 100644 index 000000000..b8d48617f --- /dev/null +++ b/etc/import/third_party/utf8_range/TARGETS.grpc @@ -0,0 +1,8 @@ +{ "": + { "type": ["@", "rules", "CC", "library"] + , "name": ["utf8_range"] + , "srcs": ["naive.c", "range2-neon.c", "range2-sse.c"] + , "hdrs": ["utf8_range.h"] + , "pure C": ["YES"] + } +} diff --git a/etc/repos.json b/etc/repos.json index 73001b941..8272fe8c6 100644 --- a/etc/repos.json +++ b/etc/repos.json @@ -377,11 +377,11 @@ , "com_github_grpc_grpc": { "repository": { "type": "archive" - , "content": "a60418787a6e6f98c584d8de42d93b8983254939" - , "fetch": "https://github.com/grpc/grpc/archive/refs/tags/v1.53.0.tar.gz" - , "sha256": "9717ffc52120861136e478155c2ff3a9c21740e2244de52fa966f376d7471adf" - , "sha512": "a02fe1969787cd39061269b8c7c1313c1f599a31aa4e11760a744a9e0f85202e33832ee669b9d0d07df4af1a45470d605a69d9e4a4df89251e9b0093aee89eca" - , "subdir": "grpc-1.53.0" + , "content": "eb5007bb7fb6575db8c1467090719d3262a05868" + , "fetch": "https://github.com/grpc/grpc/archive/refs/tags/v1.56.2.tar.gz" + , "sha256": "931f07db9d48cff6a6007c1033ba6d691fe655bea2765444bc1ad974dfc840aa" + , "sha512": "bb4be3b8a128566581b7ea69d1faa6d08a28624656c2f4afdd8bb8fa804e362b7d8f2bb6c52789081b48f299ddffe18e1795c8dce9b9bc304a5e6d3a4274b11e" + , "subdir": "grpc-1.56.2" } , "target_root": "import targets" , "target_file_name": "TARGETS.grpc" @@ -389,7 +389,7 @@ , "bindings": { "rules": "rules-grpc" , "protobuf": "protobuf" - , "libssl": "ssl" + , "ssl": "ssl" , "absl": "com_google_absl" , "zlib": "zlib" , "re2": "re2" From d9d4b607ab30186a424480e3e274cb439983561f Mon Sep 17 00:00:00 2001 From: Alberto Sartori Date: Fri, 4 Aug 2023 12:49:12 +0200 Subject: [PATCH 36/97] grpc and protobuf: use c++17 Due to a possible clang issue, protobuf v23.4 fails on debian-12 built with c++20. As such, protobuf and its consumer grpc have been set to build with c++17 until this is resolved upstream. Co-authored: Paul Cristian Sarbu --- etc/defaults/CC/TARGETS.grpc | 2 +- etc/defaults/CC/TARGETS.protobuf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/defaults/CC/TARGETS.grpc b/etc/defaults/CC/TARGETS.grpc index b562de406..fbafa1361 100644 --- a/etc/defaults/CC/TARGETS.grpc +++ b/etc/defaults/CC/TARGETS.grpc @@ -97,7 +97,7 @@ , "body": { "type": "++" , "$1": - [ ["-std=c++20"] + [ ["-std=c++17"] , { "type": "case" , "expr": {"type": "var", "name": "COMPILER_FAMILY"} , "case": {"msvc": []} diff --git a/etc/defaults/CC/TARGETS.protobuf b/etc/defaults/CC/TARGETS.protobuf index 2bde69ea6..f2ae56d66 100644 --- a/etc/defaults/CC/TARGETS.protobuf +++ b/etc/defaults/CC/TARGETS.protobuf @@ -13,7 +13,7 @@ , "body": { "type": "++" , "$1": - [ ["-std=c++20", "-DHAVE_PTHREAD=1", "-DHAVE_ZLIB=1"] + [ ["-std=c++17", "-DHAVE_PTHREAD=1", "-DHAVE_ZLIB=1"] , { "type": "case" , "expr": {"type": "var", "name": "COMPILER_FAMILY"} , "case": From 588b3d2e2828339f95cc332157dd35232da95ce9 Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Tue, 8 Aug 2023 16:14:50 +0200 Subject: [PATCH 37/97] absl: export reused library "raw_logging_internal" --- etc/import/absl/base/TARGETS.absl | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/etc/import/absl/base/TARGETS.absl b/etc/import/absl/base/TARGETS.absl index a4d5b0555..9d6a6f14c 100644 --- a/etc/import/absl/base/TARGETS.absl +++ b/etc/import/absl/base/TARGETS.absl @@ -224,6 +224,26 @@ , "deps": ["config", "core_headers"] } , "raw_logging_internal": + { "type": "export" + , "target": "raw_logging_internal (unexported)" + , "flexible_config": + [ "ADD_CFLAGS" + , "ADD_CXXFLAGS" + , "AR" + , "ARCH" + , "CC" + , "CFLAGS" + , "COMPILER_FAMILY" + , "CXX" + , "CXXFLAGS" + , "DEBUG" + , "ENV" + , "HOST_ARCH" + , "OS" + , "TARGET_ARCH" + ] + } +, "raw_logging_internal (unexported)": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["ARCH", "TARGET_ARCH"] , "name": ["raw_logging_internal"] From abdbeab309f5ae69a20cf2953c514a30450774ed Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Tue, 8 Aug 2023 15:25:45 +0200 Subject: [PATCH 38/97] Unlink files before creation on install-cas If install-cas finds a file in the target location, unlink it before writing (in the same way as install does already). In this way, we avoid changing other file locations in the presence of hard links. --- CHANGELOG.md | 2 ++ src/buildtool/main/install_cas.cpp | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97b2f7308..17df375a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,8 @@ A feature release on top of `1.1.0`, backwards compatible. the local build root. - `just install-cas` now correctly handles `--raw-tree` also for remote-execution endpoints. +- `just install-cas` now, like `just install`, removes an existing + destination file before installing instead of overwriting. ## Release `1.1.0` (2023-05-19) diff --git a/src/buildtool/main/install_cas.cpp b/src/buildtool/main/install_cas.cpp index 1c0290df4..99867de87 100644 --- a/src/buildtool/main/install_cas.cpp +++ b/src/buildtool/main/install_cas.cpp @@ -180,8 +180,21 @@ auto FetchAndInstallArtifacts( output_path /= object_info.digest.hash(); } - if (not FileSystemManager::CreateDirectory(output_path.parent_path()) or - not api->RetrieveToPaths( + if (not FileSystemManager::CreateDirectory(output_path.parent_path())) { + Logger::Log(LogLevel::Error, + "failed to create parent directory {}.", + output_path.parent_path().string()); + return false; + } + if (FileSystemManager::Exists(output_path)) { + if (not FileSystemManager::RemoveFile(output_path)) { + Logger::Log(LogLevel::Error, + "Failed to remote target location {}.", + output_path.string()); + return false; + } + } + if (not api->RetrieveToPaths( {object_info}, {output_path}, alternative_api)) { Logger::Log(LogLevel::Error, "failed to retrieve artifact."); return false; From 7e44903a7e1bb30947867b267faae959888e214e Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Tue, 8 Aug 2023 10:53:39 +0200 Subject: [PATCH 39/97] Add a test verifying the overwrite property of install and install-cas Verify that install and install-cas do not overwrite and instead replace existing files. This behaviour is less surprising for the user in the presence of hard links. --- test/end-to-end/cli/TARGETS | 11 +++++- test/end-to-end/cli/install.sh | 69 ++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 test/end-to-end/cli/install.sh diff --git a/test/end-to-end/cli/TARGETS b/test/end-to-end/cli/TARGETS index 9439767bc..2306af70e 100644 --- a/test/end-to-end/cli/TARGETS +++ b/test/end-to-end/cli/TARGETS @@ -10,6 +10,15 @@ , "test": ["pwd.sh"] , "deps": [["end-to-end", "tool-under-test"]] } +, "install": + { "type": ["@", "rules", "shell/test", "script"] + , "name": ["install"] + , "test": ["install.sh"] + , "deps": [["end-to-end", "tool-under-test"]] + } , "TESTS": - {"type": "install", "tainted": ["test"], "deps": ["defaults", "pwd"]} + { "type": "install" + , "tainted": ["test"] + , "deps": ["defaults", "pwd", "install"] + } } diff --git a/test/end-to-end/cli/install.sh b/test/end-to-end/cli/install.sh new file mode 100644 index 000000000..ee2eb2712 --- /dev/null +++ b/test/end-to-end/cli/install.sh @@ -0,0 +1,69 @@ +#!/bin/sh +# Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +ROOT=$(pwd) +TOOL=$(realpath ./bin/tool-under-test) +BUILDROOT="${TEST_TMPDIR}/build-root" + +mkdir src +touch src/ROOT +cat > src/TARGETS <<'EOF' +{ "": + { "type": "generic" + , "outs": ["hello.txt"] + , "cmds": ["echo Hello World > hello.txt"] + } +} +EOF +SRCDIR=$(realpath src) + +mkdir out +echo 'Original' > out/unrelated.txt +ln out/unrelated.txt out/hello.txt +OUTDIR=$(realpath out) +ls -al "${OUTDIR}" +echo + +# Verify non-interference of install +cd "${SRCDIR}" +"${TOOL}" install --local-build-root "${BUILDROOT}" -o "${OUTDIR}" 2>&1 + +echo +ls -al "${OUTDIR}" +cd "${OUTDIR}" +grep World hello.txt +grep Original unrelated.txt + +# Verify non-interference of install +cd "${SRCDIR}" +"${TOOL}" build --local-build-root "${BUILDROOT}" \ + --dump-artifacts "${OUTDIR}/artifacts.json" 2>&1 +echo +ID=$(jq -rM '."hello.txt".id' "${OUTDIR}/artifacts.json") +ln "${OUTDIR}/unrelated.txt" "${OUTDIR}/${ID}" +ls -al "${OUTDIR}" +echo + +"${TOOL}" install-cas --local-build-root "${BUILDROOT}" \ + -o "${OUTDIR}" "${ID}" 2>&1 +echo +ls -al "${OUTDIR}" +cd "${OUTDIR}" +grep World "${ID}" +grep Original unrelated.txt + +echo OK From a58110bb7a3fa2a112085ce378f508b39e4f7096 Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Wed, 9 Aug 2023 12:26:37 +0200 Subject: [PATCH 40/97] Update fmt library --- etc/repos.json | 10 +++++----- src/buildtool/execution_api/remote/config.hpp | 3 ++- test/bootstrap/test-mixed-bootstrap.sh | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/etc/repos.json b/etc/repos.json index 8272fe8c6..7bad15595 100644 --- a/etc/repos.json +++ b/etc/repos.json @@ -211,11 +211,11 @@ , "fmt": { "repository": { "type": "zip" - , "content": "558d83d0fedb1f1ee3c3c4f1576d549cae7614ed" - , "fetch": "https://github.com/fmtlib/fmt/releases/download/9.1.0/fmt-9.1.0.zip" - , "sha256": "cceb4cb9366e18a5742128cb3524ce5f50e88b476f1e54737a47ffdf4df4c996" - , "sha512": "482d1ceaf042e97c66ff570babe9bd6f9cab7e4eec3dc56eb7d5e075b6b9b16f8eb333e96de9627fa33324d32b88d2aacebd0c34c3c706650eaa1729b0dd7fd6" - , "subdir": "fmt-9.1.0" + , "content": "3af42ee467295259c9bd6533f1092268136b226d" + , "fetch": "https://github.com/fmtlib/fmt/releases/download/10.0.0/fmt-10.0.0.zip" + , "sha256": "4943cb165f3f587f26da834d3056ee8733c397e024145ca7d2a8a96bb71ac281" + , "sha512": "e4f80c34a78490a6bb3fa79fe097dac3d660dc7900fdeac9f8e744c1f1ff414c0a54a9b8a61fc9631702b12aaaa77000673357545e5771872076154810ebbbfc" + , "subdir": "fmt-10.0.0" } , "target_root": "import targets" , "target_file_name": "TARGETS.fmt" diff --git a/src/buildtool/execution_api/remote/config.hpp b/src/buildtool/execution_api/remote/config.hpp index 55c96a463..449e95067 100644 --- a/src/buildtool/execution_api/remote/config.hpp +++ b/src/buildtool/execution_api/remote/config.hpp @@ -68,7 +68,8 @@ class RemoteExecutionConfig { Port port{}; [[nodiscard]] auto ToJson() const noexcept -> nlohmann::json { - return nlohmann::json{fmt::format("{}:{}", host, port)}; + return nlohmann::json{ + fmt::format("{}:{}", host, static_cast(port))}; } }; diff --git a/test/bootstrap/test-mixed-bootstrap.sh b/test/bootstrap/test-mixed-bootstrap.sh index d43a6a58d..981116c49 100755 --- a/test/bootstrap/test-mixed-bootstrap.sh +++ b/test/bootstrap/test-mixed-bootstrap.sh @@ -44,7 +44,7 @@ cp distdir/v4.0.0.tar.gz "${DISTDIR}" # - fmt rm -rf "${LOCALBASE}/include/fmt*" rm -rf "${LOCALBASE}/lib/libfmt*" -cp distdir/fmt-9.1.0.zip "${DISTDIR}" +cp distdir/fmt-10.0.0.zip "${DISTDIR}" # bootstrap command From f9626c99d092774e362c0e8ac19589c7155d4923 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Thu, 10 Aug 2023 12:00:57 +0200 Subject: [PATCH 41/97] built-in: Add 'symlink' rule The rule generates a non-upwards symbolic link with given target path. --- .../target_map/built_in_rules.cpp | 93 +++++++++++++++---- 1 file changed, 75 insertions(+), 18 deletions(-) diff --git a/src/buildtool/build_engine/target_map/built_in_rules.cpp b/src/buildtool/build_engine/target_map/built_in_rules.cpp index 257006f58..29eba7bff 100644 --- a/src/buildtool/build_engine/target_map/built_in_rules.cpp +++ b/src/buildtool/build_engine/target_map/built_in_rules.cpp @@ -49,7 +49,7 @@ auto const kGenericRuleFields = "out_dirs", "outs"}; -auto const kFileGenRuleFields = +auto const kBlobGenRuleFields = std::unordered_set{"arguments_config", "data", "deps", @@ -78,14 +78,15 @@ auto const kConfigureRuleFields = "target", "type"}; -void FileGenRuleWithDeps( +void BlobGenRuleWithDeps( const std::vector& transition_keys, const std::vector& dependency_values, const BuildMaps::Base::FieldReader::Ptr& desc, const BuildMaps::Target::ConfiguredTarget& key, const BuildMaps::Target::TargetMap::SetterPtr& setter, const BuildMaps::Target::TargetMap::LoggerPtr& logger, - const gsl::not_null& result_map) { + const gsl::not_null& result_map, + const ObjectType& blob_type) { // Associate keys and values std::unordered_map deps_by_transition; @@ -159,21 +160,21 @@ void FileGenRuleWithDeps( } } - auto file_name_exp = desc->ReadOptionalExpression( + auto name_exp = desc->ReadOptionalExpression( "name", ExpressionPtr{std::string{"out.txt"}}); - if (not file_name_exp) { + if (not name_exp) { return; } - auto file_name_val = file_name_exp.Evaluate( + auto name_val = name_exp.Evaluate( param_config, string_fields_fcts, [logger](auto const& msg) { (*logger)(fmt::format("While evaluating name:\n{}", msg), true); }); - if (not file_name_val) { + if (not name_val) { return; } - if (not file_name_val->IsString()) { + if (not name_val->IsString()) { (*logger)(fmt::format("name should evaluate to a string, but got {}", - file_name_val->ToString()), + name_val->ToString()), true); return; } @@ -195,11 +196,22 @@ void FileGenRuleWithDeps( true); return; } + + // if symlink target, we only accept non-upwards + if (IsSymlinkObject(blob_type) and + not PathIsNonUpwards(data_val->String())) { + (*logger)(fmt::format("data string {} does not constitute a " + "non-upwards symlink target path", + data_val->String()), + true); + return; + } + auto stage = ExpressionPtr{Expression::map_t{ - file_name_val->String(), + name_val->String(), ExpressionPtr{ArtifactDescription{ ArtifactDigest::Create(data_val->String()), - ObjectType::File}}}}; + blob_type}}}}; auto analysis_result = std::make_shared( TargetResult{.artifact_stage = stage, @@ -216,16 +228,21 @@ void FileGenRuleWithDeps( (*setter)(std::move(analysis_result)); } -void FileGenRule( +void BlobGenRule( const nlohmann::json& desc_json, const BuildMaps::Target::ConfiguredTarget& key, const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller, const BuildMaps::Target::TargetMap::SetterPtr& setter, const BuildMaps::Target::TargetMap::LoggerPtr& logger, - const gsl::not_null& result_map) { + const gsl::not_null& result_map, + const ObjectType& blob_type) { auto desc = BuildMaps::Base::FieldReader::CreatePtr( - desc_json, key.target, "file-generation target", logger); - desc->ExpectFields(kFileGenRuleFields); + desc_json, + key.target, + IsSymlinkObject(blob_type) ? "symlink target" + : "file-generation target", + logger); + desc->ExpectFields(kBlobGenRuleFields); auto param_vars = desc->ReadStringList("arguments_config"); if (not param_vars) { return; @@ -282,13 +299,52 @@ void FileGenRule( setter, logger, key, - result_map](auto const& values) { - FileGenRuleWithDeps( - transition_keys, values, desc, key, setter, logger, result_map); + result_map, + blob_type](auto const& values) { + BlobGenRuleWithDeps(transition_keys, + values, + desc, + key, + setter, + logger, + result_map, + blob_type); }, logger); } +void FileGenRule( + const nlohmann::json& desc_json, + const BuildMaps::Target::ConfiguredTarget& key, + const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller, + const BuildMaps::Target::TargetMap::SetterPtr& setter, + const BuildMaps::Target::TargetMap::LoggerPtr& logger, + const gsl::not_null& result_map) { + BlobGenRule(desc_json, + key, + subcaller, + setter, + logger, + result_map, + ObjectType::File); +} + +void SymlinkRule( + const nlohmann::json& desc_json, + const BuildMaps::Target::ConfiguredTarget& key, + const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller, + const BuildMaps::Target::TargetMap::SetterPtr& setter, + const BuildMaps::Target::TargetMap::LoggerPtr& logger, + const gsl::not_null& result_map) { + BlobGenRule(desc_json, + key, + subcaller, + setter, + logger, + result_map, + ObjectType::Symlink); +} + void TreeRuleWithDeps( const std::vector& dependency_values, const std::string& name, @@ -1357,6 +1413,7 @@ auto const kBuiltIns = std::unordered_map< {"export", ExportRule}, {"file_gen", FileGenRule}, {"tree", TreeRule}, + {"symlink", SymlinkRule}, {"generic", GenericRule}, {"install", InstallRule}, {"configure", ConfigureRule}}; From 588d61a178874fe48a32ebba87a278169b8d3636 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Thu, 10 Aug 2023 12:02:52 +0200 Subject: [PATCH 42/97] test: Add checks for the 'symlink' built-in rule --- .../data_targets/simple_targets/TARGETS | 2 + .../target_map/target_map.test.cpp | 26 +++++++++ test/end-to-end/built-in-rules/TARGETS | 8 ++- .../built-in-rules/symlink_config.sh | 55 +++++++++++++++++++ 4 files changed, 90 insertions(+), 1 deletion(-) create mode 100755 test/end-to-end/built-in-rules/symlink_config.sh diff --git a/test/buildtool/build_engine/target_map/data_targets/simple_targets/TARGETS b/test/buildtool/build_engine/target_map/data_targets/simple_targets/TARGETS index 0001a223a..b549a8a12 100644 --- a/test/buildtool/build_engine/target_map/data_targets/simple_targets/TARGETS +++ b/test/buildtool/build_engine/target_map/data_targets/simple_targets/TARGETS @@ -70,4 +70,6 @@ } , "generate file": {"type": "file_gen", "name": "generated.txt", "data": "Hello World!"} +, "generate symlink": + {"type": "symlink", "name": "generated_link", "data": "dummy_link_target"} } diff --git a/test/buildtool/build_engine/target_map/target_map.test.cpp b/test/buildtool/build_engine/target_map/target_map.test.cpp index 98d3a9382..bd868b484 100644 --- a/test/buildtool/build_engine/target_map/target_map.test.cpp +++ b/test/buildtool/build_engine/target_map/target_map.test.cpp @@ -629,6 +629,32 @@ TEST_CASE("built-in rules") { CHECK(result->Blobs()[0] == "Hello World!"); } + SECTION("symlink") { + error = false; + error_msg = "NONE"; + { + TaskSystem ts; + target_map.ConsumeAfterKeysReady( + &ts, + {BuildMaps::Target::ConfiguredTarget{ + .target = + BuildMaps::Base::EntityName{ + "", "simple_targets", "generate symlink"}, + .config = empty_config}}, + [&result](auto values) { result = *values[0]; }, + [&error, &error_msg](std::string const& msg, bool /*unused*/) { + error = true; + error_msg = msg; + }); + } + CHECK(!error); + CHECK(error_msg == "NONE"); + CHECK(result->Artifacts()->ToJson()["generated_link"]["type"] == + "KNOWN"); + CHECK(result->Blobs().size() == 1); + CHECK(result->Blobs()[0] == "dummy_link_target"); + } + SECTION("configure") { auto target = BuildMaps::Base::EntityName{"", "config_targets", "bar in foo"}; diff --git a/test/end-to-end/built-in-rules/TARGETS b/test/end-to-end/built-in-rules/TARGETS index e4322e15d..f37849ae0 100644 --- a/test/end-to-end/built-in-rules/TARGETS +++ b/test/end-to-end/built-in-rules/TARGETS @@ -16,9 +16,15 @@ , "test": ["tree.sh"] , "deps": [["end-to-end", "tool-under-test"]] } +, "symlink_config": + { "type": ["@", "rules", "shell/test", "script"] + , "name": ["symlink_config"] + , "test": ["symlink_config.sh"] + , "deps": [["end-to-end", "tool-under-test"]] + } , "TESTS": { "type": "install" , "tainted": ["test"] - , "deps": ["generic_out_dirs", "filegen_config", "tree"] + , "deps": ["generic_out_dirs", "filegen_config", "tree", "symlink_config"] } } diff --git a/test/end-to-end/built-in-rules/symlink_config.sh b/test/end-to-end/built-in-rules/symlink_config.sh new file mode 100755 index 000000000..2af9240f0 --- /dev/null +++ b/test/end-to-end/built-in-rules/symlink_config.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -e + +readonly LBRDIR="$TMPDIR/local-build-root" + +touch ROOT + +cat <<'EOF' > TARGETS +{ "": + { "type": "symlink" + , "name": "read-runfiles" + , "data": + { "type": "join" + , "separator": ";" + , "$1": {"type": "runfiles", "dep": "uses config"} + } + , "deps": ["uses config"] + } +, "uses config": + { "type": "symlink" + , "arguments_config": ["NAME"] + , "name": {"type": "var", "name": "NAME", "default": "null"} + , "data": "irrelevant" + } +} +EOF + +bin/tool-under-test analyse -D '{"NAME": "here/../be/../dragons"}' \ + --local-build-root "$LBRDIR" --dump-targets targets.json --dump-blobs blobs.json 2>&1 +echo +echo "Blobs" +cat blobs.json +[ $(jq '. == ["here/../be/../dragons"]' blobs.json) = "true" ] +echo + +echo "Targets" +cat targets.json +[ $(jq '."@"."".""."" == [{"NAME": "here/../be/../dragons"}]' targets.json) = "true" ] + +echo OK From dd576b8730b1aa9ea90fba20367471b4a09da5d3 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Thu, 10 Aug 2023 12:03:33 +0200 Subject: [PATCH 43/97] doc: Extend documentation with 'symlink' built-in rule --- doc/concepts/built-in-rules.md | 17 +++++++++++++++++ doc/concepts/symlinks.md | 3 +++ 2 files changed, 20 insertions(+) diff --git a/doc/concepts/built-in-rules.md b/doc/concepts/built-in-rules.md index 319d0d82c..5591c1744 100644 --- a/doc/concepts/built-in-rules.md +++ b/doc/concepts/built-in-rules.md @@ -140,6 +140,23 @@ of the `"tree"` target are a singleton map with the key the result of evaluating `"name"` (which has to evaluate to a single string) and value that tree. +`"symlink"` +------------ + +The `"symlink"` rule allows to specify a non-upwards symbolic link with a +given link target. To be able to accurately report about file names of +artifacts or runfiles of other targets, they can be specified in the field +`"deps"` which has to evaluate to a list of targets. The names of the +artifacts and runfiles of a target specified in `"deps"` can be accessed +through the functions `"outs"` and `"runfiles"`, respectively, during the +evaluation of the arguments `"name"` and `"data"` which have to evaluate to +a single string. + +Artifacts and runfiles of a `"symlink"` target are a singleton map with +key the result of evaluating `"name"` and value a non-upwards symbolic link +with target path the result of evaluating `"data"` (which must evaluate to +a non-upwards path). The provides map is empty. + `"configure"` ------------- diff --git a/doc/concepts/symlinks.md b/doc/concepts/symlinks.md index 2d33e3886..5a7114e4c 100644 --- a/doc/concepts/symlinks.md +++ b/doc/concepts/symlinks.md @@ -118,6 +118,9 @@ permissions. As such, they locally use the existing file CAS. Remotely, the existing execution protocol already allows the handling of symbolic links via corresponding Protobuf messages, therefore no extensions are needed. +Additionally, the built-in rules are extended with a `"symlink"` target, +allowing the generation of a symlink with given non-upwards target path. + ### Import resolved `git`-trees Finally, to be as flexible as possible in handling external repositories with From 24124411aecdeebb4182ad857756265df0764363 Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Thu, 10 Aug 2023 11:56:15 +0200 Subject: [PATCH 44/97] Move downstepping to subobjects to a library ... in order to make that functionality reusable --- src/buildtool/execution_api/utils/TARGETS | 18 +++ .../execution_api/utils/subobject.cpp | 120 ++++++++++++++++++ .../execution_api/utils/subobject.hpp | 34 +++++ src/buildtool/main/TARGETS | 2 +- src/buildtool/main/install_cas.cpp | 99 ++------------- 5 files changed, 181 insertions(+), 92 deletions(-) create mode 100644 src/buildtool/execution_api/utils/TARGETS create mode 100644 src/buildtool/execution_api/utils/subobject.cpp create mode 100644 src/buildtool/execution_api/utils/subobject.hpp diff --git a/src/buildtool/execution_api/utils/TARGETS b/src/buildtool/execution_api/utils/TARGETS new file mode 100644 index 000000000..20b65c182 --- /dev/null +++ b/src/buildtool/execution_api/utils/TARGETS @@ -0,0 +1,18 @@ +{ "subobject": + { "type": ["@", "rules", "CC", "library"] + , "name": ["subobject"] + , "hdrs": ["subobject.hpp"] + , "srcs": ["subobject.cpp"] + , "deps": + [ ["src/buildtool/common", "common"] + , ["src/buildtool/execution_api/common", "common"] + , ["@", "gsl", "", "gsl"] + ] + , "private-deps": + [ ["src/buildtool/compatibility", "compatibility"] + , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"] + , ["src/buildtool/logging", "logging"] + ] + , "stage": ["src", "buildtool", "execution_api", "utils"] + } +} diff --git a/src/buildtool/execution_api/utils/subobject.cpp b/src/buildtool/execution_api/utils/subobject.cpp new file mode 100644 index 000000000..46ec58a78 --- /dev/null +++ b/src/buildtool/execution_api/utils/subobject.cpp @@ -0,0 +1,120 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/buildtool/execution_api/utils/subobject.hpp" +#ifndef BOOTSTRAP_BUILD_TOOL + +#include "src/buildtool/compatibility/compatibility.hpp" +#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" +#include "src/buildtool/logging/log_level.hpp" +#include "src/buildtool/logging/logger.hpp" + +auto RetrieveSubPathId(Artifact::ObjectInfo object_info, + gsl::not_null const& api, + const std::filesystem::path& sub_path) + -> std::optional { + + std::filesystem::path sofar{}; + for (auto const& segment : sub_path) { + if (object_info.type != ObjectType::Tree) { + Logger::Log(LogLevel::Warning, + "Non-tree found at path '{}', cannot follow to '{}'", + sofar.string(), + segment.string()); + break; + } + auto data = api->RetrieveToMemory(object_info); + if (not data) { + Logger::Log(LogLevel::Error, + "Failed to retrieve artifact {} at path '{}'", + object_info.ToString(), + sofar.string()); + return std::nullopt; + } + if (Compatibility::IsCompatible()) { + auto directory = + BazelMsgFactory::MessageFromString(*data); + if (not directory) { + Logger::Log(LogLevel::Warning, + "Failed to parse directory message at path '{}'", + sofar.string()); + break; + } + std::optional new_object_info{}; + if (not BazelMsgFactory::ReadObjectInfosFromDirectory( + *directory, + [&new_object_info, &segment](auto path, auto info) { + if (path == segment) { + new_object_info = info; + } + return true; + })) { + Logger::Log(LogLevel::Warning, + "Failed to process directory message at path '{}'", + sofar.string()); + break; + } + if (not new_object_info) { + Logger::Log(LogLevel::Warning, + "Entry {} not found at path '{}'", + segment.string(), + sofar.string()); + break; + } + object_info = *new_object_info; + } + else { + auto entries = GitRepo::ReadTreeData( + *data, + HashFunction::ComputeTreeHash(*data).Bytes(), + [](auto const& /*unused*/) { return true; }, + /*is_hex_id=*/false); + if (not entries) { + Logger::Log(LogLevel::Warning, + "Failed to parse tree {} at path '{}'", + object_info.ToString(), + sofar.string()); + break; + } + std::optional new_object_info{}; + if (not BazelMsgFactory::ReadObjectInfosFromGitTree( + *entries, + [&new_object_info, &segment](auto path, auto info) { + if (path == segment) { + new_object_info = info; + } + return true; + })) { + Logger::Log(LogLevel::Warning, + "Failed to process tree entries at path '{}'", + sofar.string()); + break; + } + + if (not new_object_info) { + Logger::Log(LogLevel::Warning, + "Entry {} not found at path '{}'", + segment.string(), + sofar.string()); + break; + } + object_info = *new_object_info; + } + sofar /= segment; + } + + return object_info; +} + +#endif diff --git a/src/buildtool/execution_api/utils/subobject.hpp b/src/buildtool/execution_api/utils/subobject.hpp new file mode 100644 index 000000000..85fc7bdb6 --- /dev/null +++ b/src/buildtool/execution_api/utils/subobject.hpp @@ -0,0 +1,34 @@ +// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_UTILS_SUBOBJECT_HPP +#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_UTILS_SUBOBJECT_HPP + +#ifndef BOOTSTRAP_BUILD_TOOL + +#include +#include + +#include "gsl/gsl" +#include "src/buildtool/common/artifact.hpp" +#include "src/buildtool/execution_api/common/execution_api.hpp" + +auto RetrieveSubPathId(Artifact::ObjectInfo object_info, + gsl::not_null const& api, + const std::filesystem::path& sub_path) + -> std::optional; + +#endif + +#endif diff --git a/src/buildtool/main/TARGETS b/src/buildtool/main/TARGETS index 8ff214d0b..b74a5fe49 100644 --- a/src/buildtool/main/TARGETS +++ b/src/buildtool/main/TARGETS @@ -83,9 +83,9 @@ , "private-deps": [ ["src/buildtool/compatibility", "compatibility"] , ["src/buildtool/crypto", "hash_function"] - , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"] , ["src/buildtool/execution_api/remote", "config"] , ["src/buildtool/logging", "logging"] + , ["src/buildtool/execution_api/utils", "subobject"] ] } , "analyse": diff --git a/src/buildtool/main/install_cas.cpp b/src/buildtool/main/install_cas.cpp index 99867de87..622e7a1f0 100644 --- a/src/buildtool/main/install_cas.cpp +++ b/src/buildtool/main/install_cas.cpp @@ -17,7 +17,7 @@ #include "src/buildtool/compatibility/compatibility.hpp" #include "src/buildtool/crypto/hash_function.hpp" #ifndef BOOTSTRAP_BUILD_TOOL -#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" +#include "src/buildtool/execution_api/utils/subobject.hpp" #endif #include "src/buildtool/execution_api/remote/config.hpp" @@ -81,96 +81,13 @@ auto FetchAndInstallArtifacts( if (clargs.sub_path) { std::filesystem::path sofar{}; - for (auto const& segment : *clargs.sub_path) { - if (object_info.type != ObjectType::Tree) { - Logger::Log( - LogLevel::Warning, - "Non-tree found at path '{}', cannot follow to '{}'", - sofar.string(), - segment.string()); - break; - } - auto data = api->RetrieveToMemory(object_info); - if (not data) { - Logger::Log(LogLevel::Error, - "Failed to retrieve artifact {} at path '{}'", - object_info.ToString(), - sofar.string()); - return false; - } - if (Compatibility::IsCompatible()) { - auto directory = - BazelMsgFactory::MessageFromString( - *data); - if (not directory) { - Logger::Log( - LogLevel::Warning, - "Failed to parse directory message at path '{}'", - sofar.string()); - break; - } - std::optional new_object_info{}; - if (not BazelMsgFactory::ReadObjectInfosFromDirectory( - *directory, - [&new_object_info, &segment](auto path, auto info) { - if (path == segment) { - new_object_info = info; - } - return true; - })) { - Logger::Log( - LogLevel::Warning, - "Failed to process directory message at path '{}'", - sofar.string()); - break; - } - if (not new_object_info) { - Logger::Log(LogLevel::Warning, - "Entry {} not found at path '{}'", - segment.string(), - sofar.string()); - break; - } - object_info = *new_object_info; - } - else { - auto entries = GitRepo::ReadTreeData( - *data, - HashFunction::ComputeTreeHash(*data).Bytes(), - [](auto const& /*unused*/) { return true; }, - /*is_hex_id=*/false); - if (not entries) { - Logger::Log(LogLevel::Warning, - "Failed to parse tree {} at path '{}'", - object_info.ToString(), - sofar.string()); - break; - } - std::optional new_object_info{}; - if (not BazelMsgFactory::ReadObjectInfosFromGitTree( - *entries, - [&new_object_info, &segment](auto path, auto info) { - if (path == segment) { - new_object_info = info; - } - return true; - })) { - Logger::Log(LogLevel::Warning, - "Failed to process tree entries at path '{}'", - sofar.string()); - break; - } - - if (not new_object_info) { - Logger::Log(LogLevel::Warning, - "Entry {} not found at path '{}'", - segment.string(), - sofar.string()); - break; - } - object_info = *new_object_info; - } - sofar /= segment; + auto new_object_info = + RetrieveSubPathId(object_info, api, *clargs.sub_path); + if (new_object_info) { + object_info = *new_object_info; + } + else { + return false; } } From 79f400a1dd1da6710312e33844b1ae04bdc21832 Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Thu, 10 Aug 2023 09:55:57 +0200 Subject: [PATCH 45/97] just build: allow -P to reach into trees --- src/buildtool/graph_traverser/TARGETS | 1 + .../graph_traverser/graph_traverser.hpp | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/src/buildtool/graph_traverser/TARGETS b/src/buildtool/graph_traverser/TARGETS index 04b5f7467..b99116334 100644 --- a/src/buildtool/graph_traverser/TARGETS +++ b/src/buildtool/graph_traverser/TARGETS @@ -11,6 +11,7 @@ , ["src/buildtool/execution_api/local", "local"] , ["src/buildtool/execution_api/remote", "bazel"] , ["src/buildtool/execution_api/remote", "config"] + , ["src/buildtool/execution_api/utils", "subobject"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/file_system", "jsonfs"] diff --git a/src/buildtool/graph_traverser/graph_traverser.hpp b/src/buildtool/graph_traverser/graph_traverser.hpp index 48f5c2816..9f99e6f2a 100644 --- a/src/buildtool/graph_traverser/graph_traverser.hpp +++ b/src/buildtool/graph_traverser/graph_traverser.hpp @@ -36,6 +36,7 @@ #include "src/buildtool/execution_api/local/local_api.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_api.hpp" #include "src/buildtool/execution_api/remote/config.hpp" +#include "src/buildtool/execution_api/utils/subobject.hpp" #include "src/buildtool/execution_engine/dag/dag.hpp" #include "src/buildtool/execution_engine/executor/executor.hpp" #include "src/buildtool/execution_engine/traverser/traverser.hpp" @@ -675,6 +676,50 @@ class GraphTraverser { return; } } + // Not directly an artifact, hence check if the path is contained in + // some artifact + auto target_path = ToNormalPath(std::filesystem::path{ + *clargs_.build.print_to_stdout}) + .relative_path(); + auto remote = GetRemoteApi(); + for (size_t i = 0; i < paths.size(); i++) { + auto const& path = paths[i]; + auto relpath = target_path.lexically_relative(path); + if ((not relpath.empty()) and *relpath.begin() != "..") { + Logger::Log( + LogLevel::Info, + "'{}' not a direct logical path of the specified " + "target; will take subobject '{}' of '{}'", + *(clargs_.build.print_to_stdout), + relpath.string(), + path.string()); + auto info = artifacts[i]->Content().Info(); + if (info) { + auto new_info = + RetrieveSubPathId(*info, remote, relpath); + if (new_info) { + if (not remote_api_->RetrieveToFds( + {*new_info}, + {dup(fileno(stdout))}, + /*raw_tree=*/false)) { + Logger::Log(LogLevel::Error, + "Failed to retrieve artifact {} at " + "path '{}' of '{}'", + new_info->ToString(), + relpath.string(), + path.string()); + } + } + } + else { + Logger::Log( + LogLevel::Error, + "Failed to obtain object information for {}", + *(clargs_.build.print_to_stdout)); + } + return; + } + } Logger::Log(LogLevel::Warning, "{} not a logical path of the specified target", *(clargs_.build.print_to_stdout)); From bd4805a543e1e8b5a01812f1d91720e7b45bb738 Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Thu, 10 Aug 2023 15:40:39 +0200 Subject: [PATCH 46/97] Add basic end-to-end test for build -P ... also verifying that we can look into tree outputs --- test/end-to-end/cli/TARGETS | 8 ++++- test/end-to-end/cli/build-p.sh | 53 ++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 test/end-to-end/cli/build-p.sh diff --git a/test/end-to-end/cli/TARGETS b/test/end-to-end/cli/TARGETS index 2306af70e..b4a8f9962 100644 --- a/test/end-to-end/cli/TARGETS +++ b/test/end-to-end/cli/TARGETS @@ -16,9 +16,15 @@ , "test": ["install.sh"] , "deps": [["end-to-end", "tool-under-test"]] } +, "build -P": + { "type": ["@", "rules", "shell/test", "script"] + , "name": ["build-P"] + , "test": ["build-p.sh"] + , "deps": [["end-to-end", "tool-under-test"]] + } , "TESTS": { "type": "install" , "tainted": ["test"] - , "deps": ["defaults", "pwd", "install"] + , "deps": ["defaults", "pwd", "install", "build -P"] } } diff --git a/test/end-to-end/cli/build-p.sh b/test/end-to-end/cli/build-p.sh new file mode 100644 index 000000000..853ab3e3a --- /dev/null +++ b/test/end-to-end/cli/build-p.sh @@ -0,0 +1,53 @@ +#!/bin/sh +# Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e + +ROOT=$(pwd) +JUST=$(realpath ./bin/tool-under-test) +BUILDROOT="${TEST_TMPDIR}/build-root" + +touch ROOT +cat > TARGETS <<'EOF' +{ "": + { "type": "generic" + , "out_dirs": ["foo/bar"] + , "cmds": + [ "mkdir -p foo/bar/baz/greeting" + , "echo Hello World > foo/bar/baz/greeting/hello.txt" + ] + } +} +EOF + +"${JUST}" build --local-build-root "${BUILDROOT}" --dump-artifacts out.json 2>&1 +echo +cat out.json +# foo/bar is the output artifact, and it is a tree +[ $(jq -rM '."foo/bar"."file_type"' out.json) = "t" ] + +# Therefore, foo is not an output artifact, so requesting -P leaves stdout empty +"${JUST}" build --local-build-root "${BUILDROOT}" -P foo > foo.txt +[ -f foo.txt ] && [ -z "$(cat foo.txt)" ] + +# Requesting foo/bar gives a human-readable description of the tree +"${JUST}" build --local-build-root "${BUILDROOT}" -P foo/bar | grep baz + +# going deepter into the tree we stil can get human-readable tree descriptions +"${JUST}" build --local-build-root "${BUILDROOT}" -P foo/bar/baz/greeting | grep hello.txt + +# Files inside the tree can be retrieved +"${JUST}" build --local-build-root "${BUILDROOT}" -P foo/bar/baz/greeting/hello.txt | grep World + +echo OK From b6c119047a39bc97f7912723226d790182c038d1 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Fri, 4 Aug 2023 11:14:55 +0200 Subject: [PATCH 47/97] external: Update c-ares to v1.19.1 --- etc/import/include/TARGETS.cares | 8 +++++++- etc/import/src/lib/TARGETS.cares | 6 ++++-- etc/repos.json | 10 +++++----- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/etc/import/include/TARGETS.cares b/etc/import/include/TARGETS.cares index d93c2929f..39f0c8d7d 100644 --- a/etc/import/include/TARGETS.cares +++ b/etc/import/include/TARGETS.cares @@ -1,5 +1,11 @@ { "ares_include_headers": { "type": ["@", "rules", "data", "staged"] - , "srcs": ["ares.h", "ares_dns.h", "ares_rules.h", "ares_version.h"] + , "srcs": + [ "ares.h" + , "ares_dns.h" + , "ares_nameser.h" + , "ares_rules.h" + , "ares_version.h" + ] } } diff --git a/etc/import/src/lib/TARGETS.cares b/etc/import/src/lib/TARGETS.cares index 94431fd13..fd6934b04 100644 --- a/etc/import/src/lib/TARGETS.cares +++ b/etc/import/src/lib/TARGETS.cares @@ -59,9 +59,11 @@ , "ares_parse_soa_reply.c" , "ares_parse_srv_reply.c" , "ares_parse_txt_reply.c" + , "ares_parse_uri_reply.c" , "ares_platform.c" , "ares_process.c" , "ares_query.c" + , "ares_rand.c" , "ares_search.c" , "ares_send.c" , "ares_strcasecmp.c" @@ -71,6 +73,8 @@ , "ares_timeout.c" , "ares_version.c" , "ares_writev.c" + , "ares__addrinfo2hostent.c" + , "ares__addrinfo_localhost.c" , "ares__close_sockets.c" , "ares__get_hostent.c" , "ares__parse_into_addrinfo.c" @@ -93,9 +97,7 @@ , "ares_inet_net_pton.h" , "ares_iphlpapi.h" , "ares_ipv6.h" - , "ares_library_init.h" , "ares_llist.h" - , "ares_nameser.h" , "ares_nowarn.h" , "ares_platform.h" , "ares_private.h" diff --git a/etc/repos.json b/etc/repos.json index 7bad15595..5207e3b68 100644 --- a/etc/repos.json +++ b/etc/repos.json @@ -364,11 +364,11 @@ , "com_github_cares_cares": { "repository": { "type": "archive" - , "content": "998c90eda4410f965ca086ce59811e2cf16c8d6b" - , "fetch": "https://github.com/c-ares/c-ares/archive/refs/tags/cares-1_17_2.tar.gz" - , "sha256": "444b7dbf33cb6e8a764d069e3485cbaaa28386d60eace6988275c3cb9534d300" - , "sha512": "1111f1e7eeb0e5d9e70d1a7c8566145d0a5e6e71b020f3fcaa02ecdf1931553ddeff83fdc152a1f9c5a780078e8afe3670164b631df56eecd2b638210cc59bb3" - , "subdir": "c-ares-cares-1_17_2" + , "content": "4237a53f7a5dc1cfb4d04a6b6374f8674c32e271" + , "fetch": "https://github.com/c-ares/c-ares/releases/download/cares-1_19_1/c-ares-1.19.1.tar.gz" + , "sha256": "321700399b72ed0e037d0074c629e7741f6b2ec2dda92956abe3e9671d3e268e" + , "sha512": "466a94efda626e815a6ef7a890637056339f883d549ea6055e289fd8cd2391130e5682c905c0fb3bd7e955af7f6deb793562c170eb0ee066a4a62085a82ba470" + , "subdir": "c-ares-1.19.1" } , "target_root": "import targets" , "target_file_name": "TARGETS.cares" From 22d5f3af715af3d79ec6c002e8609ab8a35e69b9 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Thu, 10 Aug 2023 09:50:59 +0200 Subject: [PATCH 48/97] external: Update libarchive to v3.7.1 --- etc/defaults/CC/TARGETS.archive | 4 ++- etc/import/libarchive/TARGETS.archive | 45 ++++++++++++++++++++------- etc/repos.json | 10 +++--- 3 files changed, 41 insertions(+), 18 deletions(-) diff --git a/etc/defaults/CC/TARGETS.archive b/etc/defaults/CC/TARGETS.archive index a5b1cd4ce..f5a792a81 100644 --- a/etc/defaults/CC/TARGETS.archive +++ b/etc/defaults/CC/TARGETS.archive @@ -84,7 +84,9 @@ ] } ] - , ["mingw", ["-D__USE_MINGW_ANSI_STDIO"]] + , [ "mingw" + , ["-D__USE_MINGW_ANSI_STDIO", "-D__MINGW_USE_VC2005_COMPAT"] + ] ] } , { "type": "case*" diff --git a/etc/import/libarchive/TARGETS.archive b/etc/import/libarchive/TARGETS.archive index cc4de4f22..de471234a 100644 --- a/etc/import/libarchive/TARGETS.archive +++ b/etc/import/libarchive/TARGETS.archive @@ -58,6 +58,7 @@ , "errno.h" , "ext2fs/ext2_fs.h" , "fcntl.h" + , "fnmatch.h" , "grp.h" , "inttypes.h" , "io.h" @@ -92,6 +93,7 @@ , "sys/mount.h" , "sys/param.h" , "sys/poll.h" + , "sys/queue.h" , "sys/richacl.h" , "sys/select.h" , "sys/stat.h" @@ -112,7 +114,7 @@ , "windows.h" , "wincrypt.h" , "winioctl.h" - , "Bcrypt.h" + , "bcrypt.h" , "mbedtls/aes.h" , "mbedtls/md.h" , "mbedtls/pkcs5.h" @@ -141,6 +143,7 @@ , "libxml/xmlwriter.h" , "assert.h" , "sys/richacl.h" + , "getopt.h" ] ] , [ "defines1" @@ -259,6 +262,7 @@ , ["HAVE_ERRNO_H", "errno.h"] , ["HAVE_EXT2FS_EXT2_FS_H", "ext2fs/ext2_fs.h"] , ["HAVE_FCNTL_H", "fcntl.h"] + , ["HAVE_FNMATCH_H", "fnmatch.h"] , ["HAVE_GRP_H", "grp.h"] , ["HAVE_INTTYPES_H", "inttypes.h"] , ["HAVE_IO_H", "io.h"] @@ -293,6 +297,7 @@ , ["HAVE_SYS_MOUNT_H", "sys/mount.h"] , ["HAVE_SYS_PARAM_H", "sys/param.h"] , ["HAVE_SYS_POLL_H", "sys/poll.h"] + , ["HAVE_SYS_QUEUE_H", "sys/queue.h"] , ["HAVE_SYS_RICHACL_H", "sys/richacl.h"] , ["HAVE_SYS_SELECT_H", "sys/select.h"] , ["HAVE_SYS_STAT_H", "sys/stat.h"] @@ -318,6 +323,10 @@ , "cond": {"type": "var", "name": "ENABLE_ZLIB"} , "then": [["HAVE_ZLIB_H", "zlib.h"]] } + , { "type": "if" + , "cond": {"type": "var", "name": "ENABLE_OPENSSL"} + , "then": [["HAVE_OPENSSL_EVP_H", "openssl/evp.h"]] + } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_BZip2"} , "then": [["HAVE_BZLIB_H", "bzlib.h"]] @@ -347,7 +356,7 @@ } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_CNG"} - , "then": [["HAVE_BCRYPT_H", "Bcrypt.h"]] + , "then": [["HAVE_BCRYPT_H", "bcrypt.h"]] } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_MBEDTLS"} @@ -475,6 +484,9 @@ , [ "HAVE_FDOPENDIR" , ["fdopendir", {"type": "var", "name": "sys_hdrs"}] ] + , [ "HAVE_FNMATCH" + , ["fnmatch", {"type": "var", "name": "sys_hdrs"}] + ] , ["HAVE_FORK", ["fork", {"type": "var", "name": "sys_hdrs"}]] , ["HAVE_FSTAT", ["fstat", {"type": "var", "name": "sys_hdrs"}]] , [ "HAVE_FSTATAT" @@ -507,6 +519,9 @@ , [ "HAVE_GETGRNAM_R" , ["getgrnam_r", {"type": "var", "name": "sys_hdrs"}] ] + , [ "HAVE_GETGRNAM_R" + , ["getgrnam_r", {"type": "var", "name": "sys_hdrs"}] + ] , [ "HAVE_GETPWNAM_R" , ["getpwnam_r", {"type": "var", "name": "sys_hdrs"}] ] @@ -654,15 +669,10 @@ , [ "HAVE__GET_TIMEZONE" , ["_get_timezone", {"type": "var", "name": "sys_hdrs"}] ] - , [ "HAVE__GMTIME64_S" - , ["_gmtime64_s", {"type": "var", "name": "sys_hdrs"}] - ] - , [ "HAVE__LOCALTIME64_S" - , ["_localtime64_s", {"type": "var", "name": "sys_hdrs"}] - ] - , [ "HAVE__MKGMTIME64" - , ["_mkgmtime64", {"type": "var", "name": "sys_hdrs"}] - ] + , ["HAVE_CTIME_S", ["ctime_s", ["time.h"]]] + , ["HAVE_GMTIME_S", ["gmtime_s", ["time.h"]]] + , ["HAVE_LOCALTIME_S", ["localtime_s", ["time.h"]]] + , ["HAVE__MKGMTIME", ["_mkgmtime", ["time.h"]]] ] , { "type": "if" , "cond": @@ -717,6 +727,7 @@ , ["HAVE_READDIR_R", ["readdir_r", ["dirent.h"]]] , ["HAVE_DIRFD", ["dirfd", ["dirent.h"]]] , ["HAVE_READLINKAT", ["readlinkat", ["fcntl.h", "unistd.h"]]] + , ["HAVE_GETOPT_OPTRESET", ["optreset", ["getopt.h"]]] , ["MAJOR_IN_MKDEV", ["major", ["sys/mkdev.h"]]] , ["MAJOR_IN_SYSMACROS", ["major", ["sys/sysmacros.h"]]] ] @@ -825,12 +836,22 @@ , {"type": "var", "name": "sys_hdrs"} ] ] - , [ "TIME_WITH_SYS_TIME" + , [ "HAVE_SYS_TIME_H" , [ "((struct tm*)NULL)->tm_sec" , {"type": "var", "name": "sys_hdrs"} ] ] ] + , { "type": "if" + , "cond": {"type": "var", "name": "ENABLE_OPENSSL"} + , "then": + [ [ "HAVE_PKCS5_PBKDF2_HMAC_SHA1" + , [ "PKCS5_PBKDF2_HMAC_SHA1" + , {"type": "var", "name": "sys_hdrs"} + ] + ] + ] + } , { "type": "if" , "cond": {"type": "var", "name": "ENABLE_LZMA"} , "then": diff --git a/etc/repos.json b/etc/repos.json index 5207e3b68..476342862 100644 --- a/etc/repos.json +++ b/etc/repos.json @@ -456,11 +456,11 @@ , "com_github_libarchive_libarchive": { "repository": { "type": "archive" - , "content": "2dafab016d6338086925fd04dbaa6a18bd24cb31" - , "fetch": "https://github.com/libarchive/libarchive/releases/download/v3.6.2/libarchive-3.6.2.tar.gz" - , "sha256": "ba6d02f15ba04aba9c23fd5f236bb234eab9d5209e95d1c4df85c44d5f19b9b3" - , "sha512": "24e476465054a29a2d48adf3c197a171b5361fa5038729a1f14d578c6701424de4e5dd6a2b20a6b697969ab43bdd8afc1585f8de0465c266f455d7eaa19e5048" - , "subdir": "libarchive-3.6.2" + , "content": "922b6c729d391c010ffa6228134c164886bac0f1" + , "fetch": "https://github.com/libarchive/libarchive/releases/download/v3.7.1/libarchive-3.7.1.tar.gz" + , "sha256": "5d24e40819768f74daf846b99837fc53a3a9dcdf3ce1c2003fe0596db850f0f0" + , "sha512": "ccd36fc5e24e5feaad15b08496c4f6373f1bbb6ba657af5dc633b9d1454c98aba9a59ccffc48b41afc1219119d46c15d4db09c5855950422946e4dc058f17b88" + , "subdir": "libarchive-3.7.1" } , "target_root": "import targets" , "target_file_name": "TARGETS.archive" From a7ca2cd11667bfd4f06061b362c162e6380532f3 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Mon, 7 Aug 2023 17:00:01 +0200 Subject: [PATCH 49/97] external: Update libcurl to v8.2.1 --- etc/import/TARGETS.curl | 8 ++++++++ etc/import/lib/TARGETS.curl | 13 +++++++++++++ etc/repos.json | 10 +++++----- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/etc/import/TARGETS.curl b/etc/import/TARGETS.curl index 9f77fef4c..988f905dc 100644 --- a/etc/import/TARGETS.curl +++ b/etc/import/TARGETS.curl @@ -56,6 +56,7 @@ , "USE_GNU_STRERROR_R" , "HAVE_RAND_EGD" , "HAVE_BORINGSSL" + , "HAVE_AWSLC" , "HAVE_SSL_CTX_SET_QUIC_METHOD" , "HAVE_QUICHE_CONN_SET_QLOG_FD" , "HAVE_ZSTD_CREATEDSTREAM" @@ -173,6 +174,10 @@ [ "Boolean. OpenSSL is BoringSSL." , "If true, corresponding symbol detection logic is skipped." ] + , "HAVE_AWSLC": + [ "Boolean. OpenSSL is AWSLC." + , "If true, corresponding symbol detection logic is skipped." + ] , "HAVE_SSL_CTX_SET_QUIC_METHOD": [ "Boolean. `SSL_CTX_set_quic_method` present in OpenSSL." , "If true, corresponding symbol detection logic is skipped." @@ -234,6 +239,7 @@ , "USE_GNU_STRERROR_R" , "HAVE_RAND_EGD" , "HAVE_BORINGSSL" + , "HAVE_AWSLC" , "HAVE_SSL_CTX_SET_QUIC_METHOD" , "HAVE_QUICHE_CONN_SET_QLOG_FD" , "HAVE_ZSTD_CREATEDSTREAM" @@ -402,6 +408,7 @@ , [ "HAVE_BORINGSSL" , {"type": "var", "name": "HAVE_BORINGSSL", "default": false} ] + , ["HAVE_AWSLC", {"type": "var", "name": "HAVE_AWSLC", "default": false}] , [ "HAVE_SSL_CTX_SET_QUIC_METHOD" , { "type": "var" , "name": "HAVE_SSL_CTX_SET_QUIC_METHOD" @@ -459,6 +466,7 @@ , "USE_GNU_STRERROR_R" , "HAVE_RAND_EGD" , "HAVE_BORINGSSL" + , "HAVE_AWSLC" , "HAVE_SSL_CTX_SET_QUIC_METHOD" , "HAVE_QUICHE_CONN_SET_QLOG_FD" , "HAVE_ZSTD_CREATEDSTREAM" diff --git a/etc/import/lib/TARGETS.curl b/etc/import/lib/TARGETS.curl index f26139c61..397f14741 100644 --- a/etc/import/lib/TARGETS.curl +++ b/etc/import/lib/TARGETS.curl @@ -42,6 +42,7 @@ , "USE_GNU_STRERROR_R" , "HAVE_RAND_EGD" , "HAVE_BORINGSSL" + , "HAVE_AWSLC" , "HAVE_SSL_CTX_SET_QUIC_METHOD" , "HAVE_QUICHE_CONN_SET_QLOG_FD" , "HAVE_ZSTD_CREATEDSTREAM" @@ -126,6 +127,7 @@ , ["HAVE_VARIADIC_MACROS_GCC", 1] , ["HAVE_RAND_EGD", {"type": "var", "name": "HAVE_RAND_EGD"}] , ["HAVE_BORINGSSL", {"type": "var", "name": "HAVE_BORINGSSL"}] + , ["HAVE_AWSLC", {"type": "var", "name": "HAVE_AWSLC"}] , ["HAVE_LIBZ", {"type": "var", "name": "USE_ZLIB"}] ] , { "type": "if" @@ -438,6 +440,16 @@ ] } } + , { "type": "if" + , "cond": {"type": "var", "name": "CURL_ENABLE_SSL"} + , "then": + { "type": "if" + , "cond": {"type": "var", "name": "HAVE_AWSLC"} + , "then": [] + , "else": + [["HAVE_AWSLC", ["OPENSSL_IS_AWSLC", ["openssl/base.h"]]]] + } + } , { "type": "if" , "cond": {"type": "var", "name": "USE_QUICHE"} , "then": @@ -921,6 +933,7 @@ } } , [["SIZEOF_CURL_OFF_T", "8"]] + , [["SIZEOF_CURL_SOCKET_T", "8"]] ] } ] diff --git a/etc/repos.json b/etc/repos.json index 476342862..a73989bbf 100644 --- a/etc/repos.json +++ b/etc/repos.json @@ -437,11 +437,11 @@ , "com_github_curl_curl": { "repository": { "type": "archive" - , "content": "540b4de2edfa015b894ad86d51b33a55f451ac99" - , "fetch": "https://github.com/curl/curl/releases/download/curl-8_0_1/curl-8.0.1.tar.gz" - , "sha256": "5fd29000a4089934f121eff456101f0a5d09e2a3e89da1d714adf06c4be887cb" - , "sha512": "87d945e65176613c6296f8b130bd6c5355e3ca3c62ee4010edd481950cad9760482fd3be8a84e51501c118c29ebc284b8f87c49b06f9d248e9da5819fda38f8f" - , "subdir": "curl-8.0.1" + , "content": "23e33f9883e014ced35cc09f66b6b79a0b24e18d" + , "fetch": "https://github.com/curl/curl/releases/download/curl-8_2_1/curl-8.2.1.tar.gz" + , "sha256": "f98bdb06c0f52bdd19e63c4a77b5eb19b243bcbbd0f5b002b9f3cba7295a3a42" + , "sha512": "d0a906f4dff4c485e6dae930d9a7530147f4c0a0cbb46a83cb9be9d7bd6b9c320386c8be5bcdd3749f2d468b0daf39d06e8c581bab1fa792fd26da409a575cbd" + , "subdir": "curl-8.2.1" } , "target_root": "import targets" , "target_file_name": "TARGETS.curl" From aa0c3c98f43e5a455f01bd9a7a472d62a3096865 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Fri, 11 Aug 2023 16:58:33 +0200 Subject: [PATCH 50/97] external: Clean up absl/random targets --- etc/import/absl/random/TARGETS.absl | 177 ++++++++++++++-------------- 1 file changed, 87 insertions(+), 90 deletions(-) diff --git a/etc/import/absl/random/TARGETS.absl b/etc/import/absl/random/TARGETS.absl index 9df9c0194..bd5b778b6 100644 --- a/etc/import/absl/random/TARGETS.absl +++ b/etc/import/absl/random/TARGETS.absl @@ -25,17 +25,17 @@ , "stage": ["absl", "random"] , "hdrs": ["random.h"] , "deps": - [ "random_distributions" - , "random_seed_sequences" - , "random_internal_nonsecure_base" - , "random_internal_pcg_engine" - , "random_internal_pool_urbg" - , "random_internal_randen_engine" + [ "distributions" + , "seed_sequences" + , "internal_nonsecure_base" + , "internal_pcg_engine" + , "internal_pool_urbg" + , "internal_randen_engine" ] } -, "random_distributions": +, "distributions": { "type": ["@", "rules", "CC", "library"] - , "name": ["random_distributions"] + , "name": ["distributions"] , "stage": ["absl", "random"] , "hdrs": [ "bernoulli_distribution.h" @@ -57,107 +57,107 @@ , ["absl/base", "core_headers"] , ["absl/meta", "type_traits"] , ["absl/numeric", "bits"] - , "random_internal_distribution_caller" - , "random_internal_fast_uniform_bits" - , "random_internal_fastmath" - , "random_internal_generate_real" - , "random_internal_iostream_state_saver" - , "random_internal_traits" - , "random_internal_uniform_helper" - , "random_internal_wide_multiply" + , "internal_distribution_caller" + , "internal_fast_uniform_bits" + , "internal_fastmath" + , "internal_generate_real" + , "internal_iostream_state_saver" + , "internal_traits" + , "internal_uniform_helper" + , "internal_wide_multiply" , ["absl/strings", "strings"] ] } -, "random_internal_nonsecure_base": +, "internal_nonsecure_base": { "type": ["@", "rules", "CC", "library"] - , "name": ["random_internal_nonsecure_base"] + , "name": ["internal_nonsecure_base"] , "stage": ["absl", "random"] , "hdrs": ["internal/nonsecure_base.h"] , "deps": - [ "random_internal_pool_urbg" - , "random_internal_salted_seed_seq" - , "random_internal_seed_material" + [ "internal_pool_urbg" + , "internal_salted_seed_seq" + , "internal_seed_material" , ["absl/base", "core_headers"] , ["absl/container", "inlined_vector"] , ["absl/meta", "type_traits"] , ["absl/types", "span"] ] } -, "random_internal_pcg_engine": +, "internal_pcg_engine": { "type": ["@", "rules", "CC", "library"] - , "name": ["random_internal_pcg_engine"] + , "name": ["internal_pcg_engine"] , "stage": ["absl", "random"] , "hdrs": ["internal/pcg_engine.h"] , "deps": [ ["absl/base", "config"] , ["absl/numeric", "int128"] - , "random_internal_fastmath" - , "random_internal_iostream_state_saver" + , "internal_fastmath" + , "internal_iostream_state_saver" , ["absl/meta", "type_traits"] ] } -, "random_internal_pool_urbg": +, "internal_pool_urbg": { "type": ["@", "rules", "CC", "library"] - , "name": ["random_internal_pool_urbg"] + , "name": ["internal_pool_urbg"] , "stage": ["absl", "random"] , "hdrs": ["internal/pool_urbg.h"] , "srcs": ["internal/pool_urbg.cc"] , "private-ldflags": ["-pthread"] , "deps": - [ "random_internal_randen" - , "random_internal_seed_material" - , "random_internal_traits" + [ "internal_randen" + , "internal_seed_material" + , "internal_traits" , ["absl/base", "base"] , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "endian"] , ["absl/base", "raw_logging_internal"] - , "random_seed_gen_exception" + , "seed_gen_exception" , ["absl/types", "span"] ] } -, "random_internal_randen_engine": +, "internal_randen_engine": { "type": ["@", "rules", "CC", "library"] - , "name": ["random_internal_randen_engine"] + , "name": ["internal_randen_engine"] , "stage": ["absl", "random"] , "hdrs": ["internal/randen_engine.h"] , "deps": - [ "random_internal_iostream_state_saver" - , "random_internal_randen" + [ "internal_iostream_state_saver" + , "internal_randen" , ["absl/base", "endian"] , ["absl/meta", "type_traits"] ] } -, "random_seed_sequences": +, "seed_sequences": { "type": ["@", "rules", "CC", "library"] - , "name": ["random_seed_sequences"] + , "name": ["seed_sequences"] , "stage": ["absl", "random"] , "hdrs": ["seed_sequences.h"] , "srcs": ["seed_sequences.cc"] , "deps": - [ "random_seed_gen_exception" + [ "seed_gen_exception" , ["absl/base", "config"] - , "random_internal_pool_urbg" - , "random_internal_salted_seed_seq" - , "random_internal_seed_material" + , "internal_pool_urbg" + , "internal_salted_seed_seq" + , "internal_seed_material" , ["absl/types", "span"] ] } -, "random_internal_generate_real": +, "internal_generate_real": { "type": ["@", "rules", "CC", "library"] - , "name": ["random_internal_generate_real"] + , "name": ["internal_generate_real"] , "stage": ["absl", "random"] , "hdrs": ["internal/generate_real.h"] , "deps": - [ "random_internal_fastmath" - , "random_internal_traits" + [ "internal_fastmath" + , "internal_traits" , ["absl/meta", "type_traits"] , ["absl/numeric", "bits"] ] } -, "random_internal_distribution_caller": +, "internal_distribution_caller": { "type": ["@", "rules", "CC", "library"] - , "name": ["random_internal_distribution_caller"] + , "name": ["internal_distribution_caller"] , "stage": ["absl", "random"] , "hdrs": ["internal/distribution_caller.h"] , "deps": @@ -166,34 +166,31 @@ , ["absl/utility", "utility"] ] } -, "random_internal_fast_uniform_bits": +, "internal_fast_uniform_bits": { "type": ["@", "rules", "CC", "library"] - , "name": ["random_internal_fast_uniform_bits"] + , "name": ["internal_fast_uniform_bits"] , "stage": ["absl", "random"] , "hdrs": ["internal/fast_uniform_bits.h"] , "deps": - [ "random_internal_traits" - , ["absl/base", "config"] - , ["absl/meta", "type_traits"] - ] + ["internal_traits", ["absl/base", "config"], ["absl/meta", "type_traits"]] } -, "random_internal_fastmath": +, "internal_fastmath": { "type": ["@", "rules", "CC", "library"] - , "name": ["random_internal_fastmath"] + , "name": ["internal_fastmath"] , "stage": ["absl", "random"] , "hdrs": ["internal/fastmath.h"] , "deps": [["absl/numeric", "bits"]] } -, "random_internal_iostream_state_saver": +, "internal_iostream_state_saver": { "type": ["@", "rules", "CC", "library"] - , "name": ["random_internal_iostream_state_saver"] + , "name": ["internal_iostream_state_saver"] , "stage": ["absl", "random"] , "hdrs": ["internal/iostream_state_saver.h"] , "deps": [["absl/meta", "type_traits"], ["absl/numeric", "int128"]] } -, "random_internal_traits": +, "internal_traits": { "type": ["@", "rules", "CC", "library"] - , "name": ["random_internal_traits"] + , "name": ["internal_traits"] , "stage": ["absl", "random"] , "hdrs": ["internal/traits.h"] , "deps": @@ -202,47 +199,47 @@ , ["absl/numeric", "int128"] ] } -, "random_internal_uniform_helper": +, "internal_uniform_helper": { "type": ["@", "rules", "CC", "library"] - , "name": ["random_internal_uniform_helper"] + , "name": ["internal_uniform_helper"] , "stage": ["absl", "random"] , "hdrs": ["internal/uniform_helper.h"] , "deps": - [ "random_internal_traits" + [ "internal_traits" , ["absl/base", "config"] , ["absl/meta", "type_traits"] , ["absl/numeric", "int128"] ] } -, "random_internal_wide_multiply": +, "internal_wide_multiply": { "type": ["@", "rules", "CC", "library"] - , "name": ["random_internal_wide_multiply"] + , "name": ["internal_wide_multiply"] , "stage": ["absl", "random"] , "hdrs": ["internal/wide_multiply.h"] , "deps": - [ "random_internal_traits" + [ "internal_traits" , ["absl/base", "config"] , ["absl/numeric", "bits"] , ["absl/numeric", "int128"] ] } -, "random_internal_salted_seed_seq": +, "internal_salted_seed_seq": { "type": ["@", "rules", "CC", "library"] - , "name": ["random_internal_salted_seed_seq"] + , "name": ["internal_salted_seed_seq"] , "stage": ["absl", "random"] , "hdrs": ["internal/salted_seed_seq.h"] , "deps": - [ "random_internal_seed_material" + [ "internal_seed_material" , ["absl/container", "inlined_vector"] , ["absl/meta", "type_traits"] , ["absl/types", "optional"] , ["absl/types", "span"] ] } -, "random_internal_seed_material": +, "internal_seed_material": { "type": ["@", "rules", "CC", "library"] , "arguments_config": ["USE_SYSTEM_LIBS", "OS"] - , "name": ["random_internal_seed_material"] + , "name": ["internal_seed_material"] , "stage": ["absl", "random"] , "hdrs": ["internal/seed_material.h"] , "srcs": ["internal/seed_material.cc"] @@ -258,7 +255,7 @@ , "deps": { "type": "++" , "$1": - [ [ "random_internal_fast_uniform_bits" + [ [ "internal_fast_uniform_bits" , ["absl/base", "core_headers"] , ["absl/base", "dynamic_annotations"] , ["absl/base", "raw_logging_internal"] @@ -278,65 +275,65 @@ ] } } -, "random_internal_randen": +, "internal_randen": { "type": ["@", "rules", "CC", "library"] - , "name": ["random_internal_randen"] + , "name": ["internal_randen"] , "stage": ["absl", "random"] , "hdrs": ["internal/randen.h"] , "srcs": ["internal/randen.cc"] , "deps": - [ "random_internal_platform" - , "random_internal_randen_hwaes" - , "random_internal_randen_slow" + [ "internal_platform" + , "internal_randen_hwaes" + , "internal_randen_slow" , ["absl/base", "raw_logging_internal"] ] } -, "random_seed_gen_exception": +, "seed_gen_exception": { "type": ["@", "rules", "CC", "library"] - , "name": ["random_seed_gen_exception"] + , "name": ["seed_gen_exception"] , "stage": ["absl", "random"] , "hdrs": ["seed_gen_exception.h"] , "srcs": ["seed_gen_exception.cc"] , "deps": [["absl/base", "config"]] } -, "random_internal_platform": +, "internal_platform": { "type": ["@", "rules", "CC", "library"] - , "name": ["random_internal_platform"] + , "name": ["internal_platform"] , "stage": ["absl", "random"] , "hdrs": ["internal/randen_traits.h", "internal/platform.h"] , "srcs": ["internal/randen_round_keys.cc"] , "deps": [["absl/base", "config"]] } -, "random_internal_randen_hwaes": +, "internal_randen_hwaes": { "type": ["@", "rules", "CC", "library"] - , "name": ["random_internal_randen_hwaes"] + , "name": ["internal_randen_hwaes"] , "arguments_config": ["TARGET_ARCH", "COMPILER_FAMILY"] , "stage": ["absl", "random"] , "hdrs": ["internal/randen_detect.h", "internal/randen_hwaes.h"] , "srcs": ["internal/randen_detect.cc"] , "deps": - [ "random_internal_platform" - , "random_internal_randen_hwaes_impl" + [ "internal_platform" + , "internal_randen_hwaes_impl" , ["absl/base", "config"] ] } -, "random_internal_randen_slow": +, "internal_randen_slow": { "type": ["@", "rules", "CC", "library"] - , "name": ["random_internal_randen_slow"] + , "name": ["internal_randen_slow"] , "stage": ["absl", "random"] , "hdrs": ["internal/randen_slow.h"] , "srcs": ["internal/randen_slow.cc"] , "deps": - [ "random_internal_platform" + [ "internal_platform" , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/base", "endian"] , ["absl/numeric", "int128"] ] } -, "random_internal_randen_hwaes_impl": +, "internal_randen_hwaes_impl": { "type": ["@", "rules", "CC", "library"] - , "name": ["random_internal_randen_hwaes_impl"] + , "name": ["internal_randen_hwaes_impl"] , "arguments_config": ["TARGET_ARCH", "COMPILER_FAMILY"] , "stage": ["absl", "random"] , "private-hdrs": ["internal/randen_hwaes.h"] @@ -357,7 +354,7 @@ ] } , "deps": - [ "random_internal_platform" + [ "internal_platform" , ["absl/base", "config"] , ["absl/base", "core_headers"] , ["absl/numeric", "int128"] From e294c501a76e9388afe4e19d5985484dc50ca905 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Fri, 11 Aug 2023 17:01:12 +0200 Subject: [PATCH 51/97] external: Fix missing abseil export targets --- etc/import/absl/flags/TARGETS.absl | 20 ++++++++++++++++++++ etc/import/absl/numeric/TARGETS.absl | 20 ++++++++++++++++++++ etc/import/absl/random/TARGETS.absl | 21 +++++++++++++++++++++ 3 files changed, 61 insertions(+) diff --git a/etc/import/absl/flags/TARGETS.absl b/etc/import/absl/flags/TARGETS.absl index d3f9d2c39..0ebca782a 100644 --- a/etc/import/absl/flags/TARGETS.absl +++ b/etc/import/absl/flags/TARGETS.absl @@ -35,6 +35,26 @@ ] } , "marshalling": + { "type": "export" + , "target": "marshalling_internal" + , "flexible_config": + [ "ADD_CFLAGS" + , "ADD_CXXFLAGS" + , "AR" + , "ARCH" + , "CC" + , "CFLAGS" + , "COMPILER_FAMILY" + , "CXX" + , "CXXFLAGS" + , "DEBUG" + , "ENV" + , "HOST_ARCH" + , "OS" + , "TARGET_ARCH" + ] + } +, "marshalling_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["marshalling"] , "stage": ["absl", "flags"] diff --git a/etc/import/absl/numeric/TARGETS.absl b/etc/import/absl/numeric/TARGETS.absl index f38f11011..4ebf47a6e 100644 --- a/etc/import/absl/numeric/TARGETS.absl +++ b/etc/import/absl/numeric/TARGETS.absl @@ -1,4 +1,24 @@ { "int128": + { "type": "export" + , "target": "int128_internal" + , "flexible_config": + [ "ADD_CFLAGS" + , "ADD_CXXFLAGS" + , "AR" + , "ARCH" + , "CC" + , "CFLAGS" + , "COMPILER_FAMILY" + , "CXX" + , "CXXFLAGS" + , "DEBUG" + , "ENV" + , "HOST_ARCH" + , "OS" + , "TARGET_ARCH" + ] + } +, "int128_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["int128"] , "stage": ["absl", "numeric"] diff --git a/etc/import/absl/random/TARGETS.absl b/etc/import/absl/random/TARGETS.absl index bd5b778b6..2407f1f61 100644 --- a/etc/import/absl/random/TARGETS.absl +++ b/etc/import/absl/random/TARGETS.absl @@ -34,6 +34,27 @@ ] } , "distributions": + { "type": "export" + , "target": "distributions_internal" + , "flexible_config": + [ "ADD_CFLAGS" + , "ADD_CXXFLAGS" + , "AR" + , "ARCH" + , "CC" + , "CFLAGS" + , "COMPILER_FAMILY" + , "CXX" + , "CXXFLAGS" + , "DEBUG" + , "ENV" + , "HOST_ARCH" + , "OS" + , "TARGET_ARCH" + , "USE_SYSTEM_LIBS" + ] + } +, "distributions_internal": { "type": ["@", "rules", "CC", "library"] , "name": ["distributions"] , "stage": ["absl", "random"] From 0fe237e2e9c23e8b482cdc5ff6efc8b8b2a4feda Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Mon, 14 Aug 2023 11:28:10 +0200 Subject: [PATCH 52/97] Update CHANGELOG --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17df375a1..90259af6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ A feature release on top of `1.1.0`, backwards compatible. - Relative non-upwards symbolic links are now treated as first-class objects. This introduces a new artifact type and allows the free use of such symbolic links throughout the build process. +- `just-mr` can now optionally resolve symlinks contained in archives. ### Other changes @@ -24,6 +25,8 @@ A feature release on top of `1.1.0`, backwards compatible. garbage collection. - `just-mr` now supports unpacking tar archives compressed with bzip2 and xz. +- The option `-P` of `build` and `install-cas` can be used to + inspect parts of a tree. ### Fixes From 755360ab129879c8981c84262ef1f1d3e697a5e0 Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Mon, 14 Aug 2023 11:45:49 +0200 Subject: [PATCH 53/97] expressions: add constant for true --- src/buildtool/build_engine/expression/expression.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/buildtool/build_engine/expression/expression.hpp b/src/buildtool/build_engine/expression/expression.hpp index 86b6cc384..9915c0894 100644 --- a/src/buildtool/build_engine/expression/expression.hpp +++ b/src/buildtool/build_engine/expression/expression.hpp @@ -243,6 +243,7 @@ class Expression { inline static ExpressionPtr const kEmptyMapExpr = Expression::FromJson(R"({"type": "empty_map"})"_json); inline static ExpressionPtr const kOne = Expression::FromJson("1.0"_json); + inline static ExpressionPtr const kTrue = Expression::FromJson("true"_json); private: std::variant Date: Mon, 14 Aug 2023 11:43:28 +0200 Subject: [PATCH 54/97] expression: add new built in "set" ... to obtain from a list of strings a map with those entries as keys and true as value. In this way, repeated membership tests in lists can be implemented more efficiently. --- doc/concepts/expressions.md | 4 ++++ .../build_engine/expression/evaluator.cpp | 19 +++++++++++++++++++ .../expression/expression.test.cpp | 18 ++++++++++++++++++ 3 files changed, 41 insertions(+) diff --git a/doc/concepts/expressions.md b/doc/concepts/expressions.md index b6aa09458..c66de8bcb 100644 --- a/doc/concepts/expressions.md +++ b/doc/concepts/expressions.md @@ -249,6 +249,10 @@ those) argument(s) to obtain the final result. same iteration order as the list for all lists indexable by 32-bit integers. +- `"set"` The argument has to be a list of strings. The result is + a map with the members of the list as keys, and all values being + `true`. + - `"++"` The argument has to be a list of lists. The result is the concatenation of those lists. diff --git a/src/buildtool/build_engine/expression/evaluator.cpp b/src/buildtool/build_engine/expression/evaluator.cpp index 6aeea5912..431792e8c 100644 --- a/src/buildtool/build_engine/expression/evaluator.cpp +++ b/src/buildtool/build_engine/expression/evaluator.cpp @@ -167,6 +167,24 @@ auto Enumerate(ExpressionPtr const& expr) -> ExpressionPtr { return ExpressionPtr{Expression::map_t{result}}; } +auto Set(ExpressionPtr const& expr) -> ExpressionPtr { + if (not expr->IsList()) { + throw Evaluator::EvaluationError{ + fmt::format("set expects list of strings but instead got: {}.", + expr->ToString())}; + } + auto result = Expression::map_t::underlying_map_t{}; + for (auto const& entry : expr->List()) { + if (not entry->IsString()) { + throw Evaluator::EvaluationError{ + fmt::format("set expects list of strings found entry: {}.", + entry->ToString())}; + } + result[entry->String()] = Expression::kTrue; + } + return ExpressionPtr{Expression::map_t{result}}; +} + auto NubRight(ExpressionPtr const& expr) -> ExpressionPtr { if (not expr->IsList()) { throw Evaluator::EvaluationError{fmt::format( @@ -938,6 +956,7 @@ auto const kBuiltInFunctions = {"escape_chars", EscapeCharsExpr}, {"keys", UnaryExpr(Keys)}, {"enumerate", UnaryExpr(Enumerate)}, + {"set", UnaryExpr(Set)}, {"values", UnaryExpr(Values)}, {"lookup", LookupExpr}, {"empty_map", EmptyMapExpr}, diff --git a/test/buildtool/build_engine/expression/expression.test.cpp b/test/buildtool/build_engine/expression/expression.test.cpp index 10edbcbd4..7f3df7c54 100644 --- a/test/buildtool/build_engine/expression/expression.test.cpp +++ b/test/buildtool/build_engine/expression/expression.test.cpp @@ -924,6 +924,24 @@ TEST_CASE("Expression Evaluation", "[expression]") { // NOLINT )"_json)); } + SECTION("set expression") { + auto expr = Expression::FromJson(R"( + { "type": "set" + , "$1": ["foo", "bar", "baz"] + } + )"_json); + REQUIRE(expr); + + auto result = expr.Evaluate(env, fcts); + REQUIRE(result); + CHECK(result == Expression::FromJson(R"( + { "foo": true + , "bar": true + , "baz": true + } + )"_json)); + } + SECTION("keys expression") { auto expr = Expression::FromJson(R"( { "type": "keys" From cbe7bde952b4ed5d910e9a152f953ba172c5f654 Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Mon, 14 Aug 2023 12:13:05 +0200 Subject: [PATCH 55/97] expression: add new built in "reverse" While a foldl is enough to implement a reverse functionality, adding it as a built in allows doing so in linear time. --- doc/concepts/expressions.md | 3 +++ .../build_engine/expression/evaluator.cpp | 11 +++++++++++ .../build_engine/expression/expression.test.cpp | 15 +++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/doc/concepts/expressions.md b/doc/concepts/expressions.md index c66de8bcb..dcd567afa 100644 --- a/doc/concepts/expressions.md +++ b/doc/concepts/expressions.md @@ -253,6 +253,9 @@ those) argument(s) to obtain the final result. a map with the members of the list as keys, and all values being `true`. +- `"reverse"` The argument has to be a list. The result is a new list + with the entries in reverse order. + - `"++"` The argument has to be a list of lists. The result is the concatenation of those lists. diff --git a/src/buildtool/build_engine/expression/evaluator.cpp b/src/buildtool/build_engine/expression/evaluator.cpp index 431792e8c..1d207b3b4 100644 --- a/src/buildtool/build_engine/expression/evaluator.cpp +++ b/src/buildtool/build_engine/expression/evaluator.cpp @@ -185,6 +185,16 @@ auto Set(ExpressionPtr const& expr) -> ExpressionPtr { return ExpressionPtr{Expression::map_t{result}}; } +auto Reverse(ExpressionPtr const& expr) -> ExpressionPtr { + if (not expr->IsList()) { + throw Evaluator::EvaluationError{fmt::format( + "reverse expects list but instead got: {}.", expr->ToString())}; + } + auto reverse_result = Expression::list_t(expr->List()); + std::reverse(reverse_result.begin(), reverse_result.end()); + return ExpressionPtr{reverse_result}; +} + auto NubRight(ExpressionPtr const& expr) -> ExpressionPtr { if (not expr->IsList()) { throw Evaluator::EvaluationError{fmt::format( @@ -957,6 +967,7 @@ auto const kBuiltInFunctions = {"keys", UnaryExpr(Keys)}, {"enumerate", UnaryExpr(Enumerate)}, {"set", UnaryExpr(Set)}, + {"reverse", UnaryExpr(Reverse)}, {"values", UnaryExpr(Values)}, {"lookup", LookupExpr}, {"empty_map", EmptyMapExpr}, diff --git a/test/buildtool/build_engine/expression/expression.test.cpp b/test/buildtool/build_engine/expression/expression.test.cpp index 7f3df7c54..242c0f4a3 100644 --- a/test/buildtool/build_engine/expression/expression.test.cpp +++ b/test/buildtool/build_engine/expression/expression.test.cpp @@ -942,6 +942,21 @@ TEST_CASE("Expression Evaluation", "[expression]") { // NOLINT )"_json)); } + SECTION("reverse expression") { + auto expr = Expression::FromJson(R"( + { "type": "reverse" + , "$1": ["foo", "bar", "baz"] + } + )"_json); + REQUIRE(expr); + + auto result = expr.Evaluate(env, fcts); + REQUIRE(result); + CHECK(result == Expression::FromJson(R"( + ["baz", "bar", "foo"] + )"_json)); + } + SECTION("keys expression") { auto expr = Expression::FromJson(R"( { "type": "keys" From aee1a0c7df5b86c08c4ac3188b3635f8f9879129 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Mon, 14 Aug 2023 14:45:19 +0200 Subject: [PATCH 56/97] doc: Fix typos in tutorial --- doc/tutorial/cross-compiling.md | 18 +++++++++--------- doc/tutorial/just-execute.org | 12 ++++++------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/doc/tutorial/cross-compiling.md b/doc/tutorial/cross-compiling.md index 88f930214..4bdeadbb8 100644 --- a/doc/tutorial/cross-compiling.md +++ b/doc/tutorial/cross-compiling.md @@ -1,4 +1,4 @@ -Cros compiling and testing cross-compiled targets +Cross compiling and testing cross-compiled targets ================================================== So far, we were always building for the platform on which we were @@ -31,7 +31,7 @@ Let's start a new project. $ touch ROOT ``` -We create a file `reposs.template.json` specifying the one local repository. +We create a file `repos.template.json` specifying the one local repository. ``` {.jsonc srcname="repos.template.json"} { "repositories": { "": @@ -90,7 +90,7 @@ To cross compile, we simply add `TARGET_ARCH`. $ just-mr build -D '{"COMPILER_FAMILY": "gnu", "OS": "linux", "ARCH": "x86_64", "TARGET_ARCH": "arm64"}' INFO: Performing repositories setup INFO: Found 21 repositories to set up -INFO: Setup finished, exec ["just","build","-C","/home/aehlig/.cache/just/protocol-dependent/generation-0/git-sha1/casf/8c/c5d925c6d230e233f0e89107c8edb22d699063","-D","{\"COMPILER_FAMILY\": \"gnu\", \"OS\": \"linux\", \"ARCH\": \"x86_64\", \"TARGET_ARCH\": \"arm64\"}"] +INFO: Setup finished, exec ["just","build","-C","...","-D","{\"COMPILER_FAMILY\": \"gnu\", \"OS\": \"linux\", \"ARCH\": \"x86_64\", \"TARGET_ARCH\": \"arm64\"}"] INFO: Requested target is [["@","","","helloworld"],{"ARCH":"x86_64","COMPILER_FAMILY":"gnu","OS":"linux","TARGET_ARCH":"arm64"}] INFO: Analysed target [["@","","","helloworld"],{"ARCH":"x86_64","COMPILER_FAMILY":"gnu","OS":"linux","TARGET_ARCH":"arm64"}] INFO: Discovered 2 actions, 1 trees, 0 blobs @@ -185,14 +185,14 @@ $ ``` The reason is that a test actually has to run the created binary -and that requires a build enviroment of the target architecture. -So, if not being told how to obtain such an environemt, they carry -out the test in the best mannor they can, i.e., by transitioning +and that requires a build environment of the target architecture. +So, if not being told how to obtain such an environment, they carry +out the test in the best manner they can, i.e., by transitioning everything to host. So, in order to properly test the cross-compiled binary, we need to do two things. - We need to setup remote execution on the correct architecture, - either by buying the apropriate hardware, or by running an emulator. + either by buying the appropriate hardware, or by running an emulator. - We need to tell `justbuild` on how to reach that endpoint. To continue the example, let's say we set up an `arm64` machine, @@ -220,7 +220,7 @@ map have the same value in the remote-execution properties of the action (in particular, `{}` matches everything); so more specific dispatches have to be specified earlier in the list. In our case, we have a single endpoint in our private network that we should -use whenever the propterty `"runnner"` has value `"arm64-worker"`. +use whenever the property `"runner"` has value `"arm64-worker"`. The IP/port specification might differ in your setup. The path to this file is passed by the `--endpoint-configuration` option. @@ -267,7 +267,7 @@ $ The resulting command line might look complicated, but the authentication-related options, as well as the dispatch-related -otpions (including setting `ARGUMENTS_DISPATCH` via `-D`) can simply +options (including setting `ARGUMENTS_DISPATCH` via `-D`) can simply be set in the `"just args"` entry of the `.just-mrrc` file. When inspecting the result, we can use `just install-cas` as usual, diff --git a/doc/tutorial/just-execute.org b/doc/tutorial/just-execute.org index c92c5108a..6afac092b 100644 --- a/doc/tutorial/just-execute.org +++ b/doc/tutorial/just-execute.org @@ -77,7 +77,7 @@ Hello from \LaTeX! In this first example, we simply call ~just execute~ and the environment of the caller is made available. We therefore recommend to -have a dedicated non-priviledged ~build~ user to run the execution +have a dedicated non-privileged ~build~ user to run the execution service. In the following, we will use ~%~ to indicate the prompt of the ~build~ user, ~$~ for a /normal/ user. @@ -98,7 +98,7 @@ to a different build tool. To use it, as a /normal/ user, on a different shell type #+BEGIN_SRC bash -$ just [...] -r localhost: +$ just [...] -r localhost: #+END_SRC Let's run these commands to understand the output. @@ -155,7 +155,7 @@ INFO: execution service started: {"interface":"127.0.0.1","pid":7680,"port":4411 #+END_SRC #+BEGIN_SRC bash -$ cat /tmp/foo.json +$ cat /tmp/foo.json {"interface":"127.0.0.1","pid":7680,"port":44115} #+END_SRC @@ -210,7 +210,7 @@ On the other hand, if the clients will connect from a different machine, and ~just execute~ will use a different interface (see [[Expose a particular interface]] below), the steps are a bit more involved. We need an additional configuration file where we state the ip address of -the used interface. For example, if the interface ip address is +the used interface. For example, if the interface ip address is ~192.168.1.14~, we will write #+BEGIN_SRC bash % cat << EOF > ssl-ext-x509.cnf @@ -275,7 +275,7 @@ scenarios. This short tutorial will use ~debootstrap~ and ~schroot~ to create and enter the chroot environment. Of course, different strategies/programs -can be used. +can be used. ***** Prepare the root file system Install debian bullseye in directory ~/chroot/bullseye-latex~ @@ -343,7 +343,7 @@ which should report #+BEGIN_SRC bash INFO: Performing repositories setup INFO: Found 2 repositories to set up -INFO: Setup finished, exec ["just","install","-C","/home/alberto/.cache/just/protocol-dependent/generation-0/git-sha1/casf/c0/086f5dc35e084224781becfedb11baee8e9e24","-o",".","-r","localhost:8080"] +INFO: Setup finished, exec ["just","install","-C","...","-o",".","-r","localhost:8080"] INFO: Requested target is [["@","tutorial","doc/just-execute/latex-hello-world","tutorial"],{}] INFO: Analysed target [["@","tutorial","doc/just-execute/latex-hello-world","tutorial"],{}] INFO: Export targets found: 0 cached, 0 uncached, 0 not eligible for caching From a9694ba06acc9ea225f83bbd31f93cbedec4ff0c Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Thu, 10 Aug 2023 15:44:16 +0200 Subject: [PATCH 57/97] ArchiveOps: reduce verbosity in archive type enum --- src/other_tools/root_maps/content_git_map.cpp | 6 +-- src/other_tools/utils/archive_ops.cpp | 36 ++++++++-------- src/other_tools/utils/archive_ops.hpp | 8 +--- .../just-mr/create_test_archives.cpp | 6 +-- test/other_tools/utils/archive_usage.test.cpp | 42 ++++++++----------- 5 files changed, 42 insertions(+), 56 deletions(-) diff --git a/src/other_tools/root_maps/content_git_map.cpp b/src/other_tools/root_maps/content_git_map.cpp index 0c991ef14..7f0b30e09 100644 --- a/src/other_tools/root_maps/content_git_map.cpp +++ b/src/other_tools/root_maps/content_git_map.cpp @@ -31,12 +31,10 @@ namespace { std::filesystem::path const& dst_dir) noexcept -> std::optional { if (repo_type == "archive") { - return ArchiveOps::ExtractArchive( - ArchiveType::kArchiveTypeTar, archive, dst_dir); + return ArchiveOps::ExtractArchive(ArchiveType::Tar, archive, dst_dir); } if (repo_type == "zip") { - return ArchiveOps::ExtractArchive( - ArchiveType::kArchiveTypeZip, archive, dst_dir); + return ArchiveOps::ExtractArchive(ArchiveType::Zip, archive, dst_dir); } return "unrecognized repository type"; } diff --git a/src/other_tools/utils/archive_ops.cpp b/src/other_tools/utils/archive_ops.cpp index c080c4e36..94c000c79 100644 --- a/src/other_tools/utils/archive_ops.cpp +++ b/src/other_tools/utils/archive_ops.cpp @@ -51,13 +51,13 @@ void archive_read_closer(archive* a_in) { auto enable_write_filter(archive* aw, ArchiveType type) -> bool { switch (type) { - case ArchiveType::kArchiveTypeTar: + case ArchiveType::Tar: return true; // no compression filter - case ArchiveType::kArchiveTypeTarGz: + case ArchiveType::TarGz: return (archive_write_add_filter_gzip(aw) == ARCHIVE_OK); - case ArchiveType::kArchiveTypeTarBz2: + case ArchiveType::TarBz2: return (archive_write_add_filter_bzip2(aw) == ARCHIVE_OK); - case ArchiveType::kArchiveTypeTarXz: + case ArchiveType::TarXz: return (archive_write_add_filter_xz(aw) == ARCHIVE_OK); default: return false; @@ -66,15 +66,15 @@ auto enable_write_filter(archive* aw, ArchiveType type) -> bool { auto enable_read_filter(archive* ar, ArchiveType type) -> bool { switch (type) { - case ArchiveType::kArchiveTypeTar: + case ArchiveType::Tar: // autodetect from supported compression filters return (archive_read_support_filter_xz(ar) == ARCHIVE_OK) and (archive_read_support_filter_gzip(ar) == ARCHIVE_OK) and (archive_read_support_filter_bzip2(ar) == ARCHIVE_OK); - case ArchiveType::kArchiveTypeTarGz: + case ArchiveType::TarGz: return (archive_read_support_filter_gzip(ar) == ARCHIVE_OK); - case ArchiveType::kArchiveTypeTarBz2: + case ArchiveType::TarBz2: return (archive_read_support_filter_bzip2(ar) == ARCHIVE_OK); - case ArchiveType::kArchiveTypeTarXz: + case ArchiveType::TarXz: return (archive_read_support_filter_xz(ar) == ARCHIVE_OK); default: return false; @@ -128,16 +128,16 @@ auto ArchiveOps::CopyData(archive* ar, archive* aw) auto ArchiveOps::EnableWriteFormats(archive* aw, ArchiveType type) -> std::optional { switch (type) { - case ArchiveType::kArchiveTypeZip: { + case ArchiveType::Zip: { if (archive_write_set_format_zip(aw) != ARCHIVE_OK) { return std::string("ArchiveOps: ") + std::string(archive_error_string(aw)); } } break; - case ArchiveType::kArchiveTypeTar: - case ArchiveType::kArchiveTypeTarGz: - case ArchiveType::kArchiveTypeTarBz2: - case ArchiveType::kArchiveTypeTarXz: { + case ArchiveType::Tar: + case ArchiveType::TarGz: + case ArchiveType::TarBz2: + case ArchiveType::TarXz: { if ((archive_write_set_format_pax_restricted(aw) != ARCHIVE_OK) or not enable_write_filter(aw, type)) { return std::string("ArchiveOps: ") + @@ -151,16 +151,16 @@ auto ArchiveOps::EnableWriteFormats(archive* aw, ArchiveType type) auto ArchiveOps::EnableReadFormats(archive* ar, ArchiveType type) -> std::optional { switch (type) { - case ArchiveType::kArchiveTypeZip: { + case ArchiveType::Zip: { if (archive_read_support_format_zip(ar) != ARCHIVE_OK) { return std::string("ArchiveOps: ") + std::string(archive_error_string(ar)); } } break; - case ArchiveType::kArchiveTypeTar: - case ArchiveType::kArchiveTypeTarGz: - case ArchiveType::kArchiveTypeTarBz2: - case ArchiveType::kArchiveTypeTarXz: { + case ArchiveType::Tar: + case ArchiveType::TarGz: + case ArchiveType::TarBz2: + case ArchiveType::TarXz: { if ((archive_read_support_format_tar(ar) != ARCHIVE_OK) or not enable_read_filter(ar, type)) { return std::string("ArchiveOps: ") + diff --git a/src/other_tools/utils/archive_ops.hpp b/src/other_tools/utils/archive_ops.hpp index 22cf7e4b5..6ad5b7c46 100644 --- a/src/other_tools/utils/archive_ops.hpp +++ b/src/other_tools/utils/archive_ops.hpp @@ -25,13 +25,7 @@ using archive = struct archive; using archive_entry = struct archive_entry; } -enum class ArchiveType : size_t { - kArchiveTypeZip, - kArchiveTypeTar, - kArchiveTypeTarGz, - kArchiveTypeTarBz2, - kArchiveTypeTarXz -}; +enum class ArchiveType : size_t { Zip, Tar, TarGz, TarBz2, TarXz }; /// \brief Class handling archiving and unarchiving operations via libarchive class ArchiveOps { diff --git a/test/end-to-end/just-mr/create_test_archives.cpp b/test/end-to-end/just-mr/create_test_archives.cpp index 3fa6759a1..f7223410a 100644 --- a/test/end-to-end/just-mr/create_test_archives.cpp +++ b/test/end-to-end/just-mr/create_test_archives.cpp @@ -120,8 +120,8 @@ auto main() -> int { auto anchor = FileSystemManager::ChangeDirectory(tmp_dir->GetPath()); // 2. create the archives wrt to current directory std::optional res{std::nullopt}; - res = ArchiveOps::CreateArchive( - ArchiveType::kArchiveTypeZip, "zip_repo.zip", "root"); + res = + ArchiveOps::CreateArchive(ArchiveType::Zip, "zip_repo.zip", "root"); if (res) { Logger::Log(LogLevel::Error, "Creating test zip archive failed with:\n{}", @@ -129,7 +129,7 @@ auto main() -> int { return 1; } res = ArchiveOps::CreateArchive( - ArchiveType::kArchiveTypeTarGz, "tgz_repo.tar.gz", "root"); + ArchiveType::TarGz, "tgz_repo.tar.gz", "root"); if (res) { Logger::Log(LogLevel::Error, "Creating test tar.gz archive failed with:\n{}", diff --git a/test/other_tools/utils/archive_usage.test.cpp b/test/other_tools/utils/archive_usage.test.cpp index 732798a8b..0c55ad7c7 100644 --- a/test/other_tools/utils/archive_usage.test.cpp +++ b/test/other_tools/utils/archive_usage.test.cpp @@ -348,7 +348,7 @@ TEST_CASE("ArchiveOps", "[archive_ops]") { create_files(test_dir_tar); res = ArchiveOps::CreateArchive( - ArchiveType::kArchiveTypeTar, filename_tar, test_dir_tar, "."); + ArchiveType::Tar, filename_tar, test_dir_tar, "."); if (res != std::nullopt) { FAIL(*res); } @@ -357,8 +357,8 @@ TEST_CASE("ArchiveOps", "[archive_ops]") { REQUIRE(FileSystemManager::RemoveDirectory(test_dir_tar, /*recursively=*/true)); REQUIRE(FileSystemManager::CreateDirectory(test_dir_tar)); - res = ArchiveOps::ExtractArchive( - ArchiveType::kArchiveTypeTar, filename_tar, "."); + res = + ArchiveOps::ExtractArchive(ArchiveType::Tar, filename_tar, "."); if (res != std::nullopt) { FAIL(*res); } @@ -389,10 +389,8 @@ TEST_CASE("ArchiveOps", "[archive_ops]") { create_files(test_dir_tar_gz); - res = ArchiveOps::CreateArchive(ArchiveType::kArchiveTypeTarGz, - filename_tar_gz, - test_dir_tar_gz, - "."); + res = ArchiveOps::CreateArchive( + ArchiveType::TarGz, filename_tar_gz, test_dir_tar_gz, "."); if (res != std::nullopt) { FAIL(*res); } @@ -402,7 +400,7 @@ TEST_CASE("ArchiveOps", "[archive_ops]") { /*recursively=*/true)); REQUIRE(FileSystemManager::CreateDirectory(test_dir_tar_gz)); res = ArchiveOps::ExtractArchive( - ArchiveType::kArchiveTypeTarGz, filename_tar_gz, "."); + ArchiveType::TarGz, filename_tar_gz, "."); if (res != std::nullopt) { FAIL(*res); } @@ -414,7 +412,7 @@ TEST_CASE("ArchiveOps", "[archive_ops]") { /*recursively=*/true)); REQUIRE(FileSystemManager::CreateDirectory(test_dir_tar_gz)); res = ArchiveOps::ExtractArchive( - ArchiveType::kArchiveTypeTar, filename_tar_gz, "."); + ArchiveType::Tar, filename_tar_gz, "."); if (res != std::nullopt) { FAIL(*res); } @@ -446,10 +444,8 @@ TEST_CASE("ArchiveOps", "[archive_ops]") { create_files(test_dir_tar_bz2); - res = ArchiveOps::CreateArchive(ArchiveType::kArchiveTypeTarBz2, - filename_tar_bz2, - test_dir_tar_bz2, - "."); + res = ArchiveOps::CreateArchive( + ArchiveType::TarBz2, filename_tar_bz2, test_dir_tar_bz2, "."); if (res != std::nullopt) { FAIL(*res); } @@ -459,7 +455,7 @@ TEST_CASE("ArchiveOps", "[archive_ops]") { /*recursively=*/true)); REQUIRE(FileSystemManager::CreateDirectory(test_dir_tar_bz2)); res = ArchiveOps::ExtractArchive( - ArchiveType::kArchiveTypeTarBz2, filename_tar_bz2, "."); + ArchiveType::TarBz2, filename_tar_bz2, "."); if (res != std::nullopt) { FAIL(*res); } @@ -471,7 +467,7 @@ TEST_CASE("ArchiveOps", "[archive_ops]") { /*recursively=*/true)); REQUIRE(FileSystemManager::CreateDirectory(test_dir_tar_bz2)); res = ArchiveOps::ExtractArchive( - ArchiveType::kArchiveTypeTar, filename_tar_bz2, "."); + ArchiveType::Tar, filename_tar_bz2, "."); if (res != std::nullopt) { FAIL(*res); } @@ -503,10 +499,8 @@ TEST_CASE("ArchiveOps", "[archive_ops]") { create_files(test_dir_tar_xz); - res = ArchiveOps::CreateArchive(ArchiveType::kArchiveTypeTarXz, - filename_tar_xz, - test_dir_tar_xz, - "."); + res = ArchiveOps::CreateArchive( + ArchiveType::TarXz, filename_tar_xz, test_dir_tar_xz, "."); if (res != std::nullopt) { FAIL(*res); } @@ -516,7 +510,7 @@ TEST_CASE("ArchiveOps", "[archive_ops]") { /*recursively=*/true)); REQUIRE(FileSystemManager::CreateDirectory(test_dir_tar_xz)); res = ArchiveOps::ExtractArchive( - ArchiveType::kArchiveTypeTarXz, filename_tar_xz, "."); + ArchiveType::TarXz, filename_tar_xz, "."); if (res != std::nullopt) { FAIL(*res); } @@ -528,7 +522,7 @@ TEST_CASE("ArchiveOps", "[archive_ops]") { /*recursively=*/true)); REQUIRE(FileSystemManager::CreateDirectory(test_dir_tar_xz)); res = ArchiveOps::ExtractArchive( - ArchiveType::kArchiveTypeTar, filename_tar_xz, "."); + ArchiveType::Tar, filename_tar_xz, "."); if (res != std::nullopt) { FAIL(*res); } @@ -561,7 +555,7 @@ TEST_CASE("ArchiveOps", "[archive_ops]") { create_files(test_dir_zip); res = ArchiveOps::CreateArchive( - ArchiveType::kArchiveTypeZip, filename_zip, test_dir_zip, "."); + ArchiveType::Zip, filename_zip, test_dir_zip, "."); if (res != std::nullopt) { FAIL(*res); } @@ -570,8 +564,8 @@ TEST_CASE("ArchiveOps", "[archive_ops]") { REQUIRE(FileSystemManager::RemoveDirectory(test_dir_zip, /*recursively=*/true)); REQUIRE(FileSystemManager::CreateDirectory(test_dir_zip)); - res = ArchiveOps::ExtractArchive( - ArchiveType::kArchiveTypeZip, filename_zip, "."); + res = + ArchiveOps::ExtractArchive(ArchiveType::Zip, filename_zip, "."); if (res != std::nullopt) { FAIL(*res); } From 26fe1ac82a8437b292e5c17e791c02d5a6813a26 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Thu, 10 Aug 2023 17:04:10 +0200 Subject: [PATCH 58/97] test: Refactoring the archive usage tests Tests for various archive types are by design repetitive and thus can be streamlined to avoid excessive code duplication. As a positive consequence, this change also adds previously missing .xz archive usage checks. --- test/other_tools/utils/archive_usage.test.cpp | 500 +++++------------- 1 file changed, 143 insertions(+), 357 deletions(-) diff --git a/test/other_tools/utils/archive_usage.test.cpp b/test/other_tools/utils/archive_usage.test.cpp index 0c55ad7c7..68fe436dd 100644 --- a/test/other_tools/utils/archive_usage.test.cpp +++ b/test/other_tools/utils/archive_usage.test.cpp @@ -14,8 +14,10 @@ #include #include +#include #include "catch2/catch_test_macros.hpp" +#include "catch2/generators/catch_generators_all.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/other_tools/utils/archive_ops.hpp" @@ -37,6 +39,47 @@ auto const kExpected = filetree_t{{"foo", {"foo", AE_IFREG}}, {"bar/", {"", AE_IFDIR}}, {"bar/baz", {"baz", AE_IFREG}}}; +struct archive_test_info_t { + std::string test_name; + ArchiveType type; + std::string test_dir; + std::string filename; + std::vector tools; + std::string cmd; +}; + +std::vector const kTestScenarios = { + {.test_name = "tar", + .type = ArchiveType::Tar, + .test_dir = "test_tar", + .filename = "test.tar", + .tools = {"tar"}, + .cmd = "/usr/bin/tar xf"}, + {.test_name = "tar.gz", + .type = ArchiveType::TarGz, + .test_dir = "test_tar_gz", + .filename = "test.tar.gz", + .tools = {"tar", "gzip"}, + .cmd = "/usr/bin/tar xzf"}, + {.test_name = "tar.bz2", + .type = ArchiveType::TarBz2, + .test_dir = "test_tar_bz2", + .filename = "test.tar.bz2", + .tools = {"tar", "bzip2"}, + .cmd = "/usr/bin/tar xjf"}, + {.test_name = "tar.xz", + .type = ArchiveType::TarXz, + .test_dir = "test_tar_xz", + .filename = "test.tar.xz", + .tools = {"tar", "xz"}, + .cmd = "/usr/bin/tar xJf"}, + {.test_name = "zip", + .type = ArchiveType::Zip, + .test_dir = "test_zip", + .filename = "test.zip", + .tools = {"unzip"}, + .cmd = "/usr/bin/unzip"}}; + [[nodiscard]] auto read_archive(archive* a, std::string const& path) -> filetree_t { filetree_t result{}; @@ -94,6 +137,7 @@ void extract_archive(std::string const& path) { REQUIRE(archive_read_support_format_zip(a) == ARCHIVE_OK); REQUIRE(archive_read_support_filter_gzip(a) == ARCHIVE_OK); REQUIRE(archive_read_support_filter_bzip2(a) == ARCHIVE_OK); + REQUIRE(archive_read_support_filter_xz(a) == ARCHIVE_OK); REQUIRE(archive_read_open_filename(a, path.c_str(), kBlockSize) == ARCHIVE_OK); @@ -160,6 +204,52 @@ void create_files(std::filesystem::path const& destDir = ".") noexcept { } } +void enable_write_format_and_filter(archive* aw, ArchiveType type) { + switch (type) { + case ArchiveType::Zip: { + REQUIRE(archive_write_set_format_zip(aw) == ARCHIVE_OK); + } break; + case ArchiveType::Tar: { + REQUIRE(archive_write_set_format_pax_restricted(aw) == ARCHIVE_OK); + } break; + case ArchiveType::TarGz: { + REQUIRE(archive_write_set_format_pax_restricted(aw) == ARCHIVE_OK); + REQUIRE(archive_write_add_filter_gzip(aw) == ARCHIVE_OK); + } break; + case ArchiveType::TarBz2: { + REQUIRE(archive_write_set_format_pax_restricted(aw) == ARCHIVE_OK); + REQUIRE(archive_write_add_filter_bzip2(aw) == ARCHIVE_OK); + } break; + case ArchiveType::TarXz: { + REQUIRE(archive_write_set_format_pax_restricted(aw) == ARCHIVE_OK); + REQUIRE(archive_write_add_filter_xz(aw) == ARCHIVE_OK); + } break; + } +} + +void enable_read_format_and_filter(archive* ar, ArchiveType type) { + switch (type) { + case ArchiveType::Zip: { + REQUIRE(archive_read_support_format_zip(ar) == ARCHIVE_OK); + } break; + case ArchiveType::Tar: { + REQUIRE(archive_read_support_format_tar(ar) == ARCHIVE_OK); + } break; + case ArchiveType::TarGz: { + REQUIRE(archive_read_support_format_tar(ar) == ARCHIVE_OK); + REQUIRE(archive_read_support_filter_gzip(ar) == ARCHIVE_OK); + } break; + case ArchiveType::TarBz2: { + REQUIRE(archive_read_support_format_tar(ar) == ARCHIVE_OK); + REQUIRE(archive_read_support_filter_bzip2(ar) == ARCHIVE_OK); + } break; + case ArchiveType::TarXz: { + REQUIRE(archive_read_support_format_tar(ar) == ARCHIVE_OK); + REQUIRE(archive_read_support_filter_xz(ar) == ARCHIVE_OK); + } break; + } +} + } // namespace TEST_CASE("Archive read context", "[archive_context]") { @@ -180,154 +270,48 @@ TEST_CASE("Archive write disk context", "[archive_context]") { CHECK(archive_read_free(a) == ARCHIVE_OK); } -TEST_CASE("Read-write tar", "[archive_read_write]") { - std::string test_dir{"test_tar"}; - std::string filename{"test.tar"}; - - REQUIRE(FileSystemManager::RemoveDirectory(test_dir, /*recursively=*/true)); - REQUIRE(FileSystemManager::CreateDirectory(test_dir)); - auto anchor = FileSystemManager::ChangeDirectory(test_dir); - - SECTION("Write tar") { - auto* out = archive_write_new(); - REQUIRE(out != nullptr); - REQUIRE(archive_write_set_format_pax_restricted(out) == ARCHIVE_OK); - write_archive(out, filename, kExpected); - REQUIRE(archive_write_free(out) == ARCHIVE_OK); - - SECTION("Read tar") { - auto* in = archive_read_new(); - REQUIRE(in != nullptr); - REQUIRE(archive_read_support_format_tar(in) == ARCHIVE_OK); - CHECK(read_archive(in, filename) == kExpected); - REQUIRE(archive_read_free(in) == ARCHIVE_OK); - } - - SECTION("Extract tar to disk") { - extract_archive(filename); - compare_extracted(); - } - - if (FileSystemManager::IsExecutable("/usr/bin/tar")) { - SECTION("Extract via system tar") { - REQUIRE(system(("/usr/bin/tar xf " + filename).c_str()) == 0); - compare_extracted(); - } - } - } -} - -TEST_CASE("Read-write tar.gz", "[archive_read_write]") { - std::string test_dir{"test_tar_gz"}; - std::string filename{"test.tar.gz"}; +TEST_CASE("Read-write archives", "[archive_read_write]") { + // get the scenario + auto test_index = + GENERATE(Catch::Generators::range(0, kTestScenarios.size())); + auto const& scenario = kTestScenarios[test_index]; - REQUIRE(FileSystemManager::RemoveDirectory(test_dir, /*recursively=*/true)); - REQUIRE(FileSystemManager::CreateDirectory(test_dir)); - auto anchor = FileSystemManager::ChangeDirectory(test_dir); + // perform the test + REQUIRE(FileSystemManager::RemoveDirectory(scenario.test_dir, + /*recursively=*/true)); + REQUIRE(FileSystemManager::CreateDirectory(scenario.test_dir)); + auto anchor = FileSystemManager::ChangeDirectory(scenario.test_dir); - SECTION("Write tar.gz") { + SECTION(std::string("Write ") + scenario.test_name) { auto* out = archive_write_new(); REQUIRE(out != nullptr); - REQUIRE(archive_write_set_format_pax_restricted(out) == ARCHIVE_OK); - REQUIRE(archive_write_add_filter_gzip(out) == ARCHIVE_OK); - write_archive(out, filename, kExpected); + enable_write_format_and_filter(out, scenario.type); + write_archive(out, scenario.filename, kExpected); REQUIRE(archive_write_free(out) == ARCHIVE_OK); - SECTION("Read tar.gz") { + SECTION(std::string("Read ") + scenario.test_name) { auto* in = archive_read_new(); REQUIRE(in != nullptr); - REQUIRE(archive_read_support_format_tar(in) == ARCHIVE_OK); - REQUIRE(archive_read_support_filter_gzip(in) == ARCHIVE_OK); - CHECK(read_archive(in, filename) == kExpected); + enable_read_format_and_filter(in, scenario.type); + CHECK(read_archive(in, scenario.filename) == kExpected); REQUIRE(archive_read_free(in) == ARCHIVE_OK); } - SECTION("Extract tar.gz to disk") { - extract_archive(filename); + SECTION(std::string("Extract ") + scenario.test_name + " to disk") { + extract_archive(scenario.filename); compare_extracted(); } - if (FileSystemManager::IsExecutable("/usr/bin/tar") and - FileSystemManager::IsExecutable("/usr/bin/gzip")) { - SECTION("Extract via system tar and gzip") { - REQUIRE(system(("/usr/bin/tar xzf " + filename).c_str()) == 0); - compare_extracted(); - } + bool tools_exist{true}; + for (auto const& tool : scenario.tools) { + tools_exist &= FileSystemManager::IsExecutable( + std::string("/usr/bin/") + tool); } - } -} - -TEST_CASE("Read-write tar.bz2", "[archive_read_write]") { - std::string test_dir{"test_tar_bz2"}; - std::string filename{"test.tar.bz2"}; - - REQUIRE(FileSystemManager::RemoveDirectory(test_dir, /*recursively=*/true)); - REQUIRE(FileSystemManager::CreateDirectory(test_dir)); - auto anchor = FileSystemManager::ChangeDirectory(test_dir); - - SECTION("Write tar.bz2") { - auto* out = archive_write_new(); - REQUIRE(out != nullptr); - REQUIRE(archive_write_set_format_pax_restricted(out) == ARCHIVE_OK); - REQUIRE(archive_write_add_filter_bzip2(out) == ARCHIVE_OK); - write_archive(out, filename, kExpected); - REQUIRE(archive_write_free(out) == ARCHIVE_OK); - - SECTION("Read tar.bz2") { - auto* in = archive_read_new(); - REQUIRE(in != nullptr); - REQUIRE(archive_read_support_format_tar(in) == ARCHIVE_OK); - REQUIRE(archive_read_support_filter_bzip2(in) == ARCHIVE_OK); - CHECK(read_archive(in, filename) == kExpected); - REQUIRE(archive_read_free(in) == ARCHIVE_OK); - } - - SECTION("Extract tar.bz2 to disk") { - extract_archive(filename); - compare_extracted(); - } - - if (FileSystemManager::IsExecutable("/usr/bin/tar") and - FileSystemManager::IsExecutable("/usr/bin/bzip2")) { - SECTION("Extract via system tar and bzip2") { - REQUIRE(system(("/usr/bin/tar xjf " + filename).c_str()) == 0); - compare_extracted(); - } - } - } -} - -TEST_CASE("Read-write zip", "[archive_read_write]") { - std::string test_dir{"test_zip"}; - std::string filename{"test.zip"}; - - REQUIRE(FileSystemManager::RemoveDirectory(test_dir, /*recursively=*/true)); - REQUIRE(FileSystemManager::CreateDirectory(test_dir)); - auto anchor = FileSystemManager::ChangeDirectory(test_dir); - - SECTION("Write zip") { - auto* out = archive_write_new(); - REQUIRE(out != nullptr); - REQUIRE(archive_write_set_format_zip(out) == ARCHIVE_OK); - write_archive(out, filename, kExpected); - REQUIRE(archive_write_free(out) == ARCHIVE_OK); - - SECTION("Read zip") { - auto* in = archive_read_new(); - REQUIRE(in != nullptr); - REQUIRE(archive_read_support_format_zip(in) == ARCHIVE_OK); - CHECK(read_archive(in, filename) == kExpected); - REQUIRE(archive_read_free(in) == ARCHIVE_OK); - } - - SECTION("Extract zip to disk") { - extract_archive(filename); - compare_extracted(); - } - - if (FileSystemManager::IsExecutable("/usr/bin/unzip")) { - SECTION("Extract via system unzip") { - REQUIRE(system(("/usr/bin/unzip " + filename).c_str()) == 0); + if (tools_exist) { + SECTION("Extract via system tools") { + REQUIRE( + system((scenario.cmd + " " + scenario.filename).c_str()) == + 0); compare_extracted(); } } @@ -335,253 +319,55 @@ TEST_CASE("Read-write zip", "[archive_read_write]") { } TEST_CASE("ArchiveOps", "[archive_ops]") { - std::optional res{std::nullopt}; - - SECTION("Write tar") { - std::string test_dir_tar{"ops_test_tar"}; - std::string filename_tar{"test.tar"}; - - REQUIRE(FileSystemManager::RemoveDirectory(test_dir_tar, - /*recursively=*/true)); - REQUIRE(FileSystemManager::CreateDirectory(test_dir_tar)); - - create_files(test_dir_tar); - - res = ArchiveOps::CreateArchive( - ArchiveType::Tar, filename_tar, test_dir_tar, "."); - if (res != std::nullopt) { - FAIL(*res); - } - - SECTION("Extract tar to disk") { - REQUIRE(FileSystemManager::RemoveDirectory(test_dir_tar, - /*recursively=*/true)); - REQUIRE(FileSystemManager::CreateDirectory(test_dir_tar)); - res = - ArchiveOps::ExtractArchive(ArchiveType::Tar, filename_tar, "."); - if (res != std::nullopt) { - FAIL(*res); - } - compare_extracted(test_dir_tar); - } - - if (FileSystemManager::IsExecutable("/usr/bin/tar")) { - SECTION("Extract via system tar") { - REQUIRE( - FileSystemManager::RemoveDirectory(test_dir_tar, - /*recursively=*/true)); - REQUIRE(FileSystemManager::CreateDirectory(test_dir_tar)); - - REQUIRE(system(("/usr/bin/tar xf " + filename_tar).c_str()) == - 0); - compare_extracted(test_dir_tar); - } - } - } + // get the scenario + auto test_index = + GENERATE(Catch::Generators::range(0, kTestScenarios.size())); + auto const& scenario = kTestScenarios[test_index]; - SECTION("Write tar.gz") { - std::string test_dir_tar_gz{"ops_test_tar_gz"}; - std::string filename_tar_gz{"test.tar.gz"}; - - REQUIRE(FileSystemManager::RemoveDirectory(test_dir_tar_gz, - /*recursively=*/true)); - REQUIRE(FileSystemManager::CreateDirectory(test_dir_tar_gz)); - - create_files(test_dir_tar_gz); - - res = ArchiveOps::CreateArchive( - ArchiveType::TarGz, filename_tar_gz, test_dir_tar_gz, "."); - if (res != std::nullopt) { - FAIL(*res); - } - - SECTION("Extract tar.gz to disk") { - REQUIRE(FileSystemManager::RemoveDirectory(test_dir_tar_gz, - /*recursively=*/true)); - REQUIRE(FileSystemManager::CreateDirectory(test_dir_tar_gz)); - res = ArchiveOps::ExtractArchive( - ArchiveType::TarGz, filename_tar_gz, "."); - if (res != std::nullopt) { - FAIL(*res); - } - compare_extracted(test_dir_tar_gz); - } - - SECTION("Extract tar.gz to disk via compression auto detect") { - REQUIRE(FileSystemManager::RemoveDirectory(test_dir_tar_gz, - /*recursively=*/true)); - REQUIRE(FileSystemManager::CreateDirectory(test_dir_tar_gz)); - res = ArchiveOps::ExtractArchive( - ArchiveType::Tar, filename_tar_gz, "."); - if (res != std::nullopt) { - FAIL(*res); - } - compare_extracted(test_dir_tar_gz); - } - - if (FileSystemManager::IsExecutable("/usr/bin/tar") and - FileSystemManager::IsExecutable("/usr/bin/gzip")) { - REQUIRE(FileSystemManager::RemoveDirectory(test_dir_tar_gz, - /*recursively=*/true)); - REQUIRE(FileSystemManager::CreateDirectory(test_dir_tar_gz)); - - SECTION("Extract via system tar and gzip") { - REQUIRE( - system(("/usr/bin/tar xzf " + filename_tar_gz).c_str()) == - 0); - compare_extracted(test_dir_tar_gz); - } - } - } - - SECTION("Write tar.bz2") { - std::string test_dir_tar_bz2{"ops_test_tar_bz2"}; - std::string filename_tar_bz2{"test.tar.bz2"}; + // perform the test + std::optional res{std::nullopt}; - REQUIRE(FileSystemManager::RemoveDirectory(test_dir_tar_bz2, + SECTION(std::string("Write ") + scenario.test_name) { + REQUIRE(FileSystemManager::RemoveDirectory(scenario.test_dir, /*recursively=*/true)); - REQUIRE(FileSystemManager::CreateDirectory(test_dir_tar_bz2)); + REQUIRE(FileSystemManager::CreateDirectory(scenario.test_dir)); - create_files(test_dir_tar_bz2); + create_files(scenario.test_dir); res = ArchiveOps::CreateArchive( - ArchiveType::TarBz2, filename_tar_bz2, test_dir_tar_bz2, "."); + scenario.type, scenario.filename, scenario.test_dir, "."); if (res != std::nullopt) { FAIL(*res); } - SECTION("Extract tar.bz2 to disk") { - REQUIRE(FileSystemManager::RemoveDirectory(test_dir_tar_bz2, + SECTION(std::string("Extract ") + scenario.test_name + " to disk") { + REQUIRE(FileSystemManager::RemoveDirectory(scenario.test_dir, /*recursively=*/true)); - REQUIRE(FileSystemManager::CreateDirectory(test_dir_tar_bz2)); + REQUIRE(FileSystemManager::CreateDirectory(scenario.test_dir)); res = ArchiveOps::ExtractArchive( - ArchiveType::TarBz2, filename_tar_bz2, "."); + scenario.type, scenario.filename, "."); if (res != std::nullopt) { FAIL(*res); } - compare_extracted(test_dir_tar_bz2); + compare_extracted(scenario.test_dir); } - SECTION("Extract tar.bz2 to disk via compression auto detect") { - REQUIRE(FileSystemManager::RemoveDirectory(test_dir_tar_bz2, - /*recursively=*/true)); - REQUIRE(FileSystemManager::CreateDirectory(test_dir_tar_bz2)); - res = ArchiveOps::ExtractArchive( - ArchiveType::Tar, filename_tar_bz2, "."); - if (res != std::nullopt) { - FAIL(*res); - } - compare_extracted(test_dir_tar_bz2); + bool tools_exist{true}; + for (auto const& tool : scenario.tools) { + tools_exist &= FileSystemManager::IsExecutable( + std::string("/usr/bin/") + tool); } - - if (FileSystemManager::IsExecutable("/usr/bin/tar") and - FileSystemManager::IsExecutable("/usr/bin/bzip2")) { - REQUIRE(FileSystemManager::RemoveDirectory(test_dir_tar_bz2, - /*recursively=*/true)); - REQUIRE(FileSystemManager::CreateDirectory(test_dir_tar_bz2)); - - SECTION("Extract via system tar and bzip2") { + if (tools_exist) { + SECTION("Extract via system tools") { REQUIRE( - system(("/usr/bin/tar xjf " + filename_tar_bz2).c_str()) == - 0); - compare_extracted(test_dir_tar_bz2); - } - } - } - - SECTION("Write tar.xz") { - std::string test_dir_tar_xz{"ops_test_tar_xz"}; - std::string filename_tar_xz{"test.tar.xz"}; - - REQUIRE(FileSystemManager::RemoveDirectory(test_dir_tar_xz, - /*recursively=*/true)); - REQUIRE(FileSystemManager::CreateDirectory(test_dir_tar_xz)); - - create_files(test_dir_tar_xz); - - res = ArchiveOps::CreateArchive( - ArchiveType::TarXz, filename_tar_xz, test_dir_tar_xz, "."); - if (res != std::nullopt) { - FAIL(*res); - } - - SECTION("Extract tar.xz to disk") { - REQUIRE(FileSystemManager::RemoveDirectory(test_dir_tar_xz, - /*recursively=*/true)); - REQUIRE(FileSystemManager::CreateDirectory(test_dir_tar_xz)); - res = ArchiveOps::ExtractArchive( - ArchiveType::TarXz, filename_tar_xz, "."); - if (res != std::nullopt) { - FAIL(*res); - } - compare_extracted(test_dir_tar_xz); - } - - SECTION("Extract tar.xz to disk via compression auto detect") { - REQUIRE(FileSystemManager::RemoveDirectory(test_dir_tar_xz, - /*recursively=*/true)); - REQUIRE(FileSystemManager::CreateDirectory(test_dir_tar_xz)); - res = ArchiveOps::ExtractArchive( - ArchiveType::Tar, filename_tar_xz, "."); - if (res != std::nullopt) { - FAIL(*res); - } - compare_extracted(test_dir_tar_xz); - } - - if (FileSystemManager::IsExecutable("/usr/bin/tar") and - FileSystemManager::IsExecutable("/usr/bin/xz")) { - REQUIRE(FileSystemManager::RemoveDirectory(test_dir_tar_xz, + FileSystemManager::RemoveDirectory(scenario.test_dir, /*recursively=*/true)); - REQUIRE(FileSystemManager::CreateDirectory(test_dir_tar_xz)); + REQUIRE(FileSystemManager::CreateDirectory(scenario.test_dir)); - SECTION("Extract via system tar and bzip2") { REQUIRE( - system(("/usr/bin/tar xJf " + filename_tar_xz).c_str()) == + system((scenario.cmd + " " + scenario.filename).c_str()) == 0); - compare_extracted(test_dir_tar_xz); - } - } - } - - SECTION("Write zip") { - std::string test_dir_zip{"ops_test_zip"}; - std::string filename_zip{"test.zip"}; - - REQUIRE(FileSystemManager::RemoveDirectory(test_dir_zip, - /*recursively=*/true)); - REQUIRE(FileSystemManager::CreateDirectory(test_dir_zip)); - - create_files(test_dir_zip); - - res = ArchiveOps::CreateArchive( - ArchiveType::Zip, filename_zip, test_dir_zip, "."); - if (res != std::nullopt) { - FAIL(*res); - } - - SECTION("Extract zip to disk") { - REQUIRE(FileSystemManager::RemoveDirectory(test_dir_zip, - /*recursively=*/true)); - REQUIRE(FileSystemManager::CreateDirectory(test_dir_zip)); - res = - ArchiveOps::ExtractArchive(ArchiveType::Zip, filename_zip, "."); - if (res != std::nullopt) { - FAIL(*res); - } - compare_extracted(test_dir_zip); - } - - if (FileSystemManager::IsExecutable("/usr/bin/unzip")) { - SECTION("Extract via system unzip") { - REQUIRE( - FileSystemManager::RemoveDirectory(test_dir_zip, - /*recursively=*/true)); - REQUIRE(FileSystemManager::CreateDirectory(test_dir_zip)); - - REQUIRE(system(("/usr/bin/unzip " + filename_zip).c_str()) == - 0); - compare_extracted(test_dir_zip); + compare_extracted(scenario.test_dir); } } } From 24ad2ce7d070ae684159b6445981017dd7a40f8a Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Mon, 14 Aug 2023 10:00:29 +0200 Subject: [PATCH 59/97] ArchiveOps: Add proper autodetection option for tarballs We shouldn't exclude the possibility of receiving uncompressed tarballs as repositories in just-mr. Therefore, we introduce an explicit type that performs the autodetection (default behaviour in just-mr). This is done to also be more in line with our implementation which allows the granular handling of various archive types (currently used only for testing purposes). --- src/other_tools/root_maps/content_git_map.cpp | 3 ++- src/other_tools/utils/archive_ops.cpp | 12 ++++++++---- src/other_tools/utils/archive_ops.hpp | 9 ++++++++- test/other_tools/utils/archive_usage.test.cpp | 4 ++++ 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/other_tools/root_maps/content_git_map.cpp b/src/other_tools/root_maps/content_git_map.cpp index 7f0b30e09..5fb1bd882 100644 --- a/src/other_tools/root_maps/content_git_map.cpp +++ b/src/other_tools/root_maps/content_git_map.cpp @@ -31,7 +31,8 @@ namespace { std::filesystem::path const& dst_dir) noexcept -> std::optional { if (repo_type == "archive") { - return ArchiveOps::ExtractArchive(ArchiveType::Tar, archive, dst_dir); + return ArchiveOps::ExtractArchive( + ArchiveType::TarAuto, archive, dst_dir); } if (repo_type == "zip") { return ArchiveOps::ExtractArchive(ArchiveType::Zip, archive, dst_dir); diff --git a/src/other_tools/utils/archive_ops.cpp b/src/other_tools/utils/archive_ops.cpp index 94c000c79..7115680fc 100644 --- a/src/other_tools/utils/archive_ops.cpp +++ b/src/other_tools/utils/archive_ops.cpp @@ -66,16 +66,16 @@ auto enable_write_filter(archive* aw, ArchiveType type) -> bool { auto enable_read_filter(archive* ar, ArchiveType type) -> bool { switch (type) { - case ArchiveType::Tar: // autodetect from supported compression filters - return (archive_read_support_filter_xz(ar) == ARCHIVE_OK) and - (archive_read_support_filter_gzip(ar) == ARCHIVE_OK) and - (archive_read_support_filter_bzip2(ar) == ARCHIVE_OK); + case ArchiveType::Tar: + return true; // no compression filter case ArchiveType::TarGz: return (archive_read_support_filter_gzip(ar) == ARCHIVE_OK); case ArchiveType::TarBz2: return (archive_read_support_filter_bzip2(ar) == ARCHIVE_OK); case ArchiveType::TarXz: return (archive_read_support_filter_xz(ar) == ARCHIVE_OK); + case ArchiveType::TarAuto: + return (archive_read_support_filter_all(ar) == ARCHIVE_OK); default: return false; } @@ -144,6 +144,9 @@ auto ArchiveOps::EnableWriteFormats(archive* aw, ArchiveType type) std::string(archive_error_string(aw)); } } break; + case ArchiveType::TarAuto: + return std::string( + "ArchiveOps: Writing a tarball-type archive must be explicit!"); } return std::nullopt; // success! } @@ -157,6 +160,7 @@ auto ArchiveOps::EnableReadFormats(archive* ar, ArchiveType type) std::string(archive_error_string(ar)); } } break; + case ArchiveType::TarAuto: case ArchiveType::Tar: case ArchiveType::TarGz: case ArchiveType::TarBz2: diff --git a/src/other_tools/utils/archive_ops.hpp b/src/other_tools/utils/archive_ops.hpp index 6ad5b7c46..74fcf39c2 100644 --- a/src/other_tools/utils/archive_ops.hpp +++ b/src/other_tools/utils/archive_ops.hpp @@ -25,7 +25,14 @@ using archive = struct archive; using archive_entry = struct archive_entry; } -enum class ArchiveType : size_t { Zip, Tar, TarGz, TarBz2, TarXz }; +enum class ArchiveType : size_t { + Zip, + Tar, // uncompressed + TarGz, + TarBz2, + TarXz, + TarAuto, // autodetect tarball-type archives +}; /// \brief Class handling archiving and unarchiving operations via libarchive class ArchiveOps { diff --git a/test/other_tools/utils/archive_usage.test.cpp b/test/other_tools/utils/archive_usage.test.cpp index 68fe436dd..59b09d8c5 100644 --- a/test/other_tools/utils/archive_usage.test.cpp +++ b/test/other_tools/utils/archive_usage.test.cpp @@ -224,6 +224,8 @@ void enable_write_format_and_filter(archive* aw, ArchiveType type) { REQUIRE(archive_write_set_format_pax_restricted(aw) == ARCHIVE_OK); REQUIRE(archive_write_add_filter_xz(aw) == ARCHIVE_OK); } break; + case ArchiveType::TarAuto: + return; // unused } } @@ -247,6 +249,8 @@ void enable_read_format_and_filter(archive* ar, ArchiveType type) { REQUIRE(archive_read_support_format_tar(ar) == ARCHIVE_OK); REQUIRE(archive_read_support_filter_xz(ar) == ARCHIVE_OK); } break; + case ArchiveType::TarAuto: + return; // unused } } From 89757d717dd93b38ee5fe1d282b5946641fc6f29 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Thu, 10 Aug 2023 17:17:42 +0200 Subject: [PATCH 60/97] ArchiveOps: Allow lzip and lzma compression filters for tarballs These are provided by the recently added liblzma external dependency. --- src/other_tools/utils/archive_ops.cpp | 16 ++++++++++++++-- src/other_tools/utils/archive_ops.hpp | 2 ++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/other_tools/utils/archive_ops.cpp b/src/other_tools/utils/archive_ops.cpp index 7115680fc..dd291036d 100644 --- a/src/other_tools/utils/archive_ops.cpp +++ b/src/other_tools/utils/archive_ops.cpp @@ -59,6 +59,10 @@ auto enable_write_filter(archive* aw, ArchiveType type) -> bool { return (archive_write_add_filter_bzip2(aw) == ARCHIVE_OK); case ArchiveType::TarXz: return (archive_write_add_filter_xz(aw) == ARCHIVE_OK); + case ArchiveType::TarLz: + return (archive_write_add_filter_lzip(aw) == ARCHIVE_OK); + case ArchiveType::TarLzma: + return (archive_write_add_filter_lzma(aw) == ARCHIVE_OK); default: return false; } @@ -74,6 +78,10 @@ auto enable_read_filter(archive* ar, ArchiveType type) -> bool { return (archive_read_support_filter_bzip2(ar) == ARCHIVE_OK); case ArchiveType::TarXz: return (archive_read_support_filter_xz(ar) == ARCHIVE_OK); + case ArchiveType::TarLz: + return (archive_read_support_filter_lzip(ar) == ARCHIVE_OK); + case ArchiveType::TarLzma: + return (archive_read_support_filter_lzma(ar) == ARCHIVE_OK); case ArchiveType::TarAuto: return (archive_read_support_filter_all(ar) == ARCHIVE_OK); default: @@ -137,7 +145,9 @@ auto ArchiveOps::EnableWriteFormats(archive* aw, ArchiveType type) case ArchiveType::Tar: case ArchiveType::TarGz: case ArchiveType::TarBz2: - case ArchiveType::TarXz: { + case ArchiveType::TarXz: + case ArchiveType::TarLz: + case ArchiveType::TarLzma: { if ((archive_write_set_format_pax_restricted(aw) != ARCHIVE_OK) or not enable_write_filter(aw, type)) { return std::string("ArchiveOps: ") + @@ -164,7 +174,9 @@ auto ArchiveOps::EnableReadFormats(archive* ar, ArchiveType type) case ArchiveType::Tar: case ArchiveType::TarGz: case ArchiveType::TarBz2: - case ArchiveType::TarXz: { + case ArchiveType::TarXz: + case ArchiveType::TarLz: + case ArchiveType::TarLzma: { if ((archive_read_support_format_tar(ar) != ARCHIVE_OK) or not enable_read_filter(ar, type)) { return std::string("ArchiveOps: ") + diff --git a/src/other_tools/utils/archive_ops.hpp b/src/other_tools/utils/archive_ops.hpp index 74fcf39c2..91595867d 100644 --- a/src/other_tools/utils/archive_ops.hpp +++ b/src/other_tools/utils/archive_ops.hpp @@ -31,6 +31,8 @@ enum class ArchiveType : size_t { TarGz, TarBz2, TarXz, + TarLz, + TarLzma, TarAuto, // autodetect tarball-type archives }; From 149b74e00eb0937231087251d7644be0aa478929 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Thu, 10 Aug 2023 17:18:44 +0200 Subject: [PATCH 61/97] test: Add checks for tarballs with lzip and lzma compression --- test/other_tools/utils/archive_usage.test.cpp | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/test/other_tools/utils/archive_usage.test.cpp b/test/other_tools/utils/archive_usage.test.cpp index 59b09d8c5..3131e7a2c 100644 --- a/test/other_tools/utils/archive_usage.test.cpp +++ b/test/other_tools/utils/archive_usage.test.cpp @@ -73,6 +73,18 @@ std::vector const kTestScenarios = { .filename = "test.tar.xz", .tools = {"tar", "xz"}, .cmd = "/usr/bin/tar xJf"}, + {.test_name = "tar.lz", + .type = ArchiveType::TarLz, + .test_dir = "test_tar_lz", + .filename = "test.tar.lz", + .tools = {"tar", "lzip"}, + .cmd = "/usr/bin/tar --lzip -x -f"}, + {.test_name = "tar.lzma", + .type = ArchiveType::TarLzma, + .test_dir = "test_tar_lzma", + .filename = "test.tar.lzma", + .tools = {"tar", "lzma"}, + .cmd = "/usr/bin/tar --lzma -x -f"}, {.test_name = "zip", .type = ArchiveType::Zip, .test_dir = "test_zip", @@ -138,6 +150,8 @@ void extract_archive(std::string const& path) { REQUIRE(archive_read_support_filter_gzip(a) == ARCHIVE_OK); REQUIRE(archive_read_support_filter_bzip2(a) == ARCHIVE_OK); REQUIRE(archive_read_support_filter_xz(a) == ARCHIVE_OK); + REQUIRE(archive_read_support_filter_lzip(a) == ARCHIVE_OK); + REQUIRE(archive_read_support_filter_lzma(a) == ARCHIVE_OK); REQUIRE(archive_read_open_filename(a, path.c_str(), kBlockSize) == ARCHIVE_OK); @@ -224,6 +238,14 @@ void enable_write_format_and_filter(archive* aw, ArchiveType type) { REQUIRE(archive_write_set_format_pax_restricted(aw) == ARCHIVE_OK); REQUIRE(archive_write_add_filter_xz(aw) == ARCHIVE_OK); } break; + case ArchiveType::TarLz: { + REQUIRE(archive_write_set_format_pax_restricted(aw) == ARCHIVE_OK); + REQUIRE(archive_write_add_filter_lzip(aw) == ARCHIVE_OK); + } break; + case ArchiveType::TarLzma: { + REQUIRE(archive_write_set_format_pax_restricted(aw) == ARCHIVE_OK); + REQUIRE(archive_write_add_filter_lzma(aw) == ARCHIVE_OK); + } break; case ArchiveType::TarAuto: return; // unused } @@ -249,6 +271,14 @@ void enable_read_format_and_filter(archive* ar, ArchiveType type) { REQUIRE(archive_read_support_format_tar(ar) == ARCHIVE_OK); REQUIRE(archive_read_support_filter_xz(ar) == ARCHIVE_OK); } break; + case ArchiveType::TarLz: { + REQUIRE(archive_read_support_format_tar(ar) == ARCHIVE_OK); + REQUIRE(archive_read_support_filter_lzip(ar) == ARCHIVE_OK); + } break; + case ArchiveType::TarLzma: { + REQUIRE(archive_read_support_format_tar(ar) == ARCHIVE_OK); + REQUIRE(archive_read_support_filter_lzma(ar) == ARCHIVE_OK); + } break; case ArchiveType::TarAuto: return; // unused } @@ -311,6 +341,7 @@ TEST_CASE("Read-write archives", "[archive_read_write]") { tools_exist &= FileSystemManager::IsExecutable( std::string("/usr/bin/") + tool); } + if (tools_exist) { SECTION("Extract via system tools") { REQUIRE( From 65886ff172190648c1114a9953c154f3f4821119 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Thu, 10 Aug 2023 17:46:49 +0200 Subject: [PATCH 62/97] ArchiveOps: Add handling of 7zip archives Also updates the archive usage tests accordingly. --- src/other_tools/utils/archive_ops.cpp | 14 +++++++++++++- src/other_tools/utils/archive_ops.hpp | 3 ++- test/other_tools/utils/archive_usage.test.cpp | 15 ++++++++++++++- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/other_tools/utils/archive_ops.cpp b/src/other_tools/utils/archive_ops.cpp index dd291036d..a0804abcf 100644 --- a/src/other_tools/utils/archive_ops.cpp +++ b/src/other_tools/utils/archive_ops.cpp @@ -71,7 +71,7 @@ auto enable_write_filter(archive* aw, ArchiveType type) -> bool { auto enable_read_filter(archive* ar, ArchiveType type) -> bool { switch (type) { case ArchiveType::Tar: - return true; // no compression filter + return true; // no outside compression filter case ArchiveType::TarGz: return (archive_read_support_filter_gzip(ar) == ARCHIVE_OK); case ArchiveType::TarBz2: @@ -142,6 +142,12 @@ auto ArchiveOps::EnableWriteFormats(archive* aw, ArchiveType type) std::string(archive_error_string(aw)); } } break; + case ArchiveType::_7Zip: { + if (archive_write_set_format_7zip(aw) != ARCHIVE_OK) { + return std::string("ArchiveOps: ") + + std::string(archive_error_string(aw)); + } + } break; case ArchiveType::Tar: case ArchiveType::TarGz: case ArchiveType::TarBz2: @@ -170,6 +176,12 @@ auto ArchiveOps::EnableReadFormats(archive* ar, ArchiveType type) std::string(archive_error_string(ar)); } } break; + case ArchiveType::_7Zip: { + if (archive_read_support_format_7zip(ar) != ARCHIVE_OK) { + return std::string("ArchiveOps: ") + + std::string(archive_error_string(ar)); + } + } break; case ArchiveType::TarAuto: case ArchiveType::Tar: case ArchiveType::TarGz: diff --git a/src/other_tools/utils/archive_ops.hpp b/src/other_tools/utils/archive_ops.hpp index 91595867d..94013f306 100644 --- a/src/other_tools/utils/archive_ops.hpp +++ b/src/other_tools/utils/archive_ops.hpp @@ -27,13 +27,14 @@ using archive_entry = struct archive_entry; enum class ArchiveType : size_t { Zip, + _7Zip, Tar, // uncompressed TarGz, TarBz2, TarXz, TarLz, TarLzma, - TarAuto, // autodetect tarball-type archives + TarAuto // autodetect tarball-type archives }; /// \brief Class handling archiving and unarchiving operations via libarchive diff --git a/test/other_tools/utils/archive_usage.test.cpp b/test/other_tools/utils/archive_usage.test.cpp index 3131e7a2c..542f05e1b 100644 --- a/test/other_tools/utils/archive_usage.test.cpp +++ b/test/other_tools/utils/archive_usage.test.cpp @@ -90,7 +90,13 @@ std::vector const kTestScenarios = { .test_dir = "test_zip", .filename = "test.zip", .tools = {"unzip"}, - .cmd = "/usr/bin/unzip"}}; + .cmd = "/usr/bin/unzip"}, + {.test_name = "7zip", + .type = ArchiveType::_7Zip, + .test_dir = "test_7zip", + .filename = "test.7z", + .tools = {"7z"}, // 7z comes with its own lzma-type compression + .cmd = "/usr/bin/7z x"}}; [[nodiscard]] auto read_archive(archive* a, std::string const& path) -> filetree_t { @@ -147,6 +153,7 @@ void extract_archive(std::string const& path) { REQUIRE(a != nullptr); REQUIRE(archive_read_support_format_tar(a) == ARCHIVE_OK); REQUIRE(archive_read_support_format_zip(a) == ARCHIVE_OK); + REQUIRE(archive_read_support_format_7zip(a) == ARCHIVE_OK); REQUIRE(archive_read_support_filter_gzip(a) == ARCHIVE_OK); REQUIRE(archive_read_support_filter_bzip2(a) == ARCHIVE_OK); REQUIRE(archive_read_support_filter_xz(a) == ARCHIVE_OK); @@ -223,6 +230,9 @@ void enable_write_format_and_filter(archive* aw, ArchiveType type) { case ArchiveType::Zip: { REQUIRE(archive_write_set_format_zip(aw) == ARCHIVE_OK); } break; + case ArchiveType::_7Zip: { + REQUIRE(archive_write_set_format_7zip(aw) == ARCHIVE_OK); + } break; case ArchiveType::Tar: { REQUIRE(archive_write_set_format_pax_restricted(aw) == ARCHIVE_OK); } break; @@ -256,6 +266,9 @@ void enable_read_format_and_filter(archive* ar, ArchiveType type) { case ArchiveType::Zip: { REQUIRE(archive_read_support_format_zip(ar) == ARCHIVE_OK); } break; + case ArchiveType::_7Zip: { + REQUIRE(archive_read_support_format_7zip(ar) == ARCHIVE_OK); + } break; case ArchiveType::Tar: { REQUIRE(archive_read_support_format_tar(ar) == ARCHIVE_OK); } break; From d95a7a4bb8d00a2e994bee82f4f64b2ed1d9ec12 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Mon, 14 Aug 2023 11:05:34 +0200 Subject: [PATCH 63/97] just-mr archives: Add 7zip support via autodetection option Similarly to tarballs, an autodetection option for zip-like archives is added to enable a unified handling of both traditional zip and 7zip formats. Thus, for "zip" archives just-mr uses now this autodetection option. --- src/other_tools/root_maps/content_git_map.cpp | 3 ++- src/other_tools/utils/archive_ops.cpp | 14 ++++++++++++++ src/other_tools/utils/archive_ops.hpp | 3 ++- test/other_tools/utils/archive_usage.test.cpp | 2 ++ 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/other_tools/root_maps/content_git_map.cpp b/src/other_tools/root_maps/content_git_map.cpp index 5fb1bd882..64c3db758 100644 --- a/src/other_tools/root_maps/content_git_map.cpp +++ b/src/other_tools/root_maps/content_git_map.cpp @@ -35,7 +35,8 @@ namespace { ArchiveType::TarAuto, archive, dst_dir); } if (repo_type == "zip") { - return ArchiveOps::ExtractArchive(ArchiveType::Zip, archive, dst_dir); + return ArchiveOps::ExtractArchive( + ArchiveType::ZipAuto, archive, dst_dir); } return "unrecognized repository type"; } diff --git a/src/other_tools/utils/archive_ops.cpp b/src/other_tools/utils/archive_ops.cpp index a0804abcf..f0d025318 100644 --- a/src/other_tools/utils/archive_ops.cpp +++ b/src/other_tools/utils/archive_ops.cpp @@ -148,6 +148,10 @@ auto ArchiveOps::EnableWriteFormats(archive* aw, ArchiveType type) std::string(archive_error_string(aw)); } } break; + case ArchiveType::ZipAuto: { + return std::string( + "ArchiveOps: Writing a zip-like archive must be explicit"); + } case ArchiveType::Tar: case ArchiveType::TarGz: case ArchiveType::TarBz2: @@ -182,6 +186,16 @@ auto ArchiveOps::EnableReadFormats(archive* ar, ArchiveType type) std::string(archive_error_string(ar)); } } break; + case ArchiveType::ZipAuto: { + if (archive_read_support_format_7zip(ar) != ARCHIVE_OK) { + return std::string("ArchiveOps: ") + + std::string(archive_error_string(ar)); + } + if (archive_read_support_format_zip(ar) != ARCHIVE_OK) { + return std::string("ArchiveOps: ") + + std::string(archive_error_string(ar)); + } + } break; case ArchiveType::TarAuto: case ArchiveType::Tar: case ArchiveType::TarGz: diff --git a/src/other_tools/utils/archive_ops.hpp b/src/other_tools/utils/archive_ops.hpp index 94013f306..773115bc9 100644 --- a/src/other_tools/utils/archive_ops.hpp +++ b/src/other_tools/utils/archive_ops.hpp @@ -28,7 +28,8 @@ using archive_entry = struct archive_entry; enum class ArchiveType : size_t { Zip, _7Zip, - Tar, // uncompressed + ZipAuto, // autodetect zip-like archives + Tar, // uncompressed TarGz, TarBz2, TarXz, diff --git a/test/other_tools/utils/archive_usage.test.cpp b/test/other_tools/utils/archive_usage.test.cpp index 542f05e1b..09d080c58 100644 --- a/test/other_tools/utils/archive_usage.test.cpp +++ b/test/other_tools/utils/archive_usage.test.cpp @@ -256,6 +256,7 @@ void enable_write_format_and_filter(archive* aw, ArchiveType type) { REQUIRE(archive_write_set_format_pax_restricted(aw) == ARCHIVE_OK); REQUIRE(archive_write_add_filter_lzma(aw) == ARCHIVE_OK); } break; + case ArchiveType::ZipAuto: case ArchiveType::TarAuto: return; // unused } @@ -292,6 +293,7 @@ void enable_read_format_and_filter(archive* ar, ArchiveType type) { REQUIRE(archive_read_support_format_tar(ar) == ARCHIVE_OK); REQUIRE(archive_read_support_filter_lzma(ar) == ARCHIVE_OK); } break; + case ArchiveType::ZipAuto: case ArchiveType::TarAuto: return; // unused } From 89f854e1c5533bc49c59e7b5ab839463615df5c9 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Mon, 14 Aug 2023 11:38:45 +0200 Subject: [PATCH 64/97] just-mr py: Add support for 7zip archives --- bin/just-mr.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/bin/just-mr.py b/bin/just-mr.py index 216c1390e..b97e8dc76 100755 --- a/bin/just-mr.py +++ b/bin/just-mr.py @@ -449,9 +449,21 @@ def archive_checkout(desc, repo_type="archive"): try_rmtree(target_tmp) os.makedirs(target_tmp) if repo_type == "zip": - run_cmd(["unzip", "-d", ".", cas_path(content_id)], cwd=target_tmp) + try: + run_cmd(["unzip", "-d", ".", cas_path(content_id)], cwd=target_tmp) + except: + try: + run_cmd(["7z", "x", cas_path(content_id)], cwd=target_tmp) + except: + print("Failed to extract zip-like archive %s" % + (cas_path(content_id), )) + sys.exit(1) else: - run_cmd(["tar", "xf", cas_path(content_id)], cwd=target_tmp) + try: + run_cmd(["tar", "xf", cas_path(content_id)], cwd=target_tmp) + except: + print("Failed to extract tarball %s" % (cas_path(content_id), )) + sys.exit(1) if ALWAYS_FILE: move_to_place(target_tmp, target) return ["file", subdir_path(target, desc)] From 4085ab0ee56a628e83003a90f25f60892770c214 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Mon, 14 Aug 2023 11:39:26 +0200 Subject: [PATCH 65/97] man-5: Add plain tar and 7zip support to just-mr repository config --- share/man/just-mr-repository-config.5.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/share/man/just-mr-repository-config.5.md b/share/man/just-mr-repository-config.5.md index 60a115ef0..dfa40b494 100644 --- a/share/man/just-mr-repository-config.5.md +++ b/share/man/just-mr-repository-config.5.md @@ -43,9 +43,9 @@ The following fields are supported: ### *`"archive"`* / *`"zip"`* They define as workspace root a remote archive. The only difference -between the two types is the nature of the archive: a compressed tarball -in the case of *`"archive"`*, or a compressed zip file in the case of -*`"zip"`*. +between the two types is the nature of the archive: a tarball (preferably +compressed) in the case of *`"archive"`*, or a compressed zip or 7zip file +in the case of *`"zip"`*. The following fields are supported: From 973009e78bed4d0f57c75c42de1ac62033603155 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Mon, 14 Aug 2023 14:08:11 +0200 Subject: [PATCH 66/97] Update CHANGELOG --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90259af6b..3f1fd23ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,9 +24,11 @@ A feature release on top of `1.1.0`, backwards compatible. directories of `just-mr`, will eventually get cleaned up by garbage collection. - `just-mr` now supports unpacking tar archives compressed with - bzip2 and xz. + bzip2, xz, lzip, and lzma. - The option `-P` of `build` and `install-cas` can be used to inspect parts of a tree. +- `just-mr` now supports unpacking 7zip archives (with default + compression) when provided as `"zip"` type repositories. ### Fixes From 75147177e4d6886d0bcb51c650f5f69deb14a632 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Mon, 14 Aug 2023 17:52:17 +0200 Subject: [PATCH 67/97] just-mr: More consistent error messages in parsing RC file --- src/other_tools/just_mr/main.cpp | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/other_tools/just_mr/main.cpp b/src/other_tools/just_mr/main.cpp index b4a7b554f..ec3329486 100644 --- a/src/other_tools/just_mr/main.cpp +++ b/src/other_tools/just_mr/main.cpp @@ -357,6 +357,13 @@ void SetupLogging(MultiRepoLogArguments const& clargs) { // read distdirs; user can append, but does not overwrite auto distdirs = rc_config["distdirs"]; if (distdirs.IsNotNull()) { + if (not distdirs->IsList()) { + Logger::Log(LogLevel::Error, + "Configuration-file provided distdirs has to be a list " + "of strings, but found {}", + distdirs->ToString()); + std::exit(kExitConfigError); + } auto const& distdirs_list = distdirs->List(); for (auto const& l : distdirs_list) { auto paths = @@ -393,9 +400,25 @@ void SetupLogging(MultiRepoLogArguments const& clargs) { // read additional just args; user can append, but does not overwrite auto just_args = rc_config["just args"]; if (just_args.IsNotNull()) { + if (not just_args->IsMap()) { + Logger::Log(LogLevel::Error, + "Configuration-file provided 'just' arguments has to " + "be a map, but found {}", + just_args->ToString()); + std::exit(kExitConfigError); + } for (auto const& [cmd_name, cmd_args] : just_args->Map()) { // get list of string args for current command std::vector args{}; + if (not cmd_args->IsList()) { + Logger::Log( + LogLevel::Error, + "Configuration-file provided 'just' argument key {} has to " + "have as value a list of strings, but found {}", + cmd_name, + cmd_args->ToString()); + std::exit(kExitConfigError); + } auto const& args_list = cmd_args->List(); args.reserve(args_list.size()); for (auto const& arg : args_list) { @@ -410,8 +433,8 @@ void SetupLogging(MultiRepoLogArguments const& clargs) { if (launcher.IsNotNull()) { if (not launcher->IsList()) { Logger::Log(LogLevel::Error, - "Configuration-file provided launcher {} is not a " - "list of strings", + "Configuration-file provided launcher has to be a " + "list of strings, but found {}", launcher->ToString()); std::exit(kExitConfigError); } From 0574233eee954e1869db0d476ff7106ee72b12b5 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Wed, 16 Aug 2023 10:46:15 +0200 Subject: [PATCH 68/97] man: Fix typos --- share/man/just-import-git.1.md | 4 ++-- share/man/just.1.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/share/man/just-import-git.1.md b/share/man/just-import-git.1.md index afc25d7ea..7ac7489e8 100644 --- a/share/man/just-import-git.1.md +++ b/share/man/just-import-git.1.md @@ -55,7 +55,7 @@ Use the specified file as the configuration to import into. The string *`-`* is treated as a request to take the config from stdin; so a file called *`-`* has to be described as *`.`*/*`-`*, or similar. If not specified, for a config file is searched in the same way, as **`just-mr`** -does, when invoved with **`--norc`**. +does, when invoked with **`--norc`**. **`-h`**, **`--help`** Output a usage message and exit. @@ -70,7 +70,7 @@ compatible repository already exists in the configuration. **`-R`** *`RELPATH`* Use the file, specified by path relative to the repository root, as multi-repository specification in the imported repository. If not -specifed, for a config file is searched in the same way as **`just-mr`** +specified, for a config file is searched in the same way as **`just-mr`** does, when invoked with **`--no-rc`**, however leaving out searches relative to global roots (*`"home"`* and *`"system"`*). In other words, *`repos.json`* and *`etc/repos.json`* are tried if this option is not diff --git a/share/man/just.1.md b/share/man/just.1.md index e587da7a8..125df37e0 100644 --- a/share/man/just.1.md +++ b/share/man/just.1.md @@ -506,7 +506,7 @@ remote-execution endpoints based on the the execution properties. The format is a JSON list of pairs (lists of length two) of an object of strings and a string. The first entry describes a condition (the remote-execution properties have to agree on the domain of this -object), the is remote-exeuction address in the NAME:PORT format as +object), the is remote-execution address in the NAME:PORT format as for the **`-r`** option. The first matching entry (if any) is taken; in none matches the default execution endpoint is taken (either as specified by **`-r`**, or local execution if no endpoint is From aa9007a411dbbcf423613d3cc5beb10658c05356 Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Wed, 16 Aug 2023 12:27:23 +0200 Subject: [PATCH 69/97] just.1 man: fix typos --- share/man/just.1.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/share/man/just.1.md b/share/man/just.1.md index 125df37e0..529a9df58 100644 --- a/share/man/just.1.md +++ b/share/man/just.1.md @@ -506,9 +506,9 @@ remote-execution endpoints based on the the execution properties. The format is a JSON list of pairs (lists of length two) of an object of strings and a string. The first entry describes a condition (the remote-execution properties have to agree on the domain of this -object), the is remote-execution address in the NAME:PORT format as -for the **`-r`** option. The first matching entry (if any) is taken; -in none matches the default execution endpoint is taken (either +object), the second entry is a remote-execution address in the NAME:PORT +format as for the **`-r`** option. The first matching entry (if any) is taken; +if none matches, the default execution endpoint is taken (either as specified by **`-r`**, or local execution if no endpoint is specified). Supported by: analyse|build|install-cas|install|rebuild|traverse. From a9f68b1f9b2445fe42d72f03c44d084e8ef0d444 Mon Sep 17 00:00:00 2001 From: Oliver Reiche Date: Wed, 16 Aug 2023 15:30:11 +0200 Subject: [PATCH 70/97] tutorial: Add missing language specifier --- doc/tutorial/cross-compiling.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tutorial/cross-compiling.md b/doc/tutorial/cross-compiling.md index 4bdeadbb8..4e1a9fce2 100644 --- a/doc/tutorial/cross-compiling.md +++ b/doc/tutorial/cross-compiling.md @@ -86,7 +86,7 @@ $ ``` To cross compile, we simply add `TARGET_ARCH`. -``` +``` sh $ just-mr build -D '{"COMPILER_FAMILY": "gnu", "OS": "linux", "ARCH": "x86_64", "TARGET_ARCH": "arm64"}' INFO: Performing repositories setup INFO: Found 21 repositories to set up From 5e71dca741099d9587fa4b705242673d45316162 Mon Sep 17 00:00:00 2001 From: Oliver Reiche Date: Wed, 16 Aug 2023 12:48:17 +0200 Subject: [PATCH 71/97] rules: Remove pointless defaults --- rules/CC/EXPRESSIONS | 32 ++++++++------------------------ rules/CC/proto/EXPRESSIONS | 20 +++++--------------- 2 files changed, 13 insertions(+), 39 deletions(-) diff --git a/rules/CC/EXPRESSIONS b/rules/CC/EXPRESSIONS index cad28e8c7..0ac44d279 100644 --- a/rules/CC/EXPRESSIONS +++ b/rules/CC/EXPRESSIONS @@ -6,9 +6,7 @@ , "bindings": [ ["fieldname", "defaults"] , ["provider", "CC"] - , [ "transition" - , {"type": "var", "name": "defaults-transition", "default": null} - ] + , ["transition", {"type": "var", "name": "defaults-transition"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} } @@ -21,9 +19,7 @@ , "bindings": [ ["fieldname", "defaults"] , ["provider", "CXX"] - , [ "transition" - , {"type": "var", "name": "defaults-transition", "default": null} - ] + , ["transition", {"type": "var", "name": "defaults-transition"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} } @@ -36,9 +32,7 @@ , "bindings": [ ["fieldname", "defaults"] , ["provider", "AR"] - , [ "transition" - , {"type": "var", "name": "defaults-transition", "default": null} - ] + , ["transition", {"type": "var", "name": "defaults-transition"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} } @@ -51,9 +45,7 @@ , "bindings": [ ["fieldname", "defaults"] , ["provider", "CFLAGS"] - , [ "transition" - , {"type": "var", "name": "defaults-transition", "default": null} - ] + , ["transition", {"type": "var", "name": "defaults-transition"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} } @@ -66,9 +58,7 @@ , "bindings": [ ["fieldname", "defaults"] , ["provider", "CXXFLAGS"] - , [ "transition" - , {"type": "var", "name": "defaults-transition", "default": null} - ] + , ["transition", {"type": "var", "name": "defaults-transition"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} } @@ -81,9 +71,7 @@ , "bindings": [ ["fieldname", "defaults"] , ["provider", "LDFLAGS"] - , [ "transition" - , {"type": "var", "name": "defaults-transition", "default": null} - ] + , ["transition", {"type": "var", "name": "defaults-transition"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} } @@ -96,9 +84,7 @@ , "bindings": [ ["fieldname", "defaults"] , ["provider", "ENV"] - , [ "transition" - , {"type": "var", "name": "defaults-transition", "default": null} - ] + , ["transition", {"type": "var", "name": "defaults-transition"}] , ["default", {"type": "empty_map"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "map_provider"} @@ -112,9 +98,7 @@ , "bindings": [ ["fieldname", "defaults"] , ["provider", "TOOLCHAIN"] - , [ "transition" - , {"type": "var", "name": "defaults-transition", "default": null} - ] + , ["transition", {"type": "var", "name": "defaults-transition"}] , ["default", {"type": "empty_map"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "map_provider"} diff --git a/rules/CC/proto/EXPRESSIONS b/rules/CC/proto/EXPRESSIONS index 6be74df95..84d3d55ec 100644 --- a/rules/CC/proto/EXPRESSIONS +++ b/rules/CC/proto/EXPRESSIONS @@ -6,9 +6,7 @@ , "bindings": [ ["fieldname", "proto-defaults"] , ["provider", "PROTOC"] - , [ "transition" - , {"type": "var", "name": "defaults-transition", "default": null} - ] + , ["transition", {"type": "var", "name": "defaults-transition"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} } @@ -21,9 +19,7 @@ , "bindings": [ ["fieldname", "proto-defaults"] , ["provider", "LDFLAGS"] - , [ "transition" - , {"type": "var", "name": "defaults-transition", "default": null} - ] + , ["transition", {"type": "var", "name": "defaults-transition"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} } @@ -36,9 +32,7 @@ , "bindings": [ ["fieldname", "proto-defaults"] , ["provider", "GRPC_PLUGIN"] - , [ "transition" - , {"type": "var", "name": "defaults-transition", "default": null} - ] + , ["transition", {"type": "var", "name": "defaults-transition"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} } @@ -51,9 +45,7 @@ , "bindings": [ ["fieldname", "proto-defaults"] , ["provider", "ENV"] - , [ "transition" - , {"type": "var", "name": "defaults-transition", "default": null} - ] + , ["transition", {"type": "var", "name": "defaults-transition"}] , ["default", {"type": "empty_map"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "map_provider"} @@ -67,9 +59,7 @@ , "bindings": [ ["fieldname", "proto-defaults"] , ["provider", "TOOLCHAIN"] - , [ "transition" - , {"type": "var", "name": "defaults-transition", "default": null} - ] + , ["transition", {"type": "var", "name": "defaults-transition"}] , ["default", {"type": "empty_map"}] ] , "body": {"type": "CALL_EXPRESSION", "name": "map_provider"} From f9ae3b52d5ea07b5f360c338671595808aae9231 Mon Sep 17 00:00:00 2001 From: Oliver Reiche Date: Thu, 10 Aug 2023 10:59:20 +0200 Subject: [PATCH 72/97] rules: Expression to get base providers as list --- rules/CC/EXPRESSIONS | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/rules/CC/EXPRESSIONS b/rules/CC/EXPRESSIONS index 0ac44d279..2ebe7c171 100644 --- a/rules/CC/EXPRESSIONS +++ b/rules/CC/EXPRESSIONS @@ -126,29 +126,39 @@ } } } -, "defaults-base-provides": - { "doc": ["Query list-provider from 'base' targets (last wins)"] +, "defaults-base-provides-list": + { "doc": ["Query list of providers from 'base' targets"] , "vars": ["provider", "default"] , "vars_doc": - { "provider": ["The name of the list-provider in the provides map."] + { "provider": ["The name of the provider in the provides map."] , "default": ["The default if provider is missing (default: [])."] } , "imports": {"provider_list": ["./", "..", "field_provider_list"]} , "expression": { "type": "let*" , "bindings": [["fieldname", "base"]] + , "body": {"type": "CALL_EXPRESSION", "name": "provider_list"} + } + } +, "defaults-base-provides": + { "doc": ["Query provider from 'base' targets (last wins)"] + , "vars": ["provider", "default"] + , "vars_doc": + { "provider": ["The name of the provider in the provides map."] + , "default": ["The default if provider is missing (default: [])."] + } + , "imports": {"base-provides-list": "defaults-base-provides-list"} + , "expression": + { "type": "foldl" + , "var": "next" + , "start": {"type": "var", "name": "default", "default": []} + , "accum_var": "curr" + , "range": {"type": "CALL_EXPRESSION", "name": "base-provides-list"} , "body": - { "type": "foldl" - , "var": "next" - , "start": {"type": "var", "name": "default", "default": []} - , "accum_var": "curr" - , "range": {"type": "CALL_EXPRESSION", "name": "provider_list"} - , "body": - { "type": "if" - , "cond": {"type": "var", "name": "next"} - , "then": {"type": "var", "name": "next"} - , "else": {"type": "var", "name": "curr"} - } + { "type": "if" + , "cond": {"type": "var", "name": "next"} + , "then": {"type": "var", "name": "next"} + , "else": {"type": "var", "name": "curr"} } } } From 5f9a850dc64a4363cc83629812439d6fca9b5187 Mon Sep 17 00:00:00 2001 From: Oliver Reiche Date: Thu, 10 Aug 2023 11:01:21 +0200 Subject: [PATCH 73/97] rules: Extend toolchains via defaults from base --- rules/CC/RULES | 39 ++++++++++++++++++++++----------------- rules/CC/proto/RULES | 39 ++++++++++++++++++++++----------------- 2 files changed, 44 insertions(+), 34 deletions(-) diff --git a/rules/CC/RULES b/rules/CC/RULES index 98c5cb469..8af70bebb 100644 --- a/rules/CC/RULES +++ b/rules/CC/RULES @@ -26,7 +26,7 @@ [ "Optional toolchain directory. A collection of artifacts that provide" , "the tools CC, CXX, and AR (if needed). Note that only artifacts of" , "the specified targets are considered (no runfiles etc.). Specifying" - , "this field overlays artifacts from \"base\"." + , "this field extends artifacts from \"base\"." ] , "deps": [ "Optional CC libraries any CC library and CC binary implicitly depend" @@ -72,7 +72,8 @@ , "config_vars": ["ARCH", "HOST_ARCH"] , "imports": { "base-provides": "defaults-base-provides" - , "artifacts": ["./", "..", "field_artifacts"] + , "base-provides-list": "defaults-base-provides-list" + , "artifacts_list": ["./", "..", "field_artifacts_list"] , "compile-deps": "compile-deps" , "compile-args-deps": "compile-args-deps" , "link-deps": "link-deps" @@ -224,24 +225,28 @@ , ["provider", "TOOLCHAIN"] , ["default", {"type": "empty_map"}] , [ "TOOLCHAIN" - , { "type": "map_union" + , { "type": "disjoint_map_union" + , "msg": "toolchain artifacts may not overlap" , "$1": - [ {"type": "CALL_EXPRESSION", "name": "base-provides"} - , { "type": "if" - , "cond": {"type": "FIELD", "name": "toolchain"} - , "then": - { "type": "let*" - , "bindings": - [ ["fieldname", "toolchain"] - , [ "transition" - , {"type": "CALL_EXPRESSION", "name": "for host"} + { "type": "++" + , "$1": + [ {"type": "CALL_EXPRESSION", "name": "base-provides-list"} + , { "type": "if" + , "cond": {"type": "FIELD", "name": "toolchain"} + , "then": + { "type": "let*" + , "bindings": + [ ["fieldname", "toolchain"] + , [ "transition" + , {"type": "CALL_EXPRESSION", "name": "for host"} + ] ] - ] - , "body": {"type": "CALL_EXPRESSION", "name": "artifacts"} + , "body": + {"type": "CALL_EXPRESSION", "name": "artifacts_list"} + } } - , "else": {"type": "empty_map"} - } - ] + ] + } } ] , [ "CFLAGS" diff --git a/rules/CC/proto/RULES b/rules/CC/proto/RULES index 6d2e7251a..cfc3786e8 100644 --- a/rules/CC/proto/RULES +++ b/rules/CC/proto/RULES @@ -19,7 +19,7 @@ [ "Optional toolchain directory. A collection of artifacts that provide" , "the protobuf compiler and the GRPC plugin (if needed). Note that only" , "artifacts of the specified targets are considered (no runfiles etc.)." - , "Specifying this field overlays artifacts from \"base\"." + , "Specifying this field extends artifacts from \"base\"." ] , "deps": [ "Optional CC libraries the resulting CC proto libraries implicitly" @@ -56,7 +56,8 @@ } , "imports": { "base-provides": ["./", "..", "defaults-base-provides"] - , "artifacts": ["", "field_artifacts"] + , "base-provides-list": ["./", "..", "defaults-base-provides-list"] + , "artifacts_list": ["", "field_artifacts_list"] , "compile-deps": ["CC", "compile-deps"] , "compile-args-deps": ["CC", "compile-args-deps"] , "link-deps": ["CC", "link-deps"] @@ -158,24 +159,28 @@ , ["provider", "TOOLCHAIN"] , ["default", {"type": "empty_map"}] , [ "TOOLCHAIN" - , { "type": "map_union" + , { "type": "disjoint_map_union" + , "msg": "toolchain artifacts may not overlap" , "$1": - [ {"type": "CALL_EXPRESSION", "name": "base-provides"} - , { "type": "if" - , "cond": {"type": "FIELD", "name": "toolchain"} - , "then": - { "type": "let*" - , "bindings": - [ ["fieldname", "toolchain"] - , [ "transition" - , {"type": "CALL_EXPRESSION", "name": "for host"} + { "type": "++" + , "$1": + [ {"type": "CALL_EXPRESSION", "name": "base-provides-list"} + , { "type": "if" + , "cond": {"type": "FIELD", "name": "toolchain"} + , "then": + { "type": "let*" + , "bindings": + [ ["fieldname", "toolchain"] + , [ "transition" + , {"type": "CALL_EXPRESSION", "name": "for host"} + ] ] - ] - , "body": {"type": "CALL_EXPRESSION", "name": "artifacts"} + , "body": + {"type": "CALL_EXPRESSION", "name": "artifacts_list"} + } } - , "else": {"type": "empty_map"} - } - ] + ] + } } ] , [ "LDFLAGS" From dbdf331b6211d957f26b562a82641e0524a9cd02 Mon Sep 17 00:00:00 2001 From: Oliver Reiche Date: Thu, 10 Aug 2023 11:58:18 +0200 Subject: [PATCH 74/97] rules: Implement nub_left for PATH --- rules/EXPRESSIONS | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/rules/EXPRESSIONS b/rules/EXPRESSIONS index c84ebf465..59efead39 100644 --- a/rules/EXPRESSIONS +++ b/rules/EXPRESSIONS @@ -273,4 +273,35 @@ } } } +, "reverse_list": + { "doc": ["Returns the list in reverse order."] + , "vars": ["list"] + , "vars_doc": {"list": ["The list to reverse."]} + , "expression": + { "type": "foldl" + , "var": "item" + , "start": [] + , "accum_var": "out" + , "range": {"type": "var", "name": "list"} + , "body": + { "type": "++" + , "$1": + [[{"type": "var", "name": "item"}], {"type": "var", "name": "out"}] + } + } + } +, "nub_left": + { "doc": ["Removes all but the leftmost duplicate from the list."] + , "vars": ["list"] + , "vars_doc": {"list": ["The list to remove duplicates from."]} + , "imports": {"reverse": "reverse_list"} + , "expression": + { "type": "let*" + , "bindings": + [ ["list", {"type": "CALL_EXPRESSION", "name": "reverse"}] + , ["list", {"type": "nub_right", "$1": {"type": "var", "name": "list"}}] + ] + , "body": {"type": "CALL_EXPRESSION", "name": "reverse"} + } + } } From 7e4f5b993e70c160d7e177d8429a197215ddd1ed Mon Sep 17 00:00:00 2001 From: Oliver Reiche Date: Fri, 21 Jul 2023 15:20:51 +0200 Subject: [PATCH 75/97] rules: Propagate PATH list from defaults ... instead of creating an ENV from it and propagating this. The idea is to keep PATH entries separate and only join them with ":" when we need it. In this way, we can accumulate PATHs from multiple defaults later, and perform a deduplication ("nub_left") to shorten the final string length. --- rules/CC/EXPRESSIONS | 75 ++++++++++++++++++++++++++++++++++ rules/CC/RULES | 35 +++++++++------- rules/CC/auto/RULES | 31 ++++++++++++++ rules/CC/pkgconfig/EXPRESSIONS | 35 +++++++++++++++- rules/CC/proto/EXPRESSIONS | 46 ++++++++++++++++++++- rules/CC/proto/RULES | 35 +++++++++------- 6 files changed, 225 insertions(+), 32 deletions(-) diff --git a/rules/CC/EXPRESSIONS b/rules/CC/EXPRESSIONS index 2ebe7c171..cec5b8a21 100644 --- a/rules/CC/EXPRESSIONS +++ b/rules/CC/EXPRESSIONS @@ -90,6 +90,19 @@ , "body": {"type": "CALL_EXPRESSION", "name": "map_provider"} } } +, "default-PATH": + { "vars": ["defaults-transition"] + , "imports": {"list_provider": ["./", "..", "field_list_provider"]} + , "expression": + { "type": "let*" + , "bindings": + [ ["fieldname", "defaults"] + , ["provider", "PATH"] + , ["transition", {"type": "var", "name": "defaults-transition"}] + ] + , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} + } + } , "default-TOOLCHAIN": { "vars": ["defaults-transition"] , "imports": {"map_provider": ["./", "..", "field_map_provider"]} @@ -811,6 +824,7 @@ , "objects": "objects" , "default-AR": "default-AR" , "default-ENV": "default-ENV" + , "default-PATH": "default-PATH" , "default-TOOLCHAIN": "default-TOOLCHAIN" , "default-NON_SYSTEM_TOOLS": "default-NON_SYSTEM_TOOLS" } @@ -868,6 +882,36 @@ ] } ] + , [ "ENV_PATH" + , { "type": "lookup" + , "map": {"type": "var", "name": "ENV"} + , "key": "PATH" + } + ] + , [ "ENV" + , { "type": "map_union" + , "$1": + [ {"type": "var", "name": "ENV"} + , { "type": "singleton_map" + , "key": "PATH" + , "value": + { "type": "join" + , "separator": ":" + , "$1": + { "type": "++" + , "$1": + [ {"type": "CALL_EXPRESSION", "name": "default-PATH"} + , { "type": "if" + , "cond": {"type": "var", "name": "ENV_PATH"} + , "then": [{"type": "var", "name": "ENV_PATH"}] + } + ] + } + } + } + ] + } + ] , ["objects", {"type": "CALL_EXPRESSION", "name": "objects"}] , [ "base name" , { "type": "if" @@ -1079,6 +1123,7 @@ , "flags": "flags" , "objects": "objects" , "default-ENV": "default-ENV" + , "default-PATH": "default-PATH" , "default-LDFLAGS": "default-LDFLAGS" , "default-TOOLCHAIN": "default-TOOLCHAIN" , "default-NON_SYSTEM_TOOLS": "default-NON_SYSTEM_TOOLS" @@ -1115,6 +1160,36 @@ ] } ] + , [ "ENV_PATH" + , { "type": "lookup" + , "map": {"type": "var", "name": "ENV"} + , "key": "PATH" + } + ] + , [ "ENV" + , { "type": "map_union" + , "$1": + [ {"type": "var", "name": "ENV"} + , { "type": "singleton_map" + , "key": "PATH" + , "value": + { "type": "join" + , "separator": ":" + , "$1": + { "type": "++" + , "$1": + [ {"type": "CALL_EXPRESSION", "name": "default-PATH"} + , { "type": "if" + , "cond": {"type": "var", "name": "ENV_PATH"} + , "then": [{"type": "var", "name": "ENV_PATH"}] + } + ] + } + } + } + ] + } + ] , ["hdrs", {"type": "empty_map"}] , ["objects", {"type": "CALL_EXPRESSION", "name": "objects"}] , [ "link-args" diff --git a/rules/CC/RULES b/rules/CC/RULES index 8af70bebb..33b8520e0 100644 --- a/rules/CC/RULES +++ b/rules/CC/RULES @@ -66,7 +66,7 @@ ] , "PATH": [ "Path for looking up the compilers. Individual paths are joined" - , "with \":\"." + , "with \":\". Specifying this field extends values from \"base\"." ] } , "config_vars": ["ARCH", "HOST_ARCH"] @@ -74,6 +74,7 @@ { "base-provides": "defaults-base-provides" , "base-provides-list": "defaults-base-provides-list" , "artifacts_list": ["./", "..", "field_artifacts_list"] + , "nub_left": ["", "nub_left"] , "compile-deps": "compile-deps" , "compile-args-deps": "compile-args-deps" , "link-deps": "link-deps" @@ -142,23 +143,26 @@ , "else": {"type": "CALL_EXPRESSION", "name": "base-provides"} } ] - , ["provider", "ENV"] - , ["default", {"type": "empty_map"}] - , [ "ENV" - , { "type": "if" - , "cond": {"type": "var", "name": "PATH"} - , "then": - { "type": "singleton_map" - , "key": "PATH" - , "value": - { "type": "join" - , "separator": ":" - , "$1": {"type": "var", "name": "PATH"} + , ["provider", "PATH"] + , [ "PATH" + , { "type": "++" + , "$1": + [ {"type": "var", "name": "PATH"} + , { "type": "++" + , "$1": {"type": "CALL_EXPRESSION", "name": "base-provides-list"} } - } - , "else": {"type": "CALL_EXPRESSION", "name": "base-provides"} + ] + } + ] + , [ "PATH" + , { "type": "let*" + , "bindings": [["list", {"type": "var", "name": "PATH"}]] + , "body": {"type": "CALL_EXPRESSION", "name": "nub_left"} } ] + , ["provider", "ENV"] + , ["default", {"type": "empty_map"}] + , ["ENV", {"type": "CALL_EXPRESSION", "name": "base-provides"}] , ["provider", "NON_SYSTEM_TOOLS"] , ["default", {"type": "empty_map"}] , [ "NON_SYSTEM_TOOLS" @@ -299,6 +303,7 @@ , "CXXFLAGS" , "LDFLAGS" , "AR" + , "PATH" , "ENV" , "TOOLCHAIN" , "NON_SYSTEM_TOOLS" diff --git a/rules/CC/auto/RULES b/rules/CC/auto/RULES index 247ee0163..0e156c989 100644 --- a/rules/CC/auto/RULES +++ b/rules/CC/auto/RULES @@ -156,6 +156,7 @@ , "flags-cc": ["./", "..", "flags-cc"] , "flags-cxx": ["./", "..", "flags-cxx"] , "default-ENV": ["./", "..", "default-ENV"] + , "default-PATH": ["./", "..", "default-PATH"] , "default-TOOLCHAIN": ["./", "..", "default-TOOLCHAIN"] , "default-NON_SYSTEM_TOOLS": ["./", "..", "default-NON_SYSTEM_TOOLS"] } @@ -222,6 +223,36 @@ ] } ] + , [ "ENV_PATH" + , { "type": "lookup" + , "map": {"type": "var", "name": "ENV"} + , "key": "PATH" + } + ] + , [ "ENV" + , { "type": "map_union" + , "$1": + [ {"type": "var", "name": "ENV"} + , { "type": "singleton_map" + , "key": "PATH" + , "value": + { "type": "join" + , "separator": ":" + , "$1": + { "type": "++" + , "$1": + [ {"type": "CALL_EXPRESSION", "name": "default-PATH"} + , { "type": "if" + , "cond": {"type": "var", "name": "ENV_PATH"} + , "then": [{"type": "var", "name": "ENV_PATH"}] + } + ] + } + } + } + ] + } + ] , [ "c.flags" , { "type": "BLOB" , "data": diff --git a/rules/CC/pkgconfig/EXPRESSIONS b/rules/CC/pkgconfig/EXPRESSIONS index 4c174c401..535faf6bf 100644 --- a/rules/CC/pkgconfig/EXPRESSIONS +++ b/rules/CC/pkgconfig/EXPRESSIONS @@ -1,6 +1,9 @@ { "pkgconfig result": { "vars": ["ENV", "name", "args", "stage"] - , "imports": {"default-ENV": ["./", "..", "default-ENV"]} + , "imports": + { "default-ENV": ["./", "..", "default-ENV"] + , "default-PATH": ["./", "..", "default-PATH"] + } , "expression": { "type": "let*" , "bindings": @@ -12,6 +15,36 @@ ] } ] + , [ "ENV_PATH" + , { "type": "lookup" + , "map": {"type": "var", "name": "ENV"} + , "key": "PATH" + } + ] + , [ "ENV" + , { "type": "map_union" + , "$1": + [ {"type": "var", "name": "ENV"} + , { "type": "singleton_map" + , "key": "PATH" + , "value": + { "type": "join" + , "separator": ":" + , "$1": + { "type": "++" + , "$1": + [ {"type": "CALL_EXPRESSION", "name": "default-PATH"} + , { "type": "if" + , "cond": {"type": "var", "name": "ENV_PATH"} + , "then": [{"type": "var", "name": "ENV_PATH"}] + } + ] + } + } + } + ] + } + ] , [ "cflags-filename" , {"type": "join", "$1": [{"type": "var", "name": "name"}, ".cflags"]} ] diff --git a/rules/CC/proto/EXPRESSIONS b/rules/CC/proto/EXPRESSIONS index 84d3d55ec..29c205c2b 100644 --- a/rules/CC/proto/EXPRESSIONS +++ b/rules/CC/proto/EXPRESSIONS @@ -51,6 +51,19 @@ , "body": {"type": "CALL_EXPRESSION", "name": "map_provider"} } } +, "default-PATH": + { "vars": ["defaults-transition"] + , "imports": {"list_provider": ["./", "../..", "field_list_provider"]} + , "expression": + { "type": "let*" + , "bindings": + [ ["fieldname", "proto-defaults"] + , ["provider", "PATH"] + , ["transition", {"type": "var", "name": "defaults-transition"}] + ] + , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} + } + } , "default-TOOLCHAIN": { "vars": ["defaults-transition"] , "imports": {"map_provider": ["./", "../..", "field_map_provider"]} @@ -118,6 +131,7 @@ , "default-LDFLAGS": "default-LDFLAGS" , "default-GRPC_PLUGIN": "default-GRPC_PLUGIN" , "default-ENV": "default-ENV" + , "default-PATH": "default-PATH" , "default-TOOLCHAIN": "default-TOOLCHAIN" , "default-NON_SYSTEM_TOOLS": "default-NON_SYSTEM_TOOLS" } @@ -172,7 +186,7 @@ } } ] - , [ "protoc-ENV" + , [ "ENV" , { "type": "map_union" , "$1": [ {"type": "CALL_EXPRESSION", "name": "default-ENV"} @@ -180,6 +194,36 @@ ] } ] + , [ "ENV_PATH" + , { "type": "lookup" + , "map": {"type": "var", "name": "ENV"} + , "key": "PATH" + } + ] + , [ "protoc-ENV" + , { "type": "map_union" + , "$1": + [ {"type": "var", "name": "ENV"} + , { "type": "singleton_map" + , "key": "PATH" + , "value": + { "type": "join" + , "separator": ":" + , "$1": + { "type": "++" + , "$1": + [ {"type": "CALL_EXPRESSION", "name": "default-PATH"} + , { "type": "if" + , "cond": {"type": "var", "name": "ENV_PATH"} + , "then": [{"type": "var", "name": "ENV_PATH"}] + } + ] + } + } + } + ] + } + ] , ["protoc-deps", {"type": "CALL_EXPRESSION", "name": "protoc-deps"}] , [ "proto srcs" , { "type": "disjoint_map_union" diff --git a/rules/CC/proto/RULES b/rules/CC/proto/RULES index cfc3786e8..4ffc73b2a 100644 --- a/rules/CC/proto/RULES +++ b/rules/CC/proto/RULES @@ -51,13 +51,14 @@ ] , "PATH": [ "Path for looking up the proto compiler. Individual paths are joined" - , "with \":\"." + , "with \":\". Specifying this field extends values from \"base\"." ] } , "imports": { "base-provides": ["./", "..", "defaults-base-provides"] , "base-provides-list": ["./", "..", "defaults-base-provides-list"] , "artifacts_list": ["", "field_artifacts_list"] + , "nub_left": ["", "nub_left"] , "compile-deps": ["CC", "compile-deps"] , "compile-args-deps": ["CC", "compile-args-deps"] , "link-deps": ["CC", "link-deps"] @@ -99,23 +100,26 @@ , "else": {"type": "CALL_EXPRESSION", "name": "base-provides"} } ] - , ["provider", "ENV"] - , ["default", {"type": "empty_map"}] - , [ "ENV" - , { "type": "if" - , "cond": {"type": "var", "name": "PATH"} - , "then": - { "type": "singleton_map" - , "key": "PATH" - , "value": - { "type": "join" - , "separator": ":" - , "$1": {"type": "var", "name": "PATH"} + , ["provider", "PATH"] + , [ "PATH" + , { "type": "++" + , "$1": + [ {"type": "var", "name": "PATH"} + , { "type": "++" + , "$1": {"type": "CALL_EXPRESSION", "name": "base-provides-list"} } - } - , "else": {"type": "CALL_EXPRESSION", "name": "base-provides"} + ] + } + ] + , [ "PATH" + , { "type": "let*" + , "bindings": [["list", {"type": "var", "name": "PATH"}]] + , "body": {"type": "CALL_EXPRESSION", "name": "nub_left"} } ] + , ["provider", "ENV"] + , ["default", {"type": "empty_map"}] + , ["ENV", {"type": "CALL_EXPRESSION", "name": "base-provides"}] , ["provider", "NON_SYSTEM_TOOLS"] , ["default", {"type": "empty_map"}] , [ "NON_SYSTEM_TOOLS" @@ -214,6 +218,7 @@ [ "PROTOC" , "LDFLAGS" , "GRPC_PLUGIN" + , "PATH" , "ENV" , "TOOLCHAIN" , "NON_SYSTEM_TOOLS" From e9812e49f3e2d8f8decb26e9a33d9160d199380d Mon Sep 17 00:00:00 2001 From: Oliver Reiche Date: Wed, 9 Aug 2023 12:17:58 +0200 Subject: [PATCH 76/97] patch: Drop patch-part ... because it was using tool that are not part of coreutils. Using patch parts can still be done via patching patches. --- rules/patch/RULES | 68 ----------------------------------------------- 1 file changed, 68 deletions(-) diff --git a/rules/patch/RULES b/rules/patch/RULES index b1ba95e3b..4de3ddbe4 100644 --- a/rules/patch/RULES +++ b/rules/patch/RULES @@ -1,18 +1,12 @@ { "file": { "doc": ["Replace a file, logically in place, by a patched version"] , "target_fields": ["src", "patch"] - , "string_fields": ["patch-part"] , "config_vars": ["PATCH", "ENV"] , "field_doc": { "src": [ "The single source file to patch, typically an explicit file reference." ] , "patch": ["The patch to apply."] - , "patch-part": - [ "If the patch contains hunks for multiple files, only apply hunks for" - , "the specified file path. Individual directory components are joined" - , "with \"/\". Note that the patch must be provided in unified format." - ] } , "artifacts_doc": ["The patched file, staged to the position the of the original file"] @@ -36,68 +30,6 @@ , "body": {"type": "CALL_EXPRESSION", "name": "stage_field"} } ] - , [ "patch-part" - , { "type": "join" - , "separator": "\\/" - , "$1": {"type": "FIELD", "name": "patch-part"} - } - ] - , [ "splitpatch" - , { "type": "BLOB" - , "data": - { "type": "join" - , "separator": "\n" - , "$1": - [ "set -e" - , { "type": "join" - , "$1": - [ { "type": "join_cmd" - , "$1": - [ "sed" - , "-n" - , { "type": "join" - , "$1": - [ "/^--- " - , {"type": "var", "name": "patch-part"} - , "/," - , "/^[^-+@\\ ]\\|^---\\ /p" - ] - } - ] - } - , " $1 | sed '$d' > patch" - ] - } - ] - } - } - ] - , [ "patch" - , { "type": "if" - , "cond": {"type": "var", "name": "patch-part"} - , "then": - { "type": "ACTION" - , "inputs": - { "type": "map_union" - , "$1": - [ { "type": "to_subdir" - , "subdir": "in" - , "$1": {"type": "var", "name": "patch"} - } - , { "type": "singleton_map" - , "key": "splitpatch" - , "value": {"type": "var", "name": "splitpatch"} - } - ] - } - , "outs": ["patch"] - , "cmd": ["sh", "./splitpatch", "in/patch"] - , "env": - {"type": "var", "name": "ENV", "default": {"type": "empty_map"}} - } - , "else": {"type": "var", "name": "patch"} - } - ] , [ "inputs" , { "type": "map_union" , "$1": From e1bfd8422b150ddf136607da15e339bf46dceda2 Mon Sep 17 00:00:00 2001 From: Oliver Reiche Date: Thu, 10 Aug 2023 11:01:49 +0200 Subject: [PATCH 77/97] patch: Support defaults for patch rules ... and extend compatibility by not relying on arguments that are only supported by GNU patch implementations. --- rules/patch/EXPRESSIONS | 77 ++++++++++++ rules/patch/RULES | 272 ++++++++++++++++++++++++++++++++++++++-- rules/patch/TARGETS | 6 + 3 files changed, 342 insertions(+), 13 deletions(-) create mode 100644 rules/patch/EXPRESSIONS create mode 100644 rules/patch/TARGETS diff --git a/rules/patch/EXPRESSIONS b/rules/patch/EXPRESSIONS new file mode 100644 index 000000000..7f8ce192b --- /dev/null +++ b/rules/patch/EXPRESSIONS @@ -0,0 +1,77 @@ +{ "default-PATCH": + { "vars": ["defaults-transition"] + , "imports": {"list_provider": ["./", "..", "field_list_provider"]} + , "expression": + { "type": "let*" + , "bindings": + [ ["fieldname", "patch-defaults"] + , ["provider", "PATCH"] + , ["transition", {"type": "var", "name": "defaults-transition"}] + ] + , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} + } + } +, "default-ENV": + { "vars": ["defaults-transition"] + , "imports": {"map_provider": ["./", "..", "field_map_provider"]} + , "expression": + { "type": "let*" + , "bindings": + [ ["fieldname", "patch-defaults"] + , ["provider", "ENV"] + , ["transition", {"type": "var", "name": "defaults-transition"}] + , ["default", {"type": "empty_map"}] + ] + , "body": {"type": "CALL_EXPRESSION", "name": "map_provider"} + } + } +, "default-PATH": + { "vars": ["defaults-transition"] + , "imports": {"list_provider": ["./", "..", "field_list_provider"]} + , "expression": + { "type": "let*" + , "bindings": + [ ["fieldname", "patch-defaults"] + , ["provider", "PATH"] + , ["transition", {"type": "var", "name": "defaults-transition"}] + ] + , "body": {"type": "CALL_EXPRESSION", "name": "list_provider"} + } + } +, "default-TOOLCHAIN": + { "vars": ["defaults-transition"] + , "imports": {"map_provider": ["./", "..", "field_map_provider"]} + , "expression": + { "type": "let*" + , "bindings": + [ ["fieldname", "patch-defaults"] + , ["provider", "TOOLCHAIN"] + , ["transition", {"type": "var", "name": "defaults-transition"}] + , ["default", {"type": "empty_map"}] + ] + , "body": {"type": "CALL_EXPRESSION", "name": "map_provider"} + } + } +, "default-NON_SYSTEM_TOOLS": + { "vars": ["defaults-transition"] + , "expression": + { "type": "map_union" + , "$1": + { "type": "foreach" + , "var": "x" + , "range": {"type": "FIELD", "name": "patch-defaults"} + , "body": + { "type": "DEP_PROVIDES" + , "dep": {"type": "var", "name": "x"} + , "provider": "NON_SYSTEM_TOOLS" + , "transition": + { "type": "var" + , "name": "defaults-transition" + , "default": {"type": "empty_map"} + } + , "default": {"type": "empty_map"} + } + } + } + } +} diff --git a/rules/patch/RULES b/rules/patch/RULES index 4de3ddbe4..712a4877a 100644 --- a/rules/patch/RULES +++ b/rules/patch/RULES @@ -1,7 +1,151 @@ -{ "file": +{ "defaults": + { "doc": + [ "A rule to provide defaults." + , "All targets take their defaults for PATCH from the target" + , "[\"\", \"defaults\"]. This is probably the only sensible" + , "use of this rule. As targets form a different root, the defaults" + , "can be provided without changing this directory." + ] + , "target_fields": ["base", "toolchain"] + , "string_fields": ["PATCH", "PATH", "SYSTEM_TOOLS"] + , "field_doc": + { "base": ["Other targets (using the same rule) to inherit values from."] + , "toolchain": + [ "Optional toolchain directory. A collection of artifacts that provide" + , "the tool PATCH. Note that only artifacts of" + , "the specified targets are considered (no runfiles etc.). Specifying" + , "this field extends artifacts from \"base\"." + ] + , "PATCH": ["The patch binary to use"] + , "SYSTEM_TOOLS": + [ "List of tools (\"PATCH\") that should be taken from" + , "the system instead of from \"toolchain\" (if specified)." + ] + , "PATH": + [ "Path for looking up the compilers. Individual paths are joined" + , "with \":\". Specifying this field extends values from \"base\"." + ] + } + , "config_vars": ["ARCH", "HOST_ARCH", "TARGET_ARCH"] + , "imports": + { "base-provides": ["CC", "defaults-base-provides"] + , "base-provides-list": ["CC", "defaults-base-provides-list"] + , "artifacts_list": ["", "field_artifacts_list"] + , "nub_left": ["", "nub_left"] + , "for host": ["transitions", "for host"] + } + , "config_transitions": + {"toolchain": [{"type": "CALL_EXPRESSION", "name": "for host"}]} + , "expression": + { "type": "let*" + , "bindings": + [ ["PATCH", {"type": "FIELD", "name": "PATCH"}] + , ["PATH", {"type": "FIELD", "name": "PATH"}] + , ["provider", "PATCH"] + , [ "PATCH" + , { "type": "if" + , "cond": {"type": "var", "name": "PATCH"} + , "then": {"type": "var", "name": "PATCH"} + , "else": {"type": "CALL_EXPRESSION", "name": "base-provides"} + } + ] + , ["provider", "PATH"] + , [ "PATH" + , { "type": "++" + , "$1": + [ {"type": "var", "name": "PATH"} + , { "type": "++" + , "$1": {"type": "CALL_EXPRESSION", "name": "base-provides-list"} + } + ] + } + ] + , [ "PATH" + , { "type": "let*" + , "bindings": [["list", {"type": "var", "name": "PATH"}]] + , "body": {"type": "CALL_EXPRESSION", "name": "nub_left"} + } + ] + , ["provider", "ENV"] + , ["default", {"type": "empty_map"}] + , ["ENV", {"type": "CALL_EXPRESSION", "name": "base-provides"}] + , ["provider", "NON_SYSTEM_TOOLS"] + , ["default", {"type": "empty_map"}] + , [ "NON_SYSTEM_TOOLS" + , { "type": "map_union" + , "$1": + { "type": "++" + , "$1": + [ [{"type": "CALL_EXPRESSION", "name": "base-provides"}] + , { "type": "if" + , "cond": {"type": "FIELD", "name": "PATCH"} + , "then": + [ { "type": "singleton_map" + , "key": "PATCH" + , "value": + { "type": "if" + , "cond": {"type": "FIELD", "name": "toolchain"} + , "then": true + , "else": false + } + } + ] + } + , { "type": "foreach" + , "range": {"type": "FIELD", "name": "SYSTEM_TOOLS"} + , "var": "tool" + , "body": + { "type": "singleton_map" + , "key": {"type": "var", "name": "tool"} + , "value": false + } + } + ] + } + } + ] + , ["provider", "TOOLCHAIN"] + , ["default", {"type": "empty_map"}] + , [ "TOOLCHAIN" + , { "type": "disjoint_map_union" + , "msg": "toolchain artifacts may not overlap" + , "$1": + { "type": "++" + , "$1": + [ {"type": "CALL_EXPRESSION", "name": "base-provides-list"} + , { "type": "if" + , "cond": {"type": "FIELD", "name": "toolchain"} + , "then": + { "type": "let*" + , "bindings": + [ ["fieldname", "toolchain"] + , [ "transition" + , {"type": "CALL_EXPRESSION", "name": "for host"} + ] + ] + , "body": + {"type": "CALL_EXPRESSION", "name": "artifacts_list"} + } + } + ] + } + } + ] + ] + , "body": + { "type": "RESULT" + , "provides": + { "type": "env" + , "vars": ["PATCH", "PATH", "ENV", "TOOLCHAIN", "NON_SYSTEM_TOOLS"] + } + } + } + } +, "file": { "doc": ["Replace a file, logically in place, by a patched version"] , "target_fields": ["src", "patch"] , "config_vars": ["PATCH", "ENV"] + , "implicit": {"patch-defaults": ["defaults"]} , "field_doc": { "src": [ "The single source file to patch, typically an explicit file reference." @@ -14,11 +158,88 @@ , "imports": { "stage_field": ["./", "..", "stage_singleton_field"] , "stage_artifact": ["./", "..", "stage_artifact_to_singleton_field"] + , "default-PATCH": "default-PATCH" + , "default-TOOLCHAIN": "default-TOOLCHAIN" + , "default-ENV": "default-ENV" + , "default-PATH": "default-PATH" + , "default-NON_SYSTEM_TOOLS": "default-NON_SYSTEM_TOOLS" } , "expression": { "type": "let*" , "bindings": - [ [ "orig" + [ ["TOOLCHAIN_DIR", "toolchain"] + , ["TOOLCHAIN", {"type": "CALL_EXPRESSION", "name": "default-TOOLCHAIN"}] + , [ "TOOLCHAIN" + , { "type": "to_subdir" + , "subdir": {"type": "var", "name": "TOOLCHAIN_DIR"} + , "$1": {"type": "var", "name": "TOOLCHAIN"} + } + ] + , [ "NON_SYSTEM_TOOLS" + , {"type": "CALL_EXPRESSION", "name": "default-NON_SYSTEM_TOOLS"} + ] + , [ "PATCH" + , { "type": "var" + , "name": "PATCH" + , "default": + { "type": "join" + , "$1": + { "type": "++" + , "$1": + [ { "type": "if" + , "cond": + { "type": "lookup" + , "key": "PATCH" + , "map": {"type": "var", "name": "NON_SYSTEM_TOOLS"} + } + , "then": + ["./", {"type": "var", "name": "TOOLCHAIN_DIR"}, "/"] + } + , {"type": "CALL_EXPRESSION", "name": "default-PATCH"} + ] + } + } + } + ] + , [ "ENV" + , { "type": "map_union" + , "$1": + [ {"type": "CALL_EXPRESSION", "name": "default-ENV"} + , {"type": "var", "name": "ENV", "default": {"type": "empty_map"}} + ] + } + ] + , [ "ENV_PATH" + , { "type": "lookup" + , "map": {"type": "var", "name": "ENV"} + , "key": "PATH" + } + ] + , [ "ENV" + , { "type": "map_union" + , "$1": + [ {"type": "var", "name": "ENV"} + , { "type": "singleton_map" + , "key": "PATH" + , "value": + { "type": "join" + , "separator": ":" + , "$1": + { "type": "++" + , "$1": + [ {"type": "CALL_EXPRESSION", "name": "default-PATH"} + , { "type": "if" + , "cond": {"type": "var", "name": "ENV_PATH"} + , "then": [{"type": "var", "name": "ENV_PATH"}] + } + ] + } + } + } + ] + } + ] + , [ "orig" , { "type": "let*" , "bindings": [["fieldname", "src"], ["location", "orig"]] , "body": {"type": "CALL_EXPRESSION", "name": "stage_field"} @@ -30,26 +251,51 @@ , "body": {"type": "CALL_EXPRESSION", "name": "stage_field"} } ] + , [ "script" + , { "type": "singleton_map" + , "key": "run_patch.sh" + , "value": + { "type": "BLOB" + , "data": + { "type": "join" + , "separator": "\n" + , "$1": + [ "set -e" + , "cp orig patched" + , "chmod +w patched" + , { "type": "join" + , "$1": + [ { "type": "join_cmd" + , "$1": + [ {"type": "var", "name": "PATCH", "default": "patch"} + , "patched" + , "patch" + ] + } + , " >log || (cat log && exit 1)" + ] + } + ] + } + } + } + ] , [ "inputs" , { "type": "map_union" , "$1": - [{"type": "var", "name": "orig"}, {"type": "var", "name": "patch"}] + [ {"type": "var", "name": "orig"} + , {"type": "var", "name": "patch"} + , {"type": "var", "name": "TOOLCHAIN"} + , {"type": "var", "name": "script"} + ] } ] , [ "patched" , { "type": "ACTION" , "inputs": {"type": "var", "name": "inputs"} , "outs": ["patched"] - , "cmd": - [ {"type": "var", "name": "PATCH", "default": "patch"} - , "-s" - , "--read-only=ignore" - , "--follow-symlinks" - , "-o" - , "patched" - , "orig" - , "patch" - ] + , "cmd": ["sh", "./run_patch.sh"] + , "env": {"type": "var", "name": "ENV"} } ] , [ "result" diff --git a/rules/patch/TARGETS b/rules/patch/TARGETS new file mode 100644 index 000000000..9a0ad77a2 --- /dev/null +++ b/rules/patch/TARGETS @@ -0,0 +1,6 @@ +{ "defaults": + { "type": ["patch", "defaults"] + , "PATCH": ["patch"] + , "PATH": ["/bin", "/usr/bin"] + } +} From ddc7e2c87d5b92f6fefce3f68d0d78d2d022044a Mon Sep 17 00:00:00 2001 From: Oliver Reiche Date: Fri, 11 Aug 2023 17:05:16 +0200 Subject: [PATCH 78/97] just: Add defaults for patch --- etc/defaults/patch/TARGETS | 5 +++++ etc/toolchain/patch/TARGETS | 6 ++++++ 2 files changed, 11 insertions(+) create mode 100644 etc/defaults/patch/TARGETS create mode 100644 etc/toolchain/patch/TARGETS diff --git a/etc/defaults/patch/TARGETS b/etc/defaults/patch/TARGETS new file mode 100644 index 000000000..7f35ee5a5 --- /dev/null +++ b/etc/defaults/patch/TARGETS @@ -0,0 +1,5 @@ +{ "defaults": + { "type": ["patch", "defaults"] + , "base": [["@", "toolchain", "patch", "defaults"]] + } +} diff --git a/etc/toolchain/patch/TARGETS b/etc/toolchain/patch/TARGETS new file mode 100644 index 000000000..9a0ad77a2 --- /dev/null +++ b/etc/toolchain/patch/TARGETS @@ -0,0 +1,6 @@ +{ "defaults": + { "type": ["patch", "defaults"] + , "PATCH": ["patch"] + , "PATH": ["/bin", "/usr/bin"] + } +} From e17365cb6d1af6ae12fdaaf97f28a02c7ee61bed Mon Sep 17 00:00:00 2001 From: Oliver Reiche Date: Mon, 7 Aug 2023 15:14:46 +0200 Subject: [PATCH 79/97] rules: Support BUILD_ARCH for CC defaults --- rules/CC/RULES | 7 +++++-- rules/transitions/EXPRESSIONS | 25 +++++++++++++++++-------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/rules/CC/RULES b/rules/CC/RULES index 33b8520e0..c30aaee16 100644 --- a/rules/CC/RULES +++ b/rules/CC/RULES @@ -26,7 +26,10 @@ [ "Optional toolchain directory. A collection of artifacts that provide" , "the tools CC, CXX, and AR (if needed). Note that only artifacts of" , "the specified targets are considered (no runfiles etc.). Specifying" - , "this field extends artifacts from \"base\"." + , "this field extends artifacts from \"base\". If the toolchain" + , "supports cross-compilation, it should perform a dispatch on the" + , "configuration variable \"BUILD_ARCH\" to determine for which" + , "architecture to generate code for." ] , "deps": [ "Optional CC libraries any CC library and CC binary implicitly depend" @@ -69,7 +72,7 @@ , "with \":\". Specifying this field extends values from \"base\"." ] } - , "config_vars": ["ARCH", "HOST_ARCH"] + , "config_vars": ["ARCH", "HOST_ARCH", "TARGET_ARCH"] , "imports": { "base-provides": "defaults-base-provides" , "base-provides-list": "defaults-base-provides-list" diff --git a/rules/transitions/EXPRESSIONS b/rules/transitions/EXPRESSIONS index 4fd0d1e9e..596b20ef8 100644 --- a/rules/transitions/EXPRESSIONS +++ b/rules/transitions/EXPRESSIONS @@ -1,13 +1,22 @@ { "for host": - { "vars": ["ARCH", "HOST_ARCH"] + { "vars": ["ARCH", "HOST_ARCH", "TARGET_ARCH"] , "expression": - { "type": "singleton_map" - , "key": "TARGET_ARCH" - , "value": - { "type": "var" - , "name": "HOST_ARCH" - , "default": {"type": "var", "name": "ARCH"} - } + { "type": "let*" + , "bindings": + [ [ "BUILD_ARCH" + , { "type": "var" + , "name": "TARGET_ARCH" + , "default": {"type": "var", "name": "ARCH"} + } + ] + , [ "TARGET_ARCH" + , { "type": "var" + , "name": "HOST_ARCH" + , "default": {"type": "var", "name": "ARCH"} + } + ] + ] + , "body": {"type": "env", "vars": ["BUILD_ARCH", "TARGET_ARCH"]} } } , "target properties": From ffbb5ab230d1307e10f72bf5d94441b2a134c5af Mon Sep 17 00:00:00 2001 From: Oliver Reiche Date: Thu, 6 Jul 2023 14:21:27 +0200 Subject: [PATCH 80/97] Introduce configuration variable TOOLCHAIN_CONFIG ... to pass along toolchain settings for current and future toolchain definitions. Configuration variable COMPILER_FAMILY is replaced by TOOLCHAIN_CONFIG["FAMILY"]. --- CHANGELOG.md | 4 +++ INSTALL.md | 4 +-- TARGETS | 31 ++++++------------- bin/bootstrap.py | 6 ++-- etc/defaults/CC/TARGETS | 24 ++++++++++++-- etc/defaults/CC/TARGETS.absl | 12 +++++-- etc/defaults/CC/TARGETS.archive | 12 +++++-- etc/defaults/CC/TARGETS.curl | 12 +++++-- etc/defaults/CC/TARGETS.git2 | 12 +++++-- etc/defaults/CC/TARGETS.grpc | 22 +++++++++++-- etc/defaults/CC/TARGETS.protobuf | 12 +++++-- etc/import/TARGETS.archive | 17 +++++++--- etc/import/TARGETS.boringssl | 4 +-- etc/import/TARGETS.bzip2 | 6 ++-- etc/import/TARGETS.cares | 2 +- etc/import/TARGETS.catch2 | 2 +- etc/import/TARGETS.curl | 3 +- etc/import/TARGETS.fmt | 2 +- etc/import/TARGETS.git2 | 2 +- etc/import/TARGETS.grpc | 8 ++--- etc/import/TARGETS.lzma | 2 +- etc/import/TARGETS.protobuf | 8 ++--- etc/import/TARGETS.re2 | 2 +- etc/import/TARGETS.zlib | 2 +- etc/import/absl/algorithm/TARGETS.absl | 2 +- etc/import/absl/base/TARGETS.absl | 22 ++++++++----- etc/import/absl/cleanup/TARGETS.absl | 2 +- etc/import/absl/container/TARGETS.absl | 8 ++--- etc/import/absl/debugging/TARGETS.absl | 12 +++++-- etc/import/absl/flags/TARGETS.absl | 4 +-- etc/import/absl/functional/TARGETS.absl | 6 ++-- etc/import/absl/hash/TARGETS.absl | 2 +- etc/import/absl/log/TARGETS.absl | 8 ++--- etc/import/absl/memory/TARGETS.absl | 2 +- etc/import/absl/meta/TARGETS.absl | 2 +- etc/import/absl/numeric/TARGETS.absl | 4 +-- etc/import/absl/random/TARGETS.absl | 17 +++++++--- etc/import/absl/status/TARGETS.absl | 4 +-- etc/import/absl/strings/TARGETS.absl | 8 ++--- etc/import/absl/synchronization/TARGETS.absl | 2 +- etc/import/absl/time/TARGETS.absl | 2 +- etc/import/absl/types/TARGETS.absl | 6 ++-- etc/import/absl/utility/TARGETS.absl | 2 +- etc/import/deps/http-parser/TARGETS.git2 | 13 ++++++-- etc/import/lib/TARGETS.curl | 14 +++++++-- etc/import/libarchive/TARGETS.archive | 14 +++++++-- etc/import/src/TARGETS.archive | 22 +++++++++++-- etc/import/src/TARGETS.curl | 22 +++++++++++-- .../src/google/protobuf/TARGETS.protobuf | 13 ++++++-- .../third_party/utf8_range/TARGETS.protobuf | 2 +- etc/toolchain/CC/TARGETS | 14 +++++++-- test/TARGETS | 19 ++---------- test/bootstrap/TARGETS | 2 +- test/bootstrap/test-bootstrap-pkgconfig.sh | 2 +- test/bootstrap/test-mixed-bootstrap.sh | 2 +- test/bootstrap/test-symlink-bootstrap.sh | 2 +- 56 files changed, 314 insertions(+), 153 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f1fd23ca..f65503348 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,10 @@ A feature release on top of `1.1.0`, backwards compatible. inspect parts of a tree. - `just-mr` now supports unpacking 7zip archives (with default compression) when provided as `"zip"` type repositories. +- The configuration variable `COMPILER_FAMILY` is replaced by the more + flexible `TOOLCHAIN_CONFIG`, an object which may contain the field + `FAMILY`. From now on, this object is used to set the compiler family + (e.g., for GNU, set `{"TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}`). ### Fixes diff --git a/INSTALL.md b/INSTALL.md index 86a234678..3e93f0855 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -31,7 +31,7 @@ variables. The full list can be obtained via `just-mr describe`. | ARCH | x86, x86_64, arm, arm64 | x86_64 | | HOST_ARCH | x86, x86_64, arm, arm64 | *derived from ARCH* | | TARGET_ARCH | x86, x86_64, arm, arm64 | *derived from ARCH* | -| COMPILER_FAMILY | gnu, clang, unknown | unknown | +| TOOLCHAIN_CONFIG["FAMILY"] | gnu, clang, unknown | unknown | | DEBUG | true, false | false | | BUILD_STATIC_BINARY | true, false | false | @@ -92,7 +92,7 @@ Additionally, if `SOURCE_DATE_EPOCH` is set in the build environment, it is forwarded to the build configuration as well. If, on the other hand, `CC` or `CXX` are set in the build configuration, those are also used for the initial steps of the bootstrap procedure. Remember that setting -one of those variables also requires the `COMPILER_FAMILY` to +one of those variables also requires the `TOOLCHAIN_CONFIG["FAMILY"]` to ensure the proper flags are used (if in doubt, set to `"unknown"`). In any case, the resulting binary is self contained and can be moved diff --git a/TARGETS b/TARGETS index 884655470..7dc8e547a 100644 --- a/TARGETS +++ b/TARGETS @@ -10,7 +10,6 @@ , "BUILD_STATIC_BINARY" , "FINAL_LDFLAGS" , "CC" - , "COMPILER_FAMILY" , "CXX" , "DEBUG" , "ENV" @@ -18,6 +17,7 @@ , "OS" , "SOURCE_DATE_EPOCH" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" , "VERSION_EXTRA_SUFFIX" , "PKG_CONFIG_ARGS" ] @@ -30,9 +30,10 @@ , "HOST_ARCH": ["The architecture on which the build actions are carried out."] , "TARGET_ARCH": ["The architecture for which to build the binary."] - , "COMPILER_FAMILY": - [ "The compiler family to use as (cross) toolchain." + , "TOOLCHAIN_CONFIG": + [ "The toolchain configuration." , "" + , "Use field \"FAMILY\" to specify the compiler family." , "Supported values are \"unknown\" (the default), \"clang\", and \"gnu\"." , "The unknown compiler family tries to not make any assumptions on the" , "used C and C++ compilers and uses the generic \"cc\" and \"c++\" as" @@ -77,7 +78,7 @@ , "HOST_ARCH" , "TARGET_ARCH" , "DEBUG" - , "COMPILER_FAMILY" + , "TOOLCHAIN_CONFIG" , "CC" , "CXX" , "ADD_CXXFLAGS" @@ -93,8 +94,7 @@ } , "just": { "type": "configure" - , "arguments_config": - ["OS", "ARCH", "HOST_ARCH", "TARGET_ARCH", "COMPILER_FAMILY"] + , "arguments_config": ["OS", "ARCH", "HOST_ARCH", "TARGET_ARCH"] , "target": "exported-just" , "config": { "type": "let*" @@ -113,14 +113,9 @@ , "default": {"type": "var", "name": "ARCH"} } ] - , [ "COMPILER_FAMILY" - , {"type": "var", "name": "COMPILER_FAMILY", "default": "unknown"} - ] ] , "body": - { "type": "env" - , "vars": ["OS", "ARCH", "HOST_ARCH", "TARGET_ARCH", "COMPILER_FAMILY"] - } + {"type": "env", "vars": ["OS", "ARCH", "HOST_ARCH", "TARGET_ARCH"]} } } , "installed just-mr": @@ -134,7 +129,7 @@ , "HOST_ARCH" , "TARGET_ARCH" , "DEBUG" - , "COMPILER_FAMILY" + , "TOOLCHAIN_CONFIG" , "CC" , "CXX" , "ADD_CXXFLAGS" @@ -149,8 +144,7 @@ } , "just-mr": { "type": "configure" - , "arguments_config": - ["OS", "ARCH", "HOST_ARCH", "TARGET_ARCH", "COMPILER_FAMILY"] + , "arguments_config": ["OS", "ARCH", "HOST_ARCH", "TARGET_ARCH"] , "target": "exported-just-mr" , "config": { "type": "let*" @@ -169,14 +163,9 @@ , "default": {"type": "var", "name": "ARCH"} } ] - , [ "COMPILER_FAMILY" - , {"type": "var", "name": "COMPILER_FAMILY", "default": "clang"} - ] ] , "body": - { "type": "env" - , "vars": ["OS", "ARCH", "HOST_ARCH", "TARGET_ARCH", "COMPILER_FAMILY"] - } + {"type": "env", "vars": ["OS", "ARCH", "HOST_ARCH", "TARGET_ARCH"]} } } , "libgit2": diff --git a/bin/bootstrap.py b/bin/bootstrap.py index 17c60e489..654f931f1 100755 --- a/bin/bootstrap.py +++ b/bin/bootstrap.py @@ -91,11 +91,11 @@ CXXFLAGS = [] FINAL_LDFLAGS = ["-Wl,-z,stack-size=8388608"] -if "COMPILER_FAMILY" in CONF: - if CONF["COMPILER_FAMILY"] == "gnu": +if "TOOLCHAIN_CONFIG" in CONF and "FAMILY" in CONF["TOOLCHAIN_CONFIG"]: + if CONF["TOOLCHAIN_CONFIG"]["FAMILY"] == "gnu": CC="gcc" CXX="g++" - elif CONF["COMPILER_FAMILY"] == "clang": + elif CONF["TOOLCHAIN_CONFIG"]["FAMILY"] == "clang": CC="clang" CXX="clang++" diff --git a/etc/defaults/CC/TARGETS b/etc/defaults/CC/TARGETS index 1b3d6a1fa..f6e13ea6c 100644 --- a/etc/defaults/CC/TARGETS +++ b/etc/defaults/CC/TARGETS @@ -1,10 +1,19 @@ { "defaults": { "type": ["CC", "defaults"] - , "arguments_config": ["COMPILER_FAMILY", "DEBUG"] + , "arguments_config": ["TOOLCHAIN_CONFIG", "DEBUG"] , "base": [["@", "toolchain", "CC", "defaults"]] , "ADD_CFLAGS": { "type": "case" - , "expr": {"type": "var", "name": "COMPILER_FAMILY", "default": "unknown"} + , "expr": + { "type": "lookup" + , "map": + { "type": "var" + , "name": "TOOLCHAIN_CONFIG" + , "default": {"type": "empty_map"} + } + , "key": "FAMILY" + , "default": "unknown" + } , "case": { "msvc": { "type": "if" @@ -22,7 +31,16 @@ } , "ADD_CXXFLAGS": { "type": "case" - , "expr": {"type": "var", "name": "COMPILER_FAMILY", "default": "unknown"} + , "expr": + { "type": "lookup" + , "map": + { "type": "var" + , "name": "TOOLCHAIN_CONFIG" + , "default": {"type": "empty_map"} + } + , "key": "FAMILY" + , "default": "unknown" + } , "case": { "msvc": { "type": "if" diff --git a/etc/defaults/CC/TARGETS.absl b/etc/defaults/CC/TARGETS.absl index 7985d990b..266709c9e 100644 --- a/etc/defaults/CC/TARGETS.absl +++ b/etc/defaults/CC/TARGETS.absl @@ -1,12 +1,20 @@ { "defaults": { "type": ["CC", "defaults"] - , "arguments_config": ["COMPILER_FAMILY", "OS", "ARCH", "TARGET_ARCH"] + , "arguments_config": ["TOOLCHAIN_CONFIG", "OS", "ARCH", "TARGET_ARCH"] , "base": [["@", "base", "CC", "defaults"]] , "ADD_CXXFLAGS": { "type": "let*" , "bindings": [ [ "COMPILER_FAMILY" - , {"type": "var", "name": "COMPILER_FAMILY", "default": "unknown"} + , { "type": "lookup" + , "map": + { "type": "var" + , "name": "TOOLCHAIN_CONFIG" + , "default": {"type": "empty_map"} + } + , "key": "FAMILY" + , "default": "unknown" + } ] , [ "OS" , { "type": "var" diff --git a/etc/defaults/CC/TARGETS.archive b/etc/defaults/CC/TARGETS.archive index f5a792a81..1aa19fb6d 100644 --- a/etc/defaults/CC/TARGETS.archive +++ b/etc/defaults/CC/TARGETS.archive @@ -1,7 +1,7 @@ { "defaults": { "type": ["CC", "defaults"] , "arguments_config": - ["OS", "COMPILER_FAMILY", "DEBUG", "ENABLE_BZip2", "HIDE_SYMBOLS"] + ["OS", "TOOLCHAIN_CONFIG", "DEBUG", "ENABLE_BZip2", "HIDE_SYMBOLS"] , "base": [["@", "base", "CC", "defaults"]] , "ADD_CFLAGS": { "type": "let*" @@ -14,7 +14,15 @@ } ] , [ "COMPILER_FAMILY" - , {"type": "var", "name": "COMPILER_FAMILY", "default": "unknown"} + , { "type": "lookup" + , "map": + { "type": "var" + , "name": "TOOLCHAIN_CONFIG" + , "default": {"type": "empty_map"} + } + , "key": "FAMILY" + , "default": "unknown" + } ] , ["DEBUG", {"type": "var", "name": "DEBUG", "default": false}] ] diff --git a/etc/defaults/CC/TARGETS.curl b/etc/defaults/CC/TARGETS.curl index 233c3b6f6..e57d38bc9 100644 --- a/etc/defaults/CC/TARGETS.curl +++ b/etc/defaults/CC/TARGETS.curl @@ -1,13 +1,21 @@ { "defaults": { "type": ["CC", "defaults"] , "arguments_config": - ["COMPILER_FAMILY", "CURL_ENABLE_SSL", "ARCH", "TARGET_ARCH"] + ["TOOLCHAIN_CONFIG", "CURL_ENABLE_SSL", "ARCH", "TARGET_ARCH"] , "base": [["@", "base", "CC", "defaults"]] , "ADD_CFLAGS": { "type": "let*" , "bindings": [ [ "COMPILER_FAMILY" - , {"type": "var", "name": "COMPILER_FAMILY", "default": "unknown"} + , { "type": "lookup" + , "map": + { "type": "var" + , "name": "TOOLCHAIN_CONFIG" + , "default": {"type": "empty_map"} + } + , "key": "FAMILY" + , "default": "unknown" + } ] , [ "CURL_ENABLE_SSL" , {"type": "var", "name": "CURL_ENABLE_SSL", "default": true} diff --git a/etc/defaults/CC/TARGETS.git2 b/etc/defaults/CC/TARGETS.git2 index 595cee1f9..12320267f 100644 --- a/etc/defaults/CC/TARGETS.git2 +++ b/etc/defaults/CC/TARGETS.git2 @@ -1,12 +1,20 @@ { "defaults": { "type": ["CC", "defaults"] - , "arguments_config": ["COMPILER_FAMILY", "OS"] + , "arguments_config": ["TOOLCHAIN_CONFIG", "OS"] , "base": [["@", "base", "CC", "defaults"]] , "ADD_CFLAGS": { "type": "let*" , "bindings": [ [ "COMPILER_FAMILY" - , {"type": "var", "name": "COMPILER_FAMILY", "default": "unknown"} + , { "type": "lookup" + , "map": + { "type": "var" + , "name": "TOOLCHAIN_CONFIG" + , "default": {"type": "empty_map"} + } + , "key": "FAMILY" + , "default": "unknown" + } ] , [ "OS" , { "type": "var" diff --git a/etc/defaults/CC/TARGETS.grpc b/etc/defaults/CC/TARGETS.grpc index fbafa1361..86cf6e196 100644 --- a/etc/defaults/CC/TARGETS.grpc +++ b/etc/defaults/CC/TARGETS.grpc @@ -1,13 +1,21 @@ { "defaults": { "type": ["CC", "defaults"] , "arguments_config": - ["DEBUG", "OS", "ARCH", "TARGET_ARCH", "COMPILER_FAMILY"] + ["DEBUG", "OS", "ARCH", "TARGET_ARCH", "TOOLCHAIN_CONFIG"] , "base": [["@", "base", "CC", "defaults"]] , "ADD_CFLAGS": { "type": "let*" , "bindings": [ [ "COMPILER_FAMILY" - , {"type": "var", "name": "COMPILER_FAMILY", "default": "unknown"} + , { "type": "lookup" + , "map": + { "type": "var" + , "name": "TOOLCHAIN_CONFIG" + , "default": {"type": "empty_map"} + } + , "key": "FAMILY" + , "default": "unknown" + } ] , [ "OS" , { "type": "var" @@ -73,7 +81,15 @@ { "type": "let*" , "bindings": [ [ "COMPILER_FAMILY" - , {"type": "var", "name": "COMPILER_FAMILY", "default": "unknown"} + , { "type": "lookup" + , "map": + { "type": "var" + , "name": "TOOLCHAIN_CONFIG" + , "default": {"type": "empty_map"} + } + , "key": "FAMILY" + , "default": "unknown" + } ] , [ "OS" , { "type": "var" diff --git a/etc/defaults/CC/TARGETS.protobuf b/etc/defaults/CC/TARGETS.protobuf index f2ae56d66..229c8f311 100644 --- a/etc/defaults/CC/TARGETS.protobuf +++ b/etc/defaults/CC/TARGETS.protobuf @@ -1,13 +1,21 @@ { "defaults": { "type": ["CC", "defaults"] - , "arguments_config": ["ARCH", "TARGET_ARCH", "COMPILER_FAMILY"] + , "arguments_config": ["ARCH", "TARGET_ARCH", "TOOLCHAIN_CONFIG"] , "base": [["@", "base", "CC", "defaults"]] , "ADD_CFLAGS": ["-std=gnu17"] , "ADD_CXXFLAGS": { "type": "let*" , "bindings": [ [ "COMPILER_FAMILY" - , {"type": "var", "name": "COMPILER_FAMILY", "default": "unknown"} + , { "type": "lookup" + , "map": + { "type": "var" + , "name": "TOOLCHAIN_CONFIG" + , "default": {"type": "empty_map"} + } + , "key": "FAMILY" + , "default": "unknown" + } ] ] , "body": diff --git a/etc/import/TARGETS.archive b/etc/import/TARGETS.archive index af2dc82b4..3b1cc98ce 100644 --- a/etc/import/TARGETS.archive +++ b/etc/import/TARGETS.archive @@ -6,7 +6,7 @@ [ "OS" , "ARCH" , "TARGET_ARCH" - , "COMPILER_FAMILY" + , "TOOLCHAIN_CONFIG" , "USE_SYSTEM_LIBS" , "DEBUG" , "CC" @@ -109,7 +109,8 @@ , "HIDE_SYMBOLS": [ "Boolean. Default value: true." , "Hide all symbols not officially external." - , "Has effect if ~COMPILER_FAMILY~==\"clang\"|\"gnu\"|\"intel\"." + , "Has effect if" + , "TOOLCHAIN_CONFIG[\"FAMILY\"] == \"clang\"|\"gnu\"|\"intel\"." ] } } @@ -119,7 +120,7 @@ [ "OS" , "ARCH" , "TARGET_ARCH" - , "COMPILER_FAMILY" + , "TOOLCHAIN_CONFIG" , "USE_SYSTEM_LIBS" , "DEBUG" , "HIDE_SYMBOLS" @@ -160,7 +161,15 @@ } ] , [ "COMPILER_FAMILY" - , {"type": "var", "name": "COMPILER_FAMILY", "default": "unknown"} + , { "type": "lookup" + , "map": + { "type": "var" + , "name": "TOOLCHAIN_CONFIG" + , "default": {"type": "empty_map"} + } + , "key": "FAMILY" + , "default": "unknown" + } ] , [ "TARGET_ARCH" , { "type": "var" diff --git a/etc/import/TARGETS.boringssl b/etc/import/TARGETS.boringssl index 62defd2ef..2c95bc0fb 100644 --- a/etc/import/TARGETS.boringssl +++ b/etc/import/TARGETS.boringssl @@ -22,7 +22,7 @@ [ "OS" , "ARCH" , "TARGET_ARCH" - , "COMPILER_FAMILY" + , "TOOLCHAIN_CONFIG" , "DEBUG" , "CC" , "CFLAGS" @@ -38,7 +38,7 @@ [ "OS" , "ARCH" , "TARGET_ARCH" - , "COMPILER_FAMILY" + , "TOOLCHAIN_CONFIG" , "DEBUG" , "CXX" , "CXXFLAGS" diff --git a/etc/import/TARGETS.bzip2 b/etc/import/TARGETS.bzip2 index 0da3c9f77..d125001fa 100644 --- a/etc/import/TARGETS.bzip2 +++ b/etc/import/TARGETS.bzip2 @@ -6,7 +6,7 @@ [ "OS" , "ARCH" , "TARGET_ARCH" - , "COMPILER_FAMILY" + , "TOOLCHAIN_CONFIG" , "DEBUG" , "CC" , "CFLAGS" @@ -22,7 +22,7 @@ [ "OS" , "ARCH" , "TARGET_ARCH" - , "COMPILER_FAMILY" + , "TOOLCHAIN_CONFIG" , "DEBUG" , "CC" , "CFLAGS" @@ -40,7 +40,7 @@ [ "OS" , "ARCH" , "TARGET_ARCH" - , "COMPILER_FAMILY" + , "TOOLCHAIN_CONFIG" , "DEBUG" , "CC" , "CFLAGS" diff --git a/etc/import/TARGETS.cares b/etc/import/TARGETS.cares index ac27e6985..7889b3c9d 100644 --- a/etc/import/TARGETS.cares +++ b/etc/import/TARGETS.cares @@ -5,7 +5,7 @@ [ "OS" , "ARCH" , "TARGET_ARCH" - , "COMPILER_FAMILY" + , "TOOLCHAIN_CONFIG" , "DEBUG" , "CC" , "CFLAGS" diff --git a/etc/import/TARGETS.catch2 b/etc/import/TARGETS.catch2 index 9be5ce167..1182d8545 100644 --- a/etc/import/TARGETS.catch2 +++ b/etc/import/TARGETS.catch2 @@ -30,7 +30,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -38,6 +37,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "catch2_internal": diff --git a/etc/import/TARGETS.curl b/etc/import/TARGETS.curl index 988f905dc..230066f3f 100644 --- a/etc/import/TARGETS.curl +++ b/etc/import/TARGETS.curl @@ -6,7 +6,7 @@ [ "OS" , "ARCH" , "TARGET_ARCH" - , "COMPILER_FAMILY" + , "TOOLCHAIN_CONFIG" , "USE_SYSTEM_LIBS" , "DEBUG" , "CC" @@ -198,7 +198,6 @@ [ "OS" , "ARCH" , "TARGET_ARCH" - , "COMPILER_FAMILY" , "USE_SYSTEM_LIBS" , "CURL_HIDDEN_SYMBOLS" , "USE_ZLIB" diff --git a/etc/import/TARGETS.fmt b/etc/import/TARGETS.fmt index df4f426a4..799863985 100644 --- a/etc/import/TARGETS.fmt +++ b/etc/import/TARGETS.fmt @@ -11,7 +11,7 @@ [ "OS" , "ARCH" , "TARGET_ARCH" - , "COMPILER_FAMILY" + , "TOOLCHAIN_CONFIG" , "DEBUG" , "CXX" , "CXXFLAGS" diff --git a/etc/import/TARGETS.git2 b/etc/import/TARGETS.git2 index 486eb0e07..b712223e7 100644 --- a/etc/import/TARGETS.git2 +++ b/etc/import/TARGETS.git2 @@ -6,7 +6,7 @@ [ "OS" , "ARCH" , "TARGET_ARCH" - , "COMPILER_FAMILY" + , "TOOLCHAIN_CONFIG" , "DEBUG" , "CC" , "CFLAGS" diff --git a/etc/import/TARGETS.grpc b/etc/import/TARGETS.grpc index b6ef81bbf..186dce7ca 100644 --- a/etc/import/TARGETS.grpc +++ b/etc/import/TARGETS.grpc @@ -92,7 +92,7 @@ , "ARCH" , "HOST_ARCH" , "TARGET_ARCH" - , "COMPILER_FAMILY" + , "TOOLCHAIN_CONFIG" , "DEBUG" , "CC" , "CXX" @@ -189,7 +189,7 @@ , "ARCH" , "HOST_ARCH" , "TARGET_ARCH" - , "COMPILER_FAMILY" + , "TOOLCHAIN_CONFIG" , "DEBUG" , "CC" , "CXX" @@ -361,7 +361,7 @@ , "ARCH" , "HOST_ARCH" , "TARGET_ARCH" - , "COMPILER_FAMILY" + , "TOOLCHAIN_CONFIG" , "DEBUG" , "CC" , "CXX" @@ -2883,7 +2883,7 @@ , "ARCH" , "HOST_ARCH" , "TARGET_ARCH" - , "COMPILER_FAMILY" + , "TOOLCHAIN_CONFIG" , "DEBUG" , "CC" , "CXX" diff --git a/etc/import/TARGETS.lzma b/etc/import/TARGETS.lzma index d760e48ef..ddb87d414 100644 --- a/etc/import/TARGETS.lzma +++ b/etc/import/TARGETS.lzma @@ -6,7 +6,7 @@ [ "OS" , "ARCH" , "TARGET_ARCH" - , "COMPILER_FAMILY" + , "TOOLCHAIN_CONFIG" , "DEBUG" , "CC" , "CFLAGS" diff --git a/etc/import/TARGETS.protobuf b/etc/import/TARGETS.protobuf index 6492a383a..76466ebc8 100644 --- a/etc/import/TARGETS.protobuf +++ b/etc/import/TARGETS.protobuf @@ -12,7 +12,7 @@ , "ARCH" , "HOST_ARCH" , "TARGET_ARCH" - , "COMPILER_FAMILY" + , "TOOLCHAIN_CONFIG" , "DEBUG" , "CXX" , "CXXFLAGS" @@ -34,7 +34,7 @@ , "ARCH" , "HOST_ARCH" , "TARGET_ARCH" - , "COMPILER_FAMILY" + , "TOOLCHAIN_CONFIG" , "DEBUG" , "CXX" , "CXXFLAGS" @@ -55,7 +55,7 @@ , "ARCH" , "HOST_ARCH" , "TARGET_ARCH" - , "COMPILER_FAMILY" + , "TOOLCHAIN_CONFIG" , "DEBUG" , "CXX" , "CXXFLAGS" @@ -76,7 +76,7 @@ , "ARCH" , "HOST_ARCH" , "TARGET_ARCH" - , "COMPILER_FAMILY" + , "TOOLCHAIN_CONFIG" , "DEBUG" , "CXX" , "CXXFLAGS" diff --git a/etc/import/TARGETS.re2 b/etc/import/TARGETS.re2 index 95775dea5..83f12755f 100644 --- a/etc/import/TARGETS.re2 +++ b/etc/import/TARGETS.re2 @@ -8,7 +8,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -16,6 +15,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "re2_internal": diff --git a/etc/import/TARGETS.zlib b/etc/import/TARGETS.zlib index 0b85bea6f..7c18eab14 100644 --- a/etc/import/TARGETS.zlib +++ b/etc/import/TARGETS.zlib @@ -22,7 +22,7 @@ [ "OS" , "ARCH" , "TARGET_ARCH" - , "COMPILER_FAMILY" + , "TOOLCHAIN_CONFIG" , "DEBUG" , "CC" , "CFLAGS" diff --git a/etc/import/absl/algorithm/TARGETS.absl b/etc/import/absl/algorithm/TARGETS.absl index 810dfa02c..b01f66679 100644 --- a/etc/import/absl/algorithm/TARGETS.absl +++ b/etc/import/absl/algorithm/TARGETS.absl @@ -15,7 +15,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -23,6 +22,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "container_internal": diff --git a/etc/import/absl/base/TARGETS.absl b/etc/import/absl/base/TARGETS.absl index 9d6a6f14c..90e1b5790 100644 --- a/etc/import/absl/base/TARGETS.absl +++ b/etc/import/absl/base/TARGETS.absl @@ -8,7 +8,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -16,11 +15,12 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "base_internal_": { "type": ["@", "rules", "CC", "library"] - , "arguments_config": ["OS", "COMPILER_FAMILY", "ARCH", "TARGET_ARCH"] + , "arguments_config": ["OS", "TOOLCHAIN_CONFIG", "ARCH", "TARGET_ARCH"] , "name": ["base"] , "stage": ["absl", "base"] , "hdrs": @@ -49,7 +49,15 @@ , "$1": [ ["-pthread"] , { "type": "case*" - , "expr": {"type": "var", "name": "COMPILER_FAMILY"} + , "expr": + { "type": "lookup" + , "map": + { "type": "var" + , "name": "TOOLCHAIN_CONFIG" + , "default": {"type": "empty_map"} + } + , "key": "FAMILY" + } , "case": [["mingw", ["-ladvapi32"]]] } , { "type": "case*" @@ -135,7 +143,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -143,6 +150,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "core_headers_internal": @@ -178,7 +186,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -186,6 +193,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "dynamic_annotations_internal": @@ -205,7 +213,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -213,6 +220,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "log_severity_internal": @@ -233,7 +241,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -241,6 +248,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "raw_logging_internal (unexported)": diff --git a/etc/import/absl/cleanup/TARGETS.absl b/etc/import/absl/cleanup/TARGETS.absl index f6f9b1112..5aa16c750 100644 --- a/etc/import/absl/cleanup/TARGETS.absl +++ b/etc/import/absl/cleanup/TARGETS.absl @@ -8,7 +8,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -16,6 +15,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "cleanup_internal_": diff --git a/etc/import/absl/container/TARGETS.absl b/etc/import/absl/container/TARGETS.absl index e74c658d1..96165d82c 100644 --- a/etc/import/absl/container/TARGETS.absl +++ b/etc/import/absl/container/TARGETS.absl @@ -8,7 +8,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -16,6 +15,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "inlined_vector_internal_": @@ -103,7 +103,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -111,6 +110,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "flat_hash_map_internal": @@ -237,7 +237,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -245,6 +244,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "flat_hash_set_internal": @@ -271,7 +271,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -279,6 +278,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "btree_internal": diff --git a/etc/import/absl/debugging/TARGETS.absl b/etc/import/absl/debugging/TARGETS.absl index 658ec06d3..56cd8da61 100644 --- a/etc/import/absl/debugging/TARGETS.absl +++ b/etc/import/absl/debugging/TARGETS.absl @@ -1,6 +1,6 @@ { "symbolize": { "type": ["@", "rules", "CC", "library"] - , "arguments_config": ["COMPILER_FAMILY", "ARCH", "TARGET_ARCH"] + , "arguments_config": ["TOOLCHAIN_CONFIG", "ARCH", "TARGET_ARCH"] , "name": ["symbolize"] , "stage": ["absl", "debugging"] , "hdrs": ["internal/symbolize.h", "symbolize.h"] @@ -30,7 +30,15 @@ { "type": "++" , "$1": [ { "type": "case*" - , "expr": {"type": "var", "name": "COMPILER_FAMILY"} + , "expr": + { "type": "lookup" + , "map": + { "type": "var" + , "name": "TOOLCHAIN_CONFIG" + , "default": {"type": "empty_map"} + } + , "key": "FAMILY" + } , "case": [["mingw", ["-ladvapi32"]]] } , { "type": "case" diff --git a/etc/import/absl/flags/TARGETS.absl b/etc/import/absl/flags/TARGETS.absl index 0ebca782a..76370c9c1 100644 --- a/etc/import/absl/flags/TARGETS.absl +++ b/etc/import/absl/flags/TARGETS.absl @@ -44,7 +44,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -52,6 +51,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "marshalling_internal": @@ -155,7 +155,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -163,6 +162,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "flag_": diff --git a/etc/import/absl/functional/TARGETS.absl b/etc/import/absl/functional/TARGETS.absl index cbf5adccd..be3a96ecd 100644 --- a/etc/import/absl/functional/TARGETS.absl +++ b/etc/import/absl/functional/TARGETS.absl @@ -8,7 +8,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -16,6 +15,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "function_ref_internal": @@ -39,7 +39,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -47,6 +46,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "any_invocable_internal": @@ -72,7 +72,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -80,6 +79,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "bind_front_internal": diff --git a/etc/import/absl/hash/TARGETS.absl b/etc/import/absl/hash/TARGETS.absl index b2c56ac45..526faa708 100644 --- a/etc/import/absl/hash/TARGETS.absl +++ b/etc/import/absl/hash/TARGETS.absl @@ -8,7 +8,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -16,6 +15,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "hash_internal": diff --git a/etc/import/absl/log/TARGETS.absl b/etc/import/absl/log/TARGETS.absl index 67f0f3dc6..df213b237 100644 --- a/etc/import/absl/log/TARGETS.absl +++ b/etc/import/absl/log/TARGETS.absl @@ -8,7 +8,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -16,6 +15,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "absl_check_internal": @@ -35,7 +35,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -43,6 +42,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "absl_log_internal": @@ -62,7 +62,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -70,6 +69,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "die_if_null_internal": @@ -110,7 +110,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -118,6 +117,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "initialize_internal": diff --git a/etc/import/absl/memory/TARGETS.absl b/etc/import/absl/memory/TARGETS.absl index c8ca115ce..efb89a0f1 100644 --- a/etc/import/absl/memory/TARGETS.absl +++ b/etc/import/absl/memory/TARGETS.absl @@ -8,7 +8,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -16,6 +15,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "memory_internal": diff --git a/etc/import/absl/meta/TARGETS.absl b/etc/import/absl/meta/TARGETS.absl index 4ccf97c9c..a6199106c 100644 --- a/etc/import/absl/meta/TARGETS.absl +++ b/etc/import/absl/meta/TARGETS.absl @@ -8,7 +8,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -16,6 +15,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "type_traits_internal": diff --git a/etc/import/absl/numeric/TARGETS.absl b/etc/import/absl/numeric/TARGETS.absl index 4ebf47a6e..ed778e609 100644 --- a/etc/import/absl/numeric/TARGETS.absl +++ b/etc/import/absl/numeric/TARGETS.absl @@ -8,7 +8,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -16,6 +15,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "int128_internal": @@ -37,7 +37,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -45,6 +44,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "bits_internal": diff --git a/etc/import/absl/random/TARGETS.absl b/etc/import/absl/random/TARGETS.absl index 2407f1f61..c32ecb3ac 100644 --- a/etc/import/absl/random/TARGETS.absl +++ b/etc/import/absl/random/TARGETS.absl @@ -8,7 +8,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -16,6 +15,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" , "USE_SYSTEM_LIBS" ] } @@ -43,7 +43,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -51,6 +50,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" , "USE_SYSTEM_LIBS" ] } @@ -328,7 +328,6 @@ , "internal_randen_hwaes": { "type": ["@", "rules", "CC", "library"] , "name": ["internal_randen_hwaes"] - , "arguments_config": ["TARGET_ARCH", "COMPILER_FAMILY"] , "stage": ["absl", "random"] , "hdrs": ["internal/randen_detect.h", "internal/randen_hwaes.h"] , "srcs": ["internal/randen_detect.cc"] @@ -355,7 +354,7 @@ , "internal_randen_hwaes_impl": { "type": ["@", "rules", "CC", "library"] , "name": ["internal_randen_hwaes_impl"] - , "arguments_config": ["TARGET_ARCH", "COMPILER_FAMILY"] + , "arguments_config": ["TARGET_ARCH", "TOOLCHAIN_CONFIG"] , "stage": ["absl", "random"] , "private-hdrs": ["internal/randen_hwaes.h"] , "srcs": ["internal/randen_hwaes.cc"] @@ -367,7 +366,15 @@ , ["arm64", ["-march=armv8-a+crypto"]] , [ "x86_64" , { "type": "case*" - , "expr": {"type": "var", "name": "COMPILER_FAMILY"} + , "expr": + { "type": "lookup" + , "map": + { "type": "var" + , "name": "TOOLCHAIN_CONFIG" + , "default": {"type": "empty_map"} + } + , "key": "FAMILY" + } , "case": [["msvc", []]] , "default": ["-maes", "-msse4.1"] } diff --git a/etc/import/absl/status/TARGETS.absl b/etc/import/absl/status/TARGETS.absl index 245ea40ec..5b6a13d2e 100644 --- a/etc/import/absl/status/TARGETS.absl +++ b/etc/import/absl/status/TARGETS.absl @@ -8,7 +8,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -16,6 +15,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "status_internal": @@ -50,7 +50,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -58,6 +57,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "statusor_internal": diff --git a/etc/import/absl/strings/TARGETS.absl b/etc/import/absl/strings/TARGETS.absl index 81f5a8394..129a48f6e 100644 --- a/etc/import/absl/strings/TARGETS.absl +++ b/etc/import/absl/strings/TARGETS.absl @@ -8,7 +8,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -16,6 +15,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "strings_internal": @@ -89,7 +89,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -97,6 +96,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "internal_": @@ -130,7 +130,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -138,6 +137,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "cord_internal_": @@ -314,7 +314,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -322,6 +321,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "str_format_internal_": diff --git a/etc/import/absl/synchronization/TARGETS.absl b/etc/import/absl/synchronization/TARGETS.absl index bffd7ea65..3ca1d759a 100644 --- a/etc/import/absl/synchronization/TARGETS.absl +++ b/etc/import/absl/synchronization/TARGETS.absl @@ -8,7 +8,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -16,6 +15,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "synchronization_internal": diff --git a/etc/import/absl/time/TARGETS.absl b/etc/import/absl/time/TARGETS.absl index e08a1fac2..aafe343f7 100644 --- a/etc/import/absl/time/TARGETS.absl +++ b/etc/import/absl/time/TARGETS.absl @@ -8,7 +8,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -16,6 +15,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "time_internal": diff --git a/etc/import/absl/types/TARGETS.absl b/etc/import/absl/types/TARGETS.absl index a6de1d33b..0dc84c402 100644 --- a/etc/import/absl/types/TARGETS.absl +++ b/etc/import/absl/types/TARGETS.absl @@ -8,7 +8,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -16,6 +15,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "optional_internal": @@ -51,7 +51,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -59,6 +58,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "variant_internal": @@ -93,7 +93,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -101,6 +100,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "span_internal": diff --git a/etc/import/absl/utility/TARGETS.absl b/etc/import/absl/utility/TARGETS.absl index 91197dcc9..209cd13e4 100644 --- a/etc/import/absl/utility/TARGETS.absl +++ b/etc/import/absl/utility/TARGETS.absl @@ -8,7 +8,6 @@ , "ARCH" , "CC" , "CFLAGS" - , "COMPILER_FAMILY" , "CXX" , "CXXFLAGS" , "DEBUG" @@ -16,6 +15,7 @@ , "HOST_ARCH" , "OS" , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" ] } , "utility_internal": diff --git a/etc/import/deps/http-parser/TARGETS.git2 b/etc/import/deps/http-parser/TARGETS.git2 index ad48f2c25..80f4ffc67 100644 --- a/etc/import/deps/http-parser/TARGETS.git2 +++ b/etc/import/deps/http-parser/TARGETS.git2 @@ -1,13 +1,22 @@ { "git2_http_parser": { "type": ["@", "rules", "CC", "library"] - , "arguments_config": ["COMPILER_FAMILY"] + , "arguments_config": ["TOOLCHAIN_CONFIG"] , "name": ["git2_http_parser"] , "pure C": ["yes"] , "srcs": ["http_parser.c"] , "hdrs": ["http_parser.h"] , "private-cflags": { "type": "case" - , "expr": {"type": "var", "name": "COMPILER_FAMILY", "default": "unknown"} + , "expr": + { "type": "lookup" + , "map": + { "type": "var" + , "name": "TOOLCHAIN_CONFIG" + , "default": {"type": "empty_map"} + } + , "key": "FAMILY" + , "default": "unknown" + } , "case": {"gnu": ["-Wimplicit-fallthrough=1"]} } } diff --git a/etc/import/lib/TARGETS.curl b/etc/import/lib/TARGETS.curl index 397f14741..5e5ec9b04 100644 --- a/etc/import/lib/TARGETS.curl +++ b/etc/import/lib/TARGETS.curl @@ -3,7 +3,7 @@ , "arguments_config": [ "OS" , "TARGET_ARCH" - , "COMPILER_FAMILY" + , "TOOLCHAIN_CONFIG" , "CURL_HIDDEN_SYMBOLS" , "USE_ZLIB" , "ENABLE_ARES" @@ -51,7 +51,17 @@ , "config": { "type": "let*" , "bindings": - [ [ "sys_hdrs" + [ [ "COMPILER_FAMILY" + , { "type": "lookup" + , "map": + { "type": "var" + , "name": "TOOLCHAIN_CONFIG" + , "default": {"type": "empty_map"} + } + , "key": "FAMILY" + } + ] + , [ "sys_hdrs" , [ "windows.h" , "ws2tcpip.h" , "winsock2.h" diff --git a/etc/import/libarchive/TARGETS.archive b/etc/import/libarchive/TARGETS.archive index de471234a..cde2b465e 100644 --- a/etc/import/libarchive/TARGETS.archive +++ b/etc/import/libarchive/TARGETS.archive @@ -3,7 +3,7 @@ , "arguments_config": [ "OS" , "TARGET_ARCH" - , "COMPILER_FAMILY" + , "TOOLCHAIN_CONFIG" , "DEBUG" , "ENABLE_MBEDTLS" , "ENABLE_NETTLE" @@ -34,7 +34,17 @@ , "config": { "type": "let*" , "bindings": - [ [ "sys_hdrs" + [ [ "COMPILER_FAMILY" + , { "type": "lookup" + , "map": + { "type": "var" + , "name": "TOOLCHAIN_CONFIG" + , "default": {"type": "empty_map"} + } + , "key": "FAMILY" + } + ] + , [ "sys_hdrs" , [ "zlib.h" , "bzlib.h" , "lzma.h" diff --git a/etc/import/src/TARGETS.archive b/etc/import/src/TARGETS.archive index ee3ff78b8..b71c4f892 100644 --- a/etc/import/src/TARGETS.archive +++ b/etc/import/src/TARGETS.archive @@ -520,13 +520,21 @@ } , "archive_hidden_symbols": { "type": ["@", "rules", "CC", "library"] - , "arguments_config": ["COMPILER_FAMILY", "HIDE_SYMBOLS"] + , "arguments_config": ["TOOLCHAIN_CONFIG", "HIDE_SYMBOLS"] , "name": ["archive_hidden_symbols"] , "private-ldflags": { "type": "if" , "cond": { "type": "==" - , "$1": {"type": "var", "name": "COMPILER_FAMILY"} + , "$1": + { "type": "lookup" + , "map": + { "type": "var" + , "name": "TOOLCHAIN_CONFIG" + , "default": {"type": "empty_map"} + } + , "key": "FAMILY" + } , "$2": "msvc" } , "then": [] @@ -535,7 +543,15 @@ , "cond": {"type": "var", "name": "HIDE_SYMBOLS"} , "then": { "type": "case*" - , "expr": {"type": "var", "name": "COMPILER_FAMILY"} + , "expr": + { "type": "lookup" + , "map": + { "type": "var" + , "name": "TOOLCHAIN_CONFIG" + , "default": {"type": "empty_map"} + } + , "key": "FAMILY" + } , "case": [ ["clang", ["-fvisibility=hidden"]] , ["gnu", ["-fvisibility=hidden"]] diff --git a/etc/import/src/TARGETS.curl b/etc/import/src/TARGETS.curl index 1bfd750a9..29fd97066 100644 --- a/etc/import/src/TARGETS.curl +++ b/etc/import/src/TARGETS.curl @@ -429,13 +429,21 @@ } , "curl_hidden_symbols": { "type": ["@", "rules", "CC", "library"] - , "arguments_config": ["COMPILER_FAMILY", "CURL_HIDDEN_SYMBOLS"] + , "arguments_config": ["TOOLCHAIN_CONFIG", "CURL_HIDDEN_SYMBOLS"] , "name": ["curl_hidden_symbols"] , "private-ldflags": { "type": "if" , "cond": { "type": "==" - , "$1": {"type": "var", "name": "COMPILER_FAMILY"} + , "$1": + { "type": "lookup" + , "map": + { "type": "var" + , "name": "TOOLCHAIN_CONFIG" + , "default": {"type": "empty_map"} + } + , "key": "FAMILY" + } , "$2": "msvc" } , "then": [] @@ -444,7 +452,15 @@ , "cond": {"type": "var", "name": "CURL_HIDDEN_SYMBOLS"} , "then": { "type": "case*" - , "expr": {"type": "var", "name": "COMPILER_FAMILY"} + , "expr": + { "type": "lookup" + , "map": + { "type": "var" + , "name": "TOOLCHAIN_CONFIG" + , "default": {"type": "empty_map"} + } + , "key": "FAMILY" + } , "case": [ ["clang", ["-fvisibility=hidden"]] , ["gnu", ["-fvisibility=hidden"]] diff --git a/etc/import/src/google/protobuf/TARGETS.protobuf b/etc/import/src/google/protobuf/TARGETS.protobuf index 0589f9387..7add3c158 100644 --- a/etc/import/src/google/protobuf/TARGETS.protobuf +++ b/etc/import/src/google/protobuf/TARGETS.protobuf @@ -266,11 +266,20 @@ } , "libprotobuf_lite": { "type": ["@", "rules", "CC", "library"] - , "arguments_config": ["COMPILER_FAMILY"] + , "arguments_config": ["TOOLCHAIN_CONFIG"] , "name": ["protobuf-lite"] , "cflags": { "type": "case" - , "expr": {"type": "var", "name": "COMPILER_FAMILY", "default": "unknown"} + , "expr": + { "type": "lookup" + , "map": + { "type": "var" + , "name": "TOOLCHAIN_CONFIG" + , "default": {"type": "empty_map"} + } + , "key": "FAMILY" + , "default": "unknown" + } , "case": { "gnu": [ "-Wno-sign-compare" diff --git a/etc/import/third_party/utf8_range/TARGETS.protobuf b/etc/import/third_party/utf8_range/TARGETS.protobuf index 44c5340a1..bcd4d3dd2 100644 --- a/etc/import/third_party/utf8_range/TARGETS.protobuf +++ b/etc/import/third_party/utf8_range/TARGETS.protobuf @@ -6,7 +6,7 @@ , "ARCH" , "HOST_ARCH" , "TARGET_ARCH" - , "COMPILER_FAMILY" + , "TOOLCHAIN_CONFIG" , "DEBUG" , "CXX" , "CXXFLAGS" diff --git a/etc/toolchain/CC/TARGETS b/etc/toolchain/CC/TARGETS index 3ad0ef3d9..e5e5351c6 100644 --- a/etc/toolchain/CC/TARGETS +++ b/etc/toolchain/CC/TARGETS @@ -1,6 +1,6 @@ { "defaults": { "type": ["CC", "defaults"] - , "arguments_config": ["OS", "ARCH", "COMPILER_FAMILY"] + , "arguments_config": ["OS", "ARCH", "TOOLCHAIN_CONFIG"] , "base": { "type": "let*" , "bindings": @@ -11,7 +11,15 @@ , "$1": [{"type": "var", "name": "OS"}, {"type": "var", "name": "ARCH"}] } - , "then": {"type": "var", "name": "COMPILER_FAMILY"} + , "then": + { "type": "lookup" + , "map": + { "type": "var" + , "name": "TOOLCHAIN_CONFIG" + , "default": {"type": "empty_map"} + } + , "key": "FAMILY" + } , "else": "unknown" } ] @@ -24,7 +32,7 @@ , "default": { "type": "fail" , "msg": - [ "Unsupported COMPILER_FAMILY" + [ "Unsupported TOOLCHAIN_CONFIG[FAMILY]" , {"type": "var", "name": "COMPILER_FAMILY"} ] } diff --git a/test/TARGETS b/test/TARGETS index 0b1dbefd7..9054bd756 100644 --- a/test/TARGETS +++ b/test/TARGETS @@ -32,13 +32,7 @@ , "ALL": { "type": "configure" , "arguments_config": - [ "OS" - , "ARCH" - , "HOST_ARCH" - , "TARGET_ARCH" - , "COMPILER_FAMILY" - , "TEST_BOOTSTRAP_JUST_MR" - ] + ["OS", "ARCH", "HOST_ARCH", "TARGET_ARCH", "TEST_BOOTSTRAP_JUST_MR"] , "tainted": ["test"] , "target": "TESTS" , "config": @@ -58,20 +52,11 @@ , "default": {"type": "var", "name": "ARCH"} } ] - , [ "COMPILER_FAMILY" - , {"type": "var", "name": "COMPILER_FAMILY", "default": "clang"} - ] ] , "body": { "type": "env" , "vars": - [ "OS" - , "ARCH" - , "HOST_ARCH" - , "TARGET_ARCH" - , "COMPILER_FAMILY" - , "TEST_BOOTSTRAP_JUST_MR" - ] + ["OS", "ARCH", "HOST_ARCH", "TARGET_ARCH", "TEST_BOOTSTRAP_JUST_MR"] } } } diff --git a/test/bootstrap/TARGETS b/test/bootstrap/TARGETS index ab8ae1e69..039dec92b 100644 --- a/test/bootstrap/TARGETS +++ b/test/bootstrap/TARGETS @@ -36,7 +36,7 @@ , [ "TEST_ENV" , { "type": "singleton_map" , "key": "JUST_BUILD_CONF" - , "value": "{\"COMPILER_FAMILY\": \"gnu\", \"CC\": \"gcc\", \"CXX\": \"g++\"}" + , "value": "{\"TOOLCHAIN_CONFIG\":{\"FAMILY\": \"gnu\"}, \"CC\": \"gcc\", \"CXX\": \"g++\"}" } ] ] diff --git a/test/bootstrap/test-bootstrap-pkgconfig.sh b/test/bootstrap/test-bootstrap-pkgconfig.sh index 3087d3807..d9e0da02a 100755 --- a/test/bootstrap/test-bootstrap-pkgconfig.sh +++ b/test/bootstrap/test-bootstrap-pkgconfig.sh @@ -35,7 +35,7 @@ mkdir -p ${TEST_OUT_DIR} env LOCALBASE=${LOCALBASE} \ PACKAGE=YES \ - JUST_BUILD_CONF='{"COMPILER_FAMILY":"clang", "PKG_CONFIG_ARGS":["--define-prefix"]}' \ + JUST_BUILD_CONF='{"TOOLCHAIN_CONFIG":{"FAMILY": "clang"}, "PKG_CONFIG_ARGS":["--define-prefix"]}' \ python3 ${WRKSRC}/bin/bootstrap.py ${WRKSRC} ${WRKDIR} 2>&1 # Do some sanity checks with the binary diff --git a/test/bootstrap/test-mixed-bootstrap.sh b/test/bootstrap/test-mixed-bootstrap.sh index 981116c49..2a6654453 100755 --- a/test/bootstrap/test-mixed-bootstrap.sh +++ b/test/bootstrap/test-mixed-bootstrap.sh @@ -51,7 +51,7 @@ cp distdir/fmt-10.0.0.zip "${DISTDIR}" env LOCALBASE=${LOCALBASE} \ PACKAGE=YES \ NON_LOCAL_DEPS='["com_github_microsoft_gsl", "fmt"]' \ - JUST_BUILD_CONF='{"COMPILER_FAMILY":"clang", "PKG_CONFIG_ARGS":["--define-prefix"]}' \ + JUST_BUILD_CONF='{"TOOLCHAIN_CONFIG":{"FAMILY":"clang"}, "PKG_CONFIG_ARGS":["--define-prefix"]}' \ python3 ${WRKSRC}/bin/bootstrap.py ${WRKSRC} ${WRKDIR} ${DISTDIR} 2>&1 # Do some sanity checks with the binary diff --git a/test/bootstrap/test-symlink-bootstrap.sh b/test/bootstrap/test-symlink-bootstrap.sh index dc541fe4b..49554cf3d 100755 --- a/test/bootstrap/test-symlink-bootstrap.sh +++ b/test/bootstrap/test-symlink-bootstrap.sh @@ -50,7 +50,7 @@ ln -s ${UNRELATED_FILE} ${LOCALBASE}/lib/libfoo.so env LOCALBASE=${LOCALBASE} \ PACKAGE=YES \ - JUST_BUILD_CONF='{"COMPILER_FAMILY":"clang", "PKG_CONFIG_ARGS":["--define-prefix"]}' \ + JUST_BUILD_CONF='{"TOOLCHAIN_CONFIG":{"FAMILY":"clang"}, "PKG_CONFIG_ARGS":["--define-prefix"]}' \ python3 ${WRKSRC}/bin/bootstrap.py ${WRKSRC} ${WRKDIR} 2>&1 # Do some sanity checks with the binary From 98b6063eeeb8914e305beb7b2acad1cecae388b9 Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Wed, 16 Aug 2023 09:46:48 +0200 Subject: [PATCH 81/97] Release 1.2.0~beta1 --- CHANGELOG.md | 5 +++++ src/buildtool/main/version.cpp | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f65503348..0c8ccad34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,11 @@ A feature release on top of `1.1.0`, backwards compatible. - `just install-cas` now, like `just install`, removes an existing destination file before installing instead of overwriting. +## Release `1.2.0~beta1` (2023-08-16) + +First beta release for the upcoming `1.2.0` release; see release +notes there. + ## Release `1.1.0` (2023-05-19) A feature release on top of `1.0.0`, backwards compatible. diff --git a/src/buildtool/main/version.cpp b/src/buildtool/main/version.cpp index 5dfd3e5bf..28e63984d 100644 --- a/src/buildtool/main/version.cpp +++ b/src/buildtool/main/version.cpp @@ -21,7 +21,7 @@ auto version() -> std::string { std::size_t major = 1; std::size_t minor = 2; std::size_t revision = 0; - std::string suffix = "~alpha"; + std::string suffix = "~beta1"; #ifdef VERSION_EXTRA_SUFFIX suffix += VERSION_EXTRA_SUFFIX; #endif From 7fc304ccb797b762845a4eaaaedec3ecc50a85e6 Mon Sep 17 00:00:00 2001 From: Oliver Reiche Date: Thu, 17 Aug 2023 10:52:44 +0200 Subject: [PATCH 82/97] absl: Drop unused variables --- etc/defaults/CC/TARGETS.absl | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/etc/defaults/CC/TARGETS.absl b/etc/defaults/CC/TARGETS.absl index 266709c9e..0b353103d 100644 --- a/etc/defaults/CC/TARGETS.absl +++ b/etc/defaults/CC/TARGETS.absl @@ -1,6 +1,6 @@ { "defaults": { "type": ["CC", "defaults"] - , "arguments_config": ["TOOLCHAIN_CONFIG", "OS", "ARCH", "TARGET_ARCH"] + , "arguments_config": ["TOOLCHAIN_CONFIG"] , "base": [["@", "base", "CC", "defaults"]] , "ADD_CXXFLAGS": { "type": "let*" @@ -16,24 +16,6 @@ , "default": "unknown" } ] - , [ "OS" - , { "type": "var" - , "name": "OS" - , "default": - {"type": "fail", "msg": "Required variable 'OS' is not set."} - } - ] - , [ "TARGET_ARCH" - , { "type": "var" - , "name": "TARGET_ARCH" - , "default": - { "type": "var" - , "name": "ARCH" - , "default": - {"type": "fail", "msg": "Required variable 'ARCH' is not set."} - } - } - ] ] , "body": { "type": "++" From 83fda71af7ac89acd65515637cad6679bbacdf90 Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Mon, 14 Aug 2023 16:11:19 +0200 Subject: [PATCH 83/97] documentation: add some clarifying comments ... emphasizing that source files are just targets and that we could as well use a defined target there. --- doc/tutorial/hello-world.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/tutorial/hello-world.md b/doc/tutorial/hello-world.md index 54ed7d72c..8430775f9 100644 --- a/doc/tutorial/hello-world.md +++ b/doc/tutorial/hello-world.md @@ -92,8 +92,10 @@ strings or a list of targets, we have to specify the name as a list (this rule will simply concatenate all strings given in this field). Furthermore, at least one input to the binary is required, which can be specified via the target fields `"srcs"` or `"deps"`. In our case, the -former is used, which contains our single source file (files are -considered targets). +former is used, which contains our single source file. Source files are +also targets, but, as seen in the "Getting Started" section, not the only +ones; instead of naming a source file, we could also have specified a +`"generic"` target generating one (or many) of the sources of our binary. Now, the last file that is missing is the actual source file `main.cpp`: From 56a63969e9f397395976ffa62a0eec14df2d05ff Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Thu, 17 Aug 2023 11:40:05 +0200 Subject: [PATCH 84/97] tutorial: mention the describe subcommand ... as a way to find out more about the defining rule of a target. While there, also be a bit more explicit about target naming and the fact that the default target is a target of the rules repository. --- doc/tutorial/hello-world.md | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/doc/tutorial/hello-world.md b/doc/tutorial/hello-world.md index 8430775f9..1edfb1b6e 100644 --- a/doc/tutorial/hello-world.md +++ b/doc/tutorial/hello-world.md @@ -182,8 +182,9 @@ your project, you need to create a separate file root for providing required `TARGETS` file, which contains the `"defaults"` target that should be used by the rules. This file root is then used as the *target root* for the rules, i.e., the search path for `TARGETS` files. In this -way, the description of the `"defaults"` target is provided in a -separate file root, to keep the rules repository independent of these +way, the description of the `"defaults"` target, while logically part +of the rules repository is physically located in a separate directory +to keep the rules repository independent of these project-specific definitions. We will call the new file root `tutorial-defaults` and need to create a @@ -247,7 +248,11 @@ without specifying any external repository (e.g., full-fledged repository but merely a file root that is considered local to the `"rules-cc"` repository. In fact, the `"rules-cc"` repository cannot refer to any external repository as it does not have any defined -bindings. +bindings. The naming for rules follows the same scheme we've already +seen for targets, so a single string refers to an entity in the same +directory. As our `"defaults"` target is in the directory `"CC"` of +the rules repository we could also have written the rule `"type"` +simply as `"defaults"`. To rebuild the project, we need to rerun `just-mr` (note that due to configuration changes, rerunning only `just` would not suffice): @@ -267,6 +272,28 @@ $ Note that the output binary may have changed due to different defaults. +In this tutorial we simply set the correct parameters of the defaults target. +It is, however, not necessary to remember all the fields of a rule; we can +always ask `just` to present us the available field names and configuration +variables together with any documentation the rule author provided. For +this, we use the `describe` subcommand; as we're interested in a target of +the `rules-cc` repository, which is not the default repository, we also +have to specify the repository name. + +``` sh +$ just-mr --main rules-cc describe CC defaults +``` + +Of course, the `describe` subcommand works generically on all +targets. For example, by asking to describe our `helloworld` target, +we will get reminded about all the various fields and relevant +configuration variables of a C++ binary. + +``` sh +$ just-mr describe helloworld +``` + + Modeling target dependencies ---------------------------- From 44c0b156254e11f4c1a1bf66a6cf790a5779d774 Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Thu, 17 Aug 2023 12:42:55 +0200 Subject: [PATCH 85/97] ["shell/test", "script"] Rename implicit dependencies ... and document at the appropriate places what can be overwritten by setting those targets. --- rules/CC/test/RULES | 5 ++++- rules/shell/test/RULES | 14 ++++++++++++-- rules/shell/test/{test_runner.sh => runner} | 0 rules/shell/test/{test_summary.py => summarizer} | 0 test/end-to-end/RULES | 2 +- 5 files changed, 17 insertions(+), 4 deletions(-) rename rules/shell/test/{test_runner.sh => runner} (100%) rename rules/shell/test/{test_summary.py => summarizer} (100%) diff --git a/rules/CC/test/RULES b/rules/CC/test/RULES index 8dba3630f..41507d7ea 100644 --- a/rules/CC/test/RULES +++ b/rules/CC/test/RULES @@ -30,7 +30,7 @@ , "implicit": { "defaults": [["./", "..", "defaults"]] , "runner": ["test_runner.py"] - , "summarizer": [["./", "../../shell/test", "test_summary.py"]] + , "summarizer": [["./", "../../shell/test", "summarizer"]] } , "field_doc": { "name": @@ -88,6 +88,9 @@ , "RUNS_PER_TEST": [ "The number of times the test should be run in order to detect flakyness." , "If set, no test action will be taken from cache." + , "" + , "Test runs are summarized by the [\"shell/test\", \"summarizer\"] that" + , "is also used by shell tests." ] , "TARGET_ARCH": [ "The architecture to build the test for." diff --git a/rules/shell/test/RULES b/rules/shell/test/RULES index 117a060d3..c4f85695f 100644 --- a/rules/shell/test/RULES +++ b/rules/shell/test/RULES @@ -20,6 +20,11 @@ , "TEST_TMPDIR. The test should not assume write permissions" , "outside the working directory and the TEST_TMPDIR." , "For convenience, the environment variable TMPDIR is also set to TEST_TMPDIR." + , "" + , "This running of the test is carried out by the implicit dependency" + , "on the target \"runner\". By setting this target in the target layer" + , "of this rues repository (instead of letting it default to the" + , "respective file), the shell test environment can be modified globally." ] , "name": [ "A name for the test, used in reporting, as well as for staging" @@ -39,6 +44,12 @@ { "RUNS_PER_TEST": [ "The number of times the test should be run in order to detect flakyness." , "If set, no test action will be taken from cache." + , "" + , "The individual test runs will be summarized by the implict dependency" + , "on the target \"summarizer\". By setting this target in the target" + , "in the target layer of this rues repository (instead of letting it" + , "default to the respective file) the layout of the summary can be" + , "changed globally." ] , "TEST_ENV": ["The environment for executing the test runner."] , "TIMEOUT_SCALE": @@ -72,8 +83,7 @@ , "As the built-in \"install\" rule only takes the runfiles of its \"deps\"" , "argument, this gives an easy way of defining test suites." ] - , "implicit": - {"runner": ["test_runner.sh"], "summarizer": ["test_summary.py"]} + , "implicit": {"runner": ["runner"], "summarizer": ["summarizer"]} , "imports": { "test-result": "test-result" , "action": "test-action" diff --git a/rules/shell/test/test_runner.sh b/rules/shell/test/runner similarity index 100% rename from rules/shell/test/test_runner.sh rename to rules/shell/test/runner diff --git a/rules/shell/test/test_summary.py b/rules/shell/test/summarizer similarity index 100% rename from rules/shell/test/test_summary.py rename to rules/shell/test/summarizer diff --git a/test/end-to-end/RULES b/test/end-to-end/RULES index b94b8f8f5..aef5a6684 100644 --- a/test/end-to-end/RULES +++ b/test/end-to-end/RULES @@ -70,7 +70,7 @@ ] , "implicit": { "runner": ["with_remote_test_runner.py"] - , "summarizer": [["@", "rules", "shell/test", "test_summary.py"]] + , "summarizer": [["@", "rules", "shell/test", "summarizer"]] , "just": [["@", "src", "", "installed just"]] } , "imports": From 2cb273c2dc53e2cdacd1470425a85fceb8944cfc Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Thu, 17 Aug 2023 14:39:06 +0200 Subject: [PATCH 86/97] ["CC/auto", "config_file"] rename implicit target ... and mention it at the appropriate part of the documentation. While there, also provide a default TARGETS file. --- rules/CC/auto/RULES | 7 ++++++- rules/CC/auto/TARGETS | 1 + rules/CC/auto/{config_runner.py => runner} | 0 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 rules/CC/auto/TARGETS rename rules/CC/auto/{config_runner.py => runner} (100%) diff --git a/rules/CC/auto/RULES b/rules/CC/auto/RULES index 0e156c989..860db01c0 100644 --- a/rules/CC/auto/RULES +++ b/rules/CC/auto/RULES @@ -1015,6 +1015,11 @@ , "target configuration. In the usual case, a target using this rule is" , "configured by depending on it from a target that uses the built-in" , "\"configure\" rule." + , "" + , "The actual generation of the header file from the template" + , "is done by the implicit dependency on the \"runner\" target which" + , "can be changed globally by setting this target in the" + , "target layer of this repository." ] , "field_doc": { "output": @@ -1045,7 +1050,7 @@ , "last": "last_list_entry" , "stage_singleton_field": ["", "stage_singleton_field"] } - , "implicit": {"runner": ["config_runner.py"]} + , "implicit": {"runner": ["runner"]} , "expression": { "type": "let*" , "bindings": diff --git a/rules/CC/auto/TARGETS b/rules/CC/auto/TARGETS new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/rules/CC/auto/TARGETS @@ -0,0 +1 @@ +{} diff --git a/rules/CC/auto/config_runner.py b/rules/CC/auto/runner similarity index 100% rename from rules/CC/auto/config_runner.py rename to rules/CC/auto/runner From 070e96c28007faff99bc272d40480855e7e3837a Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Thu, 17 Aug 2023 14:08:22 +0200 Subject: [PATCH 87/97] ["CC/test", "test"] rename implict dependency ... and document at the appropriate place --- rules/CC/test/RULES | 15 +++++++++++++-- rules/CC/test/{test_runner.py => runner} | 0 2 files changed, 13 insertions(+), 2 deletions(-) rename rules/CC/test/{test_runner.py => runner} (100%) diff --git a/rules/CC/test/RULES b/rules/CC/test/RULES index 41507d7ea..af413a9b0 100644 --- a/rules/CC/test/RULES +++ b/rules/CC/test/RULES @@ -29,7 +29,7 @@ ] , "implicit": { "defaults": [["./", "..", "defaults"]] - , "runner": ["test_runner.py"] + , "runner": ["runner"] , "summarizer": [["./", "../../shell/test", "summarizer"]] } , "field_doc": @@ -39,7 +39,18 @@ , "Used to name the test binary as well as for staging the test result" ] , "args": ["Command line arguments for the test binary"] - , "srcs": ["The sources of the test binary"] + , "srcs": + [ "The sources of the test binary" + , "" + , "The resulting test binary in an environment where it can assume" + , "that the environment variable TEST_TMPDIR points to a" + , "director that may be used exclusively by this test." + , "" + , "This running of the test is carried out by the implicit dependency" + , "on the target \"runner\". By setting this target in the target layer" + , "of this rues repository (instead of letting it default to the" + , "respective file), the C/C++ test environment can be modified globally." + ] , "private-hdrs": [ "Any additional header files that need to be present when compiling" , "the test binary." diff --git a/rules/CC/test/test_runner.py b/rules/CC/test/runner similarity index 100% rename from rules/CC/test/test_runner.py rename to rules/CC/test/runner From c44b48cc9ba9d3dcfbce58d1fd2ebdfb2e5f2ceb Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Thu, 17 Aug 2023 17:49:40 +0200 Subject: [PATCH 88/97] documentation: third-party code may also be built using foreign tools ... while providing a native descirption still is the cleaner approach. Nevertheless, hinting on this fact early avoids scaring off novice users having a lot of dependencies. --- doc/tutorial/third-party-software.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/doc/tutorial/third-party-software.md b/doc/tutorial/third-party-software.md index 644b7b21d..c8653d3e3 100644 --- a/doc/tutorial/third-party-software.md +++ b/doc/tutorial/third-party-software.md @@ -3,11 +3,16 @@ Building Third-party Software Third-party projects usually ship with their own build description, which often happens to be not compatible with justbuild. Nevertheless, -it is highly desireable to include external projects via their source +it often is desireable to include external projects via their source code base, instead of relying on the integration of out-of-band binary distributions. justbuild offers a flexible approach to provide the required build description via an overlay layer without the need to -touch the original code base. +touch the original code base. This mechanism is independent of the +actual justbuild description eventually used and the latter might +well be a +[rule calling the foreign buildsystem](https://github.com/just-buildsystem/rules-cc#rule-ccforeigncmake-library). +In this section, however, we describe the cleaner approach of providing +a native build description. For the remainder of this section, we expect to have the project files available resulting from successfully completing the tutorial section on From 91dbd2751120684fba666e3b958837c96d5044bc Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Fri, 18 Aug 2023 10:23:35 +0200 Subject: [PATCH 89/97] tutorial, third party: also mention pkg-config as option While there, - remove the outdated claim that in package build of just the prebuilt library aproach of a header-only library with link dependencies is chosen, and - remove a wrong footnote separator. --- doc/tutorial/third-party-software.md | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/doc/tutorial/third-party-software.md b/doc/tutorial/third-party-software.md index c8653d3e3..815543c56 100644 --- a/doc/tutorial/third-party-software.md +++ b/doc/tutorial/third-party-software.md @@ -430,8 +430,8 @@ different build configuration (production, debug, instrumented for performance analysis; cross-compiling for a different target architecture), there are also legitimate reasons to use pre-built dependencies. The most prominent one is if your project is packaged as -part of a larger distribution. For that reason, just also has (in -`etc/import.prebuilt`) target files for all its dependencies assuming +part of a larger distribution. For that reason, just also has target files +for all its dependencies assuming they are pre-installed. The reason why target files are used at all for this situation is twofold. @@ -456,7 +456,7 @@ this situation is twofold. description maintainable, as each target still only declares its direct dependencies. -The target description for a pre-built version of the format library +A target description for a pre-built version of the format library that was used as an example in this section is shown next; with our staging mechanism the logical repository it belongs to is rooted in the `fmt` subdirectory of the `include` directory of the ambient system. @@ -472,7 +472,19 @@ staging mechanism the logical repository it belongs to is rooted in the } ``` ---- +However, even specifying all the include locations and headers can +be tedious and in the end, it is information that `pkg-config` can +provide as well. So there is a rule to import libraries that way +and the actual packaging-build version of `libfmt`, as provided in +`etc/import.pkgconfig`, looks as follows. + +``` {.jsonc srcname="etc/import.pkgconfig/TARGETS.fmt} +{ "fmt": + {"type": ["@", "rules", "CC/pkgconfig", "system_library"], "name": ["fmt"]} +} +``` + + [^1]: Explicit `TREE` references are always a list of length 3, to distinguish them from target references of length 2 (module and From d99393866f4c3f9acf942d74105a3aeea9bc95b4 Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Fri, 18 Aug 2023 10:30:49 +0200 Subject: [PATCH 90/97] Release 1.2.0~beta2 --- CHANGELOG.md | 12 ++++++++++++ src/buildtool/main/version.cpp | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c8ccad34..3581eaa21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,18 @@ A feature release on top of `1.1.0`, backwards compatible. - `just install-cas` now, like `just install`, removes an existing destination file before installing instead of overwriting. +## Release `1.2.0~beta2` (2023-08-18) + +Second beta release for the upcoming `1.2.0` release; see release +notes there. + +### Changes since `1.2.0~beta1` + +- Clean up of the internal build description of bundled dependencies. +- Clean up of the the internal rules, in particular renaming of + implict dependency targets. +- Various documentation improvements. + ## Release `1.2.0~beta1` (2023-08-16) First beta release for the upcoming `1.2.0` release; see release diff --git a/src/buildtool/main/version.cpp b/src/buildtool/main/version.cpp index 28e63984d..2a00cd211 100644 --- a/src/buildtool/main/version.cpp +++ b/src/buildtool/main/version.cpp @@ -21,7 +21,7 @@ auto version() -> std::string { std::size_t major = 1; std::size_t minor = 2; std::size_t revision = 0; - std::string suffix = "~beta1"; + std::string suffix = "~beta2"; #ifdef VERSION_EXTRA_SUFFIX suffix += VERSION_EXTRA_SUFFIX; #endif From 93b7d610816eafacdab398f548afd2b107c32804 Mon Sep 17 00:00:00 2001 From: Oliver Reiche Date: Fri, 18 Aug 2023 18:41:53 +0200 Subject: [PATCH 91/97] absl: Do not require OS and ARCH ... and use "linux" as default for OS. If TARGET_ARCH is "arm" or not set, link against -latomic (for required targets). --- etc/import/absl/base/TARGETS.absl | 42 +++++--------------- etc/import/absl/debugging/TARGETS.absl | 23 +---------- etc/import/absl/random/TARGETS.absl | 8 +++- etc/import/absl/strings/TARGETS.absl | 9 +---- etc/import/absl/synchronization/TARGETS.absl | 9 +---- etc/import/absl/time/TARGETS.absl | 9 +---- 6 files changed, 25 insertions(+), 75 deletions(-) diff --git a/etc/import/absl/base/TARGETS.absl b/etc/import/absl/base/TARGETS.absl index 90e1b5790..fcc6bc783 100644 --- a/etc/import/absl/base/TARGETS.absl +++ b/etc/import/absl/base/TARGETS.absl @@ -60,22 +60,17 @@ } , "case": [["mingw", ["-ladvapi32"]]] } - , { "type": "case*" - , "expr": {"type": "var", "name": "OS"} - , "case": [["linux", ["-lrt"]]] + , { "type": "case" + , "expr": {"type": "var", "name": "OS", "default": "linux"} + , "case": {"linux": ["-lrt"]} } , { "type": "case" , "expr": { "type": "var" , "name": "TARGET_ARCH" - , "default": - { "type": "var" - , "name": "ARCH" - , "default": - {"type": "fail", "msg": "Required variable 'ARCH' is not set."} - } + , "default": {"type": "var", "name": "ARCH", "default": "unknown"} } - , "case": {"arm": ["-latomic"]} + , "case": {"arm": ["-latomic"], "unknown": ["-latomic"]} } ] } @@ -103,14 +98,9 @@ , "expr": { "type": "var" , "name": "TARGET_ARCH" - , "default": - { "type": "var" - , "name": "ARCH" - , "default": - {"type": "fail", "msg": "Required variable 'ARCH' is not set."} - } + , "default": {"type": "var", "name": "ARCH", "default": "unknown"} } - , "case": {"arm": ["-latomic"]} + , "case": {"arm": ["-latomic"], "unknown": ["-latomic"]} } , "deps": ["config", "core_headers"] } @@ -263,14 +253,9 @@ , "expr": { "type": "var" , "name": "TARGET_ARCH" - , "default": - { "type": "var" - , "name": "ARCH" - , "default": - {"type": "fail", "msg": "Required variable 'ARCH' is not set."} - } + , "default": {"type": "var", "name": "ARCH", "default": "unknown"} } - , "case": {"arm": ["-latomic"]} + , "case": {"arm": ["-latomic"], "unknown": ["-latomic"]} } , "deps": ["atomic_hook", "config", "core_headers", "errno_saver", "log_severity"] @@ -341,14 +326,9 @@ , "expr": { "type": "var" , "name": "TARGET_ARCH" - , "default": - { "type": "var" - , "name": "ARCH" - , "default": - {"type": "fail", "msg": "Required variable 'ARCH' is not set."} - } + , "default": {"type": "var", "name": "ARCH", "default": "unknown"} } - , "case": {"arm": ["-latomic"]} + , "case": {"arm": ["-latomic"], "unknown": ["-latomic"]} } ] } diff --git a/etc/import/absl/debugging/TARGETS.absl b/etc/import/absl/debugging/TARGETS.absl index 56cd8da61..f7052bc3e 100644 --- a/etc/import/absl/debugging/TARGETS.absl +++ b/etc/import/absl/debugging/TARGETS.absl @@ -12,20 +12,6 @@ , "symbolize_unimplemented.inc" , "symbolize_win32.inc" ] - , "private-cflags": - { "type": "case" - , "expr": - { "type": "var" - , "name": "TARGET_ARCH" - , "default": - { "type": "var" - , "name": "ARCH" - , "default": - {"type": "fail", "msg": "Required variable 'ARCH' is not set."} - } - } - , "case": {"arm": ["-Wno-sign-conversion"]} - } , "private-ldflags": { "type": "++" , "$1": @@ -45,14 +31,9 @@ , "expr": { "type": "var" , "name": "TARGET_ARCH" - , "default": - { "type": "var" - , "name": "ARCH" - , "default": - {"type": "fail", "msg": "Required variable 'ARCH' is not set."} - } + , "default": {"type": "var", "name": "ARCH", "default": "unknown"} } - , "case": {"arm": ["-latomic"]} + , "case": {"arm": ["-latomic"], "unknown": ["-latomic"]} } ] } diff --git a/etc/import/absl/random/TARGETS.absl b/etc/import/absl/random/TARGETS.absl index c32ecb3ac..db5b83eeb 100644 --- a/etc/import/absl/random/TARGETS.absl +++ b/etc/import/absl/random/TARGETS.absl @@ -354,13 +354,17 @@ , "internal_randen_hwaes_impl": { "type": ["@", "rules", "CC", "library"] , "name": ["internal_randen_hwaes_impl"] - , "arguments_config": ["TARGET_ARCH", "TOOLCHAIN_CONFIG"] + , "arguments_config": ["ARCH", "TARGET_ARCH", "TOOLCHAIN_CONFIG"] , "stage": ["absl", "random"] , "private-hdrs": ["internal/randen_hwaes.h"] , "srcs": ["internal/randen_hwaes.cc"] , "private-cflags": { "type": "case*" - , "expr": {"type": "var", "name": "TARGET_ARCH"} + , "expr": + { "type": "var" + , "name": "TARGET_ARCH" + , "default": {"type": "var", "name": "ARCH"} + } , "case": [ ["arm", ["-mfpu=neon"]] , ["arm64", ["-march=armv8-a+crypto"]] diff --git a/etc/import/absl/strings/TARGETS.absl b/etc/import/absl/strings/TARGETS.absl index 129a48f6e..ea989bc16 100644 --- a/etc/import/absl/strings/TARGETS.absl +++ b/etc/import/absl/strings/TARGETS.absl @@ -239,14 +239,9 @@ , "expr": { "type": "var" , "name": "TARGET_ARCH" - , "default": - { "type": "var" - , "name": "ARCH" - , "default": - {"type": "fail", "msg": "Required variable 'ARCH' is not set."} - } + , "default": {"type": "var", "name": "ARCH", "default": "unknown"} } - , "case": {"arm": ["-latomic"]} + , "case": {"arm": ["-latomic"], "unknown": ["-latomic"]} } , "deps": [ "cord_internal" diff --git a/etc/import/absl/synchronization/TARGETS.absl b/etc/import/absl/synchronization/TARGETS.absl index 3ca1d759a..84a1a964a 100644 --- a/etc/import/absl/synchronization/TARGETS.absl +++ b/etc/import/absl/synchronization/TARGETS.absl @@ -50,14 +50,9 @@ , "expr": { "type": "var" , "name": "TARGET_ARCH" - , "default": - { "type": "var" - , "name": "ARCH" - , "default": - {"type": "fail", "msg": "Required variable 'ARCH' is not set."} - } + , "default": {"type": "var", "name": "ARCH", "default": "unknown"} } - , "case": {"arm": ["-latomic"]} + , "case": {"arm": ["-latomic"], "unknown": ["-latomic"]} } ] } diff --git a/etc/import/absl/time/TARGETS.absl b/etc/import/absl/time/TARGETS.absl index aafe343f7..54d87595d 100644 --- a/etc/import/absl/time/TARGETS.absl +++ b/etc/import/absl/time/TARGETS.absl @@ -86,14 +86,9 @@ , "expr": { "type": "var" , "name": "TARGET_ARCH" - , "default": - { "type": "var" - , "name": "ARCH" - , "default": - {"type": "fail", "msg": "Required variable 'ARCH' is not set."} - } + , "default": {"type": "var", "name": "ARCH", "default": "unknown"} } - , "case": {"arm": ["-latomic"]} + , "case": {"arm": ["-latomic"], "unknown": ["-latomic"]} } , "deps": ["civil_time", ["absl/base", "config"]] } From b127c86625b5b6751d47e21f431bc7f748ac8ded Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Mon, 21 Aug 2023 09:00:00 +0200 Subject: [PATCH 92/97] rules: Add documentation for the implict dependencies --- rules/CC/RULES | 2 ++ rules/CC/auto/RULES | 2 ++ rules/CC/pkgconfig/RULES | 1 + rules/CC/test/RULES | 10 ++++++++++ rules/patch/RULES | 1 + rules/shell/test/RULES | 10 ++++++++++ 6 files changed, 26 insertions(+) diff --git a/rules/CC/RULES b/rules/CC/RULES index c30aaee16..bea41d271 100644 --- a/rules/CC/RULES +++ b/rules/CC/RULES @@ -397,6 +397,7 @@ [ "Name to use for pkg-config files. If this field is empty, the field" , "\"name\" is used instead." ] + , "defaults": ["The C/C++ toolchain to use"] } , "config_doc": { "CC": @@ -596,6 +597,7 @@ , "duplicate work will be carried out, even if the same proto library" , "is used at various places)." ] + , "defaults": ["The C/C++ toolchain to use"] } , "config_doc": { "CC": diff --git a/rules/CC/auto/RULES b/rules/CC/auto/RULES index 860db01c0..b1419fb8e 100644 --- a/rules/CC/auto/RULES +++ b/rules/CC/auto/RULES @@ -22,6 +22,7 @@ , "include path. Useful for providing additional header files to values" , "given in \"have_{cfile,cxxfile,ctype,cxxtype,csymbol,cxxsymbol}\"." ] + , "defaults": ["The C/C++ toolchain to use"] } , "config_doc": { "CC": @@ -1032,6 +1033,7 @@ , "defined in the config field \"defines\"." ] , "@only": ["If set, only replace @VAR@ and not ${VAR}"] + , "runner": ["The program generating the header file from the template."] } , "config_doc": { "defines": diff --git a/rules/CC/pkgconfig/RULES b/rules/CC/pkgconfig/RULES index a6f6ce847..4424cb2fa 100644 --- a/rules/CC/pkgconfig/RULES +++ b/rules/CC/pkgconfig/RULES @@ -10,6 +10,7 @@ , "\"--static\"), appended to the config variable \"PKG_CONFIG_ARGS\"." ] , "stage": ["The stage of the internally created flag files."] + , "defaults": ["The C/C++ toolchain to use"] } , "config_doc": { "PKG_CONFIG_ARGS": diff --git a/rules/CC/test/RULES b/rules/CC/test/RULES index af413a9b0..e09d08eb0 100644 --- a/rules/CC/test/RULES +++ b/rules/CC/test/RULES @@ -70,6 +70,16 @@ , "Individual directory components are joined with \"/\"." ] , "data": ["Any files the test binary needs access to when running"] + , "defaults": ["The C/C++ toolchain to use"] + , "runner": + [ "The test runner which starts the actual test binrary after providing" + , "the respective environment. The runner also takes care of capturing" + , "stdout/stderr and timing information." + ] + , "summarizer": + [ "Tool to aggregate the results of individual test runs (for flakyness" + , "detection) to an overall test result." + ] } , "config_doc": { "CC": ["The name of the C compiler to be used."] diff --git a/rules/patch/RULES b/rules/patch/RULES index 712a4877a..6f4471ca9 100644 --- a/rules/patch/RULES +++ b/rules/patch/RULES @@ -151,6 +151,7 @@ [ "The single source file to patch, typically an explicit file reference." ] , "patch": ["The patch to apply."] + , "patch-defaults": ["The patch binary (and toolchain) to use"] } , "artifacts_doc": ["The patched file, staged to the position the of the original file"] diff --git a/rules/shell/test/RULES b/rules/shell/test/RULES index c4f85695f..44acb0021 100644 --- a/rules/shell/test/RULES +++ b/rules/shell/test/RULES @@ -39,6 +39,16 @@ [ "Any targets that should be staged (with artifacts and runfiles) into" , "the tests working directory" ] + , "runner": + [ "The test runner which starts the actual test script after providing" + , "the respective environment. The runner also takes care of capturing" + , "stdout/stderr, timing information, and ensure the presence of the" + , "files to keep even if the script failed to produce them." + ] + , "summarizer": + [ "Tool to aggregate the results of individual test runs (for flakyness" + , "detection) to an overall test result." + ] } , "config_doc": { "RUNS_PER_TEST": From d3071ef178586d58b234df26d4a702aaaf97d7e0 Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Mon, 21 Aug 2023 14:31:20 +0200 Subject: [PATCH 93/97] bundled absl: configure in a portable way ... by disabling auto-configuration. The problem that the result of this auto-configuration depends on the language standard used and may (and in case of libproto does) leak into public interfaces of consuming libraries; this would require every user of a library with a public dependency of absl to use the same language standard that library was built with. Such a requirement makes the rules hard to use in a portable way. --- etc/defaults/patch/TARGETS.absl | 3 +++ etc/import/absl/base/TARGETS.absl | 25 ++++++++++++++++++++ etc/patches/options.h.diff | 38 +++++++++++++++++++++++++++++++ etc/repos.json | 2 +- 4 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 etc/defaults/patch/TARGETS.absl create mode 100644 etc/patches/options.h.diff diff --git a/etc/defaults/patch/TARGETS.absl b/etc/defaults/patch/TARGETS.absl new file mode 100644 index 000000000..16a4670f8 --- /dev/null +++ b/etc/defaults/patch/TARGETS.absl @@ -0,0 +1,3 @@ +{ "defaults": + {"type": ["patch", "defaults"], "base": [["@", "base", "patch", "defaults"]]} +} diff --git a/etc/import/absl/base/TARGETS.absl b/etc/import/absl/base/TARGETS.absl index fcc6bc783..7383de50e 100644 --- a/etc/import/absl/base/TARGETS.absl +++ b/etc/import/absl/base/TARGETS.absl @@ -118,11 +118,36 @@ , "deps": ["config", ["absl/meta", "type_traits"]] } , "config": + { "type": "export" + , "target": "config_" + , "flexible_config": + [ "ADD_CFLAGS" + , "ADD_CXXFLAGS" + , "AR" + , "ARCH" + , "CC" + , "CFLAGS" + , "CXX" + , "CXXFLAGS" + , "DEBUG" + , "ENV" + , "HOST_ARCH" + , "OS" + , "TARGET_ARCH" + , "TOOLCHAIN_CONFIG" + ] + } +, "config_": { "type": ["@", "rules", "CC", "library"] , "name": ["config"] , "stage": ["absl", "base"] , "hdrs": ["config.h", "options.h", "policy_checks.h"] } +, "options.h": + { "type": ["@", "rules", "patch", "file"] + , "src": [["FILE", null, "options.h"]] + , "patch": [["@", "patches", "", "options.h.diff"]] + } , "core_headers": { "type": "export" , "target": "core_headers_internal" diff --git a/etc/patches/options.h.diff b/etc/patches/options.h.diff new file mode 100644 index 000000000..d0f27f99e --- /dev/null +++ b/etc/patches/options.h.diff @@ -0,0 +1,38 @@ +--- options.h.orig 2023-08-21 14:27:30.600786502 +0200 ++++ options.h 2023-08-21 14:28:34.951926481 +0200 +@@ -94,7 +94,7 @@ + // User code should not inspect this macro. To check in the preprocessor if + // absl::any is a typedef of std::any, use the feature macro ABSL_USES_STD_ANY. + +-#define ABSL_OPTION_USE_STD_ANY 2 ++#define ABSL_OPTION_USE_STD_ANY 0 + + + // ABSL_OPTION_USE_STD_OPTIONAL +@@ -121,7 +121,7 @@ + // absl::optional is a typedef of std::optional, use the feature macro + // ABSL_USES_STD_OPTIONAL. + +-#define ABSL_OPTION_USE_STD_OPTIONAL 2 ++#define ABSL_OPTION_USE_STD_OPTIONAL 0 + + + // ABSL_OPTION_USE_STD_STRING_VIEW +@@ -148,7 +148,7 @@ + // absl::string_view is a typedef of std::string_view, use the feature macro + // ABSL_USES_STD_STRING_VIEW. + +-#define ABSL_OPTION_USE_STD_STRING_VIEW 2 ++#define ABSL_OPTION_USE_STD_STRING_VIEW 0 + + // ABSL_OPTION_USE_STD_VARIANT + // +@@ -174,7 +174,7 @@ + // absl::variant is a typedef of std::variant, use the feature macro + // ABSL_USES_STD_VARIANT. + +-#define ABSL_OPTION_USE_STD_VARIANT 2 ++#define ABSL_OPTION_USE_STD_VARIANT 0 + + + // ABSL_OPTION_USE_INLINE_NAMESPACE diff --git a/etc/repos.json b/etc/repos.json index a73989bbf..ac368a3f2 100644 --- a/etc/repos.json +++ b/etc/repos.json @@ -305,7 +305,7 @@ } , "target_root": "import targets" , "target_file_name": "TARGETS.absl" - , "bindings": {"rules": "rules-absl"} + , "bindings": {"rules": "rules-absl", "patches": "patches"} } , "zlib": { "repository": From a0a02385438089ad927f02dccb649c0d317e4ffb Mon Sep 17 00:00:00 2001 From: Paul Cristian Sarbu Date: Mon, 21 Aug 2023 14:31:11 +0200 Subject: [PATCH 94/97] external: Update zlib to v1.3 Old K&R function definitions got cleaned upstream, which removes many warnings when building zlib with more recent compilers. --- etc/repos.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/etc/repos.json b/etc/repos.json index ac368a3f2..2d556cea8 100644 --- a/etc/repos.json +++ b/etc/repos.json @@ -310,11 +310,11 @@ , "zlib": { "repository": { "type": "archive" - , "content": "d115fc690fa59d3be51b3442158876536140b6c2" - , "fetch": "https://github.com/madler/zlib/releases/download/v1.2.13/zlib-1.2.13.tar.gz" - , "sha256": "b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30" - , "sha512": "99f0e843f52290e6950cc328820c0f322a4d934a504f66c7caa76bd0cc17ece4bf0546424fc95135de85a2656fed5115abb835fd8d8a390d60ffaf946c8887ad" - , "subdir": "zlib-1.2.13" + , "content": "88faf0fca21e0d82de44366fdd52aaadbab2e6b6" + , "fetch": "https://github.com/madler/zlib/releases/download/v1.3/zlib-1.3.tar.gz" + , "sha256": "ff0ba4c292013dbc27530b3a81e1f9a813cd39de01ca5e0f8bf355702efa593e" + , "sha512": "185795044461cd78a5545250e06f6efdb0556e8d1bfe44e657b509dd6f00ba8892c8eb3febe65f79ee0b192d6af857f0e0055326d33a881449f3833f92e5f8fb" + , "subdir": "zlib-1.3" } , "target_root": "import targets" , "target_file_name": "TARGETS.zlib" From 8eb74daa0a800ca0552ad691aef76b27288ec8cb Mon Sep 17 00:00:00 2001 From: Oliver Reiche Date: Tue, 22 Aug 2023 12:13:24 +0200 Subject: [PATCH 95/97] Release 1.2.0~beta3 --- CHANGELOG.md | 14 ++++++++++++-- src/buildtool/main/version.cpp | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3581eaa21..cfedfebaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,16 @@ A feature release on top of `1.1.0`, backwards compatible. - `just install-cas` now, like `just install`, removes an existing destination file before installing instead of overwriting. +## Release `1.2.0~beta3` (2023-08-22) + +Third beta release for the upcoming `1.2.0` release; see release +notes there. + +### Changes since `1.2.0~beta2` + +- Update and clean up of bundled dependency descriptions +- Improvement of documentation + ## Release `1.2.0~beta2` (2023-08-18) Second beta release for the upcoming `1.2.0` release; see release @@ -65,8 +75,8 @@ notes there. ### Changes since `1.2.0~beta1` - Clean up of the internal build description of bundled dependencies. -- Clean up of the the internal rules, in particular renaming of - implict dependency targets. +- Clean up of the internal rules, in particular renaming of + implicit dependency targets. - Various documentation improvements. ## Release `1.2.0~beta1` (2023-08-16) diff --git a/src/buildtool/main/version.cpp b/src/buildtool/main/version.cpp index 2a00cd211..b7306fcc5 100644 --- a/src/buildtool/main/version.cpp +++ b/src/buildtool/main/version.cpp @@ -21,7 +21,7 @@ auto version() -> std::string { std::size_t major = 1; std::size_t minor = 2; std::size_t revision = 0; - std::string suffix = "~beta2"; + std::string suffix = "~beta3"; #ifdef VERSION_EXTRA_SUFFIX suffix += VERSION_EXTRA_SUFFIX; #endif From 09292ccab76f8556be8863607983c9716ab9436e Mon Sep 17 00:00:00 2001 From: Oliver Reiche Date: Wed, 16 Aug 2023 18:54:58 +0200 Subject: [PATCH 96/97] tutorial: Use new TOOLCHAIN_CONFIG variable --- doc/tutorial/cross-compiling.md | 58 ++++++++++++++++----------------- doc/tutorial/proto.md | 20 ++++++------ 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/doc/tutorial/cross-compiling.md b/doc/tutorial/cross-compiling.md index 4e1a9fce2..60bdd3c59 100644 --- a/doc/tutorial/cross-compiling.md +++ b/doc/tutorial/cross-compiling.md @@ -67,18 +67,18 @@ Then, our `TARGETS` file describe a simple binary, as usual. } ``` -As mentioned in the introduction, we need to specify `COMPILER_FAMILY`, +As mentioned in the introduction, we need to specify `TOOLCHAIN_CONFIG`, `OS`, and `ARCH`. So the canonical building for host looks something like the following. ``` sh -$ just-mr build -D '{"COMPILER_FAMILY": "gnu", "OS": "linux", "ARCH": "x86_64"}' +$ just-mr build -D '{"TOOLCHAIN_CONFIG": {"FAMILY": "gnu"}, "OS": "linux", "ARCH": "x86_64"}' INFO: Performing repositories setup INFO: Found 21 repositories to set up -INFO: Setup finished, exec ["just","build","-C","...","-D","{\"COMPILER_FAMILY\": \"gnu\", \"OS\": \"linux\", \"ARCH\": \"x86_64\"}"] -INFO: Requested target is [["@","","","helloworld"],{"ARCH":"x86_64","COMPILER_FAMILY":"gnu","OS":"linux"}] -INFO: Analysed target [["@","","","helloworld"],{"ARCH":"x86_64","COMPILER_FAMILY":"gnu","OS":"linux"}] +INFO: Setup finished, exec ["just","build","-C","...","-D","{\"TOOLCHAIN_CONFIG\": {\"FAMILY\": \"gnu\"}, \"OS\": \"linux\", \"ARCH\": \"x86_64\"}"] +INFO: Requested target is [["@","","","helloworld"],{"ARCH":"x86_64","OS":"linux","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}] +INFO: Analysed target [["@","","","helloworld"],{"ARCH":"x86_64","OS":"linux","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}] INFO: Discovered 2 actions, 1 trees, 0 blobs -INFO: Building [["@","","","helloworld"],{"ARCH":"x86_64","COMPILER_FAMILY":"gnu","OS":"linux"}]. +INFO: Building [["@","","","helloworld"],{"ARCH":"x86_64","OS":"linux","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}]. INFO: Processed 2 actions, 0 cache hits. INFO: Artifacts built, logical paths are: helloworld [0d5754a83c7c787b1c4dd717c8588ecef203fb72:16992:x] @@ -87,14 +87,14 @@ $ To cross compile, we simply add `TARGET_ARCH`. ``` sh -$ just-mr build -D '{"COMPILER_FAMILY": "gnu", "OS": "linux", "ARCH": "x86_64", "TARGET_ARCH": "arm64"}' +$ just-mr build -D '{"TOOLCHAIN_CONFIG": {"FAMILY": "gnu"}, "OS": "linux", "ARCH": "x86_64", "TARGET_ARCH": "arm64"}' INFO: Performing repositories setup INFO: Found 21 repositories to set up -INFO: Setup finished, exec ["just","build","-C","...","-D","{\"COMPILER_FAMILY\": \"gnu\", \"OS\": \"linux\", \"ARCH\": \"x86_64\", \"TARGET_ARCH\": \"arm64\"}"] -INFO: Requested target is [["@","","","helloworld"],{"ARCH":"x86_64","COMPILER_FAMILY":"gnu","OS":"linux","TARGET_ARCH":"arm64"}] -INFO: Analysed target [["@","","","helloworld"],{"ARCH":"x86_64","COMPILER_FAMILY":"gnu","OS":"linux","TARGET_ARCH":"arm64"}] +INFO: Setup finished, exec ["just","build","-C","...","-D","{\"TOOLCHAIN_CONFIG\": {\"FAMILY\": \"gnu\"}, \"OS\": \"linux\", \"ARCH\": \"x86_64\", \"TARGET_ARCH\": \"arm64\"}"] +INFO: Requested target is [["@","","","helloworld"],{"ARCH":"x86_64","OS":"linux","TARGET_ARCH":"arm64","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}] +INFO: Analysed target [["@","","","helloworld"],{"ARCH":"x86_64","OS":"linux","TARGET_ARCH":"arm64","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}] INFO: Discovered 2 actions, 1 trees, 0 blobs -INFO: Building [["@","","","helloworld"],{"ARCH":"x86_64","COMPILER_FAMILY":"gnu","OS":"linux","TARGET_ARCH":"arm64"}]. +INFO: Building [["@","","","helloworld"],{"ARCH":"x86_64","OS":"linux","TARGET_ARCH":"arm64","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}]. INFO: Processed 2 actions, 0 cache hits. INFO: Artifacts built, logical paths are: helloworld [b45459ea3dd36c7531756a4de9aaefd6af30e417:9856:x] @@ -104,10 +104,10 @@ $ To inspect the different command lines for native and cross compilation, we can use `just analyse`. ``` sh -$ just-mr analyse -D '{"COMPILER_FAMILY": "gnu", "OS": "linux", "ARCH": "x86_64"}' --dump-actions - -$ just-mr analyse -D '{"COMPILER_FAMILY": "gnu", "OS": "linux", "ARCH": "x86_64", "TARGET_ARCH": "arm64"}' --dump-actions - -$ just-mr analyse -D '{"COMPILER_FAMILY": "clang", "OS": "linux", "ARCH": "x86_64"}' --dump-actions - -$ just-mr analyse -D '{"COMPILER_FAMILY": "clang", "OS": "linux", "ARCH": "x86_64", "TARGET_ARCH": "arm64"}' --dump-actions - +$ just-mr analyse -D '{"TOOLCHAIN_CONFIG": {"FAMILY": "gnu"}, "OS": "linux", "ARCH": "x86_64"}' --dump-actions - +$ just-mr analyse -D '{"TOOLCHAIN_CONFIG": {"FAMILY": "gnu"}, "OS": "linux", "ARCH": "x86_64", "TARGET_ARCH": "arm64"}' --dump-actions - +$ just-mr analyse -D '{"TOOLCHAIN_CONFIG": {"FAMILY": "clang"}, "OS": "linux", "ARCH": "x86_64"}' --dump-actions - +$ just-mr analyse -D '{"TOOLCHAIN_CONFIG": {"FAMILY": "clang"}, "OS": "linux", "ARCH": "x86_64", "TARGET_ARCH": "arm64"}' --dump-actions - ``` @@ -135,12 +135,12 @@ a file `test/TARGETS`. Now, if we try to run the test by simply specifying the target architecture, we find that the binary to be tested is still only built for host. ``` sh -$ just-mr analyse --dump-targets - -D '{"COMPILER_FAMILY": "gnu", "OS": "linux", "ARCH": "x86_64", "TARGET_ARCH": "arm64"}' test basic +$ just-mr analyse --dump-targets - -D '{"TOOLCHAIN_CONFIG": {"FAMILY": "gnu"}, "OS": "linux", "ARCH": "x86_64", "TARGET_ARCH": "arm64"}' test basic INFO: Performing repositories setup INFO: Found 21 repositories to set up -INFO: Setup finished, exec ["just","analyse","-C","...","--dump-targets","-","-D","{\"COMPILER_FAMILY\": \"gnu\", \"OS\": \"linux\", \"ARCH\": \"x86_64\", \"TARGET_ARCH\": \"arm64\"}","test","basic"] -INFO: Requested target is [["@","","test","basic"],{"ARCH":"x86_64","COMPILER_FAMILY":"gnu","OS":"linux","TARGET_ARCH":"arm64"}] -INFO: Result of target [["@","","test","basic"],{"ARCH":"x86_64","COMPILER_FAMILY":"gnu","OS":"linux","TARGET_ARCH":"arm64"}]: { +INFO: Setup finished, exec ["just","analyse","-C","...","--dump-targets","-","-D","{\"TOOLCHAIN_CONFIG\": {\"FAMILY\": \"gnu\"}, \"OS\": \"linux\", \"ARCH\": \"x86_64\", \"TARGET_ARCH\": \"arm64\"}","test","basic"] +INFO: Requested target is [["@","","test","basic"],{"ARCH":"x86_64","OS":"linux","TARGET_ARCH":"arm64","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}] +INFO: Result of target [["@","","test","basic"],{"ARCH":"x86_64","OS":"linux","TARGET_ARCH":"arm64","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}]: { "artifacts": { "result": {"data":{"id":"33eb2ebd2ea0d6d335dfc1f948d14d506a19f693","path":"result"},"type":"ACTION"}, "stderr": {"data":{"id":"33eb2ebd2ea0d6d335dfc1f948d14d506a19f693","path":"stderr"},"type":"ACTION"}, @@ -159,23 +159,23 @@ INFO: List of analysed targets: "@": { "": { "": { - "helloworld": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"ADD_LDFLAGS":null,"ARCH":"x86_64","BUILD_POSITION_INDEPENDENT":null,"CC":null,"CFLAGS":null,"COMPILER_FAMILY":"gnu","CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"LDFLAGS":null,"OS":"linux","TARGET_ARCH":"x86_64"}] + "helloworld": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"ADD_LDFLAGS":null,"ARCH":"x86_64","BUILD_POSITION_INDEPENDENT":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"LDFLAGS":null,"OS":"linux","TARGET_ARCH":"x86_64","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}] }, "test": { - "basic": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"ADD_LDFLAGS":null,"ARCH":"x86_64","ARCH_DISPATCH":null,"BUILD_POSITION_INDEPENDENT":null,"CC":null,"CFLAGS":null,"COMPILER_FAMILY":"gnu","CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"LDFLAGS":null,"OS":"linux","RUNS_PER_TEST":null,"TARGET_ARCH":"arm64","TEST_ENV":null,"TIMEOUT_SCALE":null}], + "basic": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"ADD_LDFLAGS":null,"ARCH":"x86_64","ARCH_DISPATCH":null,"BUILD_POSITION_INDEPENDENT":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"LDFLAGS":null,"OS":"linux","RUNS_PER_TEST":null,"TARGET_ARCH":"arm64","TEST_ENV":null,"TIMEOUT_SCALE":null,"TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}], "basic.sh": [{}] } }, "rules-cc": { "CC": { - "defaults": [{"ARCH":"x86_64","COMPILER_FAMILY":"gnu","DEBUG":null,"HOST_ARCH":null,"OS":"linux","TARGET_ARCH":"x86_64"}] + "defaults": [{"ARCH":"x86_64","DEBUG":null,"HOST_ARCH":null,"OS":"linux","TARGET_ARCH":"x86_64","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}] } }, "rules-cc/just/rules": { "CC": { - "defaults": [{"ARCH":"x86_64","COMPILER_FAMILY":"gnu","DEBUG":null,"HOST_ARCH":null,"OS":"linux","TARGET_ARCH":"x86_64"}], + "defaults": [{"ARCH":"x86_64","DEBUG":null,"HOST_ARCH":null,"OS":"linux","TARGET_ARCH":"x86_64","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}], "gcc": [{"ARCH":"x86_64","HOST_ARCH":null,"OS":"linux","TARGET_ARCH":"x86_64"}], - "toolchain": [{"ARCH":"x86_64","COMPILER_FAMILY":"gnu","HOST_ARCH":null,"OS":"linux","TARGET_ARCH":"x86_64"}] + "toolchain": [{"ARCH":"x86_64","HOST_ARCH":null,"OS":"linux","TARGET_ARCH":"x86_64","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}] } } } @@ -244,15 +244,15 @@ itself) which does not require any credentials; nevertheless, it still accepts any credentials provided. ``` sh -$ just-mr build -D '{"COMPILER_FAMILY": "gnu", "OS": "linux", "ARCH": "x86_64", "TARGET_ARCH": "arm64", "ARCH_DISPATCH": {"arm64": {"runner": "arm64-worker"}}}' --endpoint-configuration dispatch.json --tls-ca-cert ca.crt --tls-client-cert client.crt --tls-client-key client.key test basic +$ just-mr build -D '{"TOOLCHAIN_CONFIG": {"FAMILY": "gnu"}, "OS": "linux", "ARCH": "x86_64", "TARGET_ARCH": "arm64", "ARCH_DISPATCH": {"arm64": {"runner": "arm64-worker"}}}' --endpoint-configuration dispatch.json --tls-ca-cert ca.crt --tls-client-cert client.crt --tls-client-key client.key test basic INFO: Performing repositories setup INFO: Found 21 repositories to set up -INFO: Setup finished, exec ["just","build","-C","...","-D","{\"COMPILER_FAMILY\": \"gnu\", \"OS\": \"linux\", \"ARCH\": \"x86_64\", \"TARGET_ARCH\": \"arm64\", \"ARCH_DISPATCH\": {\"arm64\": {\"runner\": \"arm64-worker\"}}}","--endpoint-configuration","dispatch.json","--tls-ca-cert","ca.crt","--tls-client-cert","client.crt","--tls-client-key","client.key","test","basic"] -INFO: Requested target is [["@","","test","basic"],{"ARCH":"x86_64","ARCH_DISPATCH":{"arm64":{"runner":"arm64-worker"}},"COMPILER_FAMILY":"gnu","OS":"linux","TARGET_ARCH":"arm64"}] -INFO: Analysed target [["@","","test","basic"],{"ARCH":"x86_64","ARCH_DISPATCH":{"arm64":{"runner":"arm64-worker"}},"COMPILER_FAMILY":"gnu","OS":"linux","TARGET_ARCH":"arm64"}] +INFO: Setup finished, exec ["just","build","-C","...","-D","{\"TOOLCHAIN_CONFIG\": {\"FAMILY\": \"gnu\"}, \"OS\": \"linux\", \"ARCH\": \"x86_64\", \"TARGET_ARCH\": \"arm64\", \"ARCH_DISPATCH\": {\"arm64\": {\"runner\": \"arm64-worker\"}}}","--endpoint-configuration","dispatch.json","--tls-ca-cert","ca.crt","--tls-client-cert","client.crt","--tls-client-key","client.key","test","basic"] +INFO: Requested target is [["@","","test","basic"],{"ARCH":"x86_64","ARCH_DISPATCH":{"arm64":{"runner":"arm64-worker"}},"OS":"linux","TARGET_ARCH":"arm64","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}] +INFO: Analysed target [["@","","test","basic"],{"ARCH":"x86_64","ARCH_DISPATCH":{"arm64":{"runner":"arm64-worker"}},"OS":"linux","TARGET_ARCH":"arm64","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}] INFO: Target tainted ["test"]. INFO: Discovered 3 actions, 3 trees, 1 blobs -INFO: Building [["@","","test","basic"],{"ARCH":"x86_64","ARCH_DISPATCH":{"arm64":{"runner":"arm64-worker"}},"COMPILER_FAMILY":"gnu","OS":"linux","TARGET_ARCH":"arm64"}]. +INFO: Building [["@","","test","basic"],{"ARCH":"x86_64","ARCH_DISPATCH":{"arm64":{"runner":"arm64-worker"}},"OS":"linux","TARGET_ARCH":"arm64","TOOLCHAIN_CONFIG":{"FAMILY":"gnu"}}]. INFO: Processed 3 actions, 2 cache hits. INFO: Artifacts built, logical paths are: result [7ef22e9a431ad0272713b71fdc8794016c8ef12f:5:f] diff --git a/doc/tutorial/proto.md b/doc/tutorial/proto.md index 8a04e3737..8e7fb5c0a 100644 --- a/doc/tutorial/proto.md +++ b/doc/tutorial/proto.md @@ -278,13 +278,13 @@ INFO: List of analysed targets: { "#": { "eda46ea21de25033ff7250e6a4cdc0b2c24be0c7": { - "2a483a2de7f25c1bc066e47245f55ec9a2d4a719": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"COMPILER_FAMILY":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"TARGET_ARCH":null}] + "2a483a2de7f25c1bc066e47245f55ec9a2d4a719": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}] } }, "@": { "": { "": { - "add_person": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"COMPILER_FAMILY":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"TARGET_ARCH":null}], + "add_person": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}], "address": [{}] } }, @@ -295,8 +295,8 @@ INFO: List of analysed targets: }, "rules-cc/just/protobuf": { "": { - "C++ runtime": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"COMPILER_FAMILY":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"TARGET_ARCH":null}], - "protoc": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"COMPILER_FAMILY":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"TARGET_ARCH":null}], + "C++ runtime": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}], + "protoc": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}], "well_known_protos": [{}] } } @@ -414,16 +414,16 @@ INFO: List of analysed targets: { "#": { "eda46ea21de25033ff7250e6a4cdc0b2c24be0c7": { - "2a483a2de7f25c1bc066e47245f55ec9a2d4a719": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"COMPILER_FAMILY":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"TARGET_ARCH":null}] + "2a483a2de7f25c1bc066e47245f55ec9a2d4a719": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}] } }, "@": { "": { "": { - "add_person": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"COMPILER_FAMILY":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"TARGET_ARCH":null}], + "add_person": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}], "address": [{}], - "list_people": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"COMPILER_FAMILY":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"TARGET_ARCH":null}], - "test": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"COMPILER_FAMILY":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"RUNS_PER_TEST":null,"TEST_ENV":null}], + "list_people": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}], + "test": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"RUNS_PER_TEST":null,"TEST_ENV":null,"TOOLCHAIN_CONFIG":null}], "test.sh": [{}] } }, @@ -434,8 +434,8 @@ INFO: List of analysed targets: }, "rules-cc/just/protobuf": { "": { - "C++ runtime": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"COMPILER_FAMILY":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"TARGET_ARCH":null}], - "protoc": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"COMPILER_FAMILY":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"TARGET_ARCH":null}], + "C++ runtime": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}], + "protoc": [{"ADD_CFLAGS":null,"ADD_CXXFLAGS":null,"AR":null,"ARCH":null,"CC":null,"CFLAGS":null,"CXX":null,"CXXFLAGS":null,"DEBUG":null,"ENV":null,"HOST_ARCH":null,"OS":null,"TARGET_ARCH":null,"TOOLCHAIN_CONFIG":null}], "well_known_protos": [{}] } } From 796d9a27daba4dd16f31a7b0237fff00218d754b Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Tue, 22 Aug 2023 16:08:20 +0200 Subject: [PATCH 97/97] Update tutorial to use latest version of rules-cc --- doc/tutorial/hello-world.md | 21 ++++++-------- doc/tutorial/target-file-glob-tree.md | 41 +++++++++++++++------------ doc/tutorial/third-party-software.md | 12 ++++---- 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/doc/tutorial/hello-world.md b/doc/tutorial/hello-world.md index 1edfb1b6e..23c63d9a5 100644 --- a/doc/tutorial/hello-world.md +++ b/doc/tutorial/hello-world.md @@ -30,7 +30,7 @@ Second, we also need to create the multi-repository configuration { "repository": { "type": "git" , "branch": "master" - , "commit": "123d8b03bf2440052626151c14c54abce2726e6f" + , "commit": "307c96681e6626286804c45273082dff94127878" , "repository": "https://github.com/just-buildsystem/rules-cc.git" , "subdir": "rules" } @@ -46,7 +46,7 @@ Second, we also need to create the multi-repository configuration In that configuration, two repositories are defined: 1. The `"rules-cc"` repository located in the subdirectory `rules` of - [just-buildsystem/rules-cc:123d8b03bf2440052626151c14c54abce2726e6f](https://github.com/just-buildsystem/rules-cc/tree/123d8b03bf2440052626151c14c54abce2726e6f), + [just-buildsystem/rules-cc:307c96681e6626286804c45273082dff94127878](https://github.com/just-buildsystem/rules-cc/tree/307c96681e6626286804c45273082dff94127878), which contains the high-level concepts for building C/C++ binaries and libraries. @@ -118,12 +118,11 @@ command line: $ just-mr build helloworld INFO: Requested target is [["@","tutorial","","helloworld"],{}] INFO: Analysed target [["@","tutorial","","helloworld"],{}] -INFO: Export targets found: 0 cached, 0 uncached, 0 not eligible for caching INFO: Discovered 2 actions, 1 trees, 0 blobs -INFO: Building [["@","helloworld","","helloworld"],{}]. +INFO: Building [["@","tutorial","","helloworld"],{}]. INFO: Processed 2 actions, 0 cache hits. INFO: Artifacts built, logical paths are: - helloworld [b5cfca8b810adc4686f5cac00258a137c5d4a3ba:17088:x] + helloworld [bd36255e856ddb72c844c2010a785ab70ee75d56:17088:x] $ ``` @@ -165,12 +164,11 @@ object that sets `"CXX"` to `"clang++"`: $ just-mr build helloworld -D'{"CXX":"clang++"}' INFO: Requested target is [["@","tutorial","","helloworld"],{"CXX":"clang++"}] INFO: Analysed target [["@","tutorial","","helloworld"],{"CXX":"clang++"}] -INFO: Export targets found: 0 cached, 0 uncached, 0 not eligible for caching INFO: Discovered 2 actions, 1 trees, 0 blobs INFO: Building [["@","tutorial","","helloworld"],{"CXX":"clang++"}]. INFO: Processed 2 actions, 0 cache hits. INFO: Artifacts built, logical paths are: - helloworld [b8cf7b8579d9dc7172b61660139e2c14521cedae:16944:x] + helloworld [a1e0dc77ec6f171e118a3e6992859f68617a2c6f:16944:x] $ ``` @@ -224,7 +222,7 @@ the following content: { "repository": { "type": "git" , "branch": "master" - , "commit": "123d8b03bf2440052626151c14c54abce2726e6f" + , "commit": "307c96681e6626286804c45273082dff94127878" , "repository": "https://github.com/just-buildsystem/rules-cc.git" , "subdir": "rules" } @@ -261,12 +259,11 @@ configuration changes, rerunning only `just` would not suffice): $ just-mr build helloworld INFO: Requested target is [["@","tutorial","","helloworld"],{}] INFO: Analysed target [["@","tutorial","","helloworld"],{}] -INFO: Export targets found: 0 cached, 0 uncached, 0 not eligible for caching INFO: Discovered 2 actions, 1 trees, 0 blobs INFO: Building [["@","tutorial","","helloworld"],{}]. INFO: Processed 2 actions, 0 cache hits. INFO: Artifacts built, logical paths are: - helloworld [487dc9e47b978877ed2f7d80b3395ce84b23be92:16992:x] + helloworld [0d5754a83c7c787b1c4dd717c8588ecef203fb72:16992:x] $ ``` @@ -381,12 +378,11 @@ binary can be built with the same command as before (no need to rerun $ just-mr build helloworld INFO: Requested target is [["@","tutorial","","helloworld"],{}] INFO: Analysed target [["@","tutorial","","helloworld"],{}] -INFO: Export targets found: 0 cached, 0 uncached, 0 not eligible for caching INFO: Discovered 4 actions, 2 trees, 0 blobs INFO: Building [["@","tutorial","","helloworld"],{}]. INFO: Processed 4 actions, 0 cache hits. INFO: Artifacts built, logical paths are: - helloworld [2b81e3177afc382452a2df9f294d3df90a9ccaf0:17664:x] + helloworld [a0e593e4d52e8b3e14863b3cf1f80809143829ca:17664:x] $ ``` @@ -397,7 +393,6 @@ run the following command: $ just-mr build greet greet INFO: Requested target is [["@","tutorial","greet","greet"],{}] INFO: Analysed target [["@","tutorial","greet","greet"],{}] -INFO: Export targets found: 0 cached, 0 uncached, 0 not eligible for caching INFO: Discovered 2 actions, 1 trees, 0 blobs INFO: Building [["@","tutorial","greet","greet"],{}]. INFO: Processed 2 actions, 2 cache hits. diff --git a/doc/tutorial/target-file-glob-tree.md b/doc/tutorial/target-file-glob-tree.md index 524cf3589..ea0ad3da6 100644 --- a/doc/tutorial/target-file-glob-tree.md +++ b/doc/tutorial/target-file-glob-tree.md @@ -36,7 +36,7 @@ following content. { "repository": { "type": "git" , "branch": "master" - , "commit": "123d8b03bf2440052626151c14c54abce2726e6f" + , "commit": "307c96681e6626286804c45273082dff94127878" , "repository": "https://github.com/just-buildsystem/rules-cc.git" , "subdir": "rules" } @@ -87,12 +87,11 @@ $ echo {} > TARGETS.units $ just-mr install -o . definitions.units INFO: Requested target is [["@","units","","definitions.units"],{}] INFO: Analysed target [["@","units","","definitions.units"],{}] -INFO: Export targets found: 0 cached, 0 uncached, 0 not eligible for caching INFO: Discovered 0 actions, 0 trees, 0 blobs INFO: Building [["@","units","","definitions.units"],{}]. INFO: Processed 0 actions, 0 cache hits. INFO: Artifacts can be found in: - /tmp/work-2022-08-22/definitions.units [0f24a321694aab5c1d3676e22d01fc73492bee42:342718:f] + /tmp/work/./definitions.units [0f24a321694aab5c1d3676e22d01fc73492bee42:342718:f] $ cp definitions.units definitions.units.orig $ # interactively edit definitions.units $ echo -e "/German units\n+2a\narea_soccerfield 105 m * 68 m\narea_saarland 2570 km^2\n.\nw\nq" | ed definitions.units @@ -140,18 +139,21 @@ $ just-mr analyse definitions.units --dump-actions - INFO: Requested target is [["@","units","","definitions.units"],{}] INFO: Result of target [["@","units","","definitions.units"],{}]: { "artifacts": { - "definitions.units": {"data":{"id":"98e3c7758f5dd433c6aa7b327040be676faf6f34","path":"patched"},"type":"ACTION"} + "definitions.units": {"data":{"id":"ac620477c30dc79701cdda95ec97a06f12251b6f","path":"patched"},"type":"ACTION"} }, "provides": { }, "runfiles": { - "definitions.units": {"data":{"id":"98e3c7758f5dd433c6aa7b327040be676faf6f34","path":"patched"},"type":"ACTION"} + "definitions.units": {"data":{"id":"ac620477c30dc79701cdda95ec97a06f12251b6f","path":"patched"},"type":"ACTION"} } } INFO: Actions for target [["@","units","","definitions.units"],{}]: [ { - "command": ["patch","-s","--read-only=ignore","--follow-symlinks","-o","patched","orig","patch"], + "command": ["sh","./run_patch.sh"], + "env": { + "PATH": "/bin:/usr/bin" + }, "input": { "orig": { "data": { @@ -167,6 +169,14 @@ INFO: Actions for target [["@","units","","definitions.units"],{}]: "repository": "patches" }, "type": "LOCAL" + }, + "run_patch.sh": { + "data": { + "file_type": "f", + "id": "85786bc8f6aeac0db3be48f8ce336f906e1d78a0", + "size": 93 + }, + "type": "KNOWN" } }, "output": ["patched"] @@ -181,7 +191,6 @@ Building `"definitions.units"` we find out patch applied correctly. $ just-mr build definitions.units -P definitions.units | grep -A 5 'German units' INFO: Requested target is [["@","units","","definitions.units"],{}] INFO: Analysed target [["@","units","","definitions.units"],{}] -INFO: Export targets found: 0 cached, 0 uncached, 0 not eligible for caching INFO: Discovered 1 actions, 0 trees, 1 blobs INFO: Building [["@","units","","definitions.units"],{}]. INFO: Processed 1 actions, 0 cache hits. @@ -217,13 +226,13 @@ INFO: Requested target is [["@","units","","data-draft"],{}] INFO: Result of target [["@","units","","data-draft"],{}]: { "artifacts": { "currency.units": {"data":{"file_type":"f","id":"ac6da8afaac0f34e114e123e4ab3a41e59121b10","size":14707},"type":"KNOWN"}, - "definitions.units": {"data":{"id":"98e3c7758f5dd433c6aa7b327040be676faf6f34","path":"patched"},"type":"ACTION"} + "definitions.units": {"data":{"id":"ac620477c30dc79701cdda95ec97a06f12251b6f","path":"patched"},"type":"ACTION"} }, "provides": { }, "runfiles": { "currency.units": {"data":{"file_type":"f","id":"ac6da8afaac0f34e114e123e4ab3a41e59121b10","size":14707},"type":"KNOWN"}, - "definitions.units": {"data":{"id":"98e3c7758f5dd433c6aa7b327040be676faf6f34","path":"patched"},"type":"ACTION"} + "definitions.units": {"data":{"id":"ac620477c30dc79701cdda95ec97a06f12251b6f","path":"patched"},"type":"ACTION"} } } $ @@ -316,17 +325,16 @@ total, giving 5 compile and one link action. $ just-mr build units-draft INFO: Requested target is [["@","units","","units-draft"],{}] INFO: Analysed target [["@","units","","units-draft"],{}] -INFO: Export targets found: 0 cached, 0 uncached, 0 not eligible for caching INFO: Discovered 6 actions, 1 trees, 0 blobs INFO: Building [["@","units","","units-draft"],{}]. -INFO (action:12af248ce5737be492f7f5909284d4e3b6488807): +INFO (action:f9426e7a0c3525618ead3787872e843c86f12dd2): Stderr of command: ["cc","-I","work","-isystem","include","-c","work/strfunc.c","-o","work/strfunc.o"] work/strfunc.c:109:8: warning: extra tokens at end of #endif directive [-Wendif-labels] 109 | #endif NO_STRSPN | ^~~~~~~~~ INFO: Processed 6 actions, 0 cache hits. INFO: Artifacts built, logical paths are: - units [718cb1489bd006082f966ea73e3fba3dd072d084:124488:x] + units [40cdc2a9fa6f06004bbf290014519ba21f122e7d:124488:x] $ ``` @@ -339,12 +347,11 @@ a patch. $ just-mr install -o . strfunc.c INFO: Requested target is [["@","units","","strfunc.c"],{}] INFO: Analysed target [["@","units","","strfunc.c"],{}] -INFO: Export targets found: 0 cached, 0 uncached, 0 not eligible for caching INFO: Discovered 0 actions, 0 trees, 0 blobs INFO: Building [["@","units","","strfunc.c"],{}]. INFO: Processed 0 actions, 0 cache hits. INFO: Artifacts can be found in: - /tmp/work-2022-08-22/strfunc.c [e2aab4b825fa2822ccf33746d467a4944212abb9:2201:f] + /tmp/work/./strfunc.c [e2aab4b825fa2822ccf33746d467a4944212abb9:2201:f] $ cp strfunc.c strfunc.c.orig $ echo -e "109\ns|N|// N\nw\nq" | ed strfunc.c 2201 @@ -390,12 +397,11 @@ cache. $ just-mr build units INFO: Requested target is [["@","units","","units"],{}] INFO: Analysed target [["@","units","","units"],{}] -INFO: Export targets found: 0 cached, 0 uncached, 0 not eligible for caching INFO: Discovered 7 actions, 1 trees, 1 blobs INFO: Building [["@","units","","units"],{}]. INFO: Processed 7 actions, 5 cache hits. INFO: Artifacts built, logical paths are: - units [718cb1489bd006082f966ea73e3fba3dd072d084:124488:x] + units [40cdc2a9fa6f06004bbf290014519ba21f122e7d:124488:x] $ ``` @@ -415,12 +421,11 @@ Then things work as expected $ just-mr install -o /tmp/testinstall INFO: Requested target is [["@","units","",""],{}] INFO: Analysed target [["@","units","",""],{}] -INFO: Export targets found: 0 cached, 0 uncached, 0 not eligible for caching INFO: Discovered 8 actions, 1 trees, 1 blobs INFO: Building [["@","units","",""],{}]. INFO: Processed 8 actions, 8 cache hits. INFO: Artifacts can be found in: - /tmp/testinstall/bin/units [718cb1489bd006082f966ea73e3fba3dd072d084:124488:x] + /tmp/testinstall/bin/units [40cdc2a9fa6f06004bbf290014519ba21f122e7d:124488:x] /tmp/testinstall/share/units/currency.units [ac6da8afaac0f34e114e123e4ab3a41e59121b10:14707:f] /tmp/testinstall/share/units/definitions.units [763f3289422c296057e142f61be190ee6bef049a:342772:f] $ /tmp/testinstall/bin/units 'area_saarland' 'area_soccerfield' diff --git a/doc/tutorial/third-party-software.md b/doc/tutorial/third-party-software.md index 815543c56..6c87d08b5 100644 --- a/doc/tutorial/third-party-software.md +++ b/doc/tutorial/third-party-software.md @@ -140,7 +140,7 @@ additional binding `"format"` for it: { "repository": { "type": "git" , "branch": "master" - , "commit": "123d8b03bf2440052626151c14c54abce2726e6f" + , "commit": "307c96681e6626286804c45273082dff94127878" , "repository": "https://github.com/just-buildsystem/rules-cc.git" , "subdir": "rules" } @@ -210,7 +210,7 @@ INFO: Discovered 7 actions, 3 trees, 0 blobs INFO: Building [["@","tutorial","","helloworld"],{}]. INFO: Processed 7 actions, 1 cache hits. INFO: Artifacts built, logical paths are: - helloworld [0ec4e36cfb5f2c3efa0fff789349a46694a6d303:132736:x] + helloworld [18d25e828a0176cef6fb029bfd83e1862712ec87:132736:x] $ ``` @@ -261,7 +261,7 @@ be set for them in `repos.json`: { "repository": { "type": "git" , "branch": "master" - , "commit": "123d8b03bf2440052626151c14c54abce2726e6f" + , "commit": "307c96681e6626286804c45273082dff94127878" , "repository": "https://github.com/just-buildsystem/rules-cc.git" , "subdir": "rules" } @@ -312,7 +312,7 @@ INFO: Discovered 7 actions, 3 trees, 0 blobs INFO: Building [["@","tutorial","","helloworld"],{}]. INFO: Processed 7 actions, 7 cache hits. INFO: Artifacts built, logical paths are: - helloworld [0ec4e36cfb5f2c3efa0fff789349a46694a6d303:132736:x] + helloworld [18d25e828a0176cef6fb029bfd83e1862712ec87:132736:x] $ $ just-mr build helloworld INFO: Requested target is [["@","tutorial","","helloworld"],{}] @@ -322,7 +322,7 @@ INFO: Discovered 4 actions, 2 trees, 0 blobs INFO: Building [["@","tutorial","","helloworld"],{}]. INFO: Processed 4 actions, 4 cache hits. INFO: Artifacts built, logical paths are: - helloworld [0ec4e36cfb5f2c3efa0fff789349a46694a6d303:132736:x] + helloworld [18d25e828a0176cef6fb029bfd83e1862712ec87:132736:x] $ ``` @@ -370,7 +370,7 @@ example, the following `repos.json` defines the overlay { "repository": { "type": "git" , "branch": "master" - , "commit": "123d8b03bf2440052626151c14c54abce2726e6f" + , "commit": "307c96681e6626286804c45273082dff94127878" , "repository": "https://github.com/just-buildsystem/rules-cc.git" , "subdir": "rules" }