Merge branch 'just-buildsystem:master' into github-ci

rules/CC/EXPRESSIONS

@@ -820,12 +820,14 @@
     , "cflags-files"
     , "lint-deps fieldnames"
     , "deps-transition"
+    , "deps-fieldnames"
     , "compile-args"
     , "defaults-transition"
     ]
   , "imports":
     { "objects": "objects"
     , "list_provider": ["./", "..", "field_list_provider"]
+    , "runfiles_list": ["./", "..", "field_runfiles_list"]
     , "default-TOOLCHAIN": "default-TOOLCHAIN"
     , "default-NON_SYSTEM_TOOLS": "default-NON_SYSTEM_TOOLS"
     , "compiler": "compiler"
@@ -870,6 +872,52 @@
             ]
           }
         ]
+      , [ "direct-deps hdrs"
+        , { "type": "to_subdir"
+          , "subdir": "include"
+          , "$1":
+            { "type": "let*"
+            , "bindings":
+              [["transition", {"type": "var", "name": "deps-transition"}]]
+            , "body":
+              { "type": "map_union"
+              , "$1":
+                { "type": "++"
+                , "$1":
+                  { "type": "foreach"
+                  , "var": "fieldname"
+                  , "range": {"type": "var", "name": "deps-fieldnames"}
+                  , "body":
+                    {"type": "CALL_EXPRESSION", "name": "runfiles_list"}
+                  }
+                }
+              }
+            }
+          }
+        ]
+      , [ "own headers"
+        , { "type": "to_subdir"
+          , "subdir": "work"
+          , "$1":
+            { "type": "map_union"
+            , "$1":
+              [ {"type": "var", "name": "hdrs"}
+              , {"type": "var", "name": "private-hdrs"}
+              ]
+            }
+          }
+        ]
+      , [ "direct hdrs"
+        , { "type": "map_union"
+          , "$1":
+            [ {"type": "var", "name": "own headers"}
+            , {"type": "var", "name": "direct-deps hdrs"}
+            ]
+          }
+        ]
+      , [ "direct deps artifact names"
+        , {"type": "keys", "$1": {"type": "var", "name": "direct hdrs"}}
+        ]
       , [ "hdr lint"
         , { "type": "foreach"
           , "range":
@@ -908,7 +956,10 @@
                       }
                     ]
                   ]
-                , "body": {"type": "env", "vars": ["cmd", "src"]}
+                , "body":
+                  { "type": "env"
+                  , "vars": ["cmd", "src", "direct deps artifact names"]
+                  }
                 }
               }
             }
@@ -980,7 +1031,10 @@
                   { "type": "let*"
                   , "bindings":
                     [["src", {"type": "var", "name": "work src_name"}]]
-                  , "body": {"type": "env", "vars": ["cmd", "src"]}
+                  , "body":
+                    { "type": "env"
+                    , "vars": ["cmd", "src", "direct deps artifact names"]
+                    }
                   }
                 }
               }

rules/lint/RULES

@@ -63,6 +63,10 @@
       , "  by the field \"config\", and"
       , "- OUT pointing to a directory to which files with the lint result"
       , "  can be written."
+      , "- META pointing to a json file contaning"
+      , "  - at key \"direct deps artifact names\" a list of all input"
+      , "    artifacts that come from the target itself or are runfiles of a"
+      , "    direct dependency."
       , "It is supposed to indicate by the exit code whether the file to lint"
       , "complies with the given linting policy, with 0 meaning"
       , "compliant."
@@ -152,6 +156,12 @@
                   , "dep": {"type": "var", "name": "_"}
                   }
                 ]
+              , [ "direct deps artifact names"
+                , { "type": "DEP_PROVIDES"
+                  , "dep": {"type": "var", "name": "_"}
+                  , "provider": "direct deps artifact names"
+                  }
+                ]
               ]
             , "body":
               { "type": "TREE"
@@ -170,6 +180,19 @@
                       , "subdir": "config"
                       , "$1": {"type": "var", "name": "config"}
                       }
+                    , { "type": "singleton_map"
+                      , "key": "meta.json"
+                      , "value":
+                        { "type": "BLOB"
+                        , "data":
+                          { "type": "json_encode"
+                          , "$1":
+                            { "type": "env"
+                            , "vars": ["direct deps artifact names"]
+                            }
+                          }
+                        }
+                      }
                     ]
                   }
                 , "cmd":

rules/lint/call_lint

@@ -23,6 +23,7 @@ echo "${RESULT}" > result
 export OUT="$(pwd)/out"
 mkdir -p config
 export CONFIG="$(pwd)/config"
+export META="$(pwd)/meta.json"
 
 cd work
 

src/buildtool/build_engine/base_maps/TARGETS

@@ -12,6 +12,7 @@
   , "srcs": ["directory_map.cpp"]
   , "deps":
     [ "module_name"
+    , ["@", "gsl", "", "gsl"]
     , ["src/buildtool/common", "config"]
     , ["src/buildtool/file_system", "file_root"]
     , ["src/buildtool/multithreading", "async_map_consumer"]
@@ -91,7 +92,12 @@
     ]
   , "stage": ["src", "buildtool", "build_engine", "base_maps"]
   , "private-deps":
-    [["src/buildtool/crypto", "hash_function"], ["src/utils/cpp", "json"]]
+    [ ["src/buildtool/common", "artifact_description"]
+    , ["src/buildtool/common", "common"]
+    , ["src/buildtool/crypto", "hash_function"]
+    , ["src/buildtool/file_system", "object_type"]
+    , ["src/utils/cpp", "json"]
+    ]
   }
 , "field_reader":
   { "type": ["@", "rules", "CC", "library"]
@@ -113,7 +119,8 @@
   , "name": ["expression_function"]
   , "hdrs": ["expression_function.hpp"]
   , "deps":
-    [ ["@", "gsl", "", "gsl"]
+    [ ["@", "fmt", "", "fmt"]
+    , ["@", "gsl", "", "gsl"]
     , ["src/buildtool/build_engine/expression", "expression"]
     , ["src/buildtool/logging", "log_level"]
     , ["src/buildtool/logging", "logging"]

src/buildtool/build_engine/expression/TARGETS

@@ -43,6 +43,7 @@
     [ "expression_ptr_interface"
     , "linked_map"
     , ["@", "gsl", "", "gsl"]
+    , ["@", "json", "", "json"]
     , ["src/buildtool/build_engine/base_maps", "entity_name_data"]
     , ["src/buildtool/common", "artifact_description"]
     , ["src/buildtool/crypto", "hash_function"]
@@ -55,7 +56,8 @@
     ]
   , "stage": ["src", "buildtool", "build_engine", "expression"]
   , "private-deps":
-    [ ["src/buildtool/crypto", "hasher"]
+    [ ["@", "fmt", "", "fmt"]
+    , ["src/buildtool/crypto", "hasher"]
     , ["src/buildtool/logging", "log_level"]
     , ["src/buildtool/logging", "logging"]
     , ["src/utils/cpp", "path"]

src/buildtool/build_engine/target_map/TARGETS

@@ -17,6 +17,7 @@
   , "hdrs": ["result_map.hpp"]
   , "deps":
     [ ["@", "gsl", "", "gsl"]
+    , ["@", "json", "", "json"]
     , ["src/buildtool/build_engine/analysed_target", "target"]
     , ["src/buildtool/build_engine/base_maps", "entity_name"]
     , ["src/buildtool/build_engine/expression", "expression"]
@@ -59,6 +60,8 @@
     , ["src/buildtool/build_engine/base_maps", "entity_name"]
     , ["src/buildtool/build_engine/base_maps", "field_reader"]
     , ["src/buildtool/build_engine/expression", "expression"]
+    , ["src/buildtool/build_engine/expression", "expression_ptr_interface"]
+    , ["src/buildtool/common", "artifact_description"]
     , ["src/buildtool/common", "artifact_digest_factory"]
     , ["src/buildtool/common", "common"]
     , ["src/buildtool/common", "config"]
@@ -86,6 +89,7 @@
   , "deps":
     [ "configured_target"
     , "target_map"
+    , ["@", "gsl", "", "gsl"]
     , ["src/buildtool/build_engine/analysed_target", "target"]
     , ["src/buildtool/build_engine/base_maps", "entity_name"]
     , ["src/buildtool/build_engine/base_maps", "field_reader"]

src/buildtool/common/TARGETS

@@ -52,7 +52,11 @@
     , ["src/utils/cpp", "expected"]
     , ["src/utils/cpp", "gsl"]
     ]
-  , "private-deps": ["protocol_traits", ["src/buildtool/crypto", "hasher"]]
+  , "private-deps":
+    [ "protocol_traits"
+    , ["@", "gsl", "", "gsl"]
+    , ["src/buildtool/crypto", "hasher"]
+    ]
   , "stage": ["src", "buildtool", "common"]
   }
 , "artifact_digest_factory":
@@ -87,6 +91,7 @@
     ]
   , "deps":
     [ "bazel_types"
+    , "protocol_traits"
     , ["@", "json", "", "json"]
     , ["src/buildtool/crypto", "hash_function"]
     , ["src/buildtool/crypto", "hash_info"]
@@ -109,7 +114,11 @@
     , ["src/buildtool/file_system", "object_type"]
     , ["src/buildtool/logging", "logging"]
     ]
-  , "private-deps": ["artifact_digest_factory", ["src/utils/cpp", "json"]]
+  , "private-deps":
+    [ "artifact_digest_factory"
+    , ["src/buildtool/logging", "log_level"]
+    , ["src/utils/cpp", "json"]
+    ]
   , "stage": ["src", "buildtool", "common"]
   }
 , "action_description":

src/buildtool/common/artifact.hpp

@@ -25,6 +25,7 @@
 #include "nlohmann/json.hpp"
 #include "src/buildtool/common/artifact_digest.hpp"
 #include "src/buildtool/common/identifier.hpp"
+#include "src/buildtool/common/protocol_traits.hpp"
 #include "src/buildtool/crypto/hash_function.hpp"
 #include "src/buildtool/crypto/hash_info.hpp"
 #include "src/buildtool/file_system/object_type.hpp"
@@ -99,8 +100,14 @@ class Artifact {
             }
 
             auto const object_type = FromChar(*type.c_str());
+            // TODO(design): The logic of ArtifactDigestFactory::Create is
+            // duplicated here to avoid a cyclic dependency. A better solution
+            // is advisable.
             auto hash_info =
-                HashInfo::Create(hash_type, id, IsTreeObject(object_type));
+                HashInfo::Create(hash_type,
+                                 id,
+                                 ProtocolTraits::IsTreeAllowed(hash_type) and
+                                     IsTreeObject(object_type));
             if (not hash_info) {
                 Logger::Log(
                     LogLevel::Debug, "{}", std::move(hash_info).error());

src/buildtool/common/artifact_digest.hpp

@@ -19,6 +19,7 @@
 #include <string>
 #include <utility>  // std::move
 
+#include "src/buildtool/crypto/hash_function.hpp"
 #include "src/buildtool/crypto/hash_info.hpp"
 #include "src/utils/cpp/hash_combine.hpp"
 
@@ -50,6 +51,10 @@ class ArtifactDigest final {
         return hash_info_ == other.hash_info_;
     }
 
+    [[nodiscard]] auto GetHashType() const& noexcept -> HashFunction::Type {
+        return hash_info_.HashType();
+    }
+
   private:
     HashInfo hash_info_;
     std::size_t size_ = 0;

src/buildtool/common/remote/TARGETS

@@ -8,6 +8,7 @@
     , ["@", "grpc", "", "grpc++"]
     , ["@", "gsl", "", "gsl"]
     , ["src/buildtool/auth", "auth"]
+    , ["src/buildtool/common", "bazel_types"]
     , ["src/buildtool/common", "common"]
     , ["src/buildtool/logging", "log_level"]
     , ["src/buildtool/logging", "logging"]
@@ -57,9 +58,9 @@
   , "deps":
     [ "retry_config"
     , ["@", "grpc", "", "grpc++"]
+    , ["src/buildtool/logging", "log_level"]
     , ["src/buildtool/logging", "logging"]
     ]
   , "stage": ["src", "buildtool", "common", "remote"]
-  , "private-deps": [["src/buildtool/logging", "log_level"]]
   }
 }

src/buildtool/common/remote/retry.cpp

@@ -15,14 +15,16 @@
 #include "src/buildtool/common/remote/retry.hpp"
 #ifndef BOOTSTRAP_BUILD_TOOL
 
+#include <algorithm>
 #include <chrono>
 #include <thread>
 
 #include "src/buildtool/logging/log_level.hpp"
 
 auto WithRetry(CallableReturningRetryResponse const& f,
                RetryConfig const& retry_config,
-               Logger const& logger) noexcept -> bool {
+               Logger const& logger,
+               LogLevel fatal_log_level) noexcept -> bool {
     try {
         auto const& attempts = retry_config.GetMaxAttempts();
         for (auto attempt = 1U; attempt <= attempts; ++attempt) {
@@ -32,7 +34,7 @@ auto WithRetry(CallableReturningRetryResponse const& f,
             }
             if (fatal) {
                 if (error_msg) {
-                    logger.Emit(LogLevel::Error, "{}", *error_msg);
+                    logger.Emit(fatal_log_level, "{}", *error_msg);
                 }
                 return false;
             }
@@ -51,15 +53,16 @@ auto WithRetry(CallableReturningRetryResponse const& f,
             }
             else {
                 if (error_msg) {
-                    logger.Emit(LogLevel::Error,
+                    logger.Emit(fatal_log_level,
                                 "After {} attempts: {}",
                                 attempt,
                                 *error_msg);
                 }
             }
         }
     } catch (...) {
-        logger.Emit(LogLevel::Error, "WithRetry: caught unknown exception");
+        logger.Emit(std::min(fatal_log_level, LogLevel::Warning),
+                    "WithRetry: caught unknown exception");
     }
     return false;
 }

src/buildtool/common/remote/retry.hpp

@@ -24,6 +24,7 @@
 
 #include "grpcpp/grpcpp.h"
 #include "src/buildtool/common/remote/retry_config.hpp"
+#include "src/buildtool/logging/log_level.hpp"
 #include "src/buildtool/logging/logger.hpp"
 
 // Utility class to help detecting when exit the retry loop. This class can be
@@ -51,9 +52,11 @@ using CallableReturningRetryResponse = std::function<RetryResponse(void)>;
 /// \brief Calls a function with a retry strategy using a backoff algorithm.
 /// Retry loop interrupts when one of the two members of the function's returned
 /// RetryResponse object is set to true.
-[[nodiscard]] auto WithRetry(CallableReturningRetryResponse const& f,
-                             RetryConfig const& retry_config,
-                             Logger const& logger) noexcept -> bool;
+[[nodiscard]] auto WithRetry(
+    CallableReturningRetryResponse const& f,
+    RetryConfig const& retry_config,
+    Logger const& logger,
+    LogLevel fatal_log_level = LogLevel::Error) noexcept -> bool;
 
 using CallableReturningGrpcStatus = std::function<grpc::Status(void)>;
 

src/buildtool/crypto/hash_function.hpp

@@ -95,4 +95,15 @@ class HashFunction {
         -> std::optional<std::pair<Hasher::HashDigest, std::uintmax_t>>;
 };
 
+[[nodiscard]] constexpr auto ToString(HashFunction::Type type) noexcept -> const
+    char* {
+    switch (type) {
+        case HashFunction::Type::GitSHA1:
+            return "git-SHA1";
+        case HashFunction::Type::PlainSHA256:
+            return "plain-SHA256";
+    }
+    Ensures(false);  // unreachable
+}
+
 #endif  // INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_FUNCTION_HPP