From 3165afba0eceb1e038740e5af29d73341930b213 Mon Sep 17 00:00:00 2001 From: Grzegorz Lukasik Date: Wed, 11 Dec 2024 18:01:26 +0100 Subject: [PATCH 01/10] Add bazel info starlark-environments-proto --- .../lib/runtime/commands/InfoCommand.java | 2 + .../build/lib/runtime/commands/info/BUILD | 4 + .../StarlarkEnvironmentsProtoInfoItem.java | 398 ++++++++++++++++++ src/main/protobuf/builtin.proto | 4 + 4 files changed, 408 insertions(+) create mode 100644 src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoCommand.java index 159a85c617494b..69e9d925c97cc3 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoCommand.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoCommand.java @@ -58,6 +58,7 @@ import com.google.devtools.build.lib.runtime.commands.info.ReleaseInfoItem; import com.google.devtools.build.lib.runtime.commands.info.ServerLogInfoItem; import com.google.devtools.build.lib.runtime.commands.info.ServerPidInfoItem; +import com.google.devtools.build.lib.runtime.commands.info.StarlarkEnvironmentsProtoInfoItem; import com.google.devtools.build.lib.runtime.commands.info.StarlarkSemanticsInfoItem; import com.google.devtools.build.lib.runtime.commands.info.UsedHeapSizeAfterGcInfoItem; import com.google.devtools.build.lib.runtime.commands.info.UsedHeapSizeInfoItem; @@ -297,6 +298,7 @@ private static Map getHardwiredInfoItemMap( new BuildLanguageInfoItem(), new DefaultPackagePathInfoItem(commandOptions), new StarlarkSemanticsInfoItem(commandOptions), + new StarlarkEnvironmentsProtoInfoItem(), new WorkerMetricsInfoItem(), new LocalResourcesInfoItem()); ImmutableMap.Builder result = new ImmutableMap.Builder<>(); diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/info/BUILD b/src/main/java/com/google/devtools/build/lib/runtime/commands/info/BUILD index ef89912b8f2a39..4e2aea68771cea 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/commands/info/BUILD +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/info/BUILD @@ -18,8 +18,10 @@ java_library( "//src/main/java/com/google/devtools/build/lib:runtime", "//src/main/java/com/google/devtools/build/lib:runtime/memory_pressure", "//src/main/java/com/google/devtools/build/lib/actions:localhost_capacity", + "//src/main/java/com/google/devtools/build/lib/analysis:analysis_cluster", "//src/main/java/com/google/devtools/build/lib/analysis:blaze_version_info", "//src/main/java/com/google/devtools/build/lib/analysis:config/build_configuration", + "//src/main/java/com/google/devtools/build/lib/bazel/rules", "//src/main/java/com/google/devtools/build/lib/bugreport", "//src/main/java/com/google/devtools/build/lib/buildeventstream/proto:build_event_stream_java_proto", "//src/main/java/com/google/devtools/build/lib/cmdline", @@ -40,8 +42,10 @@ java_library( "//src/main/java/com/google/devtools/build/skyframe", "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", "//src/main/java/com/google/devtools/common/options", + "//src/main/java/net/starlark/java/annot", "//src/main/java/net/starlark/java/eval", "//src/main/protobuf:build_java_proto", + "//src/main/protobuf:builtin_java_proto", "//src/main/protobuf:failure_details_java_proto", "//third_party:flogger", "//third_party:guava", diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java new file mode 100644 index 00000000000000..f6a9c80da9f88d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java @@ -0,0 +1,398 @@ +// Copyright 2024 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime.commands.info; + +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.docgen.builtin.BuiltinProtos.Builtins; +import com.google.devtools.build.docgen.builtin.BuiltinProtos; +import com.google.devtools.build.docgen.builtin.BuiltinProtos.Value; +import com.google.devtools.build.docgen.builtin.BuiltinProtos.ApiContext; +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; +import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; +import com.google.devtools.build.lib.bazel.rules.BazelRuleClassProvider; +import com.google.devtools.build.lib.packages.*; +import com.google.devtools.build.lib.runtime.CommandEnvironment; +import com.google.devtools.build.lib.runtime.InfoItem; +import com.google.devtools.build.lib.server.FailureDetails; +import com.google.devtools.build.lib.skyframe.StarlarkBuiltinsValue; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.DetailedExitCode; +import com.google.devtools.build.skyframe.EvaluationResult; +import com.google.devtools.build.skyframe.SkyValue; +import net.starlark.java.annot.StarlarkAnnotations; +import net.starlark.java.annot.StarlarkBuiltin; +import net.starlark.java.annot.StarlarkMethod; +import net.starlark.java.eval.*; + +import java.lang.reflect.Method; +import java.util.*; +import java.util.function.Function; + +import static com.google.common.base.Preconditions.checkNotNull; + +/** + * TODO: + * - Types are not set for now + * - StarlarkFunction, like cc_proto_library (which is autoloaded), are missing doc and params or have strange values. + * Examples: java_binary, java_import, java_library, java_proto_library, proto_library, java_test, py_binary + * Documentation generated for py_binary, seems to be generated based on: https://github.com/bazelbuild/rules_python/blob/026b300d918ced0f4e9f99a22ab8407656ed20ac/python/py_binary.bzl#L28 + * doc: "Creates an executable Python program.\n\nThis is the public macro wrapping the underlying rule. Args are forwarded\non as-is unless otherwise specified. See the underlying {rule}`py_binary`\nrule for detailed attribute documentation.\n\nThis macro affects the following args:\n* `python_version`: cannot be `PY2`\n* `srcs_version`: cannot be `PY2` or `PY2ONLY`\n* `tags`: May have special marker values added, if not already present.\n\nArgs:\n **attrs: Rule attributes forwarded onto the underlying {rule}`py_binary`." + * - There is 7 symbols that report as members of com.google.devtools.build.lib.packages.AutoloadSymbols$1, possibly + * related to the issue that bazel 8.0.0 displays message: + * WARNING: Couldn't auto load rules or symbols, because no dependency on module/repository 'rules_android' found. This will result in a failure if there's a reference to those rules or symbols. + * AndroidIdeInfo, aar_import, android_* + * - Some common rule attributes are missing documentation, documentation seems to be in HTML pages for those: + * https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/docgen/PredefinedAttributes.java + * E.g. visibility, compatible_with, flaky, env, etc. + * + */ +public final class StarlarkEnvironmentsProtoInfoItem extends InfoItem { + + public StarlarkEnvironmentsProtoInfoItem() { + super("starlark-environments-proto", "TODO", true); + } + + @Override + public boolean needsSyncPackageLoading() { + // Requires CommandEnvironment.syncPackageLoading to be called in order to initialize the + // skyframe executor. + return true; + } + + @Override + public byte[] get(Supplier configurationSupplier, CommandEnvironment env) + throws AbruptExitException { + checkNotNull(env); + StarlarkBuiltinsValue builtins = loadStarlarkBuiltins(env); + return print(build(builtins)); + } + + private StarlarkBuiltinsValue loadStarlarkBuiltins(CommandEnvironment env) + throws AbruptExitException { + EvaluationResult result = + env.getSkyframeExecutor() + .evaluateSkyKeys( + env.getReporter(), + ImmutableList.of(StarlarkBuiltinsValue.key(true)), + /* keepGoing= */ false); + if (result.hasError()) { + throw new AbruptExitException( + DetailedExitCode.of( + FailureDetails.FailureDetail.newBuilder() + .setMessage("Failed to load Starlark builtins") + .setInfoCommand(FailureDetails.InfoCommand.getDefaultInstance()) + .build())); + } + return (StarlarkBuiltinsValue) result.get(StarlarkBuiltinsValue.key(true)); + } + + private byte[] build(StarlarkBuiltinsValue builtins) { + Builtins.Builder builder = Builtins.newBuilder(); + + ConfiguredRuleClassProvider provider = BazelRuleClassProvider.create(); + BazelStarlarkEnvironment env = provider.getBazelStarlarkEnvironment(); + + buildFor(builder, ApiContext.ALL, Starlark.UNIVERSE); + + // TODO: Probably there should be BUILD_BZL and WORKSPACE_BZL + ImmutableMap.Builder bzlBuilder = ImmutableMap.builder(); + bzlBuilder.putAll(builtins.predeclaredForWorkspaceBzl); + bzlBuilder.putAll(builtins.predeclaredForBuildBzl); + buildFor(builder, ApiContext.BZL, bzlBuilder.buildKeepingLast()); + + buildFor(builder, ApiContext.BUILD, builtins.predeclaredForBuild); + buildFor(builder, ApiContext.MODULE, env.getModuleBazelEnv()); + buildFor(builder, ApiContext.REPO, env.getRepoBazelEnv()); + // TODO: Add ApiContext.VENDOR + // TODO: Add ApiContext.WORKSPACE + + return builder.build().toByteArray(); + } + + private void buildFor(Builtins.Builder builtins, ApiContext env, Map symbols) { + List> entries = new ArrayList<>(symbols.entrySet()); + entries.sort(Map.Entry.comparingByKey()); + + for (var entry : entries) { + String name = entry.getKey(); + Object obj = entry.getValue(); + + if (obj instanceof GuardedValue guardedValue) { + obj = guardedValue.getObject(); + } + + Value.Builder value = Value.newBuilder(); + value.setName(name); + value.setApiContext(env); + + // TODO: Figure out if number of cases can be reduced. + if (obj instanceof BuiltinFunction builtinFunction) { + // Samples: depset, int, package, rule, str + fillForStarlarkMethod(value, builtinFunction.getAnnotation()); + } else if (obj instanceof RuleFunction fn) { + // Samples: alias, cc_binary, filegroup, toolchain, xcode_config + fillForRuleFunction(value, fn); + } else if (obj instanceof StarlarkFunction fn) { + // Samples: cc_proto_binary, java_binary, py_binary + fillForStarlarkFunction(value, fn); + } else if (obj instanceof StarlarkProvider p) { + // Samples: CcSharedLibraryInfo, JavaInfo, ProtoInfo, PyInfo + fillForStarlarkProvider(value, p); + } else if (obj instanceof StarlarkInfoNoSchema info) { + // Samples: cc_common, java_common, native + // TODO: Export as Type. + value.setType("CLASS: " + obj.getClass().getName()); + } else if (obj instanceof BuiltinProvider p) { + // Samples: CcInfo, DefaultInfo, InstrumentedFilesInfo, struct + fillForBuiltinProvider(value, p); + } else if (obj instanceof StarlarkAspect aspect) { + // Sample: cc_proto_aspect + fillForStarlarkAspect(value, aspect); + } else { + StarlarkBuiltin starlarkBuiltin = StarlarkAnnotations.getStarlarkBuiltin(obj.getClass()); + if (starlarkBuiltin != null) { + // Sample: attr, config, json, proto, + // TODO: Export as Type, use Starlark.getMethodAnnotations to collect all available methods. + value.setDoc(starlarkBuiltin.doc()); + } + // TODO: Handle more gracefully. + value.setType("CLASS: " + obj.getClass().getName()); + } + + builtins.addGlobal(value.build()); + } + } + + private static void fillForStarlarkFunction(Value.Builder value, StarlarkFunction fn) { + Signature sig = new Signature(); + sig.doc = fn.getDocumentation(); + sig.parameterNames = fn.getParameterNames(); + sig.hasVarargs = fn.hasVarargs(); + sig.hasKwargs = fn.hasKwargs(); + sig.getDefaultValue = + (i) -> { + Object v = fn.getDefaultValue(i); + return v == null ? null : Starlark.repr(v); + }; + signatureToValue(value, sig); + } + + private static void fillForStarlarkAspect(Value.Builder value, StarlarkAspect aspect) { + Signature sig = new Signature(); + + sig.parameterNames = new ArrayList<>(); + sig.paramDocs = new ArrayList<>(); + for (String fieldName : aspect.getParamAttributes()) { + sig.parameterNames.add(fieldName); + sig.paramDocs.add(""); + } + signatureToValue(value, sig); + } + + private static void fillForBuiltinProvider(Value.Builder value, BuiltinProvider p) { + Signature sig = new Signature(); + sig.parameterNames = new ArrayList<>(); + sig.paramDocs = new ArrayList<>(); + + Class valueClass = p.getValueClass(); + + Method selfCall = Starlark.getSelfCallMethod(StarlarkSemantics.DEFAULT, p.getClass()); + if (selfCall != null) { + StarlarkMethod m = StarlarkAnnotations.getStarlarkMethod(selfCall); + if (m != null) { + sig = getSignature(m); + } else { + // TODO: Handle. + value.setType("CLASS: " + valueClass); + return; + } + } else { + // TODO: Handle. + value.setType("CLASS: " + valueClass); + return; + } + + // Override doc with documentation of type itself, otherwise it will have simple + // "Constructor for type" documentation. + StarlarkBuiltin starlarkBuiltin = StarlarkAnnotations.getStarlarkBuiltin(valueClass); + sig.doc = starlarkBuiltin.doc(); + + signatureToValue(value, sig); + } + + // TODO: Order methods. + private static void fillForStarlarkProvider(Value.Builder value, StarlarkProvider p) { + Signature sig = new Signature(); + sig.doc = p.getDocumentation().orElseGet(() -> "NO DOC"); + + sig.parameterNames = new ArrayList<>(); + sig.paramDocs = new ArrayList<>(); + if (p.getFields() != null) { + Map> schema = p.getSchema(); + for (String fieldName : p.getFields()) { + sig.parameterNames.add(fieldName); + + Optional fieldDoc = schema.get(fieldName); + sig.paramDocs.add(fieldDoc.orElse(null)); + } + } + signatureToValue(value, sig); + } + + private static void fillForRuleFunction(Value.Builder value, RuleFunction fn) { + RuleClass clz = fn.getRuleClass(); + Signature sig = new Signature(); + // TODO: Documentation for some native rules is in comments, like here: + // https://github.com/bazelbuild/bazel/blob/b8073bbcaa63c9405824f94184014b19a2255a52/src/main/java/com/google/devtools/build/lib/rules/Alias.java#L110 + // and its parsed out by BuildDocCollector from source files... + sig.doc = clz.getStarlarkDocumentation(); + sig.parameterNames = new ArrayList<>(); + sig.paramDocs = new ArrayList<>(); + List defaultValues = new ArrayList<>(); + for (Attribute attr : clz.getAttributes()) { + sig.parameterNames.add(attr.getName()); + // TODO: Common attributes do not have documentation. + sig.paramDocs.add(attr.getDoc()); + if (!attr.isMandatory()) { + // TODO: Support default values. + defaultValues.add(attr.getType().getDefaultValue()); + } else { + defaultValues.add(null); + } + } + sig.getDefaultValue = i -> defaultValues.get(i) != null ? String.valueOf(defaultValues.get(i)) : null; + signatureToValue(value, sig); + } + + // ------------------------------------------------ + // ----- TODO: Code chopped from ApiExporter + private static void fillForStarlarkMethod(Value.Builder value, StarlarkMethod annot) { + signatureToValue(value, getSignature(annot)); + } + + private static Value.Builder signatureToValue(Value.Builder value, Signature sig) { + // TODO: E.g. cc_proto_library that is autoloaded does not have doc. + if (sig.doc != null) { + value.setDoc(sig.doc); + } + + int nparams = sig.parameterNames.size(); + int kwargsIndex = sig.hasKwargs ? --nparams : -1; + int varargsIndex = sig.hasVarargs ? --nparams : -1; + // Inv: nparams is number of regular parameters. + + BuiltinProtos.Callable.Builder callable = BuiltinProtos.Callable.newBuilder(); + for (int i = 0; i < sig.parameterNames.size(); i++) { + String name = sig.parameterNames.get(i); + BuiltinProtos.Param.Builder param = BuiltinProtos.Param.newBuilder(); + if (i == varargsIndex) { + // *args + param.setName("*" + name); // * seems redundant + param.setIsStarArg(true); + } else if (i == kwargsIndex) { + // **kwargs + param.setName("**" + name); // ** seems redundant + param.setIsStarStarArg(true); + } else { + // regular parameter + param.setName(name); + String v = sig.getDefaultValue.apply(i); + if (v != null) { + param.setDefaultValue(v); + } else { + param.setIsMandatory(true); // bool seems redundant + } + } + // TODO: Validate when it can be null. + if (sig.paramDocs != null && sig.paramDocs.get(i) != null) { + param.setDoc(sig.paramDocs.get(i)); + } + callable.addParam(param); + } + value.setCallable(callable); + return value; + } + + + // Extracts signature and parameter default value expressions from a StarlarkMethod annotation. + private static Signature getSignature(StarlarkMethod annot) { + // Build-time annotation processing ensures mandatory parameters do not follow optional ones. + boolean hasStar = false; + String star = null; + String starStar = null; + ArrayList params = new ArrayList<>(); + ArrayList defaults = new ArrayList<>(); + ArrayList docs = new ArrayList<>(); + + for (net.starlark.java.annot.Param param : annot.parameters()) { + // Ignore undocumented parameters + if (!param.documented()) { + docs.add(""); + continue; + } + // Implicit * or *args parameter separates transition from positional to named. + // f (..., *, ... ) or f(..., *args, ...) + // TODO(adonovan): this logic looks fishy. Clean it up. + if (param.named() && !param.positional() && !hasStar) { + hasStar = true; + if (!annot.extraPositionals().name().isEmpty()) { + star = annot.extraPositionals().name(); + } + } + params.add(param.name()); + defaults.add(param.defaultValue().isEmpty() ? null : param.defaultValue()); + docs.add(param.doc()); + } + + // f(..., *args, ...) + if (!annot.extraPositionals().name().isEmpty() && !hasStar) { + star = annot.extraPositionals().name(); + } + if (star != null) { + params.add(star); + docs.add(annot.extraPositionals().doc()); + } + + // f(..., **kwargs) + if (!annot.extraKeywords().name().isEmpty()) { + starStar = annot.extraKeywords().name(); + params.add(starStar); + docs.add(annot.extraKeywords().doc()); + } + + Signature sig = new Signature(); + sig.doc = annot.doc(); + sig.parameterNames = params; + sig.hasVarargs = star != null; + sig.hasKwargs = starStar != null; + sig.getDefaultValue = defaults::get; + sig.paramDocs = docs; + return sig; + } + + private static class Signature { + List parameterNames; + List paramDocs; + boolean hasVarargs; + boolean hasKwargs; + String doc; + + // Returns the string form of the ith default value, using the + // index, ordering, and null Conventions of StarlarkFunction.getDefaultValue. + Function getDefaultValue = (i) -> null; + } +} diff --git a/src/main/protobuf/builtin.proto b/src/main/protobuf/builtin.proto index 9e9fe1977aec55..c7754b353207c5 100644 --- a/src/main/protobuf/builtin.proto +++ b/src/main/protobuf/builtin.proto @@ -57,6 +57,10 @@ enum ApiContext { ALL = 0; BZL = 1; BUILD = 2; + MODULE = 3; + REPO = 4; + VENDOR = 5; + WORKSPACE = 6; } // Generic representation for a Starlark object. If the object is callable From d12de98a023cd640c5c663fcb8a52d787202e2d2 Mon Sep 17 00:00:00 2001 From: Grzegorz Lukasik Date: Thu, 2 Jan 2025 22:15:21 +0100 Subject: [PATCH 02/10] Support ApiContext.VENDOR --- .../commands/info/StarlarkEnvironmentsProtoInfoItem.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java index f6a9c80da9f88d..184d150ce6949c 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java @@ -117,7 +117,7 @@ private byte[] build(StarlarkBuiltinsValue builtins) { buildFor(builder, ApiContext.BUILD, builtins.predeclaredForBuild); buildFor(builder, ApiContext.MODULE, env.getModuleBazelEnv()); buildFor(builder, ApiContext.REPO, env.getRepoBazelEnv()); - // TODO: Add ApiContext.VENDOR + buildFor(builder, ApiContext.VENDOR, env.getStarlarkGlobals().getVendorToplevels()); // TODO: Add ApiContext.WORKSPACE return builder.build().toByteArray(); From 54b153fb604303823bd72ab22d52c94e53fb6b7a Mon Sep 17 00:00:00 2001 From: Grzegorz Lukasik Date: Fri, 3 Jan 2025 17:54:27 +0100 Subject: [PATCH 03/10] Support ApiContext.WORKSPACE --- .../info/StarlarkEnvironmentsProtoInfoItem.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java index 184d150ce6949c..30a57a14a55214 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java @@ -118,7 +118,22 @@ private byte[] build(StarlarkBuiltinsValue builtins) { buildFor(builder, ApiContext.MODULE, env.getModuleBazelEnv()); buildFor(builder, ApiContext.REPO, env.getRepoBazelEnv()); buildFor(builder, ApiContext.VENDOR, env.getStarlarkGlobals().getVendorToplevels()); - // TODO: Add ApiContext.WORKSPACE + + // Construct WORKSPACE symbols manually, similarly as done in WorkspaceFactory. + // As commented in WorkspaceFactory.getDefaultEnvironment, the WORKSPACE file is going away so method to get + // its symbols have not been added to the BazelStarlarkEnvironment. + ImmutableMap.Builder workspaceEnv = ImmutableMap.builder(); + for (Map.Entry entry : provider.getRuleClassMap().entrySet()) { + // Add workspace-only symbols, otherwise a lot of unsupported symbols would be added to the WORKSPACE environment. + if (entry.getValue().getWorkspaceOnly()) { + workspaceEnv.put(entry.getKey(), entry.getValue()); + } + } + WorkspaceGlobals workspaceGlobals = new WorkspaceGlobals( + /* allowWorkspaceFunction */ true, + provider.getRuleClassMap()); + Starlark.addMethods(workspaceEnv, workspaceGlobals, StarlarkSemantics.DEFAULT); + buildFor(builder, ApiContext.WORKSPACE, workspaceEnv.buildKeepingLast()); return builder.build().toByteArray(); } From 8c76621d47459f37abf2062a265abc4c98bf867f Mon Sep 17 00:00:00 2001 From: Grzegorz Lukasik Date: Sun, 5 Jan 2025 12:58:06 +0100 Subject: [PATCH 04/10] Ignore undocumented attributes (generator_*, $*, :*) --- .../commands/info/StarlarkEnvironmentsProtoInfoItem.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java index 30a57a14a55214..70595ca6f12995 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java @@ -279,6 +279,13 @@ private static void fillForRuleFunction(Value.Builder value, RuleFunction fn) { sig.paramDocs = new ArrayList<>(); List defaultValues = new ArrayList<>(); for (Attribute attr : clz.getAttributes()) { + // Remove all undocumented attributes, including e.g. generator_{name,function,location}, or attributes + // with names beggining with $ and :. + // TODO: Provide better way of marking attributes that are not writable in Stalark, undocumented does not imply + // that attribute cannot be used in Stalark files. + if (!attr.isDocumented()) { + continue; + } sig.parameterNames.add(attr.getName()); // TODO: Common attributes do not have documentation. sig.paramDocs.add(attr.getDoc()); From 59f0e3026587eb56db2aebe42fec56bec1123cff Mon Sep 17 00:00:00 2001 From: Grzegorz Lukasik Date: Sun, 5 Jan 2025 13:03:34 +0100 Subject: [PATCH 05/10] Ignore names starting with _ --- .../commands/info/StarlarkEnvironmentsProtoInfoItem.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java index 70595ca6f12995..b9b478766dc540 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java @@ -146,6 +146,10 @@ private void buildFor(Builtins.Builder builtins, ApiContext env, Map Date: Sun, 5 Jan 2025 17:41:54 +0100 Subject: [PATCH 06/10] Support True, False, None --- .../commands/info/StarlarkEnvironmentsProtoInfoItem.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java index b9b478766dc540..1fd397973026d0 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java @@ -159,7 +159,11 @@ private void buildFor(Builtins.Builder builtins, ApiContext env, Map Date: Sun, 5 Jan 2025 20:26:26 +0100 Subject: [PATCH 07/10] Order attrs --- .../StarlarkEnvironmentsProtoInfoItem.java | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java index 1fd397973026d0..d49e056fd056d8 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java @@ -61,6 +61,21 @@ * */ public final class StarlarkEnvironmentsProtoInfoItem extends InfoItem { + // TODO: Taken from DocgenConsts. + public static final ImmutableMap ATTRIBUTE_ORDERING = + ImmutableMap.builder() + .put("name", -99) + .put("deps", -98) + .put("src", -97) + .put("srcs", -96) + .put("data", -95) + .put("resource", -94) + .put("resources", -93) + .put("out", -92) + .put("outs", -91) + .put("hdrs", -90) + .buildOrThrow(); + public StarlarkEnvironmentsProtoInfoItem() { super("starlark-environments-proto", "TODO", true); @@ -286,7 +301,21 @@ private static void fillForRuleFunction(Value.Builder value, RuleFunction fn) { sig.parameterNames = new ArrayList<>(); sig.paramDocs = new ArrayList<>(); List defaultValues = new ArrayList<>(); - for (Attribute attr : clz.getAttributes()) { + + List sortedAttrs = new ArrayList<>(clz.getAttributes()); + sortedAttrs.sort( + (o1, o2) -> + { + // Logic taken from RuleDocumentationAttribute + int p1 = ATTRIBUTE_ORDERING.getOrDefault(o1.getName(), 0); + int p2 = ATTRIBUTE_ORDERING.getOrDefault(o2.getName(), 0); + if (p1 != p2) { + return p1 - p2; + } + return o1.getName().compareTo(o2.getName()); + }); + + for (Attribute attr : sortedAttrs) { // Remove all undocumented attributes, including e.g. generator_{name,function,location}, or attributes // with names beggining with $ and :. // TODO: Provide better way of marking attributes that are not writable in Stalark, undocumented does not imply From 84720d39c4562f5e766185b75186349be74ea733 Mon Sep 17 00:00:00 2001 From: Grzegorz Lukasik Date: Sun, 5 Jan 2025 22:49:44 +0100 Subject: [PATCH 08/10] Supporting modules: attr, testing --- .../StarlarkEnvironmentsProtoInfoItem.java | 43 +++++++++++-------- src/main/protobuf/builtin.proto | 6 ++- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java index d49e056fd056d8..56b127ce10646b 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java @@ -204,11 +204,11 @@ private void buildFor(Builtins.Builder builtins, ApiContext env, Map(); - sig.paramDocs = new ArrayList<>(); for (String fieldName : aspect.getParamAttributes()) { sig.parameterNames.add(fieldName); sig.paramDocs.add(""); @@ -243,8 +241,6 @@ private static void fillForStarlarkAspect(Value.Builder value, StarlarkAspect as private static void fillForBuiltinProvider(Value.Builder value, BuiltinProvider p) { Signature sig = new Signature(); - sig.parameterNames = new ArrayList<>(); - sig.paramDocs = new ArrayList<>(); Class valueClass = p.getValueClass(); @@ -272,13 +268,10 @@ private static void fillForBuiltinProvider(Value.Builder value, BuiltinProvider signatureToValue(value, sig); } - // TODO: Order methods. private static void fillForStarlarkProvider(Value.Builder value, StarlarkProvider p) { Signature sig = new Signature(); sig.doc = p.getDocumentation().orElseGet(() -> "NO DOC"); - sig.parameterNames = new ArrayList<>(); - sig.paramDocs = new ArrayList<>(); if (p.getFields() != null) { Map> schema = p.getSchema(); for (String fieldName : p.getFields()) { @@ -298,8 +291,6 @@ private static void fillForRuleFunction(Value.Builder value, RuleFunction fn) { // https://github.com/bazelbuild/bazel/blob/b8073bbcaa63c9405824f94184014b19a2255a52/src/main/java/com/google/devtools/build/lib/rules/Alias.java#L110 // and its parsed out by BuildDocCollector from source files... sig.doc = clz.getStarlarkDocumentation(); - sig.parameterNames = new ArrayList<>(); - sig.paramDocs = new ArrayList<>(); List defaultValues = new ArrayList<>(); List sortedAttrs = new ArrayList<>(clz.getAttributes()); @@ -337,6 +328,22 @@ private static void fillForRuleFunction(Value.Builder value, RuleFunction fn) { signatureToValue(value, sig); } + private static void fillForStarlarkBuiltin( + ApiContext env, Value.Builder value, Class clz, StarlarkBuiltin starlarkBuiltin) { + value.setDoc(starlarkBuiltin.doc()); + + for (var entry: Starlark.getMethodAnnotations(clz).entrySet()) { + Method method = entry.getKey(); + StarlarkMethod starlarkMethod = entry.getValue(); + + Value.Builder moduleGlobal = Value.newBuilder(); + moduleGlobal.setName(method.getName()); + moduleGlobal.setApiContext(env); + fillForStarlarkMethod(moduleGlobal, starlarkMethod); + value.addGlobal(moduleGlobal.build()); + } + } + // ------------------------------------------------ // ----- TODO: Code chopped from ApiExporter private static void fillForStarlarkMethod(Value.Builder value, StarlarkMethod annot) { @@ -357,6 +364,9 @@ private static Value.Builder signatureToValue(Value.Builder value, Signature sig BuiltinProtos.Callable.Builder callable = BuiltinProtos.Callable.newBuilder(); for (int i = 0; i < sig.parameterNames.size(); i++) { String name = sig.parameterNames.get(i); + if (name.startsWith("$") || name.startsWith("_") || name.startsWith(":")) { + continue; + } BuiltinProtos.Param.Builder param = BuiltinProtos.Param.newBuilder(); if (i == varargsIndex) { // *args @@ -376,8 +386,7 @@ private static Value.Builder signatureToValue(Value.Builder value, Signature sig param.setIsMandatory(true); // bool seems redundant } } - // TODO: Validate when it can be null. - if (sig.paramDocs != null && sig.paramDocs.get(i) != null) { + if (i < sig.paramDocs.size() && sig.paramDocs.get(i) != null) { param.setDoc(sig.paramDocs.get(i)); } callable.addParam(param); @@ -444,8 +453,8 @@ private static Signature getSignature(StarlarkMethod annot) { } private static class Signature { - List parameterNames; - List paramDocs; + List parameterNames = new ArrayList<>(); + List paramDocs = new ArrayList<>(); boolean hasVarargs; boolean hasKwargs; String doc; diff --git a/src/main/protobuf/builtin.proto b/src/main/protobuf/builtin.proto index c7754b353207c5..3f67df6879c54e 100644 --- a/src/main/protobuf/builtin.proto +++ b/src/main/protobuf/builtin.proto @@ -64,7 +64,8 @@ enum ApiContext { } // Generic representation for a Starlark object. If the object is callable -// (can act as a function), then callable will be set. +// (can act as a function), then callable will be set. If he object is a module +// itself, it will have repeated global set for all module's global variables. message Value { string name = 1; @@ -74,6 +75,9 @@ message Value { // Set when the object is a function. Callable callable = 3; + // Set when the object is a module. + repeated Value global = 6; + // Value documentation. string doc = 4; From 68e0ba55e92b61d4c2828c76dd18cba29c6ac856 Mon Sep 17 00:00:00 2001 From: Grzegorz Lukasik Date: Sun, 5 Jan 2025 23:13:05 +0100 Subject: [PATCH 09/10] Support structs - native, cc_common, java_common --- .../info/StarlarkEnvironmentsProtoInfoItem.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java index 56b127ce10646b..60a62d07d68dfe 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java @@ -190,10 +190,9 @@ private void buildFor(Builtins.Builder builtins, ApiContext env, Map Date: Mon, 6 Jan 2025 23:00:09 +0100 Subject: [PATCH 10/10] Support properly BuiltinProvider --- .../StarlarkEnvironmentsProtoInfoItem.java | 50 +++++++++++-------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java index 60a62d07d68dfe..740bc972533d81 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/info/StarlarkEnvironmentsProtoInfoItem.java @@ -14,6 +14,7 @@ package com.google.devtools.build.lib.runtime.commands.info; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.docgen.builtin.BuiltinProtos.Builtins; import com.google.devtools.build.docgen.builtin.BuiltinProtos; @@ -238,32 +239,37 @@ private static void fillForStarlarkAspect(Value.Builder value, StarlarkAspect as signatureToValue(value, sig); } - private static void fillForBuiltinProvider(Value.Builder value, BuiltinProvider p) { - Signature sig; - - Class valueClass = p.getValueClass(); + private static void fillForBuiltinProvider(Value.Builder value, BuiltinProvider p) { + // Get documentation from the provider symbol itself. If it's not documented, or the documentation is inherited from + // some builtin like Provider itself, then get documentation from the class holding actual value + // ("provider instance"). + StarlarkBuiltin starlarkBuiltin = StarlarkAnnotations.getStarlarkBuiltin(p.getClass()); + String doc = ""; + if ("PROVIDER".equals(starlarkBuiltin.category()) && !starlarkBuiltin.doc().isEmpty()) { + doc = starlarkBuiltin.doc(); + } else { + Class valueClass = p.getValueClass(); + StarlarkBuiltin valueClassBuiltin = StarlarkAnnotations.getStarlarkBuiltin(valueClass); + doc = valueClassBuiltin.doc(); + } Method selfCall = Starlark.getSelfCallMethod(StarlarkSemantics.DEFAULT, p.getClass()); - if (selfCall != null) { - StarlarkMethod m = StarlarkAnnotations.getStarlarkMethod(selfCall); - if (m != null) { - sig = getSignature(m); - } else { - // TODO: Handle. - value.setType("CLASS: " + valueClass); - return; - } - } else { - // TODO: Handle. - value.setType("CLASS: " + valueClass); + + if (selfCall == null) { + // Provider cannot be constructed in Starlark code. + // This is true for example for: Actions (deprecated), AnalysisFailureInfo, CcToolchainConfigInfo, + // InstrumentedFilesInfo, PackageSpecificationInfo. + // TODO: Error displayed by bazel when all above but Actions providers are called is: + // Error: 'Provider' object is not callable + // This is somewhat confusing as other Providers are callable. + value.setDoc(doc); return; } - // Override doc with documentation of type itself, otherwise it will have simple - // "Constructor for type" documentation. - StarlarkBuiltin starlarkBuiltin = StarlarkAnnotations.getStarlarkBuiltin(valueClass); - sig.doc = starlarkBuiltin.doc(); - + StarlarkMethod m = StarlarkAnnotations.getStarlarkMethod(selfCall); + Preconditions.checkNotNull(m); + Signature sig = getSignature(m); + sig.doc = doc; signatureToValue(value, sig); } @@ -283,7 +289,7 @@ private static void fillForStarlarkProvider(Value.Builder value, StarlarkProvide signatureToValue(value, sig); } - private void fillForStructure(Value.Builder value, Structure struct) { + private static void fillForStructure(Value.Builder value, Structure struct) { // TODO: Missing documentation for struct itself and for fields. for (String field : struct.getFieldNames()) { value.addGlobal(Value.newBuilder().setName(field));