From a20c38808cb7f1ac721bb5131c81e64f654e7536 Mon Sep 17 00:00:00 2001 From: Chirag Ramani Date: Tue, 19 Mar 2024 13:30:06 -0700 Subject: [PATCH] Feature to opt out of tracking .swiftdoc and .swiftsourceinfo Files (#1179) This PR introduces configuration options aimed at optimizing CI build processes by excluding non-essential Swift documentation and source info files, files that don't serve as inputs to other build targets. By introducing flags to control the exclusion of these files, we aim to reduce unnecessary network I/O and improve build performance in specific CI environments. (non-dev facing only) Features Introduced: - `swift.emit_swiftdoc` - This feature is enabled by default. - Opting out of this feature will prevent .swiftdoc files from being tracked by `SwiftInfo` provider, though they will still be generated by the Swift compiler. - These documentation files are generally used for IDE features and are not required in certain CI contexts, such as non-developer-facing validation suites. - `swift.emit_swiftsourceinfo` - This feature is enabled by default. - When this feature is disabled, .swiftsourceinfo files, which support source-level debugging, will also be excluded from build outputs and will not be tracked by `SwiftInfo` provider. It is important to note that, by default, the Swift compiler generates .swiftdoc and .swiftsourceinfo files as part of the output when using the `-emit-module` flag. As of now, this behavior is consistent, given the absence of compiler options in Swift that permit controlling the generation of these files. --- swift/internal/compiling.bzl | 35 ++++-- swift/internal/feature_names.bzl | 10 ++ swift/internal/swift_autoconfiguration.bzl | 4 + swift/internal/swift_toolchain.bzl | 4 + swift/internal/xcode_swift_toolchain.bzl | 4 + test/rules/provider_test.bzl | 17 +-- test/split_derived_files_tests.bzl | 120 ++++++++++++++++++++- 7 files changed, 180 insertions(+), 14 deletions(-) diff --git a/swift/internal/compiling.bzl b/swift/internal/compiling.bzl index 78646768f..1f285a40a 100644 --- a/swift/internal/compiling.bzl +++ b/swift/internal/compiling.bzl @@ -48,7 +48,9 @@ load( "SWIFT_FEATURE_EMIT_BC", "SWIFT_FEATURE_EMIT_C_MODULE", "SWIFT_FEATURE_EMIT_PRIVATE_SWIFTINTERFACE", + "SWIFT_FEATURE_EMIT_SWIFTDOC", "SWIFT_FEATURE_EMIT_SWIFTINTERFACE", + "SWIFT_FEATURE_EMIT_SWIFTSOURCEINFO", "SWIFT_FEATURE_EMIT_SYMBOL_GRAPH", "SWIFT_FEATURE_ENABLE_BATCH_MODE", "SWIFT_FEATURE_ENABLE_LIBRARY_EVOLUTION", @@ -2372,12 +2374,24 @@ def compile( swift_toolchain.generated_header_module_implicit_deps_providers.swift_infos ) + # Determine if `.swiftdoc` and `.swiftsourceinfo` files should be included. + include_swiftdoc = is_feature_enabled( + feature_configuration = feature_configuration, + feature_name = SWIFT_FEATURE_EMIT_SWIFTDOC, + ) + include_swiftsourceinfo = is_feature_enabled( + feature_configuration = feature_configuration, + feature_name = SWIFT_FEATURE_EMIT_SWIFTSOURCEINFO, + ) + compile_outputs, other_outputs = _declare_compile_outputs( srcs = srcs, actions = actions, feature_configuration = feature_configuration, generated_header_name = generated_header_name, generated_module_deps_swift_infos = generated_module_deps_swift_infos, + include_swiftdoc = include_swiftdoc, + include_swiftsourceinfo = include_swiftsourceinfo, module_name = module_name, target_name = target_name, user_compile_flags = copts, @@ -2401,11 +2415,13 @@ def compile( # various things (such as the filename prefix for param files generated # for that action). This guarantees some predictability. compile_outputs.swiftmodule_file, - compile_outputs.swiftdoc_file, - compile_outputs.swiftsourceinfo_file, compile_outputs.generated_header_file, compile_outputs.symbol_graph_directory, ]) + other_outputs + if include_swiftdoc: + all_derived_outputs.append(compile_outputs.swiftdoc_file) + if include_swiftsourceinfo: + all_derived_outputs.append(compile_outputs.swiftsourceinfo_file) else: all_compile_outputs = compact([ # The `.swiftmodule` file is explicitly listed as the first output @@ -2413,15 +2429,17 @@ def compile( # various things (such as the filename prefix for param files generated # for that action). This guarantees some predictability. compile_outputs.swiftmodule_file, - compile_outputs.swiftdoc_file, compile_outputs.swiftinterface_file, compile_outputs.private_swiftinterface_file, - compile_outputs.swiftsourceinfo_file, compile_outputs.generated_header_file, compile_outputs.indexstore_directory, compile_outputs.macro_expansion_directory, compile_outputs.symbol_graph_directory, ]) + compile_outputs.object_files + other_outputs + if include_swiftdoc: + all_compile_outputs.append(compile_outputs.swiftdoc_file) + if include_swiftsourceinfo: + all_compile_outputs.append(compile_outputs.swiftsourceinfo_file) all_derived_outputs = [] # Merge the providers from our dependencies so that we have one each for @@ -2921,6 +2939,8 @@ def _declare_compile_outputs( feature_configuration, generated_header_name, generated_module_deps_swift_infos, + include_swiftdoc, + include_swiftsourceinfo, module_name, srcs, target_name, @@ -2936,6 +2956,8 @@ def _declare_compile_outputs( generated_module_deps_swift_infos: `SwiftInfo` providers from dependencies of the module for the generated header of the target being compiled. + include_swiftdoc: If .swiftdoc file should be included or not. + include_swiftsourceinfo: If .swiftsourceinfo file should be included or not. module_name: The name of the Swift module being compiled. srcs: The list of source files that will be compiled. target_name: The name (excluding package path) of the target being @@ -2963,11 +2985,12 @@ def _declare_compile_outputs( swiftdoc_file = derived_files.swiftdoc( actions = actions, module_name = module_name, - ) + ) if include_swiftdoc else None + swiftsourceinfo_file = derived_files.swiftsourceinfo( actions = actions, module_name = module_name, - ) + ) if include_swiftsourceinfo else None if are_all_features_enabled( feature_configuration = feature_configuration, diff --git a/swift/internal/feature_names.bzl b/swift/internal/feature_names.bzl index f92ef6033..ebad02d24 100644 --- a/swift/internal/feature_names.bzl +++ b/swift/internal/feature_names.bzl @@ -343,3 +343,13 @@ SWIFT_FEATURE__SUPPORTS_MACROS = "swift._supports_macros" # Pass -warnings-as-errors to the compiler. SWIFT_FEATURE_TREAT_WARNINGS_AS_ERRORS = "swift.treat_warnings_as_errors" + +# Defines whether .swiftdoc files are included in build outputs. +# This feature is enabled by default. +# Note: If opted out of this feature, .swiftdoc are generated by the compiler but excluded from Bazel's tracking. +SWIFT_FEATURE_EMIT_SWIFTDOC = "swift.emit_swiftdoc" + +# Defines whether .swiftsourceinfo files are included in build outputs. +# This feature is enabled by default. +# Note: If opted out of this feature, .swiftsourceinfo are generated by the compiler but excluded from Bazel's tracking. +SWIFT_FEATURE_EMIT_SWIFTSOURCEINFO = "swift.emit_swiftsourceinfo" diff --git a/swift/internal/swift_autoconfiguration.bzl b/swift/internal/swift_autoconfiguration.bzl index 513964337..66c89175d 100644 --- a/swift/internal/swift_autoconfiguration.bzl +++ b/swift/internal/swift_autoconfiguration.bzl @@ -28,6 +28,8 @@ load( "@build_bazel_rules_swift//swift/internal:feature_names.bzl", "SWIFT_FEATURE_CODEVIEW_DEBUG_INFO", "SWIFT_FEATURE_DEBUG_PREFIX_MAP", + "SWIFT_FEATURE_EMIT_SWIFTDOC", + "SWIFT_FEATURE_EMIT_SWIFTSOURCEINFO", "SWIFT_FEATURE_ENABLE_BATCH_MODE", "SWIFT_FEATURE_ENABLE_SKIP_FUNCTION_BODIES", "SWIFT_FEATURE_FILE_PREFIX_MAP", @@ -365,6 +367,8 @@ def _create_windows_toolchain(repository_ctx): enabled_features = [ SWIFT_FEATURE_CODEVIEW_DEBUG_INFO, SWIFT_FEATURE_DEBUG_PREFIX_MAP, + SWIFT_FEATURE_EMIT_SWIFTDOC, + SWIFT_FEATURE_EMIT_SWIFTSOURCEINFO, SWIFT_FEATURE_ENABLE_BATCH_MODE, SWIFT_FEATURE_ENABLE_SKIP_FUNCTION_BODIES, SWIFT_FEATURE_SUPPORTS_PRIVATE_DEPS, diff --git a/swift/internal/swift_toolchain.bzl b/swift/internal/swift_toolchain.bzl index a16a1d43d..77af981f8 100644 --- a/swift/internal/swift_toolchain.bzl +++ b/swift/internal/swift_toolchain.bzl @@ -34,6 +34,8 @@ load( "SWIFT_FEATURE_CACHEABLE_SWIFTMODULES", "SWIFT_FEATURE_COVERAGE_PREFIX_MAP", "SWIFT_FEATURE_DEBUG_PREFIX_MAP", + "SWIFT_FEATURE_EMIT_SWIFTDOC", + "SWIFT_FEATURE_EMIT_SWIFTSOURCEINFO", "SWIFT_FEATURE_MODULE_MAP_HOME_IS_CWD", "SWIFT_FEATURE_NO_GENERATED_MODULE_MAP", "SWIFT_FEATURE_OPT_USES_WMO", @@ -315,6 +317,8 @@ def _swift_toolchain_impl(ctx): SWIFT_FEATURE_CACHEABLE_SWIFTMODULES, SWIFT_FEATURE_COVERAGE_PREFIX_MAP, SWIFT_FEATURE_DEBUG_PREFIX_MAP, + SWIFT_FEATURE_EMIT_SWIFTDOC, + SWIFT_FEATURE_EMIT_SWIFTSOURCEINFO, SWIFT_FEATURE_NO_GENERATED_MODULE_MAP, SWIFT_FEATURE_OPT_USES_WMO, SWIFT_FEATURE_USE_GLOBAL_MODULE_CACHE, diff --git a/swift/internal/xcode_swift_toolchain.bzl b/swift/internal/xcode_swift_toolchain.bzl index d1989827b..26009dc17 100644 --- a/swift/internal/xcode_swift_toolchain.bzl +++ b/swift/internal/xcode_swift_toolchain.bzl @@ -35,6 +35,8 @@ load( "SWIFT_FEATURE_COVERAGE", "SWIFT_FEATURE_COVERAGE_PREFIX_MAP", "SWIFT_FEATURE_DEBUG_PREFIX_MAP", + "SWIFT_FEATURE_EMIT_SWIFTDOC", + "SWIFT_FEATURE_EMIT_SWIFTSOURCEINFO", "SWIFT_FEATURE_ENABLE_BATCH_MODE", "SWIFT_FEATURE_ENABLE_SKIP_FUNCTION_BODIES", "SWIFT_FEATURE_FILE_PREFIX_MAP", @@ -614,6 +616,8 @@ def _xcode_swift_toolchain_impl(ctx): SWIFT_FEATURE_CACHEABLE_SWIFTMODULES, SWIFT_FEATURE_COVERAGE_PREFIX_MAP, SWIFT_FEATURE_DEBUG_PREFIX_MAP, + SWIFT_FEATURE_EMIT_SWIFTDOC, + SWIFT_FEATURE_EMIT_SWIFTSOURCEINFO, SWIFT_FEATURE_ENABLE_BATCH_MODE, SWIFT_FEATURE_ENABLE_SKIP_FUNCTION_BODIES, SWIFT_FEATURE_OBJC_LINK_FLAGS, diff --git a/test/rules/provider_test.bzl b/test/rules/provider_test.bzl index 468ce352e..c34b54546 100644 --- a/test/rules/provider_test.bzl +++ b/test/rules/provider_test.bzl @@ -243,6 +243,16 @@ def _compare_expected_files(env, access_description, expected, actual): """ actual = _normalize_collection(actual) + expected_is_subset = "*" in expected + expected_include = [ + s + for s in expected + if not s.startswith("-") and s != "*" + ] + + if actual == [None] and not expected_include: + return + if ( not types.is_list(actual) or any([type(item) != "File" for item in actual]) @@ -259,13 +269,6 @@ def _compare_expected_files(env, access_description, expected, actual): return remaining = list(actual) - - expected_is_subset = "*" in expected - expected_include = [ - s - for s in expected - if not s.startswith("-") and s != "*" - ] expected_exclude = [s[1:] for s in expected if s.startswith("-")] # For every expected file, pick off the first actual that we find that has diff --git a/test/split_derived_files_tests.bzl b/test/split_derived_files_tests.bzl index 5e649c06b..8808a6d4a 100644 --- a/test/split_derived_files_tests.bzl +++ b/test/split_derived_files_tests.bzl @@ -56,6 +56,36 @@ split_swiftmodule_skip_function_bodies_test = make_action_command_line_test_rule ], }, ) +default_no_split_no_emit_swiftdoc_test = make_provider_test_rule( + config_settings = { + "//command_line_option:features": [ + "-swift.emit_swiftdoc", + ], + }, +) +default_no_split_no_emit_swiftsourceinfo_test = make_provider_test_rule( + config_settings = { + "//command_line_option:features": [ + "-swift.emit_swiftsourceinfo", + ], + }, +) +split_no_emit_swiftdoc_test = make_provider_test_rule( + config_settings = { + "//command_line_option:features": [ + "-swift.emit_swiftdoc", + "swift.split_derived_files_generation", + ], + }, +) +split_no_emit_swiftsourceinfo_test = make_provider_test_rule( + config_settings = { + "//command_line_option:features": [ + "-swift.emit_swiftsourceinfo", + "swift.split_derived_files_generation", + ], + }, +) split_swiftmodule_indexing_test = make_action_command_line_test_rule( config_settings = { "//command_line_option:features": [ @@ -124,7 +154,7 @@ def split_derived_files_test_suite(name): ) default_no_split_provider_test( - name = "{}_default_no_split_provider".format(name), + name = "{}_default_no_split_provider_swiftmodule".format(name), expected_files = [ "test_fixtures_debug_settings_simple.swiftmodule", ], @@ -134,6 +164,94 @@ def split_derived_files_test_suite(name): target_under_test = "@build_bazel_rules_swift//test/fixtures/debug_settings:simple", ) + default_no_split_provider_test( + name = "{}_default_no_split_provider_swiftdoc".format(name), + expected_files = [ + "test_fixtures_debug_settings_simple.swiftdoc", + ], + field = "direct_modules.swift.swiftdoc", + provider = "SwiftInfo", + tags = [name], + target_under_test = "@build_bazel_rules_swift//test/fixtures/debug_settings:simple", + ) + + default_no_split_provider_test( + name = "{}_default_no_split_provider_swiftsourceinfo".format(name), + expected_files = [ + "test_fixtures_debug_settings_simple.swiftsourceinfo", + ], + field = "direct_modules.swift.swiftsourceinfo", + provider = "SwiftInfo", + tags = [name], + target_under_test = "@build_bazel_rules_swift//test/fixtures/debug_settings:simple", + ) + + default_no_split_no_emit_swiftdoc_test( + name = "{}_default_no_split_provider_no_emit_swiftdoc".format(name), + expected_files = [ + "-test_fixtures_debug_settings_simple.swiftdoc", + ], + field = "direct_modules.swift.swiftdoc", + provider = "SwiftInfo", + tags = [name], + target_under_test = "@build_bazel_rules_swift//test/fixtures/debug_settings:simple", + ) + + default_no_split_no_emit_swiftsourceinfo_test( + name = "{}_default_no_split_provider_no_emit_swiftsourceinfo".format(name), + expected_files = [ + "-test_fixtures_debug_settings_simple.swiftsourceinfo", + ], + field = "direct_modules.swift.swiftsourceinfo", + provider = "SwiftInfo", + tags = [name], + target_under_test = "@build_bazel_rules_swift//test/fixtures/debug_settings:simple", + ) + + split_swiftmodule_provider_test( + name = "{}_split_provider_swiftdoc".format(name), + field = "direct_modules.swift.swiftdoc", + expected_files = [ + "test_fixtures_debug_settings_simple.swiftdoc", + ], + provider = "SwiftInfo", + tags = [name], + target_under_test = "@build_bazel_rules_swift//test/fixtures/debug_settings:simple", + ) + + split_swiftmodule_provider_test( + name = "{}_split_provider_swiftsourceinfo".format(name), + field = "direct_modules.swift.swiftsourceinfo", + expected_files = [ + "test_fixtures_debug_settings_simple.swiftsourceinfo", + ], + provider = "SwiftInfo", + tags = [name], + target_under_test = "@build_bazel_rules_swift//test/fixtures/debug_settings:simple", + ) + + split_no_emit_swiftdoc_test( + name = "{}_split_provider_no_emit_swiftdoc".format(name), + field = "direct_modules.swift.swiftdoc", + expected_files = [ + "-test_fixtures_debug_settings_simple.swiftdoc", + ], + provider = "SwiftInfo", + tags = [name], + target_under_test = "@build_bazel_rules_swift//test/fixtures/debug_settings:simple", + ) + + split_no_emit_swiftsourceinfo_test( + name = "{}_split_provider_no_emit_swiftsourceinfo".format(name), + field = "direct_modules.swift.swiftsourceinfo", + expected_files = [ + "-test_fixtures_debug_settings_simple.swiftsourceinfo", + ], + provider = "SwiftInfo", + tags = [name], + target_under_test = "@build_bazel_rules_swift//test/fixtures/debug_settings:simple", + ) + default_no_split_provider_test( name = "{}_default_no_split_provider_ccinfo".format(name), expected_files = [