diff --git a/.github/workflows/pr-auto-fix.yaml b/.github/workflows/pr-auto-fix.yaml index 573de52d9cb30..654f943c1d432 100644 --- a/.github/workflows/pr-auto-fix.yaml +++ b/.github/workflows/pr-auto-fix.yaml @@ -48,8 +48,9 @@ jobs: with: script: | // If you'd like not to run this code on your commits, add your github user id here: - NO_AUTOFIX_USERS = [] + NO_AUTOFIX_USERS = ["copybara-service[bot]"] const { owner, repo } = context.repo + console.log("Actor: " + context.actor); if (NO_AUTOFIX_USERS.includes(context.actor)) { console.log('Cancelling'); const run_id = "${{ github.run_id }}"; diff --git a/BUILD b/BUILD index 6e02ac04d5297..d07866b8b9cd3 100644 --- a/BUILD +++ b/BUILD @@ -1604,6 +1604,7 @@ grpc_cc_library( "//src/core:slice_refcount", "//src/core:socket_mutator", "//src/core:stats_data", + "//src/core:status_flag", "//src/core:status_helper", "//src/core:strerror", "//src/core:thread_quota", diff --git a/CMakeLists.txt b/CMakeLists.txt index 4c8979a56ec45..b9390ecccf282 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1212,6 +1212,9 @@ if(gRPC_BUILD_TESTS) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx posix_event_engine_connect_test) endif() + if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_POSIX) + add_dependencies(buildtests_cxx posix_event_engine_native_dns_test) + endif() if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx posix_event_engine_test) endif() @@ -1229,6 +1232,7 @@ if(gRPC_BUILD_TESTS) add_dependencies(buildtests_cxx proxy_auth_test) add_dependencies(buildtests_cxx qps_json_driver) add_dependencies(buildtests_cxx qps_worker) + add_dependencies(buildtests_cxx query_extensions_test) add_dependencies(buildtests_cxx race_test) add_dependencies(buildtests_cxx random_early_detection_test) add_dependencies(buildtests_cxx raw_end2end_test) @@ -2238,12 +2242,12 @@ add_library(grpc src/core/lib/event_engine/default_event_engine_factory.cc src/core/lib/event_engine/event_engine.cc src/core/lib/event_engine/forkable.cc - src/core/lib/event_engine/memory_allocator.cc src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc src/core/lib/event_engine/posix_engine/ev_poll_posix.cc src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc src/core/lib/event_engine/posix_engine/internal_errqueue.cc src/core/lib/event_engine/posix_engine/lockfree_event.cc + src/core/lib/event_engine/posix_engine/native_dns_resolver.cc src/core/lib/event_engine/posix_engine/posix_endpoint.cc src/core/lib/event_engine/posix_engine/posix_engine.cc src/core/lib/event_engine/posix_engine/posix_engine_listener.cc @@ -2966,12 +2970,12 @@ add_library(grpc_unsecure src/core/lib/event_engine/default_event_engine_factory.cc src/core/lib/event_engine/event_engine.cc src/core/lib/event_engine/forkable.cc - src/core/lib/event_engine/memory_allocator.cc src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc src/core/lib/event_engine/posix_engine/ev_poll_posix.cc src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc src/core/lib/event_engine/posix_engine/internal_errqueue.cc src/core/lib/event_engine/posix_engine/lockfree_event.cc + src/core/lib/event_engine/posix_engine/native_dns_resolver.cc src/core/lib/event_engine/posix_engine/posix_endpoint.cc src/core/lib/event_engine/posix_engine/posix_engine.cc src/core/lib/event_engine/posix_engine/posix_engine_listener.cc @@ -4954,12 +4958,12 @@ add_library(grpc_authorization_provider src/core/lib/event_engine/default_event_engine_factory.cc src/core/lib/event_engine/event_engine.cc src/core/lib/event_engine/forkable.cc - src/core/lib/event_engine/memory_allocator.cc src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc src/core/lib/event_engine/posix_engine/ev_poll_posix.cc src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc src/core/lib/event_engine/posix_engine/internal_errqueue.cc src/core/lib/event_engine/posix_engine/lockfree_event.cc + src/core/lib/event_engine/posix_engine/native_dns_resolver.cc src/core/lib/event_engine/posix_engine/posix_endpoint.cc src/core/lib/event_engine/posix_engine/posix_engine.cc src/core/lib/event_engine/posix_engine/posix_engine_listener.cc @@ -8856,7 +8860,6 @@ add_executable(chunked_vector_test src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c src/core/ext/upb-gen/google/rpc/status.upb_minitable.c src/core/lib/debug/trace.cc - src/core/lib/event_engine/memory_allocator.cc src/core/lib/experiments/config.cc src/core/lib/experiments/experiments.cc src/core/lib/gprpp/status_helper.cc @@ -12153,7 +12156,6 @@ add_executable(flow_control_test src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c src/core/ext/upb-gen/google/rpc/status.upb_minitable.c src/core/lib/debug/trace.cc - src/core/lib/event_engine/memory_allocator.cc src/core/lib/experiments/config.cc src/core/lib/experiments/experiments.cc src/core/lib/gprpp/status_helper.cc @@ -12239,7 +12241,6 @@ add_executable(for_each_test src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c src/core/ext/upb-gen/google/rpc/status.upb_minitable.c src/core/lib/debug/trace.cc - src/core/lib/event_engine/memory_allocator.cc src/core/lib/experiments/config.cc src/core/lib/experiments/experiments.cc src/core/lib/gprpp/status_helper.cc @@ -15017,7 +15018,6 @@ add_executable(interceptor_list_test src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c src/core/ext/upb-gen/google/rpc/status.upb_minitable.c src/core/lib/debug/trace.cc - src/core/lib/event_engine/memory_allocator.cc src/core/lib/experiments/config.cc src/core/lib/experiments/experiments.cc src/core/lib/gprpp/status_helper.cc @@ -15941,7 +15941,6 @@ add_executable(map_pipe_test src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c src/core/ext/upb-gen/google/rpc/status.upb_minitable.c src/core/lib/debug/trace.cc - src/core/lib/event_engine/memory_allocator.cc src/core/lib/experiments/config.cc src/core/lib/experiments/experiments.cc src/core/lib/gprpp/status_helper.cc @@ -18274,6 +18273,48 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_POSIX) ) +endif() +endif() +if(gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_POSIX) + + add_executable(posix_event_engine_native_dns_test + test/core/event_engine/event_engine_test_utils.cc + test/core/event_engine/test_suite/event_engine_test_framework.cc + test/core/event_engine/test_suite/posix/oracle_event_engine_posix.cc + test/core/event_engine/test_suite/posix_event_engine_native_dns_test.cc + test/core/event_engine/test_suite/tests/dns_test.cc + test/core/util/fake_udp_and_tcp_server.cc + test/cpp/util/get_grpc_test_runfile_dir.cc + test/cpp/util/windows/manifest_file.cc + ) + target_compile_features(posix_event_engine_native_dns_test PUBLIC cxx_std_14) + target_include_directories(posix_event_engine_native_dns_test + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} + third_party/googletest/googletest/include + third_party/googletest/googletest + third_party/googletest/googlemock/include + third_party/googletest/googlemock + ${_gRPC_PROTO_GENS_DIR} + ) + + target_link_libraries(posix_event_engine_native_dns_test + ${_gRPC_ALLTARGETS_LIBRARIES} + gtest + grpc++_test_util + ) + + endif() endif() if(gRPC_BUILD_TESTS) @@ -18938,6 +18979,40 @@ target_link_libraries(qps_worker ) +endif() +if(gRPC_BUILD_TESTS) + +add_executable(query_extensions_test + test/core/event_engine/query_extensions_test.cc +) +target_compile_features(query_extensions_test PUBLIC cxx_std_14) +target_include_directories(query_extensions_test + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} + third_party/googletest/googletest/include + third_party/googletest/googletest + third_party/googletest/googlemock/include + third_party/googletest/googlemock + ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(query_extensions_test + ${_gRPC_ALLTARGETS_LIBRARIES} + gtest + absl::statusor + gpr +) + + endif() if(gRPC_BUILD_TESTS) @@ -24562,12 +24637,12 @@ add_executable(test_core_transport_chaotic_good_frame_test src/core/lib/event_engine/default_event_engine_factory.cc src/core/lib/event_engine/event_engine.cc src/core/lib/event_engine/forkable.cc - src/core/lib/event_engine/memory_allocator.cc src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc src/core/lib/event_engine/posix_engine/ev_poll_posix.cc src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc src/core/lib/event_engine/posix_engine/internal_errqueue.cc src/core/lib/event_engine/posix_engine/lockfree_event.cc + src/core/lib/event_engine/posix_engine/native_dns_resolver.cc src/core/lib/event_engine/posix_engine/posix_endpoint.cc src/core/lib/event_engine/posix_engine/posix_engine.cc src/core/lib/event_engine/posix_engine/posix_engine_listener.cc diff --git a/Makefile b/Makefile index 4833a2944fea3..6df49a7bac0ff 100644 --- a/Makefile +++ b/Makefile @@ -1444,12 +1444,12 @@ LIBGRPC_SRC = \ src/core/lib/event_engine/default_event_engine_factory.cc \ src/core/lib/event_engine/event_engine.cc \ src/core/lib/event_engine/forkable.cc \ - src/core/lib/event_engine/memory_allocator.cc \ src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc \ src/core/lib/event_engine/posix_engine/ev_poll_posix.cc \ src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc \ src/core/lib/event_engine/posix_engine/internal_errqueue.cc \ src/core/lib/event_engine/posix_engine/lockfree_event.cc \ + src/core/lib/event_engine/posix_engine/native_dns_resolver.cc \ src/core/lib/event_engine/posix_engine/posix_endpoint.cc \ src/core/lib/event_engine/posix_engine/posix_engine.cc \ src/core/lib/event_engine/posix_engine/posix_engine_listener.cc \ @@ -2022,12 +2022,12 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/event_engine/default_event_engine_factory.cc \ src/core/lib/event_engine/event_engine.cc \ src/core/lib/event_engine/forkable.cc \ - src/core/lib/event_engine/memory_allocator.cc \ src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc \ src/core/lib/event_engine/posix_engine/ev_poll_posix.cc \ src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc \ src/core/lib/event_engine/posix_engine/internal_errqueue.cc \ src/core/lib/event_engine/posix_engine/lockfree_event.cc \ + src/core/lib/event_engine/posix_engine/native_dns_resolver.cc \ src/core/lib/event_engine/posix_engine/posix_endpoint.cc \ src/core/lib/event_engine/posix_engine/posix_engine.cc \ src/core/lib/event_engine/posix_engine/posix_engine_listener.cc \ diff --git a/Package.swift b/Package.swift index 210d9bc8f5682..7c9d9ce10e9e2 100644 --- a/Package.swift +++ b/Package.swift @@ -1260,7 +1260,6 @@ let package = Package( "src/core/lib/event_engine/forkable.h", "src/core/lib/event_engine/grpc_polled_fd.h", "src/core/lib/event_engine/handle_containers.h", - "src/core/lib/event_engine/memory_allocator.cc", "src/core/lib/event_engine/memory_allocator_factory.h", "src/core/lib/event_engine/nameser.h", "src/core/lib/event_engine/poller.h", @@ -1277,6 +1276,8 @@ let package = Package( "src/core/lib/event_engine/posix_engine/internal_errqueue.h", "src/core/lib/event_engine/posix_engine/lockfree_event.cc", "src/core/lib/event_engine/posix_engine/lockfree_event.h", + "src/core/lib/event_engine/posix_engine/native_dns_resolver.cc", + "src/core/lib/event_engine/posix_engine/native_dns_resolver.h", "src/core/lib/event_engine/posix_engine/posix_endpoint.cc", "src/core/lib/event_engine/posix_engine/posix_endpoint.h", "src/core/lib/event_engine/posix_engine/posix_engine.cc", @@ -1303,6 +1304,7 @@ let package = Package( "src/core/lib/event_engine/posix_engine/wakeup_fd_posix.h", "src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.cc", "src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.h", + "src/core/lib/event_engine/ref_counted_dns_resolver_interface.h", "src/core/lib/event_engine/resolved_address.cc", "src/core/lib/event_engine/resolved_address_internal.h", "src/core/lib/event_engine/shim.cc", @@ -1652,6 +1654,7 @@ let package = Package( "src/core/lib/promise/seq.h", "src/core/lib/promise/sleep.cc", "src/core/lib/promise/sleep.h", + "src/core/lib/promise/status_flag.h", "src/core/lib/promise/trace.cc", "src/core/lib/promise/trace.h", "src/core/lib/promise/try_seq.h", diff --git a/bazel/experiments.bzl b/bazel/experiments.bzl index 6e0200f2fd988..e5f402426956b 100644 --- a/bazel/experiments.bzl +++ b/bazel/experiments.bzl @@ -17,14 +17,10 @@ """Dictionary of tags to experiments so we know when to test different experiments.""" EXPERIMENT_ENABLES = { - "block_excessive_requests_before_settings_ack": "block_excessive_requests_before_settings_ack", "call_status_override_on_cancellation": "call_status_override_on_cancellation", "canary_client_privacy": "canary_client_privacy", "client_idleness": "client_idleness", "client_privacy": "client_privacy", - "combiner_offload_to_event_engine": "combiner_offload_to_event_engine", - "chttp2_batch_requests": "chttp2_batch_requests,combiner_offload_to_event_engine", - "chttp2_offload_on_rst_stream": "chttp2_offload_on_rst_stream,combiner_offload_to_event_engine", "event_engine_client": "event_engine_client", "event_engine_dns": "event_engine_dns", "event_engine_listener": "event_engine_listener", @@ -32,7 +28,6 @@ EXPERIMENT_ENABLES = { "http2_stats_fix": "http2_stats_fix", "keepalive_fix": "keepalive_fix", "keepalive_server_fix": "keepalive_server_fix", - "lazier_stream_updates": "lazier_stream_updates", "memory_pressure_controller": "memory_pressure_controller", "monitoring_experiment": "monitoring_experiment", "multiping": "multiping", @@ -40,10 +35,9 @@ EXPERIMENT_ENABLES = { "peer_state_based_framing": "peer_state_based_framing", "pending_queue_cap": "pending_queue_cap", "pick_first_happy_eyeballs": "pick_first_happy_eyeballs", - "ping_on_rst_stream": "ping_on_rst_stream", "promise_based_client_call": "promise_based_client_call", "promise_based_inproc_transport": "promise_based_inproc_transport", - "promise_based_server_call": "lazier_stream_updates,promise_based_server_call", + "promise_based_server_call": "promise_based_server_call", "red_max_concurrent_streams": "red_max_concurrent_streams", "registered_method_lookup_in_transport": "registered_method_lookup_in_transport", "registered_methods_map": "registered_methods_map", @@ -51,15 +45,11 @@ EXPERIMENT_ENABLES = { "round_robin_delegate_to_pick_first": "round_robin_delegate_to_pick_first", "rstpit": "rstpit", "schedule_cancellation_over_write": "schedule_cancellation_over_write", - "separate_ping_from_keepalive": "separate_ping_from_keepalive", "server_privacy": "server_privacy", - "settings_timeout": "settings_timeout", - "tarpit": "tarpit", "tcp_frame_size_tuning": "tcp_frame_size_tuning", "tcp_rcv_lowat": "tcp_rcv_lowat", "trace_record_callops": "trace_record_callops", "unconstrained_max_quota_buffer_size": "unconstrained_max_quota_buffer_size", - "uniquely_unowned": "uniquely_unowned", "work_serializer_clears_time_cache": "work_serializer_clears_time_cache", "work_serializer_dispatch": "work_serializer_dispatch", "write_size_policy": "write_size_policy", @@ -116,17 +106,9 @@ EXPERIMENTS = { ], }, "on": { - "bad_client_test": [ - "block_excessive_requests_before_settings_ack", - "tarpit", - ], "core_end2end_test": [ "event_engine_listener", ], - "cpp_end2end_test": [ - "chttp2_batch_requests", - "chttp2_offload_on_rst_stream", - ], "cpp_lb_end2end_test": [ "pick_first_happy_eyeballs", "round_robin_delegate_to_pick_first", @@ -136,9 +118,6 @@ EXPERIMENTS = { "event_engine_listener", ], "flow_control_test": [ - "chttp2_batch_requests", - "chttp2_offload_on_rst_stream", - "lazier_stream_updates", "overload_protection", "write_size_cap", "write_size_policy", @@ -206,23 +185,12 @@ EXPERIMENTS = { ], }, "on": { - "bad_client_test": [ - "block_excessive_requests_before_settings_ack", - "tarpit", - ], - "cpp_end2end_test": [ - "chttp2_batch_requests", - "chttp2_offload_on_rst_stream", - ], "cpp_lb_end2end_test": [ "pick_first_happy_eyeballs", "round_robin_delegate_to_pick_first", "wrr_delegate_to_pick_first", ], "flow_control_test": [ - "chttp2_batch_requests", - "chttp2_offload_on_rst_stream", - "lazier_stream_updates", "overload_protection", "write_size_cap", "write_size_policy", @@ -300,17 +268,9 @@ EXPERIMENTS = { ], }, "on": { - "bad_client_test": [ - "block_excessive_requests_before_settings_ack", - "tarpit", - ], "core_end2end_test": [ "event_engine_listener", ], - "cpp_end2end_test": [ - "chttp2_batch_requests", - "chttp2_offload_on_rst_stream", - ], "cpp_lb_end2end_test": [ "pick_first_happy_eyeballs", "round_robin_delegate_to_pick_first", @@ -320,9 +280,6 @@ EXPERIMENTS = { "event_engine_listener", ], "flow_control_test": [ - "chttp2_batch_requests", - "chttp2_offload_on_rst_stream", - "lazier_stream_updates", "overload_protection", "write_size_cap", "write_size_policy", diff --git a/bazel/grpc_deps.bzl b/bazel/grpc_deps.bzl index a982d8079b5a5..1f64e9fe5b43e 100644 --- a/bazel/grpc_deps.bzl +++ b/bazel/grpc_deps.bzl @@ -512,11 +512,11 @@ def grpc_deps(): if "io_opentelemetry_cpp" not in native.existing_rules(): http_archive( name = "io_opentelemetry_cpp", - sha256 = "f30cd88bf898a5726d245eba882b8e81012021eb00df34109f4dfb203f005cea", - strip_prefix = "opentelemetry-cpp-1.11.0", + sha256 = "149f076cc7a79bbd3a3c34fb3ab61d3a3e8dcfe2b9596f79153e17123c32f897", + strip_prefix = "opentelemetry-cpp-064fef0d871c57ffac6739d3311659a5770a9db4", urls = [ - "https://storage.googleapis.com/grpc-bazel-mirror/github.com/open-telemetry/opentelemetry-cpp/archive/refs/tags/v1.11.0.tar.gz", - "https://github.com/open-telemetry/opentelemetry-cpp/archive/refs/tags/v1.11.0.tar.gz", + "https://storage.googleapis.com/grpc-bazel-mirror/github.com/open-telemetry/opentelemetry-cpp/archive/064fef0d871c57ffac6739d3311659a5770a9db4.tar.gz", + "https://github.com/open-telemetry/opentelemetry-cpp/archive/064fef0d871c57ffac6739d3311659a5770a9db4.tar.gz", ], ) diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index bd2a9aa0bbd76..f5ea18449e226 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -877,6 +877,7 @@ libs: - src/core/lib/event_engine/posix_engine/grpc_polled_fd_posix.h - src/core/lib/event_engine/posix_engine/internal_errqueue.h - src/core/lib/event_engine/posix_engine/lockfree_event.h + - src/core/lib/event_engine/posix_engine/native_dns_resolver.h - src/core/lib/event_engine/posix_engine/posix_endpoint.h - src/core/lib/event_engine/posix_engine/posix_engine.h - src/core/lib/event_engine/posix_engine/posix_engine_closure.h @@ -891,6 +892,7 @@ libs: - src/core/lib/event_engine/posix_engine/wakeup_fd_pipe.h - src/core/lib/event_engine/posix_engine/wakeup_fd_posix.h - src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.h + - src/core/lib/event_engine/ref_counted_dns_resolver_interface.h - src/core/lib/event_engine/resolved_address_internal.h - src/core/lib/event_engine/shim.h - src/core/lib/event_engine/tcp_socket_utils.h @@ -1050,6 +1052,7 @@ libs: - src/core/lib/promise/race.h - src/core/lib/promise/seq.h - src/core/lib/promise/sleep.h + - src/core/lib/promise/status_flag.h - src/core/lib/promise/trace.h - src/core/lib/promise/try_seq.h - src/core/lib/resolver/endpoint_addresses.h @@ -1692,12 +1695,12 @@ libs: - src/core/lib/event_engine/default_event_engine_factory.cc - src/core/lib/event_engine/event_engine.cc - src/core/lib/event_engine/forkable.cc - - src/core/lib/event_engine/memory_allocator.cc - src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc - src/core/lib/event_engine/posix_engine/ev_poll_posix.cc - src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc - src/core/lib/event_engine/posix_engine/internal_errqueue.cc - src/core/lib/event_engine/posix_engine/lockfree_event.cc + - src/core/lib/event_engine/posix_engine/native_dns_resolver.cc - src/core/lib/event_engine/posix_engine/posix_endpoint.cc - src/core/lib/event_engine/posix_engine/posix_engine.cc - src/core/lib/event_engine/posix_engine/posix_engine_listener.cc @@ -2330,6 +2333,7 @@ libs: - src/core/lib/event_engine/posix_engine/grpc_polled_fd_posix.h - src/core/lib/event_engine/posix_engine/internal_errqueue.h - src/core/lib/event_engine/posix_engine/lockfree_event.h + - src/core/lib/event_engine/posix_engine/native_dns_resolver.h - src/core/lib/event_engine/posix_engine/posix_endpoint.h - src/core/lib/event_engine/posix_engine/posix_engine.h - src/core/lib/event_engine/posix_engine/posix_engine_closure.h @@ -2344,6 +2348,7 @@ libs: - src/core/lib/event_engine/posix_engine/wakeup_fd_pipe.h - src/core/lib/event_engine/posix_engine/wakeup_fd_posix.h - src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.h + - src/core/lib/event_engine/ref_counted_dns_resolver_interface.h - src/core/lib/event_engine/resolved_address_internal.h - src/core/lib/event_engine/shim.h - src/core/lib/event_engine/tcp_socket_utils.h @@ -2498,6 +2503,7 @@ libs: - src/core/lib/promise/race.h - src/core/lib/promise/seq.h - src/core/lib/promise/sleep.h + - src/core/lib/promise/status_flag.h - src/core/lib/promise/trace.h - src/core/lib/promise/try_seq.h - src/core/lib/resolver/endpoint_addresses.h @@ -2769,12 +2775,12 @@ libs: - src/core/lib/event_engine/default_event_engine_factory.cc - src/core/lib/event_engine/event_engine.cc - src/core/lib/event_engine/forkable.cc - - src/core/lib/event_engine/memory_allocator.cc - src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc - src/core/lib/event_engine/posix_engine/ev_poll_posix.cc - src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc - src/core/lib/event_engine/posix_engine/internal_errqueue.cc - src/core/lib/event_engine/posix_engine/lockfree_event.cc + - src/core/lib/event_engine/posix_engine/native_dns_resolver.cc - src/core/lib/event_engine/posix_engine/posix_endpoint.cc - src/core/lib/event_engine/posix_engine/posix_engine.cc - src/core/lib/event_engine/posix_engine/posix_engine_listener.cc @@ -4463,6 +4469,7 @@ libs: - src/core/lib/event_engine/posix_engine/grpc_polled_fd_posix.h - src/core/lib/event_engine/posix_engine/internal_errqueue.h - src/core/lib/event_engine/posix_engine/lockfree_event.h + - src/core/lib/event_engine/posix_engine/native_dns_resolver.h - src/core/lib/event_engine/posix_engine/posix_endpoint.h - src/core/lib/event_engine/posix_engine/posix_engine.h - src/core/lib/event_engine/posix_engine/posix_engine_closure.h @@ -4477,6 +4484,7 @@ libs: - src/core/lib/event_engine/posix_engine/wakeup_fd_pipe.h - src/core/lib/event_engine/posix_engine/wakeup_fd_posix.h - src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.h + - src/core/lib/event_engine/ref_counted_dns_resolver_interface.h - src/core/lib/event_engine/resolved_address_internal.h - src/core/lib/event_engine/shim.h - src/core/lib/event_engine/tcp_socket_utils.h @@ -4623,6 +4631,7 @@ libs: - src/core/lib/promise/promise.h - src/core/lib/promise/race.h - src/core/lib/promise/seq.h + - src/core/lib/promise/status_flag.h - src/core/lib/promise/trace.h - src/core/lib/promise/try_seq.h - src/core/lib/resolver/endpoint_addresses.h @@ -4809,12 +4818,12 @@ libs: - src/core/lib/event_engine/default_event_engine_factory.cc - src/core/lib/event_engine/event_engine.cc - src/core/lib/event_engine/forkable.cc - - src/core/lib/event_engine/memory_allocator.cc - src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc - src/core/lib/event_engine/posix_engine/ev_poll_posix.cc - src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc - src/core/lib/event_engine/posix_engine/internal_errqueue.cc - src/core/lib/event_engine/posix_engine/lockfree_event.cc + - src/core/lib/event_engine/posix_engine/native_dns_resolver.cc - src/core/lib/event_engine/posix_engine/posix_endpoint.cc - src/core/lib/event_engine/posix_engine/posix_engine.cc - src/core/lib/event_engine/posix_engine/posix_engine_listener.cc @@ -7298,7 +7307,6 @@ targets: - src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c - src/core/ext/upb-gen/google/rpc/status.upb_minitable.c - src/core/lib/debug/trace.cc - - src/core/lib/event_engine/memory_allocator.cc - src/core/lib/experiments/config.cc - src/core/lib/experiments/experiments.cc - src/core/lib/gprpp/status_helper.cc @@ -9283,7 +9291,6 @@ targets: - src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c - src/core/ext/upb-gen/google/rpc/status.upb_minitable.c - src/core/lib/debug/trace.cc - - src/core/lib/event_engine/memory_allocator.cc - src/core/lib/experiments/config.cc - src/core/lib/experiments/experiments.cc - src/core/lib/gprpp/status_helper.cc @@ -9457,7 +9464,6 @@ targets: - src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c - src/core/ext/upb-gen/google/rpc/status.upb_minitable.c - src/core/lib/debug/trace.cc - - src/core/lib/event_engine/memory_allocator.cc - src/core/lib/experiments/config.cc - src/core/lib/experiments/experiments.cc - src/core/lib/gprpp/status_helper.cc @@ -10996,7 +11002,6 @@ targets: - src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c - src/core/ext/upb-gen/google/rpc/status.upb_minitable.c - src/core/lib/debug/trace.cc - - src/core/lib/event_engine/memory_allocator.cc - src/core/lib/experiments/config.cc - src/core/lib/experiments/experiments.cc - src/core/lib/gprpp/status_helper.cc @@ -11627,7 +11632,6 @@ targets: - src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c - src/core/ext/upb-gen/google/rpc/status.upb_minitable.c - src/core/lib/debug/trace.cc - - src/core/lib/event_engine/memory_allocator.cc - src/core/lib/experiments/config.cc - src/core/lib/experiments/experiments.cc - src/core/lib/gprpp/status_helper.cc @@ -12967,6 +12971,33 @@ targets: platforms: - linux - posix +- name: posix_event_engine_native_dns_test + gtest: true + build: test + language: c++ + headers: + - test/core/event_engine/event_engine_test_utils.h + - test/core/event_engine/test_suite/event_engine_test_framework.h + - test/core/event_engine/test_suite/posix/oracle_event_engine_posix.h + - test/core/event_engine/test_suite/tests/dns_test.h + - test/core/util/fake_udp_and_tcp_server.h + - test/cpp/util/get_grpc_test_runfile_dir.h + - test/cpp/util/windows/manifest_file.h + src: + - test/core/event_engine/event_engine_test_utils.cc + - test/core/event_engine/test_suite/event_engine_test_framework.cc + - test/core/event_engine/test_suite/posix/oracle_event_engine_posix.cc + - test/core/event_engine/test_suite/posix_event_engine_native_dns_test.cc + - test/core/event_engine/test_suite/tests/dns_test.cc + - test/core/util/fake_udp_and_tcp_server.cc + - test/cpp/util/get_grpc_test_runfile_dir.cc + - test/cpp/util/windows/manifest_file.cc + deps: + - gtest + - grpc++_test_util + platforms: + - linux + - posix - name: posix_event_engine_test gtest: true build: test @@ -13309,6 +13340,19 @@ targets: deps: - grpc++_test_config - grpc++_test_util +- name: query_extensions_test + gtest: true + build: test + language: c++ + headers: + - src/core/lib/event_engine/query_extensions.h + src: + - test/core/event_engine/query_extensions_test.cc + deps: + - gtest + - absl/status:statusor + - gpr + uses_polling: false - name: race_test gtest: true build: test @@ -16779,6 +16823,7 @@ targets: - src/core/lib/event_engine/posix_engine/grpc_polled_fd_posix.h - src/core/lib/event_engine/posix_engine/internal_errqueue.h - src/core/lib/event_engine/posix_engine/lockfree_event.h + - src/core/lib/event_engine/posix_engine/native_dns_resolver.h - src/core/lib/event_engine/posix_engine/posix_endpoint.h - src/core/lib/event_engine/posix_engine/posix_engine.h - src/core/lib/event_engine/posix_engine/posix_engine_closure.h @@ -16793,6 +16838,7 @@ targets: - src/core/lib/event_engine/posix_engine/wakeup_fd_pipe.h - src/core/lib/event_engine/posix_engine/wakeup_fd_posix.h - src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.h + - src/core/lib/event_engine/ref_counted_dns_resolver_interface.h - src/core/lib/event_engine/resolved_address_internal.h - src/core/lib/event_engine/shim.h - src/core/lib/event_engine/tcp_socket_utils.h @@ -16937,6 +16983,7 @@ targets: - src/core/lib/promise/promise.h - src/core/lib/promise/race.h - src/core/lib/promise/seq.h + - src/core/lib/promise/status_flag.h - src/core/lib/promise/trace.h - src/core/lib/promise/try_seq.h - src/core/lib/resolver/endpoint_addresses.h @@ -17106,12 +17153,12 @@ targets: - src/core/lib/event_engine/default_event_engine_factory.cc - src/core/lib/event_engine/event_engine.cc - src/core/lib/event_engine/forkable.cc - - src/core/lib/event_engine/memory_allocator.cc - src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc - src/core/lib/event_engine/posix_engine/ev_poll_posix.cc - src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc - src/core/lib/event_engine/posix_engine/internal_errqueue.cc - src/core/lib/event_engine/posix_engine/lockfree_event.cc + - src/core/lib/event_engine/posix_engine/native_dns_resolver.cc - src/core/lib/event_engine/posix_engine/posix_endpoint.cc - src/core/lib/event_engine/posix_engine/posix_engine.cc - src/core/lib/event_engine/posix_engine/posix_engine_listener.cc diff --git a/config.m4 b/config.m4 index 01455fdd842ca..f4f52a0044c52 100644 --- a/config.m4 +++ b/config.m4 @@ -534,12 +534,12 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/event_engine/default_event_engine_factory.cc \ src/core/lib/event_engine/event_engine.cc \ src/core/lib/event_engine/forkable.cc \ - src/core/lib/event_engine/memory_allocator.cc \ src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc \ src/core/lib/event_engine/posix_engine/ev_poll_posix.cc \ src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc \ src/core/lib/event_engine/posix_engine/internal_errqueue.cc \ src/core/lib/event_engine/posix_engine/lockfree_event.cc \ + src/core/lib/event_engine/posix_engine/native_dns_resolver.cc \ src/core/lib/event_engine/posix_engine/posix_endpoint.cc \ src/core/lib/event_engine/posix_engine/posix_engine.cc \ src/core/lib/event_engine/posix_engine/posix_engine_listener.cc \ diff --git a/config.w32 b/config.w32 index 0025913ecd7d3..52ac38f12049a 100644 --- a/config.w32 +++ b/config.w32 @@ -499,12 +499,12 @@ if (PHP_GRPC != "no") { "src\\core\\lib\\event_engine\\default_event_engine_factory.cc " + "src\\core\\lib\\event_engine\\event_engine.cc " + "src\\core\\lib\\event_engine\\forkable.cc " + - "src\\core\\lib\\event_engine\\memory_allocator.cc " + "src\\core\\lib\\event_engine\\posix_engine\\ev_epoll1_linux.cc " + "src\\core\\lib\\event_engine\\posix_engine\\ev_poll_posix.cc " + "src\\core\\lib\\event_engine\\posix_engine\\event_poller_posix_default.cc " + "src\\core\\lib\\event_engine\\posix_engine\\internal_errqueue.cc " + "src\\core\\lib\\event_engine\\posix_engine\\lockfree_event.cc " + + "src\\core\\lib\\event_engine\\posix_engine\\native_dns_resolver.cc " + "src\\core\\lib\\event_engine\\posix_engine\\posix_endpoint.cc " + "src\\core\\lib\\event_engine\\posix_engine\\posix_engine.cc " + "src\\core\\lib\\event_engine\\posix_engine\\posix_engine_listener.cc " + diff --git a/doc/service_config.md b/doc/service_config.md index 57f5e1989e820..6312cda7f6285 100644 --- a/doc/service_config.md +++ b/doc/service_config.md @@ -62,12 +62,12 @@ DNS](https://github.com/grpc/proposal/blob/master/A2-service-configs-in-dns.md). Here is an example service config in protobuf form: -``` +```textproto { - // Use round_robin LB policy. + # Use round_robin LB policy. load_balancing_config: { round_robin: {} } - // This method config applies to method "foo/bar" and to all methods - // of service "baz". + # This method config applies to method "foo/bar" and to all methods + # of service "baz". method_config: { name: { service: "foo" @@ -76,7 +76,7 @@ Here is an example service config in protobuf form: name: { service: "baz" } - // Default timeout for matching methods. + # Default timeout for matching methods. timeout: { seconds: 1 nanos: 1 @@ -87,7 +87,7 @@ Here is an example service config in protobuf form: Here is the same example service config in JSON form: -``` +```json { "loadBalancingConfig": [ { "round_robin": {} } ], "methodConfig": [ diff --git a/examples/cpp/cmake/common.cmake b/examples/cpp/cmake/common.cmake index 5c3d1053182d1..df9bc4db0f87f 100644 --- a/examples/cpp/cmake/common.cmake +++ b/examples/cpp/cmake/common.cmake @@ -51,6 +51,7 @@ if(GRPC_AS_SUBMODULE) # this build. set(_PROTOBUF_LIBPROTOBUF libprotobuf) set(_REFLECTION grpc++_reflection) + set(_ORCA_SERVICE grpcpp_orca_service) if(CMAKE_CROSSCOMPILING) find_program(_PROTOBUF_PROTOC protoc) else() diff --git a/examples/cpp/debugging/BUILD b/examples/cpp/debugging/BUILD new file mode 100644 index 0000000000000..c75995395d159 --- /dev/null +++ b/examples/cpp/debugging/BUILD @@ -0,0 +1,42 @@ +# Copyright 2023 the gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) + +cc_binary( + name = "crashing_greeter_client", + srcs = ["crashing_greeter_client.cc"], + defines = ["BAZEL_BUILD"], + deps = [ + "//:grpc++", + "//examples/protos:helloworld_cc_grpc", + "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/flags:parse", + ], +) + +cc_binary( + name = "greeter_callback_server_admin", + srcs = ["greeter_callback_server_admin.cc"], + defines = ["BAZEL_BUILD"], + deps = [ + "//:grpc++", + "//:grpc++_reflection", + "//:grpcpp_admin", + "//examples/protos:helloworld_cc_grpc", + "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/flags:parse", + "@com_google_absl//absl/strings:str_format", + ], +) diff --git a/examples/cpp/debugging/README.md b/examples/cpp/debugging/README.md new file mode 100644 index 0000000000000..09a75146d467c --- /dev/null +++ b/examples/cpp/debugging/README.md @@ -0,0 +1,132 @@ +# gRPC C++ Debugging Example + +This example demonstrates a handful of ways you can debug your gRPC C++ applications. + +## Enabling Trace Logs + +gRPC allows you to configure more detailed log output for various aspects of gRPC behavior. The tracing log generation might have a large overhead and result in significantly larger log file sizes, especially when you try to trace transport or timer_check. But it is a powerful tool in your debugging toolkit. + +### The Most Verbose Logging + +Specify environment variables, then run your application: + +``` +GRPC_VERBOSITY=debug +GRPC_TRACE=all +``` + +For more granularity, please see +[environment_variables](https://github.com/grpc/grpc/blob/master/doc/environment_variables.md). + +### Debug Transport Protocol + +``` +GRPC_VERBOSITY=debug +GRPC_TRACE=tcp,http,secure_endpoint,transport_security +``` + +### Debug Connection Behavior + +``` +GRPC_VERBOSITY=debug +GRPC_TRACE=call_error,connectivity_state,pick_first,round_robin,glb +``` + +## GDB and other debuggers + +`gdb` (and the like) are tools that lets you inspect your application while it is running, view stack traces on exceptions, pause and step through code at specified points or under certain conditions, etc. See https://www.sourceware.org/gdb/ + +### Inspecting errors + +``` +bazel build --config=dbg examples/cpp/debugging:crashing_greeter_client +gdb -ex run \ + --args ./bazel-bin/examples/cpp/debugging/crashing_greeter_client \ + --crash_on_errors=true \ + --target=localhork:50051 +``` + +Once the exception is thrown, you can use `bt` to see the stack trace and examine the crash, `info threads` to get the set of threads, etc. See the [GDB documentation](https://sourceware.org/gdb/current/onlinedocs/gdb.html/) for a more complete list of available features and commands. + +### Breaking inside a function + +After building the application above, this will break inside gRPC generated stub code: + +``` +gdb -ex 'b helloworld::Greeter::Stub::SayHello' \ + -ex run \ + --args ./bazel-bin/examples/cpp/debugging/crashing_greeter_client \ + --crash_on_errors=true \ + --target=localhork:50051 +``` + +## gRPC Admin Interface: Live Channel Tracing + +The [gRPC Admin Service](https://github.com/grpc/proposal/blob/master/A38-admin-interface-api.md) +provides a convenient API in each gRPC language to improve the usability of +creating a gRPC server with admin services to expose states in the gRPC library. +This includes channelz, which is a channel tracing feature; it tracks statistics +like how many messages have been sent, how many of them failed, what are the +connected sockets. See the [Channelz design doc](https://github.com/grpc/proposal/blob/master/A14-channelz.md). + +### Integrating the gRPC Admin Service Into Your Server + +As seen in the `greeter_callback_admin_server` target, you canenable admin services by using the `AddAdminServices` method. + +``` +grpc::ServerBuilder builder; +grpc::AddAdminServices(&builder); +builder.AddListeningPort(":50051", grpc::ServerCredentials(...)); +std::unique_ptr server(builder.BuildAndStart()); +``` + +### Using grpcdebug + +grpcdebug is a tool created to access the metrics from channelz and health services. + +#### Installing the grpcdebug tool + +The source code is located in a github project +[grpc-ecosystem/grpcdebug](https://github.com/grpc-ecosystem/grpcdebug). You +can either download [the latest built version] +(https://github.com/grpc-ecosystem/grpcdebug/releases/latest) (recommended) or +follow the README.md to build it yourself. + +#### Running the grpcdebug tool +##### Usage +`grpcdebug [flags] channelz [argument]` + + +| Command | Argument | Description | +| :--------- | :------------------: | :------------------------------------------------ | +| channel | \ | Display channel states in a human readable way. | +| channels | | Lists client channels for the target application. | +| server | \ | Displays server state in a human readable way. | +| servers | | Lists servers in a human readable way. | +| socket | \ | Displays socket states in a human readable way. | +| subchannel | \ | Display subchannel states in human readable way. | + +Generally, you will start with either `servers` or `channels` and then work down +to the details + +##### Getting overall server info + +To begin with, build and run the server binary in the background + +``` +bazel build --config=dbg examples/cpp/debugging:all +./bazel-bin/examples/cpp/debugging/greeter_callback_server_admin& +``` + +You can then inspect the server +```bash +grpcdebug localhost:50051 channelz servers +``` + +This will show you the server ids with their activity +```text +Server ID Listen Addresses Calls(Started/Succeeded/Failed) Last Call Started +1 [[::]:50051] 38/34/3 now +``` + +For more information about `grpcdebug` features, please see [the grpcdebug documentation](https://github.com/grpc-ecosystem/grpcdebug) diff --git a/examples/cpp/debugging/crashing_greeter_client.cc b/examples/cpp/debugging/crashing_greeter_client.cc new file mode 100644 index 0000000000000..732535539f153 --- /dev/null +++ b/examples/cpp/debugging/crashing_greeter_client.cc @@ -0,0 +1,92 @@ +// Copyright 2023 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +#include "absl/flags/flag.h" +#include "absl/flags/parse.h" + +#include + +#ifdef BAZEL_BUILD +#include "examples/protos/helloworld.grpc.pb.h" +#else +#include "helloworld.grpc.pb.h" +#endif + +ABSL_FLAG(bool, crash_on_errors, false, + "Crash the application on client errors"); +ABSL_FLAG(std::string, target, "localhost:50051", "Server address"); + +using grpc::Channel; +using grpc::ClientContext; +using grpc::Status; +using helloworld::Greeter; +using helloworld::HelloReply; +using helloworld::HelloRequest; + +class GreeterClient { + public: + GreeterClient(std::shared_ptr channel) + : stub_(Greeter::NewStub(channel)) {} + + // Assembles the client's payload, sends it and presents the response back + // from the server. + std::string SayHello(const std::string& user) { + // Data we are sending to the server. + HelloRequest request; + request.set_name(user); + + // Container for the data we expect from the server. + HelloReply reply; + + // Context for the client. It could be used to convey extra information to + // the server and/or tweak certain RPC behaviors. + ClientContext context; + + // The actual RPC. + Status status = stub_->SayHello(&context, request, &reply); + + // Act upon the status of the actual RPC. + if (absl::GetFlag(FLAGS_crash_on_errors)) { + assert(status.ok()); + } + if (status.ok()) { + return reply.message(); + } else { + return "RPC failed"; + } + } + + private: + std::unique_ptr stub_; +}; + +int main(int argc, char** argv) { + absl::ParseCommandLine(argc, argv); + // Instantiate the client. It requires a channel, out of which the actual RPCs + // are created. This channel models a connection to an endpoint specified by + // the argument "--target=" which is the only expected argument. + // We indicate that the channel isn't authenticated (use of + // InsecureChannelCredentials()). + GreeterClient greeter(grpc::CreateChannel( + absl::GetFlag(FLAGS_target), grpc::InsecureChannelCredentials())); + std::string user("world"); + std::string reply = greeter.SayHello(user); + std::cout << "Greeter received: " << reply << std::endl; + + return 0; +} diff --git a/examples/cpp/debugging/greeter_callback_server_admin.cc b/examples/cpp/debugging/greeter_callback_server_admin.cc new file mode 100644 index 0000000000000..397503f4e6d95 --- /dev/null +++ b/examples/cpp/debugging/greeter_callback_server_admin.cc @@ -0,0 +1,86 @@ +// Copyright 2023 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +#include "absl/flags/flag.h" +#include "absl/flags/parse.h" +#include "absl/strings/str_format.h" + +#include +#include +#include +#include + +#ifdef BAZEL_BUILD +#include "examples/protos/helloworld.grpc.pb.h" +#else +#include "helloworld.grpc.pb.h" +#endif + +ABSL_FLAG(uint16_t, port, 50051, "Server port for the service"); + +using grpc::CallbackServerContext; +using grpc::Server; +using grpc::ServerBuilder; +using grpc::ServerUnaryReactor; +using grpc::Status; +using helloworld::Greeter; +using helloworld::HelloReply; +using helloworld::HelloRequest; + +// Logic and data behind the server's behavior. +class GreeterServiceImpl final : public Greeter::CallbackService { + ServerUnaryReactor* SayHello(CallbackServerContext* context, + const HelloRequest* request, + HelloReply* reply) override { + std::string prefix("Hello "); + reply->set_message(prefix + request->name()); + + ServerUnaryReactor* reactor = context->DefaultReactor(); + reactor->Finish(Status::OK); + return reactor; + } +}; + +void RunServer(uint16_t port) { + std::string server_address = absl::StrFormat("0.0.0.0:%d", port); + GreeterServiceImpl service; + + grpc::EnableDefaultHealthCheckService(true); + grpc::reflection::InitProtoReflectionServerBuilderPlugin(); + ServerBuilder builder; + // Enable gRPC Admin Services + grpc::AddAdminServices(&builder); + // Listen on the given address without any authentication mechanism. + builder.AddListeningPort(server_address, grpc::InsecureServerCredentials()); + // Register "service" as the instance through which we'll communicate with + // clients. In this case it corresponds to an *synchronous* service. + builder.RegisterService(&service); + // Finally assemble the server. + std::unique_ptr server(builder.BuildAndStart()); + std::cout << "Server listening on " << server_address << std::endl; + + // Wait for the server to shutdown. Note that some other thread must be + // responsible for shutting down the server for this call to ever return. + server->Wait(); +} + +int main(int argc, char** argv) { + absl::ParseCommandLine(argc, argv); + RunServer(absl::GetFlag(FLAGS_port)); + return 0; +} diff --git a/tools/internal_ci/linux/pull_request/grpc_iwyu.cfg b/examples/cpp/orca/BUILD similarity index 55% rename from tools/internal_ci/linux/pull_request/grpc_iwyu.cfg rename to examples/cpp/orca/BUILD index 4d7fcecc9a6ca..f6641932a91b2 100644 --- a/tools/internal_ci/linux/pull_request/grpc_iwyu.cfg +++ b/examples/cpp/orca/BUILD @@ -1,4 +1,4 @@ -# Copyright 2017 gRPC authors. +# Copyright 2023 the gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,19 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Config file for the internal CI (in protobuf text format) +licenses(["notice"]) -# Location of the continuous shell script in repository. -build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" -timeout_mins: 60 -action { - define_artifacts { - regex: "**/*sponge_log.*" - regex: "github/grpc/reports/**" - } -} - -env_vars { - key: "RUN_TESTS_FLAGS" - value: "-f basictests linux iwyu --inner_jobs 16 -j 1 --internal_ci" -} +cc_binary( + name = "orca_server", + srcs = ["orca_server.cc"], + defines = ["BAZEL_BUILD"], + deps = [ + "//:grpc++", + "//:grpcpp_orca_service", + "//examples/protos:helloworld_cc_grpc", + "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/flags:parse", + "@com_google_absl//absl/strings:str_format", + ], +) diff --git a/examples/cpp/orca/README.md b/examples/cpp/orca/README.md new file mode 100644 index 0000000000000..680a4970e59c7 --- /dev/null +++ b/examples/cpp/orca/README.md @@ -0,0 +1,46 @@ +# gRPC Custom Metrics Example + +You can find a complete set of instructions for building gRPC and running the +examples in the [C++ Quick Start][]. + +This example shows how to implement a server that provides custom metrics usable +by custom load balancing policies. + +Server needs to be setup with metrics recorder and Orca service for sending +these metrics to a client: + +```c++ +GreeterServiceImpl service; +// Setup custom metrics recording +auto server_metric_recorder = + grpc::experimental::ServerMetricRecorder::Create(); +grpc::experimental::OrcaService orca_service( + server_metric_recorder.get(), + grpc::experimental::OrcaService::Options().set_min_report_duration( + absl::Seconds(0.1))); +builder.RegisterService(&orca_service); +grpc::ServerBuilder::experimental_type(&builder).EnableCallMetricRecording( + nullptr); +``` + +Afterwards per-request metrics can be reported from the gRPC service +implementation using the metric recorder from the request context: + +```c++ +auto recorder = context->ExperimentalGetCallMetricRecorder(); +if (recorder == nullptr) { + return Status(grpc::StatusCode::INTERNAL, + "Unable to access metrics recorder. Make sure " + "EnableCallMetricRecording had been called."); +} +recorder->RecordCpuUtilizationMetric(0.5); +``` + +Out of band metrics can be reported using the `server_metric_recorder` +directly: + +```c++ +server_metric_recorder->SetCpuUtilization(0.75); +``` + +[C++ Quick Start]: https://grpc.io/docs/languages/cpp/quickstart diff --git a/examples/cpp/orca/orca_server.cc b/examples/cpp/orca/orca_server.cc new file mode 100644 index 0000000000000..d05d763a6a588 --- /dev/null +++ b/examples/cpp/orca/orca_server.cc @@ -0,0 +1,101 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#include +#include +#include +#include + +#include "absl/flags/flag.h" +#include "absl/flags/parse.h" +#include "absl/strings/str_format.h" +#include "examples/protos/helloworld.grpc.pb.h" + +#include +#include +#include +#include +#include + +using grpc::CallbackServerContext; +using grpc::Server; +using grpc::ServerBuilder; +using grpc::ServerUnaryReactor; +using grpc::Status; +using helloworld::Greeter; +using helloworld::HelloReply; +using helloworld::HelloRequest; + +ABSL_FLAG(uint16_t, port, 50051, "Server port for the service"); + +// Logic and data behind the server's behavior. +class GreeterServiceImpl final : public Greeter::CallbackService { + ServerUnaryReactor* SayHello(CallbackServerContext* context, + const HelloRequest* request, + HelloReply* reply) override { + ServerUnaryReactor* reactor = context->DefaultReactor(); + // Obtain the call metric recorder and use it to report the number of + // DB queries (custom cost metric) and CPU utilization. + auto recorder = context->ExperimentalGetCallMetricRecorder(); + if (recorder == nullptr) { + reactor->Finish({grpc::StatusCode::INTERNAL, + "Unable to access metrics recorder. Make sure " + "EnableCallMetricRecording had been called."}); + return reactor; + } + recorder->RecordRequestCostMetric("db_queries", 10); + recorder->RecordCpuUtilizationMetric(0.5); + std::string prefix("Hello "); + reply->set_message(prefix + request->name()); + reactor->Finish(Status::OK); + return reactor; + } +}; + +void RunServer(uint16_t port) { + std::string server_address = absl::StrFormat("0.0.0.0:%d", port); + ServerBuilder builder; + GreeterServiceImpl service; + // Setup custom metrics recording. Note that this recorder may be use to send + // the out-of-band metrics to the client. + auto server_metric_recorder = + grpc::experimental::ServerMetricRecorder::Create(); + grpc::experimental::OrcaService orca_service( + server_metric_recorder.get(), + grpc::experimental::OrcaService::Options().set_min_report_duration( + absl::Seconds(0.1))); + builder.RegisterService(&orca_service); + grpc::ServerBuilder::experimental_type(&builder).EnableCallMetricRecording( + server_metric_recorder.get()); + // Resume setting up gRPC server as usual + grpc::EnableDefaultHealthCheckService(true); + // Listen on the given address without any authentication mechanism. + builder.AddListeningPort(server_address, grpc::InsecureServerCredentials()); + // Register "service" as the instance through which we'll communicate with + // clients. In this case it corresponds to an *synchronous* service. + builder.RegisterService(&service); + // Finally assemble the server. + std::unique_ptr server(builder.BuildAndStart()); + std::cout << "Server listening on " << server_address << std::endl; + server->Wait(); +} + +int main(int argc, char** argv) { + absl::ParseCommandLine(argc, argv); + RunServer(absl::GetFlag(FLAGS_port)); + return 0; +} diff --git a/tools/run_tests/sanity/check_do_not_submit.sh b/examples/cpp/wait_for_ready/BUILD old mode 100755 new mode 100644 similarity index 60% rename from tools/run_tests/sanity/check_do_not_submit.sh rename to examples/cpp/wait_for_ready/BUILD index 6e0438cd5cb67..00047fa2f9bd3 --- a/tools/run_tests/sanity/check_do_not_submit.sh +++ b/examples/cpp/wait_for_ready/BUILD @@ -1,5 +1,4 @@ -#! /bin/bash -# Copyright 2021 The gRPC Authors +# Copyright 2023 the gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,12 +11,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Checks if any file contains "DO NOT SUBMIT" -cd "$(dirname "$0")/../../.." || exit 1 -git grep -Irn 'DO NOT SUBMIT' -- \ - './*' \ - ':!*check_do_not_submit.sh' \ - ':!third_party/' -test $? -eq 1 || exit 1 +licenses(["notice"]) + +cc_binary( + name = "greeter_callback_client", + srcs = ["greeter_callback_client.cc"], + defines = ["BAZEL_BUILD"], + deps = [ + "//:grpc++", + "//examples/protos:helloworld_cc_grpc", + "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/flags:parse", + ], +) diff --git a/examples/cpp/wait_for_ready/CMakeLists.txt b/examples/cpp/wait_for_ready/CMakeLists.txt new file mode 100644 index 0000000000000..06024dfbc9b8b --- /dev/null +++ b/examples/cpp/wait_for_ready/CMakeLists.txt @@ -0,0 +1,70 @@ +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# cmake build file for C++ helloworld example. +# Assumes protobuf and gRPC have been installed using cmake. +# See cmake_externalproject/CMakeLists.txt for all-in-one cmake build +# that automatically builds all the dependencies before building helloworld. + +cmake_minimum_required(VERSION 3.8) + +project(HelloWorld C CXX) + +include(../cmake/common.cmake) + +# Proto file +get_filename_component(hw_proto "../../protos/helloworld.proto" ABSOLUTE) +get_filename_component(hw_proto_path "${hw_proto}" PATH) + +# Generated sources +set(hw_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/helloworld.pb.cc") +set(hw_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/helloworld.pb.h") +set(hw_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/helloworld.grpc.pb.cc") +set(hw_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/helloworld.grpc.pb.h") +add_custom_command( + OUTPUT "${hw_proto_srcs}" "${hw_proto_hdrs}" "${hw_grpc_srcs}" "${hw_grpc_hdrs}" + COMMAND ${_PROTOBUF_PROTOC} + ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}" + --cpp_out "${CMAKE_CURRENT_BINARY_DIR}" + -I "${hw_proto_path}" + --plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN_EXECUTABLE}" + "${hw_proto}" + DEPENDS "${hw_proto}") + +# Include generated *.pb.h files +include_directories("${CMAKE_CURRENT_BINARY_DIR}") + +# hw_grpc_proto +add_library(hw_grpc_proto + ${hw_grpc_srcs} + ${hw_grpc_hdrs} + ${hw_proto_srcs} + ${hw_proto_hdrs}) +target_link_libraries(hw_grpc_proto + ${_REFLECTION} + ${_GRPC_GRPCPP} + ${_PROTOBUF_LIBPROTOBUF}) + +# Targets greeter_[async_](client|server) +foreach(_target + greeter_callback_client greeter_callback_server + add_executable(${_target} "${_target}.cc") + target_link_libraries(${_target} + hw_grpc_proto + absl::flags + absl::flags_parse + ${_REFLECTION} + ${_GRPC_GRPCPP} + ${_PROTOBUF_LIBPROTOBUF}) +endforeach() diff --git a/examples/cpp/wait_for_ready/README.md b/examples/cpp/wait_for_ready/README.md new file mode 100644 index 0000000000000..1de220b99a33b --- /dev/null +++ b/examples/cpp/wait_for_ready/README.md @@ -0,0 +1,32 @@ +# gRPC C++ Wait-For-Ready Example + +The Wait-For-Ready example builds on the +[Hello World Example](https://github.com/grpc/grpc/tree/master/examples/cpp/helloworld) +and changes the gRPC client and server to show how to set wait-for-ready. + +For more information on wait-for-ready in gRPC, please refer to +[gRPC Wait For Ready Semantics](https://github.com/grpc/grpc/blob/master/doc/wait-for-ready.md). + +## Running the example + +First run the client - + +``` +$ tools/bazel run examples/cpp/wait_for_ready:greeter_callback_client +``` + +On running this, we'll see 10 RPCs failed due to "Connection refused" errors. +These RPCs did not have WAIT_FOR_READY set, resulting in the RPCs not waiting +for the channel to be connected. + +The next 10 RPCs have WAIT_FOR_READY set, so the client will be waiting for the +channel to be ready before progressing. + +Now, on a separate terminal, run the server - + +``` +$ tools/bazel run examples/cpp/helloworld:greeter_callback_server +``` + +The client channel should now be able to connect to the server, and the RPCs +should succeed. diff --git a/examples/cpp/wait_for_ready/greeter_callback_client.cc b/examples/cpp/wait_for_ready/greeter_callback_client.cc new file mode 100644 index 0000000000000..1c7865ed2bfaf --- /dev/null +++ b/examples/cpp/wait_for_ready/greeter_callback_client.cc @@ -0,0 +1,107 @@ +// +// +// Copyright 2023 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// + +#include +#include +#include +#include +#include +#include + +#include "absl/flags/flag.h" +#include "absl/flags/parse.h" + +#include + +#ifdef BAZEL_BUILD +#include "examples/protos/helloworld.grpc.pb.h" +#else +#include "helloworld.grpc.pb.h" +#endif + +ABSL_FLAG(std::string, target, "localhost:50051", "Server address"); + +using grpc::Channel; +using grpc::ClientContext; +using grpc::Status; +using helloworld::Greeter; +using helloworld::HelloReply; +using helloworld::HelloRequest; + +class GreeterClient { + public: + GreeterClient(std::shared_ptr channel) + : stub_(Greeter::NewStub(channel)) {} + + // Assembles the client's payload, sends it and presents the response back + // from the server. + std::string SayHello(const std::string& user, bool wait_for_ready) { + HelloRequest request; + request.set_name(user); + HelloReply reply; + ClientContext context; + context.set_wait_for_ready(wait_for_ready); + std::mutex mu; + std::condition_variable cv; + bool done = false; + Status status; + stub_->async()->SayHello(&context, &request, &reply, + [&mu, &cv, &done, &status](Status s) { + status = std::move(s); + std::lock_guard lock(mu); + done = true; + cv.notify_one(); + }); + + std::unique_lock lock(mu); + while (!done) { + cv.wait(lock); + } + if (status.ok()) { + return reply.message(); + } else { + std::cout << status.error_code() << ": " << status.error_message() + << "\n"; + return "RPC failed"; + } + } + + private: + std::unique_ptr stub_; +}; + +int main(int argc, char** argv) { + absl::ParseCommandLine(argc, argv); + std::string target_str = absl::GetFlag(FLAGS_target); + GreeterClient greeter( + grpc::CreateChannel(target_str, grpc::InsecureChannelCredentials())); + std::string user("world"); + // First send an RPC without wait_for_ready. If the server is not running, + // this RPC will fail immediately. + std::cout << "Greeter received: " + << greeter.SayHello(user, /*wait_for_ready=*/false) << "\n"; + std::cout << "\nWe will now send RPCs with wait_for_ready set. If the " + "server is not running already, please start it now.\n"; + // Now send RPC with wait_for_ready for set. Even if the server is not + // running, the RPC will still wait for the deadline to expire before + // failing. + std::cout << "Greeter received: " + << greeter.SayHello(user, /*wait_for_ready=*/true) << "\n"; + + return 0; +} diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 1d2d74a278292..14b9978f503b6 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -951,6 +951,7 @@ Pod::Spec.new do |s| 'src/core/lib/event_engine/posix_engine/grpc_polled_fd_posix.h', 'src/core/lib/event_engine/posix_engine/internal_errqueue.h', 'src/core/lib/event_engine/posix_engine/lockfree_event.h', + 'src/core/lib/event_engine/posix_engine/native_dns_resolver.h', 'src/core/lib/event_engine/posix_engine/posix_endpoint.h', 'src/core/lib/event_engine/posix_engine/posix_engine.h', 'src/core/lib/event_engine/posix_engine/posix_engine_closure.h', @@ -965,6 +966,7 @@ Pod::Spec.new do |s| 'src/core/lib/event_engine/posix_engine/wakeup_fd_pipe.h', 'src/core/lib/event_engine/posix_engine/wakeup_fd_posix.h', 'src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.h', + 'src/core/lib/event_engine/ref_counted_dns_resolver_interface.h', 'src/core/lib/event_engine/resolved_address_internal.h', 'src/core/lib/event_engine/shim.h', 'src/core/lib/event_engine/tcp_socket_utils.h', @@ -1147,6 +1149,7 @@ Pod::Spec.new do |s| 'src/core/lib/promise/race.h', 'src/core/lib/promise/seq.h', 'src/core/lib/promise/sleep.h', + 'src/core/lib/promise/status_flag.h', 'src/core/lib/promise/trace.h', 'src/core/lib/promise/try_seq.h', 'src/core/lib/resolver/endpoint_addresses.h', @@ -2182,6 +2185,7 @@ Pod::Spec.new do |s| 'src/core/lib/event_engine/posix_engine/grpc_polled_fd_posix.h', 'src/core/lib/event_engine/posix_engine/internal_errqueue.h', 'src/core/lib/event_engine/posix_engine/lockfree_event.h', + 'src/core/lib/event_engine/posix_engine/native_dns_resolver.h', 'src/core/lib/event_engine/posix_engine/posix_endpoint.h', 'src/core/lib/event_engine/posix_engine/posix_engine.h', 'src/core/lib/event_engine/posix_engine/posix_engine_closure.h', @@ -2196,6 +2200,7 @@ Pod::Spec.new do |s| 'src/core/lib/event_engine/posix_engine/wakeup_fd_pipe.h', 'src/core/lib/event_engine/posix_engine/wakeup_fd_posix.h', 'src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.h', + 'src/core/lib/event_engine/ref_counted_dns_resolver_interface.h', 'src/core/lib/event_engine/resolved_address_internal.h', 'src/core/lib/event_engine/shim.h', 'src/core/lib/event_engine/tcp_socket_utils.h', @@ -2378,6 +2383,7 @@ Pod::Spec.new do |s| 'src/core/lib/promise/race.h', 'src/core/lib/promise/seq.h', 'src/core/lib/promise/sleep.h', + 'src/core/lib/promise/status_flag.h', 'src/core/lib/promise/trace.h', 'src/core/lib/promise/try_seq.h', 'src/core/lib/resolver/endpoint_addresses.h', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 2e78799064665..303ac9ad5b295 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -1363,7 +1363,6 @@ Pod::Spec.new do |s| 'src/core/lib/event_engine/forkable.h', 'src/core/lib/event_engine/grpc_polled_fd.h', 'src/core/lib/event_engine/handle_containers.h', - 'src/core/lib/event_engine/memory_allocator.cc', 'src/core/lib/event_engine/memory_allocator_factory.h', 'src/core/lib/event_engine/nameser.h', 'src/core/lib/event_engine/poller.h', @@ -1380,6 +1379,8 @@ Pod::Spec.new do |s| 'src/core/lib/event_engine/posix_engine/internal_errqueue.h', 'src/core/lib/event_engine/posix_engine/lockfree_event.cc', 'src/core/lib/event_engine/posix_engine/lockfree_event.h', + 'src/core/lib/event_engine/posix_engine/native_dns_resolver.cc', + 'src/core/lib/event_engine/posix_engine/native_dns_resolver.h', 'src/core/lib/event_engine/posix_engine/posix_endpoint.cc', 'src/core/lib/event_engine/posix_engine/posix_endpoint.h', 'src/core/lib/event_engine/posix_engine/posix_engine.cc', @@ -1406,6 +1407,7 @@ Pod::Spec.new do |s| 'src/core/lib/event_engine/posix_engine/wakeup_fd_posix.h', 'src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.cc', 'src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.h', + 'src/core/lib/event_engine/ref_counted_dns_resolver_interface.h', 'src/core/lib/event_engine/resolved_address.cc', 'src/core/lib/event_engine/resolved_address_internal.h', 'src/core/lib/event_engine/shim.cc', @@ -1755,6 +1757,7 @@ Pod::Spec.new do |s| 'src/core/lib/promise/seq.h', 'src/core/lib/promise/sleep.cc', 'src/core/lib/promise/sleep.h', + 'src/core/lib/promise/status_flag.h', 'src/core/lib/promise/trace.cc', 'src/core/lib/promise/trace.h', 'src/core/lib/promise/try_seq.h', @@ -2944,6 +2947,7 @@ Pod::Spec.new do |s| 'src/core/lib/event_engine/posix_engine/grpc_polled_fd_posix.h', 'src/core/lib/event_engine/posix_engine/internal_errqueue.h', 'src/core/lib/event_engine/posix_engine/lockfree_event.h', + 'src/core/lib/event_engine/posix_engine/native_dns_resolver.h', 'src/core/lib/event_engine/posix_engine/posix_endpoint.h', 'src/core/lib/event_engine/posix_engine/posix_engine.h', 'src/core/lib/event_engine/posix_engine/posix_engine_closure.h', @@ -2958,6 +2962,7 @@ Pod::Spec.new do |s| 'src/core/lib/event_engine/posix_engine/wakeup_fd_pipe.h', 'src/core/lib/event_engine/posix_engine/wakeup_fd_posix.h', 'src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.h', + 'src/core/lib/event_engine/ref_counted_dns_resolver_interface.h', 'src/core/lib/event_engine/resolved_address_internal.h', 'src/core/lib/event_engine/shim.h', 'src/core/lib/event_engine/tcp_socket_utils.h', @@ -3140,6 +3145,7 @@ Pod::Spec.new do |s| 'src/core/lib/promise/race.h', 'src/core/lib/promise/seq.h', 'src/core/lib/promise/sleep.h', + 'src/core/lib/promise/status_flag.h', 'src/core/lib/promise/trace.h', 'src/core/lib/promise/try_seq.h', 'src/core/lib/resolver/endpoint_addresses.h', diff --git a/gRPC.podspec b/gRPC.podspec index f487dc775cf34..1ba1ac974bae9 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -32,6 +32,8 @@ Pod::Spec.new do |s| :tag => "v#{version}", } + s.resource = 'src/objective-c/PrivacyInfo.xcprivacy' + name = 'GRPCClient' s.module_name = name s.header_dir = name diff --git a/grpc.gemspec b/grpc.gemspec index ffcdbc601fcad..4b796f508bdcf 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -1266,7 +1266,6 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/event_engine/forkable.h ) s.files += %w( src/core/lib/event_engine/grpc_polled_fd.h ) s.files += %w( src/core/lib/event_engine/handle_containers.h ) - s.files += %w( src/core/lib/event_engine/memory_allocator.cc ) s.files += %w( src/core/lib/event_engine/memory_allocator_factory.h ) s.files += %w( src/core/lib/event_engine/nameser.h ) s.files += %w( src/core/lib/event_engine/poller.h ) @@ -1283,6 +1282,8 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/event_engine/posix_engine/internal_errqueue.h ) s.files += %w( src/core/lib/event_engine/posix_engine/lockfree_event.cc ) s.files += %w( src/core/lib/event_engine/posix_engine/lockfree_event.h ) + s.files += %w( src/core/lib/event_engine/posix_engine/native_dns_resolver.cc ) + s.files += %w( src/core/lib/event_engine/posix_engine/native_dns_resolver.h ) s.files += %w( src/core/lib/event_engine/posix_engine/posix_endpoint.cc ) s.files += %w( src/core/lib/event_engine/posix_engine/posix_endpoint.h ) s.files += %w( src/core/lib/event_engine/posix_engine/posix_engine.cc ) @@ -1309,6 +1310,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/event_engine/posix_engine/wakeup_fd_posix.h ) s.files += %w( src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.cc ) s.files += %w( src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.h ) + s.files += %w( src/core/lib/event_engine/ref_counted_dns_resolver_interface.h ) s.files += %w( src/core/lib/event_engine/resolved_address.cc ) s.files += %w( src/core/lib/event_engine/resolved_address_internal.h ) s.files += %w( src/core/lib/event_engine/shim.cc ) @@ -1658,6 +1660,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/promise/seq.h ) s.files += %w( src/core/lib/promise/sleep.cc ) s.files += %w( src/core/lib/promise/sleep.h ) + s.files += %w( src/core/lib/promise/status_flag.h ) s.files += %w( src/core/lib/promise/trace.cc ) s.files += %w( src/core/lib/promise/trace.h ) s.files += %w( src/core/lib/promise/try_seq.h ) diff --git a/grpc.gyp b/grpc.gyp index f81c129017235..4b043c457b2a4 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -764,12 +764,12 @@ 'src/core/lib/event_engine/default_event_engine_factory.cc', 'src/core/lib/event_engine/event_engine.cc', 'src/core/lib/event_engine/forkable.cc', - 'src/core/lib/event_engine/memory_allocator.cc', 'src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc', 'src/core/lib/event_engine/posix_engine/ev_poll_posix.cc', 'src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc', 'src/core/lib/event_engine/posix_engine/internal_errqueue.cc', 'src/core/lib/event_engine/posix_engine/lockfree_event.cc', + 'src/core/lib/event_engine/posix_engine/native_dns_resolver.cc', 'src/core/lib/event_engine/posix_engine/posix_endpoint.cc', 'src/core/lib/event_engine/posix_engine/posix_engine.cc', 'src/core/lib/event_engine/posix_engine/posix_engine_listener.cc', @@ -1284,12 +1284,12 @@ 'src/core/lib/event_engine/default_event_engine_factory.cc', 'src/core/lib/event_engine/event_engine.cc', 'src/core/lib/event_engine/forkable.cc', - 'src/core/lib/event_engine/memory_allocator.cc', 'src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc', 'src/core/lib/event_engine/posix_engine/ev_poll_posix.cc', 'src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc', 'src/core/lib/event_engine/posix_engine/internal_errqueue.cc', 'src/core/lib/event_engine/posix_engine/lockfree_event.cc', + 'src/core/lib/event_engine/posix_engine/native_dns_resolver.cc', 'src/core/lib/event_engine/posix_engine/posix_endpoint.cc', 'src/core/lib/event_engine/posix_engine/posix_engine.cc', 'src/core/lib/event_engine/posix_engine/posix_engine_listener.cc', @@ -2063,12 +2063,12 @@ 'src/core/lib/event_engine/default_event_engine_factory.cc', 'src/core/lib/event_engine/event_engine.cc', 'src/core/lib/event_engine/forkable.cc', - 'src/core/lib/event_engine/memory_allocator.cc', 'src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc', 'src/core/lib/event_engine/posix_engine/ev_poll_posix.cc', 'src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc', 'src/core/lib/event_engine/posix_engine/internal_errqueue.cc', 'src/core/lib/event_engine/posix_engine/lockfree_event.cc', + 'src/core/lib/event_engine/posix_engine/native_dns_resolver.cc', 'src/core/lib/event_engine/posix_engine/posix_endpoint.cc', 'src/core/lib/event_engine/posix_engine/posix_engine.cc', 'src/core/lib/event_engine/posix_engine/posix_engine_listener.cc', diff --git a/include/grpc/event_engine/event_engine.h b/include/grpc/event_engine/event_engine.h index 4beca657625a3..20cbc64f52fde 100644 --- a/include/grpc/event_engine/event_engine.h +++ b/include/grpc/event_engine/event_engine.h @@ -255,6 +255,45 @@ class EventEngine : public std::enable_shared_from_this { /// values are expected to remain valid for the life of the Endpoint. virtual const ResolvedAddress& GetPeerAddress() const = 0; virtual const ResolvedAddress& GetLocalAddress() const = 0; + + /// A method which allows users to query whether an Endpoint implementation + /// supports a specified extension. The name of the extension is provided + /// as an input. + /// + /// An extension could be any type with a unique string id. Each extension + /// may support additional capabilities and if the Endpoint implementation + /// supports the queried extension, it should return a valid pointer to the + /// extension type. + /// + /// E.g., use case of an EventEngine::Endpoint supporting a custom + /// extension. + /// + /// class CustomEndpointExtension { + /// public: + /// static constexpr std::string name = "my.namespace.extension_name"; + /// void Process() { ... } + /// } + /// + /// + /// class CustomEndpoint : + /// public EventEngine::Endpoint, CustomEndpointExtension { + /// public: + /// void* QueryExtension(absl::string_view id) override { + /// if (id == CustomEndpointExtension::name) { + /// return static_cast(this); + /// } + /// return nullptr; + /// } + /// ... + /// } + /// + /// auto ext_ = + /// static_cast( + /// endpoint->QueryExtension(CustomrEndpointExtension::name)); + /// if (ext_ != nullptr) { ext_->Process(); } + /// + /// + virtual void* QueryExtension(absl::string_view /*id*/) { return nullptr; } }; /// Called when a new connection is established. diff --git a/include/grpc/event_engine/internal/memory_allocator_impl.h b/include/grpc/event_engine/internal/memory_allocator_impl.h index 8397b3d970643..e34ffc7f9c2a4 100644 --- a/include/grpc/event_engine/internal/memory_allocator_impl.h +++ b/include/grpc/event_engine/internal/memory_allocator_impl.h @@ -50,6 +50,12 @@ class MemoryAllocatorImpl /// request.max() inclusively. virtual size_t Reserve(MemoryRequest request) = 0; + /// Allocate a slice, using MemoryRequest to size the number of returned + /// bytes. For a variable length request, check the returned slice length to + /// verify how much memory was allocated. Takes care of reserving memory for + /// any relevant control structures also. + virtual grpc_slice MakeSlice(MemoryRequest request) = 0; + /// Release some bytes that were previously reserved. /// If more bytes are released than were reserved, we will have undefined /// behavior. diff --git a/include/grpc/event_engine/memory_allocator.h b/include/grpc/event_engine/memory_allocator.h index b3143d8dd6a79..cd14e3a93371f 100644 --- a/include/grpc/event_engine/memory_allocator.h +++ b/include/grpc/event_engine/memory_allocator.h @@ -134,7 +134,9 @@ class MemoryAllocator { /// bytes. For a variable length request, check the returned slice length to /// verify how much memory was allocated. Takes care of reserving memory for /// any relevant control structures also. - grpc_slice MakeSlice(MemoryRequest request); + grpc_slice MakeSlice(MemoryRequest request) { + return allocator_->MakeSlice(request); + } /// A C++ allocator for containers of T. template diff --git a/include/grpc/impl/channel_arg_names.h b/include/grpc/impl/channel_arg_names.h index b4cece27c9ef6..534300f22c24d 100644 --- a/include/grpc/impl/channel_arg_names.h +++ b/include/grpc/impl/channel_arg_names.h @@ -384,6 +384,11 @@ * Defaults to 250ms. */ #define GRPC_ARG_HAPPY_EYEBALLS_CONNECTION_ATTEMPT_DELAY_MS \ "grpc.happy_eyeballs_connection_attempt_delay_ms" +/** It accepts a MemoryAllocatorFactory as input and If specified, it forces + * the default event engine to use memory allocators created using the provided + * factory. */ +#define GRPC_ARG_EVENT_ENGINE_USE_MEMORY_ALLOCATOR_FACTORY \ + "grpc.event_engine_use_memory_allocator_factory" /** \} */ #endif /* GRPC_IMPL_CHANNEL_ARG_NAMES_H */ diff --git a/include/grpc/impl/slice_type.h b/include/grpc/impl/slice_type.h index 5d618740553cc..7d9dfb78123b0 100644 --- a/include/grpc/impl/slice_type.h +++ b/include/grpc/impl/slice_type.h @@ -74,7 +74,7 @@ struct grpc_slice { } data; }; -#define GRPC_SLICE_BUFFER_INLINE_ELEMENTS 8 +#define GRPC_SLICE_BUFFER_INLINE_ELEMENTS 7 /** Represents an expandable array of slices, to be interpreted as a single item. */ diff --git a/package.xml b/package.xml index 4ff9ece3cdd35..e0e276e52c71c 100644 --- a/package.xml +++ b/package.xml @@ -1248,7 +1248,6 @@ - @@ -1265,6 +1264,8 @@ + + @@ -1291,6 +1292,7 @@ + @@ -1640,6 +1642,7 @@ + diff --git a/requirements.bazel.txt b/requirements.bazel.txt index 851dbd15a38da..532562389fd8e 100644 --- a/requirements.bazel.txt +++ b/requirements.bazel.txt @@ -16,3 +16,4 @@ xds-protos==0.0.11 opencensus==0.10.0 opencensus-ext-stackdriver==0.8.0 absl-py==1.4.0 +googleapis-common-protos==1.61.0 diff --git a/src/core/BUILD b/src/core/BUILD index 351f101b81968..449649facdcfd 100644 --- a/src/core/BUILD +++ b/src/core/BUILD @@ -1163,9 +1163,6 @@ grpc_cc_library( grpc_cc_library( name = "event_engine_memory_allocator", - srcs = [ - "lib/event_engine/memory_allocator.cc", - ], hdrs = [ "//:include/grpc/event_engine/internal/memory_allocator_impl.h", "//:include/grpc/event_engine/memory_allocator.h", @@ -1175,7 +1172,6 @@ grpc_cc_library( language = "c++", deps = [ "slice", - "slice_refcount", "//:gpr_platform", ], ) @@ -1221,6 +1217,7 @@ grpc_cc_library( "race", "resource_quota_trace", "seq", + "slice_refcount", "time", "useful", "//:gpr", @@ -1540,6 +1537,18 @@ grpc_cc_library( ], ) +grpc_cc_library( + name = "event_engine_query_extensions", + hdrs = [ + "lib/event_engine/query_extensions.h", + ], + external_deps = ["absl/strings"], + deps = [ + "//:event_engine_base_hdrs", + "//:gpr_platform", + ], +) + grpc_cc_library( name = "event_engine_work_queue", hdrs = [ @@ -2129,6 +2138,7 @@ grpc_cc_library( "forkable", "init_internally", "iomgr_port", + "native_dns_resolver", "no_destruct", "posix_event_engine_base_hdrs", "posix_event_engine_closure", @@ -2139,7 +2149,9 @@ grpc_cc_library( "posix_event_engine_tcp_socket_utils", "posix_event_engine_timer", "posix_event_engine_timer_manager", + "ref_counted_dns_resolver_interface", "useful", + "//:config_vars", "//:event_engine_base_hdrs", "//:gpr", "//:grpc_trace", @@ -2460,6 +2472,41 @@ grpc_cc_library( ], ) +grpc_cc_library( + name = "ref_counted_dns_resolver_interface", + hdrs = ["lib/event_engine/ref_counted_dns_resolver_interface.h"], + external_deps = ["absl/strings"], + deps = [ + "//:event_engine_base_hdrs", + "//:gpr_platform", + "//:orphanable", + ], +) + +grpc_cc_library( + name = "native_dns_resolver", + srcs = [ + "lib/event_engine/posix_engine/native_dns_resolver.cc", + ], + hdrs = [ + "lib/event_engine/posix_engine/native_dns_resolver.h", + ], + external_deps = [ + "absl/functional:any_invocable", + "absl/status", + "absl/status:statusor", + "absl/strings", + "absl/strings:str_format", + ], + deps = [ + "iomgr_port", + "ref_counted_dns_resolver_interface", + "useful", + "//:event_engine_base_hdrs", + "//:gpr", + ], +) + grpc_cc_library( name = "ares_resolver", srcs = [ @@ -2495,6 +2542,7 @@ grpc_cc_library( "posix_event_engine_closure", "posix_event_engine_event_poller", "posix_event_engine_tcp_socket_utils", + "ref_counted_dns_resolver_interface", "resolved_address", "slice", "windows_iocp", diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc index b5064ceb20fd9..91b4712c61ac9 100644 --- a/src/core/ext/filters/client_channel/client_channel.cc +++ b/src/core/ext/filters/client_channel/client_channel.cc @@ -315,6 +315,14 @@ class ClientChannel::PromiseBasedCallData : public ClientChannel::CallData { public: explicit PromiseBasedCallData(ClientChannel* chand) : chand_(chand) {} + ~PromiseBasedCallData() override { + if (was_queued_ && client_initial_metadata_ != nullptr) { + MutexLock lock(&chand_->resolution_mu_); + RemoveCallFromResolverQueuedCallsLocked(); + chand_->resolver_queued_calls_.erase(this); + } + } + ArenaPromise> MakeNameResolutionPromise( CallArgs call_args) { pollent_ = NowOrNever(call_args.polling_entity->WaitAndCopy()).value(); @@ -399,6 +407,7 @@ class ClientChannel::PromiseBasedCallData : public ClientChannel::CallData { const grpc_channel_filter ClientChannel::kFilterVtableWithPromises = { ClientChannel::FilterBasedCallData::StartTransportStreamOpBatch, ClientChannel::MakeCallPromise, + /* init_call: */ nullptr, ClientChannel::StartTransportOp, sizeof(ClientChannel::FilterBasedCallData), ClientChannel::FilterBasedCallData::Init, @@ -415,6 +424,7 @@ const grpc_channel_filter ClientChannel::kFilterVtableWithPromises = { const grpc_channel_filter ClientChannel::kFilterVtableWithoutPromises = { ClientChannel::FilterBasedCallData::StartTransportStreamOpBatch, nullptr, + /* init_call: */ nullptr, ClientChannel::StartTransportOp, sizeof(ClientChannel::FilterBasedCallData), ClientChannel::FilterBasedCallData::Init, @@ -562,6 +572,7 @@ class DynamicTerminationFilter::CallData { const grpc_channel_filter DynamicTerminationFilter::kFilterVtable = { DynamicTerminationFilter::CallData::StartTransportStreamOpBatch, DynamicTerminationFilter::MakeCallPromise, + /* init_call: */ nullptr, DynamicTerminationFilter::StartTransportOp, sizeof(DynamicTerminationFilter::CallData), DynamicTerminationFilter::CallData::Init, diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/cds.cc b/src/core/ext/filters/client_channel/lb_policy/xds/cds.cc index 7e42338d70d86..ea42f22e7c078 100644 --- a/src/core/ext/filters/client_channel/lb_policy/xds/cds.cc +++ b/src/core/ext/filters/client_channel/lb_policy/xds/cds.cc @@ -199,21 +199,18 @@ class CdsLb : public LoadBalancingPolicy { // The root of the tree is config_->cluster(). std::map watchers_; - // TODO(roth, yashkt): These are here because we need to handle - // pollset_set linkage as clusters are added or removed from the - // XdsCertificateProvider. However, in the aggregate cluster case, - // there may be multiple clusters in the same cert provider, and we're - // only tracking the cert providers for the most recent underlying - // cluster here. I think this is a bug that could cause us to starve - // the underlying cert providers of polling. However, it is not - // actually causing any problem in practice today, because (a) we have - // no cert provider impl that relies on gRPC's polling and (b) - // probably no one is actually configuring an aggregate cluster with - // different cert providers in different underlying clusters. - // Hopefully, this problem won't be an issue in practice until after - // the EventEngine migration is done, at which point the need for - // handling pollset_set linkage will go away, and these fields can - // simply be removed. + // TODO(roth, yashkt): These are here because XdsCertificateProvider + // does not store the actual underlying cert providers, it stores only + // their distributors, so we need to hold a ref to the cert providers + // here. However, in the aggregate cluster case, there may be multiple + // clusters in the same cert provider, and we're only tracking the cert + // providers for the most recent underlying cluster here. This is + // clearly a bug, and I think it will cause us to stop getting updates + // for all but one of the cert providers in the aggregate cluster + // case. Need to figure out the right way to fix this -- I don't + // think we want to store another map here, so ideally, we should just + // have XdsCertificateProvider actually hold the refs to the cert + // providers instead of just the distributors. RefCountedPtr root_certificate_provider_; RefCountedPtr identity_certificate_provider_; @@ -610,20 +607,7 @@ absl::Status CdsLb::UpdateXdsCertificateProvider( root_provider_instance_name, "\" not recognized.")); } } - if (root_certificate_provider_ != new_root_provider) { - if (root_certificate_provider_ != nullptr && - root_certificate_provider_->interested_parties() != nullptr) { - grpc_pollset_set_del_pollset_set( - interested_parties(), - root_certificate_provider_->interested_parties()); - } - if (new_root_provider != nullptr && - new_root_provider->interested_parties() != nullptr) { - grpc_pollset_set_add_pollset_set(interested_parties(), - new_root_provider->interested_parties()); - } - root_certificate_provider_ = std::move(new_root_provider); - } + root_certificate_provider_ = std::move(new_root_provider); xds_certificate_provider_->UpdateRootCertNameAndDistributor( cluster_name, root_provider_cert_name, root_certificate_provider_ == nullptr @@ -647,20 +631,7 @@ absl::Status CdsLb::UpdateXdsCertificateProvider( identity_provider_instance_name, "\" not recognized.")); } } - if (identity_certificate_provider_ != new_identity_provider) { - if (identity_certificate_provider_ != nullptr && - identity_certificate_provider_->interested_parties() != nullptr) { - grpc_pollset_set_del_pollset_set( - interested_parties(), - identity_certificate_provider_->interested_parties()); - } - if (new_identity_provider != nullptr && - new_identity_provider->interested_parties() != nullptr) { - grpc_pollset_set_add_pollset_set( - interested_parties(), new_identity_provider->interested_parties()); - } - identity_certificate_provider_ = std::move(new_identity_provider); - } + identity_certificate_provider_ = std::move(new_identity_provider); xds_certificate_provider_->UpdateIdentityCertNameAndDistributor( cluster_name, identity_provider_cert_name, identity_certificate_provider_ == nullptr diff --git a/src/core/ext/filters/client_channel/retry_filter.cc b/src/core/ext/filters/client_channel/retry_filter.cc index cb29fbee2a48f..f2d393f9484cb 100644 --- a/src/core/ext/filters/client_channel/retry_filter.cc +++ b/src/core/ext/filters/client_channel/retry_filter.cc @@ -143,6 +143,7 @@ const RetryMethodConfig* RetryFilter::GetRetryPolicy( const grpc_channel_filter RetryFilter::kVtable = { RetryFilter::LegacyCallData::StartTransportStreamOpBatch, nullptr, + /* init_call: */ nullptr, RetryFilter::StartTransportOp, sizeof(RetryFilter::LegacyCallData), RetryFilter::LegacyCallData::Init, diff --git a/src/core/ext/filters/deadline/deadline_filter.cc b/src/core/ext/filters/deadline/deadline_filter.cc index 94a4f65225e4e..588827b95842e 100644 --- a/src/core/ext/filters/deadline/deadline_filter.cc +++ b/src/core/ext/filters/deadline/deadline_filter.cc @@ -343,6 +343,7 @@ const grpc_channel_filter grpc_client_deadline_filter = { grpc_core::NextPromiseFactory next_promise_factory) { return next_promise_factory(std::move(call_args)); }, + /* init_call: */ nullptr, grpc_channel_next_op, sizeof(grpc_deadline_state), deadline_init_call_elem, @@ -368,6 +369,17 @@ const grpc_channel_filter grpc_server_deadline_filter = { } return next_promise_factory(std::move(call_args)); }, + [](grpc_channel_element*, grpc_core::CallSpineInterface* spine) { + spine->client_initial_metadata().receiver.InterceptAndMap( + [](grpc_core::ClientMetadataHandle md) { + auto deadline = md->get(grpc_core::GrpcTimeoutMetadata()); + if (deadline.has_value()) { + grpc_core::GetContext()->UpdateDeadline( + *deadline); + } + return md; + }); + }, grpc_channel_next_op, sizeof(server_call_data), deadline_init_call_elem, diff --git a/src/core/ext/filters/http/client/http_client_filter.cc b/src/core/ext/filters/http/client/http_client_filter.cc index 8b547bac6ee2b..ac8004cdd3e41 100644 --- a/src/core/ext/filters/http/client/http_client_filter.cc +++ b/src/core/ext/filters/http/client/http_client_filter.cc @@ -51,6 +51,10 @@ namespace grpc_core { +const NoInterceptor HttpClientFilter::Call::OnServerToClientMessage; +const NoInterceptor HttpClientFilter::Call::OnClientToServerMessage; +const NoInterceptor HttpClientFilter::Call::OnFinalize; + const grpc_channel_filter HttpClientFilter::kFilter = MakePromiseBasedFilter("http-client"); @@ -105,40 +109,27 @@ Slice UserAgentFromArgs(const ChannelArgs& args, } } // namespace -ArenaPromise HttpClientFilter::MakeCallPromise( - CallArgs call_args, NextPromiseFactory next_promise_factory) { - auto& md = call_args.client_initial_metadata; - if (test_only_use_put_requests_) { - md->Set(HttpMethodMetadata(), HttpMethodMetadata::kPut); +void HttpClientFilter::Call::OnClientInitialMetadata(ClientMetadata& md, + HttpClientFilter* filter) { + if (filter->test_only_use_put_requests_) { + md.Set(HttpMethodMetadata(), HttpMethodMetadata::kPut); } else { - md->Set(HttpMethodMetadata(), HttpMethodMetadata::kPost); + md.Set(HttpMethodMetadata(), HttpMethodMetadata::kPost); } - md->Set(HttpSchemeMetadata(), scheme_); - md->Set(TeMetadata(), TeMetadata::kTrailers); - md->Set(ContentTypeMetadata(), ContentTypeMetadata::kApplicationGrpc); - md->Set(UserAgentMetadata(), user_agent_.Ref()); - - auto* initial_metadata_err = - GetContext()->New>(); - - call_args.server_initial_metadata->InterceptAndMap( - [initial_metadata_err]( - ServerMetadataHandle md) -> absl::optional { - auto r = CheckServerMetadata(md.get()); - if (!r.ok()) { - initial_metadata_err->Set(ServerMetadataFromStatus(r)); - return absl::nullopt; - } - return std::move(md); - }); - - return Race(initial_metadata_err->Wait(), - Map(next_promise_factory(std::move(call_args)), - [](ServerMetadataHandle md) -> ServerMetadataHandle { - auto r = CheckServerMetadata(md.get()); - if (!r.ok()) return ServerMetadataFromStatus(r); - return md; - })); + md.Set(HttpSchemeMetadata(), filter->scheme_); + md.Set(TeMetadata(), TeMetadata::kTrailers); + md.Set(ContentTypeMetadata(), ContentTypeMetadata::kApplicationGrpc); + md.Set(UserAgentMetadata(), filter->user_agent_.Ref()); +} + +absl::Status HttpClientFilter::Call::OnServerInitialMetadata( + ServerMetadata& md) { + return CheckServerMetadata(&md); +} + +absl::Status HttpClientFilter::Call::OnServerTrailingMetadata( + ServerMetadata& md) { + return CheckServerMetadata(&md); } HttpClientFilter::HttpClientFilter(HttpSchemeMetadata::ValueType scheme, diff --git a/src/core/ext/filters/http/client/http_client_filter.h b/src/core/ext/filters/http/client/http_client_filter.h index 298daf03c6700..dbfa9f7f2d0e2 100644 --- a/src/core/ext/filters/http/client/http_client_filter.h +++ b/src/core/ext/filters/http/client/http_client_filter.h @@ -25,23 +25,28 @@ #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/channel_fwd.h" #include "src/core/lib/channel/promise_based_filter.h" -#include "src/core/lib/promise/arena_promise.h" #include "src/core/lib/slice/slice.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" namespace grpc_core { -class HttpClientFilter : public ChannelFilter { +class HttpClientFilter : public ImplementChannelFilter { public: static const grpc_channel_filter kFilter; static absl::StatusOr Create( const ChannelArgs& args, ChannelFilter::Args filter_args); - // Construct a promise for one call. - ArenaPromise MakeCallPromise( - CallArgs call_args, NextPromiseFactory next_promise_factory) override; + class Call { + public: + void OnClientInitialMetadata(ClientMetadata& md, HttpClientFilter* filter); + absl::Status OnServerInitialMetadata(ServerMetadata& md); + absl::Status OnServerTrailingMetadata(ServerMetadata& md); + static const NoInterceptor OnClientToServerMessage; + static const NoInterceptor OnServerToClientMessage; + static const NoInterceptor OnFinalize; + }; private: HttpClientFilter(HttpSchemeMetadata::ValueType scheme, Slice user_agent, diff --git a/src/core/ext/filters/http/server/http_server_filter.cc b/src/core/ext/filters/http/server/http_server_filter.cc index 2d4953dd26b25..38c2dda0e167e 100644 --- a/src/core/ext/filters/http/server/http_server_filter.cc +++ b/src/core/ext/filters/http/server/http_server_filter.cc @@ -49,6 +49,10 @@ namespace grpc_core { +const NoInterceptor HttpServerFilter::Call::OnClientToServerMessage; +const NoInterceptor HttpServerFilter::Call::OnServerToClientMessage; +const NoInterceptor HttpServerFilter::Call::OnFinalize; + const grpc_channel_filter HttpServerFilter::kFilter = MakePromiseBasedFilter("http-server"); @@ -71,85 +75,81 @@ ServerMetadataHandle MalformedRequest(absl::string_view explanation) { } } // namespace -ArenaPromise HttpServerFilter::MakeCallPromise( - CallArgs call_args, NextPromiseFactory next_promise_factory) { - const auto& md = call_args.client_initial_metadata; - - auto method = md->get(HttpMethodMetadata()); +ServerMetadataHandle HttpServerFilter::Call::OnClientInitialMetadata( + ClientMetadata& md, HttpServerFilter* filter) { + auto method = md.get(HttpMethodMetadata()); if (method.has_value()) { switch (*method) { case HttpMethodMetadata::kPost: break; case HttpMethodMetadata::kPut: - if (allow_put_requests_) { + if (filter->allow_put_requests_) { break; } ABSL_FALLTHROUGH_INTENDED; case HttpMethodMetadata::kInvalid: case HttpMethodMetadata::kGet: - return Immediate(MalformedRequest("Bad method header")); + return MalformedRequest("Bad method header"); } } else { - return Immediate(MalformedRequest("Missing :method header")); + return MalformedRequest("Missing :method header"); } - auto te = md->Take(TeMetadata()); + auto te = md.Take(TeMetadata()); if (te == TeMetadata::kTrailers) { // Do nothing, ok. } else if (!te.has_value()) { - return Immediate(MalformedRequest("Missing :te header")); + return MalformedRequest("Missing :te header"); } else { - return Immediate(MalformedRequest("Bad :te header")); + return MalformedRequest("Bad :te header"); } - auto scheme = md->Take(HttpSchemeMetadata()); + auto scheme = md.Take(HttpSchemeMetadata()); if (scheme.has_value()) { if (*scheme == HttpSchemeMetadata::kInvalid) { - return Immediate(MalformedRequest("Bad :scheme header")); + return MalformedRequest("Bad :scheme header"); } } else { - return Immediate(MalformedRequest("Missing :scheme header")); + return MalformedRequest("Missing :scheme header"); } - md->Remove(ContentTypeMetadata()); + md.Remove(ContentTypeMetadata()); - Slice* path_slice = md->get_pointer(HttpPathMetadata()); + Slice* path_slice = md.get_pointer(HttpPathMetadata()); if (path_slice == nullptr) { - return Immediate(MalformedRequest("Missing :path header")); + return MalformedRequest("Missing :path header"); } - if (md->get_pointer(HttpAuthorityMetadata()) == nullptr) { - absl::optional host = md->Take(HostMetadata()); + if (md.get_pointer(HttpAuthorityMetadata()) == nullptr) { + absl::optional host = md.Take(HostMetadata()); if (host.has_value()) { - md->Set(HttpAuthorityMetadata(), std::move(*host)); + md.Set(HttpAuthorityMetadata(), std::move(*host)); } } - if (md->get_pointer(HttpAuthorityMetadata()) == nullptr) { - return Immediate(MalformedRequest("Missing :authority header")); + if (md.get_pointer(HttpAuthorityMetadata()) == nullptr) { + return MalformedRequest("Missing :authority header"); } - if (!surface_user_agent_) { - md->Remove(UserAgentMetadata()); + if (!filter->surface_user_agent_) { + md.Remove(UserAgentMetadata()); } - call_args.server_initial_metadata->InterceptAndMap( - [](ServerMetadataHandle md) { - if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%s[http-server] Write metadata", - Activity::current()->DebugTag().c_str()); - } - FilterOutgoingMetadata(md.get()); - md->Set(HttpStatusMetadata(), 200); - md->Set(ContentTypeMetadata(), ContentTypeMetadata::kApplicationGrpc); - return md; - }); - - return Map(next_promise_factory(std::move(call_args)), - [](ServerMetadataHandle md) -> ServerMetadataHandle { - FilterOutgoingMetadata(md.get()); - return md; - }); + return nullptr; +} + +void HttpServerFilter::Call::OnServerInitialMetadata(ServerMetadata& md) { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, "%s[http-server] Write metadata", + Activity::current()->DebugTag().c_str()); + } + FilterOutgoingMetadata(&md); + md.Set(HttpStatusMetadata(), 200); + md.Set(ContentTypeMetadata(), ContentTypeMetadata::kApplicationGrpc); +} + +void HttpServerFilter::Call::OnServerTrailingMetadata(ServerMetadata& md) { + FilterOutgoingMetadata(&md); } absl::StatusOr HttpServerFilter::Create( diff --git a/src/core/ext/filters/http/server/http_server_filter.h b/src/core/ext/filters/http/server/http_server_filter.h index bc97bd53b8a76..a87c83518ee35 100644 --- a/src/core/ext/filters/http/server/http_server_filter.h +++ b/src/core/ext/filters/http/server/http_server_filter.h @@ -32,16 +32,23 @@ namespace grpc_core { // Processes metadata on the server side for HTTP2 transports -class HttpServerFilter : public ChannelFilter { +class HttpServerFilter : public ImplementChannelFilter { public: static const grpc_channel_filter kFilter; static absl::StatusOr Create( const ChannelArgs& args, ChannelFilter::Args filter_args); - // Construct a promise for one call. - ArenaPromise MakeCallPromise( - CallArgs call_args, NextPromiseFactory next_promise_factory) override; + class Call { + public: + ServerMetadataHandle OnClientInitialMetadata(ClientMetadata& md, + HttpServerFilter* filter); + void OnServerInitialMetadata(ServerMetadata& md); + void OnServerTrailingMetadata(ServerMetadata& md); + static const NoInterceptor OnClientToServerMessage; + static const NoInterceptor OnServerToClientMessage; + static const NoInterceptor OnFinalize; + }; private: HttpServerFilter(bool surface_user_agent, bool allow_put_requests) diff --git a/src/core/ext/filters/message_size/message_size_filter.cc b/src/core/ext/filters/message_size/message_size_filter.cc index dcfc162ab62c5..9cfc00474c49c 100644 --- a/src/core/ext/filters/message_size/message_size_filter.cc +++ b/src/core/ext/filters/message_size/message_size_filter.cc @@ -50,6 +50,15 @@ namespace grpc_core { +const NoInterceptor ClientMessageSizeFilter::Call::OnClientInitialMetadata; +const NoInterceptor ClientMessageSizeFilter::Call::OnServerInitialMetadata; +const NoInterceptor ClientMessageSizeFilter::Call::OnServerTrailingMetadata; +const NoInterceptor ClientMessageSizeFilter::Call::OnFinalize; +const NoInterceptor ServerMessageSizeFilter::Call::OnClientInitialMetadata; +const NoInterceptor ServerMessageSizeFilter::Call::OnServerInitialMetadata; +const NoInterceptor ServerMessageSizeFilter::Call::OnServerTrailingMetadata; +const NoInterceptor ServerMessageSizeFilter::Call::OnFinalize; + // // MessageSizeParsedConfig // @@ -138,60 +147,6 @@ const grpc_channel_filter ServerMessageSizeFilter::kFilter = kFilterExaminesOutboundMessages | kFilterExaminesInboundMessages>("message_size"); -class MessageSizeFilter::CallBuilder { - private: - auto Interceptor(uint32_t max_length, bool is_send) { - return [max_length, is_send, - err = err_](MessageHandle msg) -> absl::optional { - if (grpc_call_trace.enabled()) { - gpr_log(GPR_INFO, "%s[message_size] %s len:%" PRIdPTR " max:%d", - Activity::current()->DebugTag().c_str(), - is_send ? "send" : "recv", msg->payload()->Length(), - max_length); - } - if (msg->payload()->Length() > max_length) { - if (err->is_set()) return std::move(msg); - auto r = GetContext()->MakePooled( - GetContext()); - r->Set(GrpcStatusMetadata(), GRPC_STATUS_RESOURCE_EXHAUSTED); - r->Set(GrpcMessageMetadata(), - Slice::FromCopiedString( - absl::StrFormat("%s message larger than max (%u vs. %d)", - is_send ? "Sent" : "Received", - msg->payload()->Length(), max_length))); - err->Set(std::move(r)); - return absl::nullopt; - } - return std::move(msg); - }; - } - - public: - explicit CallBuilder(const MessageSizeParsedConfig& limits) - : limits_(limits) {} - - template - void AddSend(T* pipe_end) { - if (!limits_.max_send_size().has_value()) return; - pipe_end->InterceptAndMap(Interceptor(*limits_.max_send_size(), true)); - } - template - void AddRecv(T* pipe_end) { - if (!limits_.max_recv_size().has_value()) return; - pipe_end->InterceptAndMap(Interceptor(*limits_.max_recv_size(), false)); - } - - ArenaPromise Run( - CallArgs call_args, NextPromiseFactory next_promise_factory) { - return Race(err_->Wait(), next_promise_factory(std::move(call_args))); - } - - private: - Latch* const err_ = - GetContext()->ManagedNew>(); - MessageSizeParsedConfig limits_; -}; - absl::StatusOr ClientMessageSizeFilter::Create( const ChannelArgs& args, ChannelFilter::Args) { return ClientMessageSizeFilter(args); @@ -202,20 +157,40 @@ absl::StatusOr ServerMessageSizeFilter::Create( return ServerMessageSizeFilter(args); } -ArenaPromise ClientMessageSizeFilter::MakeCallPromise( - CallArgs call_args, NextPromiseFactory next_promise_factory) { +namespace { +ServerMetadataHandle CheckPayload(const Message& msg, + absl::optional max_length, + bool is_send) { + if (!max_length.has_value()) return nullptr; + if (GRPC_TRACE_FLAG_ENABLED(grpc_call_trace)) { + gpr_log(GPR_INFO, "%s[message_size] %s len:%" PRIdPTR " max:%d", + Activity::current()->DebugTag().c_str(), is_send ? "send" : "recv", + msg.payload()->Length(), *max_length); + } + if (msg.payload()->Length() <= *max_length) return nullptr; + auto r = GetContext()->MakePooled(GetContext()); + r->Set(GrpcStatusMetadata(), GRPC_STATUS_RESOURCE_EXHAUSTED); + r->Set(GrpcMessageMetadata(), Slice::FromCopiedString(absl::StrFormat( + "%s message larger than max (%u vs. %d)", + is_send ? "Sent" : "Received", + msg.payload()->Length(), *max_length))); + return r; +} +} // namespace + +ClientMessageSizeFilter::Call::Call(ClientMessageSizeFilter* filter) + : limits_(filter->parsed_config_) { // Get max sizes from channel data, then merge in per-method config values. // Note: Per-method config is only available on the client, so we // apply the max request size to the send limit and the max response // size to the receive limit. - MessageSizeParsedConfig limits = this->limits(); const MessageSizeParsedConfig* config_from_call_context = MessageSizeParsedConfig::GetFromCallContext( GetContext(), - service_config_parser_index_); + filter->service_config_parser_index_); if (config_from_call_context != nullptr) { - absl::optional max_send_size = limits.max_send_size(); - absl::optional max_recv_size = limits.max_recv_size(); + absl::optional max_send_size = limits_.max_send_size(); + absl::optional max_recv_size = limits_.max_recv_size(); if (config_from_call_context->max_send_size().has_value() && (!max_send_size.has_value() || *config_from_call_context->max_send_size() < *max_send_size)) { @@ -226,21 +201,28 @@ ArenaPromise ClientMessageSizeFilter::MakeCallPromise( *config_from_call_context->max_recv_size() < *max_recv_size)) { max_recv_size = *config_from_call_context->max_recv_size(); } - limits = MessageSizeParsedConfig(max_send_size, max_recv_size); + limits_ = MessageSizeParsedConfig(max_send_size, max_recv_size); } +} + +ServerMetadataHandle ServerMessageSizeFilter::Call::OnClientToServerMessage( + const Message& message, ServerMessageSizeFilter* filter) { + return CheckPayload(message, filter->parsed_config_.max_recv_size(), false); +} + +ServerMetadataHandle ServerMessageSizeFilter::Call::OnServerToClientMessage( + const Message& message, ServerMessageSizeFilter* filter) { + return CheckPayload(message, filter->parsed_config_.max_send_size(), true); +} - CallBuilder b(limits); - b.AddSend(call_args.client_to_server_messages); - b.AddRecv(call_args.server_to_client_messages); - return b.Run(std::move(call_args), std::move(next_promise_factory)); +ServerMetadataHandle ClientMessageSizeFilter::Call::OnClientToServerMessage( + const Message& message) { + return CheckPayload(message, limits_.max_send_size(), true); } -ArenaPromise ServerMessageSizeFilter::MakeCallPromise( - CallArgs call_args, NextPromiseFactory next_promise_factory) { - CallBuilder b(limits()); - b.AddSend(call_args.server_to_client_messages); - b.AddRecv(call_args.client_to_server_messages); - return b.Run(std::move(call_args), std::move(next_promise_factory)); +ServerMetadataHandle ClientMessageSizeFilter::Call::OnServerToClientMessage( + const Message& message) { + return CheckPayload(message, limits_.max_recv_size(), false); } namespace { diff --git a/src/core/ext/filters/message_size/message_size_filter.h b/src/core/ext/filters/message_size/message_size_filter.h index 75135a1b75e04..647aeeed94f5c 100644 --- a/src/core/ext/filters/message_size/message_size_filter.h +++ b/src/core/ext/filters/message_size/message_size_filter.h @@ -86,48 +86,60 @@ class MessageSizeParser : public ServiceConfigParser::Parser { absl::optional GetMaxRecvSizeFromChannelArgs(const ChannelArgs& args); absl::optional GetMaxSendSizeFromChannelArgs(const ChannelArgs& args); -class MessageSizeFilter : public ChannelFilter { - protected: - explicit MessageSizeFilter(const ChannelArgs& args) - : limits_(MessageSizeParsedConfig::GetFromChannelArgs(args)) {} - - class CallBuilder; - - const MessageSizeParsedConfig& limits() const { return limits_; } - - private: - MessageSizeParsedConfig limits_; -}; - -class ServerMessageSizeFilter final : public MessageSizeFilter { +class ServerMessageSizeFilter final + : public ImplementChannelFilter { public: static const grpc_channel_filter kFilter; static absl::StatusOr Create( const ChannelArgs& args, ChannelFilter::Args filter_args); - // Construct a promise for one call. - ArenaPromise MakeCallPromise( - CallArgs call_args, NextPromiseFactory next_promise_factory) override; + class Call { + public: + static const NoInterceptor OnClientInitialMetadata; + static const NoInterceptor OnServerInitialMetadata; + static const NoInterceptor OnServerTrailingMetadata; + static const NoInterceptor OnFinalize; + ServerMetadataHandle OnClientToServerMessage( + const Message& message, ServerMessageSizeFilter* filter); + ServerMetadataHandle OnServerToClientMessage( + const Message& message, ServerMessageSizeFilter* filter); + }; private: - using MessageSizeFilter::MessageSizeFilter; + explicit ServerMessageSizeFilter(const ChannelArgs& args) + : parsed_config_(MessageSizeParsedConfig::GetFromChannelArgs(args)) {} + const MessageSizeParsedConfig parsed_config_; }; -class ClientMessageSizeFilter final : public MessageSizeFilter { +class ClientMessageSizeFilter final + : public ImplementChannelFilter { public: static const grpc_channel_filter kFilter; static absl::StatusOr Create( const ChannelArgs& args, ChannelFilter::Args filter_args); - // Construct a promise for one call. - ArenaPromise MakeCallPromise( - CallArgs call_args, NextPromiseFactory next_promise_factory) override; + class Call { + public: + explicit Call(ClientMessageSizeFilter* filter); + + static const NoInterceptor OnClientInitialMetadata; + static const NoInterceptor OnServerInitialMetadata; + static const NoInterceptor OnServerTrailingMetadata; + static const NoInterceptor OnFinalize; + ServerMetadataHandle OnClientToServerMessage(const Message& message); + ServerMetadataHandle OnServerToClientMessage(const Message& message); + + private: + MessageSizeParsedConfig limits_; + }; private: + explicit ClientMessageSizeFilter(const ChannelArgs& args) + : parsed_config_(MessageSizeParsedConfig::GetFromChannelArgs(args)) {} const size_t service_config_parser_index_{MessageSizeParser::ParserIndex()}; - using MessageSizeFilter::MessageSizeFilter; + const MessageSizeParsedConfig parsed_config_; }; } // namespace grpc_core diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index a1e52fab67b92..b75d792e5721e 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -503,10 +503,8 @@ static void read_channel_args(grpc_chttp2_transport* t, if (max_requests_per_read.has_value()) { t->max_requests_per_read = grpc_core::Clamp(*max_requests_per_read, 1, 10000); - } else if (grpc_core::IsChttp2BatchRequestsEnabled()) { - t->max_requests_per_read = 32; } else { - t->max_requests_per_read = std::numeric_limits::max(); + t->max_requests_per_read = 32; } if (channel_args.GetBool(GRPC_ARG_ENABLE_CHANNELZ) @@ -523,8 +521,8 @@ static void read_channel_args(grpc_chttp2_transport* t, t->ack_pings = channel_args.GetBool("grpc.http2.ack_pings").value_or(true); - t->allow_tarpit = channel_args.GetBool(GRPC_ARG_HTTP_ALLOW_TARPIT) - .value_or(grpc_core::IsTarpitEnabled()); + t->allow_tarpit = + channel_args.GetBool(GRPC_ARG_HTTP_ALLOW_TARPIT).value_or(true); t->min_tarpit_duration_ms = channel_args .GetDurationFromIntMillis(GRPC_ARG_HTTP_TARPIT_MIN_DURATION_MS) diff --git a/src/core/ext/transport/chttp2/transport/flow_control.cc b/src/core/ext/transport/chttp2/transport/flow_control.cc index c7d1eff946f02..986c62bc427e5 100644 --- a/src/core/ext/transport/chttp2/transport/flow_control.cc +++ b/src/core/ext/transport/chttp2/transport/flow_control.cc @@ -422,23 +422,9 @@ FlowControlAction StreamFlowControl::UpdateAction(FlowControlAction action) { } // min_progress_size_ > 0 means we have a reader ready to read. if (min_progress_size_ > 0) { - if (IsLazierStreamUpdatesEnabled()) { - if (announced_window_delta_ <= - -static_cast(tfc_->sent_init_window()) / 2) { - urgency = FlowControlAction::Urgency::UPDATE_IMMEDIATELY; - } - } else { - // If we're into initial window to receive that data we should wake up - // and send an update. - if (announced_window_delta_ < 0) { - urgency = FlowControlAction::Urgency::UPDATE_IMMEDIATELY; - } else if (announced_window_delta_ == 0 && - tfc_->queued_init_window() == 0) { - // Special case when initial window size is zero, meaning that - // announced_window_delta cannot become negative (it may already be so - // however). - urgency = FlowControlAction::Urgency::UPDATE_IMMEDIATELY; - } + if (announced_window_delta_ <= + -static_cast(tfc_->sent_init_window()) / 2) { + urgency = FlowControlAction::Urgency::UPDATE_IMMEDIATELY; } } action.set_send_stream_update(urgency); diff --git a/src/core/ext/transport/chttp2/transport/frame_rst_stream.cc b/src/core/ext/transport/chttp2/transport/frame_rst_stream.cc index 6c3a56ef74fe5..71e5cd7098f89 100644 --- a/src/core/ext/transport/chttp2/transport/frame_rst_stream.cc +++ b/src/core/ext/transport/chttp2/transport/frame_rst_stream.cc @@ -126,7 +126,7 @@ grpc_error_handle grpc_chttp2_rst_stream_parser_parse(void* parser, grpc_core::StatusIntProperty::kHttp2Error, static_cast(reason)); } - if (grpc_core::IsPingOnRstStreamEnabled() && !t->is_client && + if (!t->is_client && absl::Bernoulli(t->bitgen, t->ping_on_rst_stream_percent / 100.0)) { ++t->num_pending_induced_frames; t->ping_callbacks.RequestPing(); diff --git a/src/core/ext/transport/chttp2/transport/parsing.cc b/src/core/ext/transport/chttp2/transport/parsing.cc index ade5363264a72..efa216fcf6e60 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.cc +++ b/src/core/ext/transport/chttp2/transport/parsing.cc @@ -395,8 +395,7 @@ absl::variant grpc_chttp2_perform_read( } cur += t->incoming_frame_size; t->incoming_stream = nullptr; - if (t->incoming_frame_type == GRPC_CHTTP2_FRAME_RST_STREAM && - grpc_core::IsChttp2OffloadOnRstStreamEnabled()) { + if (t->incoming_frame_type == GRPC_CHTTP2_FRAME_RST_STREAM) { requests_started = std::numeric_limits::max(); } goto dts_fh_0; // loop @@ -702,8 +701,7 @@ static grpc_error_handle init_header_frame_parser(grpc_chttp2_transport* t, t, std::string(t->peer_string.as_string_view()).c_str(), t->incoming_stream_id, t->last_new_stream_id)); return init_header_skip_frame_parser(t, priority_type, is_eoh); - } else if (grpc_core::IsBlockExcessiveRequestsBeforeSettingsAckEnabled() && - t->num_incoming_streams_before_settings_ack == 0) { + } else if (t->num_incoming_streams_before_settings_ack == 0) { GRPC_CHTTP2_IF_TRACING(gpr_log( GPR_ERROR, "transport:%p SERVER peer:%s rejecting grpc_chttp2_stream id=%d, " diff --git a/src/core/ext/transport/chttp2/transport/writing.cc b/src/core/ext/transport/chttp2/transport/writing.cc index 1d168479985d7..c81f2cf28081d 100644 --- a/src/core/ext/transport/chttp2/transport/writing.cc +++ b/src/core/ext/transport/chttp2/transport/writing.cc @@ -280,8 +280,7 @@ class WriteContext { t_->settings[GRPC_LOCAL_SETTINGS], t_->force_send_settings, GRPC_CHTTP2_NUM_SETTINGS)); - if (grpc_core::IsSettingsTimeoutEnabled() && - t_->keepalive_timeout != grpc_core::Duration::Infinity()) { + if (t_->keepalive_timeout != grpc_core::Duration::Infinity()) { GPR_ASSERT( t_->settings_ack_watchdog == grpc_event_engine::experimental::EventEngine::TaskHandle::kInvalid); @@ -734,9 +733,7 @@ void grpc_chttp2_end_write(grpc_chttp2_transport* t, grpc_error_handle error) { t->keepalive_timeout != grpc_core::Duration::Infinity()) { // Set ping timeout after finishing write so we don't measure our own send // time. - const auto timeout = grpc_core::IsSeparatePingFromKeepaliveEnabled() - ? t->ping_timeout - : t->keepalive_timeout; + const auto timeout = t->ping_timeout; auto id = t->ping_callbacks.OnPingTimeout( timeout, t->event_engine.get(), [t = t->Ref()] { grpc_core::ApplicationCallbackExecCtx callback_exec_ctx; @@ -750,8 +747,7 @@ void grpc_chttp2_end_write(grpc_chttp2_transport* t, grpc_error_handle error) { id.value()); } - if (grpc_core::IsSeparatePingFromKeepaliveEnabled() && - t->keepalive_incoming_data_wanted && + if (t->keepalive_incoming_data_wanted && t->keepalive_timeout < t->ping_timeout && t->keepalive_ping_timeout_handle != grpc_event_engine::experimental::EventEngine::TaskHandle:: diff --git a/src/core/ext/xds/certificate_provider_store.h b/src/core/ext/xds/certificate_provider_store.h index 24b172ac32d78..aba287f97892b 100644 --- a/src/core/ext/xds/certificate_provider_store.h +++ b/src/core/ext/xds/certificate_provider_store.h @@ -36,7 +36,6 @@ #include "src/core/lib/gprpp/sync.h" #include "src/core/lib/gprpp/unique_type_name.h" #include "src/core/lib/gprpp/validation_errors.h" -#include "src/core/lib/iomgr/iomgr_fwd.h" #include "src/core/lib/json/json.h" #include "src/core/lib/json/json_args.h" #include "src/core/lib/json/json_object_loader.h" @@ -96,10 +95,6 @@ class CertificateProviderStore return certificate_provider_->distributor(); } - grpc_pollset_set* interested_parties() const override { - return certificate_provider_->interested_parties(); - } - int CompareImpl(const grpc_tls_certificate_provider* other) const override { // TODO(yashykt): This should probably delegate to the `Compare` method of // the wrapped certificate_provider_ object. diff --git a/src/core/ext/xds/xds_api.cc b/src/core/ext/xds/xds_api.cc index 8b77e30ed7914..b23c446401fb3 100644 --- a/src/core/ext/xds/xds_api.cc +++ b/src/core/ext/xds/xds_api.cc @@ -57,12 +57,12 @@ namespace grpc_core { XdsApi::XdsApi(XdsClient* client, TraceFlag* tracer, - const XdsBootstrap::Node* node, upb::SymbolTable* symtab, + const XdsBootstrap::Node* node, upb::DefPool* def_pool, std::string user_agent_name, std::string user_agent_version) : client_(client), tracer_(tracer), node_(node), - symtab_(symtab), + def_pool_(def_pool), user_agent_name_(std::move(user_agent_name)), user_agent_version_(std::move(user_agent_version)) {} @@ -71,7 +71,7 @@ namespace { struct XdsApiContext { XdsClient* client; TraceFlag* tracer; - upb_DefPool* symtab; + upb_DefPool* def_pool; upb_Arena* arena; }; @@ -183,7 +183,7 @@ void MaybeLogDiscoveryRequest( if (GRPC_TRACE_FLAG_ENABLED(*context.tracer) && gpr_should_log(GPR_LOG_SEVERITY_DEBUG)) { const upb_MessageDef* msg_type = - envoy_service_discovery_v3_DiscoveryRequest_getmsgdef(context.symtab); + envoy_service_discovery_v3_DiscoveryRequest_getmsgdef(context.def_pool); char buf[10240]; upb_TextEncode(request, msg_type, nullptr, 0, buf, sizeof(buf)); gpr_log(GPR_DEBUG, "[xds_client %p] constructed ADS request: %s", @@ -207,7 +207,8 @@ std::string XdsApi::CreateAdsRequest( absl::string_view nonce, const std::vector& resource_names, absl::Status status, bool populate_node) { upb::Arena arena; - const XdsApiContext context = {client_, tracer_, symtab_->ptr(), arena.ptr()}; + const XdsApiContext context = {client_, tracer_, def_pool_->ptr(), + arena.ptr()}; // Create a request. envoy_service_discovery_v3_DiscoveryRequest* request = envoy_service_discovery_v3_DiscoveryRequest_new(arena.ptr()); @@ -270,7 +271,8 @@ void MaybeLogDiscoveryResponse( if (GRPC_TRACE_FLAG_ENABLED(*context.tracer) && gpr_should_log(GPR_LOG_SEVERITY_DEBUG)) { const upb_MessageDef* msg_type = - envoy_service_discovery_v3_DiscoveryResponse_getmsgdef(context.symtab); + envoy_service_discovery_v3_DiscoveryResponse_getmsgdef( + context.def_pool); char buf[10240]; upb_TextEncode(response, msg_type, nullptr, 0, buf, sizeof(buf)); gpr_log(GPR_DEBUG, "[xds_client %p] received response: %s", context.client, @@ -283,7 +285,8 @@ void MaybeLogDiscoveryResponse( absl::Status XdsApi::ParseAdsResponse(absl::string_view encoded_response, AdsResponseParserInterface* parser) { upb::Arena arena; - const XdsApiContext context = {client_, tracer_, symtab_->ptr(), arena.ptr()}; + const XdsApiContext context = {client_, tracer_, def_pool_->ptr(), + arena.ptr()}; // Decode the response. const envoy_service_discovery_v3_DiscoveryResponse* response = envoy_service_discovery_v3_DiscoveryResponse_parse( @@ -356,7 +359,8 @@ void MaybeLogLrsRequest( if (GRPC_TRACE_FLAG_ENABLED(*context.tracer) && gpr_should_log(GPR_LOG_SEVERITY_DEBUG)) { const upb_MessageDef* msg_type = - envoy_service_load_stats_v3_LoadStatsRequest_getmsgdef(context.symtab); + envoy_service_load_stats_v3_LoadStatsRequest_getmsgdef( + context.def_pool); char buf[10240]; upb_TextEncode(request, msg_type, nullptr, 0, buf, sizeof(buf)); gpr_log(GPR_DEBUG, "[xds_client %p] constructed LRS request: %s", @@ -377,7 +381,8 @@ std::string SerializeLrsRequest( std::string XdsApi::CreateLrsInitialRequest() { upb::Arena arena; - const XdsApiContext context = {client_, tracer_, symtab_->ptr(), arena.ptr()}; + const XdsApiContext context = {client_, tracer_, def_pool_->ptr(), + arena.ptr()}; // Create a request. envoy_service_load_stats_v3_LoadStatsRequest* request = envoy_service_load_stats_v3_LoadStatsRequest_new(arena.ptr()); @@ -447,7 +452,8 @@ void LocalityStatsPopulate( std::string XdsApi::CreateLrsRequest( ClusterLoadReportMap cluster_load_report_map) { upb::Arena arena; - const XdsApiContext context = {client_, tracer_, symtab_->ptr(), arena.ptr()}; + const XdsApiContext context = {client_, tracer_, def_pool_->ptr(), + arena.ptr()}; // Create a request. envoy_service_load_stats_v3_LoadStatsRequest* request = envoy_service_load_stats_v3_LoadStatsRequest_new(arena.ptr()); @@ -514,7 +520,8 @@ void MaybeLogLrsResponse( if (GRPC_TRACE_FLAG_ENABLED(*context.tracer) && gpr_should_log(GPR_LOG_SEVERITY_DEBUG)) { const upb_MessageDef* msg_type = - envoy_service_load_stats_v3_LoadStatsResponse_getmsgdef(context.symtab); + envoy_service_load_stats_v3_LoadStatsResponse_getmsgdef( + context.def_pool); char buf[10240]; upb_TextEncode(response, msg_type, nullptr, 0, buf, sizeof(buf)); gpr_log(GPR_DEBUG, "[xds_client %p] received LRS response: %s", @@ -537,7 +544,8 @@ absl::Status XdsApi::ParseLrsResponse(absl::string_view encoded_response, if (decoded_response == nullptr) { return absl::UnavailableError("Can't decode response."); } - const XdsApiContext context = {client_, tracer_, symtab_->ptr(), arena.ptr()}; + const XdsApiContext context = {client_, tracer_, def_pool_->ptr(), + arena.ptr()}; MaybeLogLrsResponse(context, decoded_response); // Check send_all_clusters. if (envoy_service_load_stats_v3_LoadStatsResponse_send_all_clusters( @@ -585,7 +593,8 @@ std::string XdsApi::AssembleClientConfig( // Fill-in the node information auto* node = envoy_service_status_v3_ClientConfig_mutable_node(client_config, arena.ptr()); - const XdsApiContext context = {client_, tracer_, symtab_->ptr(), arena.ptr()}; + const XdsApiContext context = {client_, tracer_, def_pool_->ptr(), + arena.ptr()}; PopulateNode(context, node_, user_agent_name_, user_agent_version_, node); // Dump each resource. std::vector type_url_storage; diff --git a/src/core/ext/xds/xds_api.h b/src/core/ext/xds/xds_api.h index 256998cc24df0..2d96892496667 100644 --- a/src/core/ext/xds/xds_api.h +++ b/src/core/ext/xds/xds_api.h @@ -148,7 +148,7 @@ class XdsApi { ""); XdsApi(XdsClient* client, TraceFlag* tracer, const XdsBootstrap::Node* node, - upb::SymbolTable* symtab, std::string user_agent_name, + upb::DefPool* def_pool, std::string user_agent_name, std::string user_agent_version); // Creates an ADS request. @@ -184,7 +184,7 @@ class XdsApi { XdsClient* client_; TraceFlag* tracer_; const XdsBootstrap::Node* node_; // Do not own. - upb::SymbolTable* symtab_; // Do not own. + upb::DefPool* def_pool_; // Do not own. const std::string user_agent_name_; const std::string user_agent_version_; }; diff --git a/src/core/ext/xds/xds_certificate_provider.cc b/src/core/ext/xds/xds_certificate_provider.cc index 0ca497939eed3..439921c648aae 100644 --- a/src/core/ext/xds/xds_certificate_provider.cc +++ b/src/core/ext/xds/xds_certificate_provider.cc @@ -374,43 +374,4 @@ void XdsCertificateProvider::WatchStatusCallback(std::string cert_name, if (it->second->IsSafeToRemove()) certificate_state_map_.erase(it); } -namespace { - -void* XdsCertificateProviderArgCopy(void* p) { - XdsCertificateProvider* xds_certificate_provider = - static_cast(p); - return xds_certificate_provider->Ref().release(); -} - -void XdsCertificateProviderArgDestroy(void* p) { - XdsCertificateProvider* xds_certificate_provider = - static_cast(p); - xds_certificate_provider->Unref(); -} - -int XdsCertificateProviderArgCmp(void* p, void* q) { - return QsortCompare(p, q); -} - -const grpc_arg_pointer_vtable kChannelArgVtable = { - XdsCertificateProviderArgCopy, XdsCertificateProviderArgDestroy, - XdsCertificateProviderArgCmp}; - -} // namespace - -grpc_arg XdsCertificateProvider::MakeChannelArg() const { - return grpc_channel_arg_pointer_create( - const_cast(GRPC_ARG_XDS_CERTIFICATE_PROVIDER), - const_cast(this), &kChannelArgVtable); -} - -RefCountedPtr -XdsCertificateProvider::GetFromChannelArgs(const grpc_channel_args* args) { - XdsCertificateProvider* xds_certificate_provider = - grpc_channel_args_find_pointer( - args, GRPC_ARG_XDS_CERTIFICATE_PROVIDER); - return xds_certificate_provider != nullptr ? xds_certificate_provider->Ref() - : nullptr; -} - } // namespace grpc_core diff --git a/src/core/ext/xds/xds_certificate_provider.h b/src/core/ext/xds/xds_certificate_provider.h index c17a7fa5cb728..44eb2421d8985 100644 --- a/src/core/ext/xds/xds_certificate_provider.h +++ b/src/core/ext/xds/xds_certificate_provider.h @@ -40,9 +40,6 @@ #include "src/core/lib/security/credentials/tls/grpc_tls_certificate_distributor.h" #include "src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.h" -#define GRPC_ARG_XDS_CERTIFICATE_PROVIDER \ - "grpc.internal.xds_certificate_provider" - namespace grpc_core { class XdsCertificateProvider : public grpc_tls_certificate_provider { @@ -50,15 +47,6 @@ class XdsCertificateProvider : public grpc_tls_certificate_provider { XdsCertificateProvider(); ~XdsCertificateProvider() override; - static absl::string_view ChannelArgName() { - return GRPC_ARG_XDS_CERTIFICATE_PROVIDER; - } - - static int ChannelArgsCompare(const XdsCertificateProvider* a, - const XdsCertificateProvider* b) { - return QsortCompare(a, b); - } - RefCountedPtr distributor() const override { return distributor_; } @@ -86,10 +74,13 @@ class XdsCertificateProvider : public grpc_tls_certificate_provider { void UpdateSubjectAlternativeNameMatchers( const std::string& cluster, std::vector matchers); - grpc_arg MakeChannelArg() const; - - static RefCountedPtr GetFromChannelArgs( - const grpc_channel_args* args); + static absl::string_view ChannelArgName() { + return "grpc.internal.xds_certificate_provider"; + } + static int ChannelArgsCompare(const XdsCertificateProvider* a, + const XdsCertificateProvider* b) { + return a->Compare(b); + } private: class ClusterCertificateState { diff --git a/src/core/ext/xds/xds_client.cc b/src/core/ext/xds/xds_client.cc index 19d2702a6d51f..e6399e98d1a30 100644 --- a/src/core/ext/xds/xds_client.cc +++ b/src/core/ext/xds/xds_client.cc @@ -756,7 +756,7 @@ void XdsClient::ChannelState::AdsCallState::AdsResponseParser::ParseResource( // Parse the resource. XdsResourceType::DecodeContext context = { xds_client(), ads_call_state_->chand()->server_, &grpc_xds_client_trace, - xds_client()->symtab_.ptr(), arena}; + xds_client()->def_pool_.ptr(), arena}; XdsResourceType::DecodeResult decode_result = result_.type->Decode(context, serialized_resource); // If we didn't already have the resource name from the Resource @@ -1490,7 +1490,7 @@ XdsClient::XdsClient( transport_factory_(std::move(transport_factory)), request_timeout_(resource_request_timeout), xds_federation_enabled_(XdsFederationEnabled()), - api_(this, &grpc_xds_client_trace, bootstrap_->node(), &symtab_, + api_(this, &grpc_xds_client_trace, bootstrap_->node(), &def_pool_, std::move(user_agent_name), std::move(user_agent_version)), work_serializer_(engine), engine_(std::move(engine)) { @@ -1722,7 +1722,7 @@ void XdsClient::MaybeRegisterResourceTypeLocked( return; } resource_types_.emplace(resource_type->type_url(), resource_type); - resource_type->InitUpbSymtab(this, symtab_.ptr()); + resource_type->InitUpbSymtab(this, def_pool_.ptr()); } const XdsResourceType* XdsClient::GetResourceTypeLocked( diff --git a/src/core/ext/xds/xds_client.h b/src/core/ext/xds/xds_client.h index bd24d8ae193b9..0831c3c768ace 100644 --- a/src/core/ext/xds/xds_client.h +++ b/src/core/ext/xds/xds_client.h @@ -317,7 +317,7 @@ class XdsClient : public DualRefCounted { // Stores resource type objects seen by type URL. std::map resource_types_ ABSL_GUARDED_BY(mu_); - upb::SymbolTable symtab_ ABSL_GUARDED_BY(mu_); + upb::DefPool def_pool_ ABSL_GUARDED_BY(mu_); // Map of existing xDS server channels. // Key is owned by the bootstrap config. diff --git a/src/core/lib/channel/channel_args.h b/src/core/lib/channel/channel_args.h index 38bb070213cab..2c10d955127f4 100644 --- a/src/core/lib/channel/channel_args.h +++ b/src/core/lib/channel/channel_args.h @@ -125,6 +125,16 @@ struct ChannelArgTypeTraits< }; }; +// Define a check for shared_ptr supported types, which must extend +// enable_shared_from_this. +template +struct SupportedSharedPtrType + : std::integral_constant< + bool, std::is_base_of, T>::value> {}; +template <> +struct SupportedSharedPtrType + : std::true_type {}; + // Specialization for shared_ptr // Incurs an allocation because shared_ptr.release is not a thing. template @@ -173,19 +183,27 @@ struct ChannelArgTypeTraits +struct ChannelArgPointerShouldBeConst { + static constexpr bool kValue = false; +}; + template -struct WrapInSharedPtr - : std::integral_constant< - bool, std::is_base_of, T>::value> {}; -template <> -struct WrapInSharedPtr - : std::true_type {}; +struct ChannelArgPointerShouldBeConst< + T, absl::void_t> { + static constexpr bool kValue = T::ChannelArgUseConstPtr(); +}; + +// GetObject support for shared_ptr and RefCountedPtr template struct GetObjectImpl; // std::shared_ptr implementation template -struct GetObjectImpl::value, void>> { +struct GetObjectImpl< + T, absl::enable_if_t::kValue && + SupportedSharedPtrType::value, + void>> { using Result = T*; using ReffedResult = std::shared_ptr; using StoredType = std::shared_ptr*; @@ -205,7 +223,10 @@ struct GetObjectImpl::value, void>> { }; // RefCountedPtr template -struct GetObjectImpl::value, void>> { +struct GetObjectImpl< + T, absl::enable_if_t::kValue && + !SupportedSharedPtrType::value, + void>> { using Result = T*; using ReffedResult = RefCountedPtr; using StoredType = Result; @@ -221,6 +242,26 @@ struct GetObjectImpl::value, void>> { }; }; +template +struct GetObjectImpl< + T, absl::enable_if_t::kValue && + !SupportedSharedPtrType::value, + void>> { + using Result = const T*; + using ReffedResult = RefCountedPtr; + using StoredType = Result; + static Result Get(StoredType p) { return p; }; + static ReffedResult GetReffed(StoredType p) { + if (p == nullptr) return nullptr; + return p->Ref(); + }; + static ReffedResult GetReffed(StoredType p, const DebugLocation& location, + const char* reason) { + if (p == nullptr) return nullptr; + return p->Ref(location, reason); + }; +}; + // Provide the canonical name for a type's channel arg key template struct ChannelArgNameTraits { @@ -237,6 +278,7 @@ struct ChannelArgNameTraits { return GRPC_INTERNAL_ARG_EVENT_ENGINE; } }; + class ChannelArgs { public: class Pointer { @@ -376,21 +418,38 @@ class ChannelArgs { GRPC_MUST_USE_RESULT auto Set(absl::string_view name, RefCountedPtr value) const -> absl::enable_if_t< - std::is_same>::VTable())>::value, + !ChannelArgPointerShouldBeConst::kValue && + std::is_same>::VTable())>::value, ChannelArgs> { return Set( name, Pointer(value.release(), ChannelArgTypeTraits>::VTable())); } template + GRPC_MUST_USE_RESULT auto Set(absl::string_view name, + RefCountedPtr value) const + -> absl::enable_if_t< + ChannelArgPointerShouldBeConst::kValue && + std::is_same>::VTable())>::value, + ChannelArgs> { + return Set( + name, Pointer(const_cast(value.release()), + ChannelArgTypeTraits>::VTable())); + } + template GRPC_MUST_USE_RESULT absl::enable_if_t< std::is_same< const grpc_arg_pointer_vtable*, decltype(ChannelArgTypeTraits>::VTable())>::value, ChannelArgs> Set(absl::string_view name, std::shared_ptr value) const { + static_assert(SupportedSharedPtrType::value, + "Type T must extend std::enable_shared_from_this to be added " + "into ChannelArgs as a shared_ptr"); auto* store_value = new std::shared_ptr(value); return Set( name, @@ -418,6 +477,8 @@ class ChannelArgs { absl::optional GetInt(absl::string_view name) const; absl::optional GetString(absl::string_view name) const; absl::optional GetOwnedString(absl::string_view name) const; + // WARNING: this is broken if `name` represents something that was stored as a + // RefCounted - we will discard the const-ness. void* GetVoidPointer(absl::string_view name) const; template typename GetObjectImpl::StoredType GetPointer( diff --git a/src/core/lib/channel/channel_stack.h b/src/core/lib/channel/channel_stack.h index 918628856b03a..5b6649a9eac87 100644 --- a/src/core/lib/channel/channel_stack.h +++ b/src/core/lib/channel/channel_stack.h @@ -128,6 +128,12 @@ struct grpc_channel_filter { grpc_core::ArenaPromise (*make_call_promise)( grpc_channel_element* elem, grpc_core::CallArgs call_args, grpc_core::NextPromiseFactory next_promise_factory); + // Register interceptors into a call. + // If this is non-null it may be used in preference to make_call_promise. + // There is an on-going migration to move all filters to providing this, and + // then to drop start_transport_stream_op_batch. + void (*init_call)(grpc_channel_element* elem, + grpc_core::CallSpineInterface* call_spine); // Called to handle channel level operations - e.g. new calls, or transport // closure. // See grpc_channel_next_op on how to call the next element in the stack diff --git a/src/core/lib/channel/connected_channel.cc b/src/core/lib/channel/connected_channel.cc index afca6eda5fc18..9ea8faeb507d0 100644 --- a/src/core/lib/channel/connected_channel.cc +++ b/src/core/lib/channel/connected_channel.cc @@ -857,6 +857,7 @@ grpc_channel_filter MakeConnectedFilter() { return { connected_channel_start_transport_stream_op_batch, make_call_promise != nullptr ? make_call_wrapper : nullptr, + /* init_call: */ nullptr, connected_channel_start_transport_op, sizeof(call_data), connected_channel_init_call_elem, @@ -882,8 +883,8 @@ grpc_channel_filter MakeConnectedFilter() { } ArenaPromise MakeTransportCallPromise( - Transport* transport, CallArgs call_args, NextPromiseFactory) { - return transport->client_transport()->MakeCallPromise(std::move(call_args)); + Transport*, CallArgs, NextPromiseFactory) { + Crash("unimplemented"); } const grpc_channel_filter kPromiseBasedTransportFilter = diff --git a/src/core/lib/channel/promise_based_filter.h b/src/core/lib/channel/promise_based_filter.h index 19efe505db29a..ad33f366378cd 100644 --- a/src/core/lib/channel/promise_based_filter.h +++ b/src/core/lib/channel/promise_based_filter.h @@ -43,6 +43,7 @@ #include #include "src/core/lib/channel/call_finalization.h" +#include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/channel_fwd.h" #include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/channel/context.h" @@ -60,6 +61,8 @@ #include "src/core/lib/promise/context.h" #include "src/core/lib/promise/pipe.h" #include "src/core/lib/promise/poll.h" +#include "src/core/lib/promise/promise.h" +#include "src/core/lib/promise/race.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/surface/call.h" @@ -122,6 +125,668 @@ class ChannelFilter { grpc_event_engine::experimental::GetDefaultEventEngine(); }; +struct NoInterceptor {}; + +namespace promise_filter_detail { + +// Determine if a list of interceptors has any that need to asyncronously error +// the promise. If so, we need to allocate a latch for the generated promise for +// the original promise stack polyfill code that's generated. + +inline constexpr bool HasAsyncErrorInterceptor() { return false; } + +inline constexpr bool HasAsyncErrorInterceptor(const NoInterceptor*) { + return false; +} + +template +inline constexpr bool HasAsyncErrorInterceptor(absl::Status (T::*)(A...)) { + return true; +} + +template +inline constexpr bool HasAsyncErrorInterceptor( + ServerMetadataHandle (T::*)(A...)) { + return true; +} + +template +inline constexpr bool HasAsyncErrorInterceptor(void (T::*)(A...)) { + return false; +} + +// For the list case we do two interceptors to avoid amiguities with the single +// argument forms above. +template +inline constexpr bool HasAsyncErrorInterceptor(I1 i1, I2 i2, + Interceptors... interceptors) { + return HasAsyncErrorInterceptor(i1) || HasAsyncErrorInterceptor(i2) || + HasAsyncErrorInterceptor(interceptors...); +} + +// Composite for a given channel type to determine if any of its interceptors +// fall into this category: later code should use this. +template +inline constexpr bool CallHasAsyncErrorInterceptor() { + return HasAsyncErrorInterceptor(&Derived::Call::OnClientToServerMessage, + &Derived::Call::OnServerInitialMetadata, + &Derived::Call::OnServerToClientMessage); +} + +// Determine if an interceptor needs to access the channel via one of its +// arguments. If so, we need to allocate a pointer to the channel for the +// generated polyfill promise for the original promise stack. + +inline constexpr bool HasChannelAccess() { return false; } + +inline constexpr bool HasChannelAccess(const NoInterceptor*) { return false; } + +template +inline constexpr bool HasChannelAccess(R (T::*)(A)) { + return false; +} + +template +inline constexpr bool HasChannelAccess(R (T::*)()) { + return false; +} + +template +inline constexpr bool HasChannelAccess(R (T::*)(A, C)) { + return true; +} + +// For the list case we do two interceptors to avoid amiguities with the single +// argument forms above. +template +inline constexpr bool HasChannelAccess(I1 i1, I2 i2, + Interceptors... interceptors) { + return HasChannelAccess(i1) || HasChannelAccess(i2) || + HasChannelAccess(interceptors...); +} + +// Composite for a given channel type to determine if any of its interceptors +// fall into this category: later code should use this. +template +inline constexpr bool CallHasChannelAccess() { + return HasChannelAccess(&Derived::Call::OnClientInitialMetadata, + &Derived::Call::OnClientToServerMessage, + &Derived::Call::OnServerInitialMetadata, + &Derived::Call::OnServerToClientMessage, + &Derived::Call::OnServerTrailingMetadata, + &Derived::Call::OnFinalize); +} + +// Given a boolean X export a type: +// either T if X is true +// or an empty type if it is false +template +struct TypeIfNeeded; + +template +struct TypeIfNeeded { + struct Type { + Type() = default; + template + explicit Type(Whatever) : Type() {} + }; +}; + +template +struct TypeIfNeeded { + using Type = T; +}; + +// For the original promise scheme polyfill: +// If a set of interceptors might fail asynchronously, wrap the main +// promise in a race with the cancellation latch. +// If not, just return the main promise. +template +struct RaceAsyncCompletion; + +template <> +struct RaceAsyncCompletion { + template + static Promise Run(Promise x, void*) { + return x; + } +}; + +template <> +struct RaceAsyncCompletion { + template + static Promise Run(Promise x, Latch* latch) { + return Race(latch->Wait(), std::move(x)); + } +}; + +// Zero-member wrapper to make sure that Call always has a constructor +// that takes a channel pointer (even if it's thrown away) +template +class CallWrapper; + +template +class CallWrapper()))>> + : public Derived::Call { + public: + explicit CallWrapper(Derived* channel) : Derived::Call(channel) {} +}; + +template +class CallWrapper> + : public Derived::Call { + public: + explicit CallWrapper(Derived*) : Derived::Call() {} +}; + +// For the original promise scheme polyfill: data associated with once call. +template +struct FilterCallData { + explicit FilterCallData(Derived* channel) : call(channel), channel(channel) {} + GPR_NO_UNIQUE_ADDRESS CallWrapper call; + GPR_NO_UNIQUE_ADDRESS + typename TypeIfNeeded, + CallHasAsyncErrorInterceptor()>::Type + error_latch; + GPR_NO_UNIQUE_ADDRESS + typename TypeIfNeeded()>::Type + channel; +}; + +template +auto MapResult(const NoInterceptor*, Promise x, void*) { + return x; +} + +template +auto MapResult(absl::Status (Derived::Call::*fn)(ServerMetadata&), Promise x, + FilterCallData* call_data) { + GPR_DEBUG_ASSERT(fn == &Derived::Call::OnServerTrailingMetadata); + return Map(std::move(x), [call_data](ServerMetadataHandle md) { + auto status = call_data->call.OnServerTrailingMetadata(*md); + if (!status.ok()) return ServerMetadataFromStatus(status); + return md; + }); +} + +template +auto MapResult(void (Derived::Call::*fn)(ServerMetadata&), Promise x, + FilterCallData* call_data) { + GPR_DEBUG_ASSERT(fn == &Derived::Call::OnServerTrailingMetadata); + return Map(std::move(x), [call_data](ServerMetadataHandle md) { + call_data->call.OnServerTrailingMetadata(*md); + return md; + }); +} + +inline auto RunCall(const NoInterceptor*, CallArgs call_args, + NextPromiseFactory next_promise_factory, void*) { + return next_promise_factory(std::move(call_args)); +} + +template +inline auto RunCall(void (Derived::Call::*fn)(ClientMetadata& md), + CallArgs call_args, NextPromiseFactory next_promise_factory, + FilterCallData* call_data) { + GPR_DEBUG_ASSERT(fn == &Derived::Call::OnClientInitialMetadata); + call_data->call.OnClientInitialMetadata(*call_args.client_initial_metadata); + return next_promise_factory(std::move(call_args)); +} + +template +inline auto RunCall( + ServerMetadataHandle (Derived::Call::*fn)(ClientMetadata& md), + CallArgs call_args, NextPromiseFactory next_promise_factory, + FilterCallData* call_data) -> ArenaPromise { + GPR_DEBUG_ASSERT(fn == &Derived::Call::OnClientInitialMetadata); + auto return_md = call_data->call.OnClientInitialMetadata( + *call_args.client_initial_metadata); + if (return_md == nullptr) return next_promise_factory(std::move(call_args)); + return Immediate(std::move(return_md)); +} + +template +inline auto RunCall(ServerMetadataHandle (Derived::Call::*fn)( + ClientMetadata& md, Derived* channel), + CallArgs call_args, NextPromiseFactory next_promise_factory, + FilterCallData* call_data) + -> ArenaPromise { + GPR_DEBUG_ASSERT(fn == &Derived::Call::OnClientInitialMetadata); + auto return_md = call_data->call.OnClientInitialMetadata( + *call_args.client_initial_metadata, call_data->channel); + if (return_md == nullptr) return next_promise_factory(std::move(call_args)); + return Immediate(std::move(return_md)); +} + +template +inline auto RunCall(void (Derived::Call::*fn)(ClientMetadata& md, + Derived* channel), + CallArgs call_args, NextPromiseFactory next_promise_factory, + FilterCallData* call_data) { + GPR_DEBUG_ASSERT(fn == &Derived::Call::OnClientInitialMetadata); + call_data->call.OnClientInitialMetadata(*call_args.client_initial_metadata, + call_data->channel); + return next_promise_factory(std::move(call_args)); +} + +inline void InterceptClientToServerMessage(const NoInterceptor*, void*, + const CallArgs&) {} + +template +inline void InterceptClientToServerMessage( + ServerMetadataHandle (Derived::Call::*fn)(const Message&), + FilterCallData* call_data, const CallArgs& call_args) { + GPR_DEBUG_ASSERT(fn == &Derived::Call::OnClientToServerMessage); + call_args.client_to_server_messages->InterceptAndMap( + [call_data](MessageHandle msg) -> absl::optional { + auto return_md = call_data->call.OnClientToServerMessage(*msg); + if (return_md == nullptr) return std::move(msg); + if (call_data->error_latch.is_set()) return absl::nullopt; + call_data->error_latch.Set(std::move(return_md)); + return absl::nullopt; + }); +} + +template +inline void InterceptClientToServerMessage( + ServerMetadataHandle (Derived::Call::*fn)(const Message&, Derived*), + FilterCallData* call_data, const CallArgs& call_args) { + GPR_DEBUG_ASSERT(fn == &Derived::Call::OnClientToServerMessage); + call_args.client_to_server_messages->InterceptAndMap( + [call_data](MessageHandle msg) -> absl::optional { + auto return_md = + call_data->call.OnClientToServerMessage(*msg, call_data->channel); + if (return_md == nullptr) return std::move(msg); + if (call_data->error_latch.is_set()) return absl::nullopt; + call_data->error_latch.Set(std::move(return_md)); + return absl::nullopt; + }); +} + +inline void InterceptClientToServerMessage(const NoInterceptor*, void*, void*, + CallSpineInterface*) {} + +template +inline void InterceptClientToServerMessage( + ServerMetadataHandle (Derived::Call::*fn)(const Message&), + typename Derived::Call* call, Derived*, CallSpineInterface* call_spine) { + GPR_DEBUG_ASSERT(fn == &Derived::Call::OnClientToServerMessage); + call_spine->server_to_client_messages().sender.InterceptAndMap( + [call, call_spine](MessageHandle msg) -> absl::optional { + auto return_md = call->OnClientToServerMessage(*msg); + if (return_md == nullptr) return std::move(msg); + return call_spine->Cancel(std::move(return_md)); + }); +} + +template +inline void InterceptClientToServerMessage( + ServerMetadataHandle (Derived::Call::*fn)(const Message&, Derived*), + typename Derived::Call* call, Derived* channel, + CallSpineInterface* call_spine) { + GPR_DEBUG_ASSERT(fn == &Derived::Call::OnClientToServerMessage); + call_spine->server_to_client_messages().sender.InterceptAndMap( + [call, call_spine, + channel](MessageHandle msg) -> absl::optional { + auto return_md = call->OnClientToServerMessage(*msg, channel); + if (return_md == nullptr) return std::move(msg); + return call_spine->Cancel(std::move(return_md)); + }); +} + +inline void InterceptClientInitialMetadata(const NoInterceptor*, void*, void*, + CallSpineInterface*) {} + +template +inline void InterceptClientInitialMetadata( + void (Derived::Call::*fn)(ClientMetadata& md), typename Derived::Call* call, + Derived*, CallSpineInterface* call_spine) { + GPR_DEBUG_ASSERT(fn == &Derived::Call::OnClientInitialMetadata); + call_spine->client_initial_metadata().receiver.InterceptAndMap( + [call](ClientMetadataHandle md) { + call->OnClientInitialMetadata(*md); + return md; + }); +} + +template +inline void InterceptClientInitialMetadata( + void (Derived::Call::*fn)(ClientMetadata& md, Derived* channel), + typename Derived::Call* call, Derived* channel, + CallSpineInterface* call_spine) { + GPR_DEBUG_ASSERT(fn == &Derived::Call::OnClientInitialMetadata); + call_spine->client_initial_metadata().receiver.InterceptAndMap( + [call, channel](ClientMetadataHandle md) { + call->OnClientInitialMetadata(*md, channel); + return md; + }); +} + +template +inline void InterceptClientInitialMetadata( + ServerMetadataHandle (Derived::Call::*fn)(ClientMetadata& md), + typename Derived::Call* call, Derived*, CallSpineInterface* call_spine) { + GPR_DEBUG_ASSERT(fn == &Derived::Call::OnClientInitialMetadata); + call_spine->client_initial_metadata().receiver.InterceptAndMap( + [call_spine, + call](ClientMetadataHandle md) -> absl::optional { + auto return_md = call->OnClientInitialMetadata(*md); + if (return_md == nullptr) return std::move(md); + return call_spine->Cancel(std::move(return_md)); + }); +} + +template +inline void InterceptClientInitialMetadata( + ServerMetadataHandle (Derived::Call::*fn)(ClientMetadata& md, + Derived* channel), + typename Derived::Call* call, Derived* channel, + CallSpineInterface* call_spine) { + GPR_DEBUG_ASSERT(fn == &Derived::Call::OnClientInitialMetadata); + call_spine->client_initial_metadata().receiver.InterceptAndMap( + [call_spine, call, channel]( + ClientMetadataHandle md) -> absl::optional { + auto return_md = call->OnClientInitialMetadata(*md, channel); + if (return_md == nullptr) return std::move(md); + return call_spine->Cancel(std::move(return_md)); + }); +} + +template +inline void InterceptServerInitialMetadata(const NoInterceptor*, void*, + const CallArgs&) {} + +template +inline void InterceptServerInitialMetadata( + void (Derived::Call::*fn)(ServerMetadata&), + FilterCallData* call_data, const CallArgs& call_args) { + GPR_DEBUG_ASSERT(fn == &Derived::Call::OnServerInitialMetadata); + call_args.server_initial_metadata->InterceptAndMap( + [call_data](ServerMetadataHandle md) { + call_data->call.OnServerInitialMetadata(*md); + return md; + }); +} + +template +inline void InterceptServerInitialMetadata( + absl::Status (Derived::Call::*fn)(ServerMetadata&), + FilterCallData* call_data, const CallArgs& call_args) { + GPR_DEBUG_ASSERT(fn == &Derived::Call::OnServerInitialMetadata); + call_args.server_initial_metadata->InterceptAndMap( + [call_data]( + ServerMetadataHandle md) -> absl::optional { + auto status = call_data->call.OnServerInitialMetadata(*md); + if (!status.ok() && !call_data->error_latch.is_set()) { + call_data->error_latch.Set(ServerMetadataFromStatus(status)); + return absl::nullopt; + } + return std::move(md); + }); +} + +inline void InterceptServerInitialMetadata(const NoInterceptor*, void*, void*, + CallSpineInterface*) {} + +template +inline void InterceptServerInitialMetadata( + void (Derived::Call::*fn)(ServerMetadata&), typename Derived::Call* call, + Derived*, CallSpineInterface* call_spine) { + GPR_DEBUG_ASSERT(fn == &Derived::Call::OnServerInitialMetadata); + call_spine->server_initial_metadata().sender.InterceptAndMap( + [call](ServerMetadataHandle md) { + call->OnServerInitialMetadata(*md); + return md; + }); +} + +template +inline void InterceptServerInitialMetadata( + absl::Status (Derived::Call::*fn)(ServerMetadata&), + typename Derived::Call* call, Derived*, CallSpineInterface* call_spine) { + GPR_DEBUG_ASSERT(fn == &Derived::Call::OnServerInitialMetadata); + call_spine->server_initial_metadata().sender.InterceptAndMap( + [call, call_spine]( + ServerMetadataHandle md) -> absl::optional { + auto status = call->OnServerInitialMetadata(*md); + if (status.ok()) return std::move(md); + return call_spine->Cancel(ServerMetadataFromStatus(status)); + }); +} + +inline void InterceptServerToClientMessage(const NoInterceptor*, void*, + const CallArgs&) {} + +template +inline void InterceptServerToClientMessage( + ServerMetadataHandle (Derived::Call::*fn)(const Message&), + FilterCallData* call_data, const CallArgs& call_args) { + GPR_DEBUG_ASSERT(fn == &Derived::Call::OnServerToClientMessage); + call_args.server_to_client_messages->InterceptAndMap( + [call_data](MessageHandle msg) -> absl::optional { + auto return_md = call_data->call.OnServerToClientMessage(*msg); + if (return_md == nullptr) return std::move(msg); + if (call_data->error_latch.is_set()) return absl::nullopt; + call_data->error_latch.Set(std::move(return_md)); + return absl::nullopt; + }); +} + +template +inline void InterceptServerToClientMessage( + ServerMetadataHandle (Derived::Call::*fn)(const Message&, Derived*), + FilterCallData* call_data, const CallArgs& call_args) { + GPR_DEBUG_ASSERT(fn == &Derived::Call::OnServerToClientMessage); + call_args.server_to_client_messages->InterceptAndMap( + [call_data](MessageHandle msg) -> absl::optional { + auto return_md = + call_data->call.OnServerToClientMessage(*msg, call_data->channel); + if (return_md == nullptr) return std::move(msg); + if (call_data->error_latch.is_set()) return absl::nullopt; + call_data->error_latch.Set(std::move(return_md)); + return absl::nullopt; + }); +} + +inline void InterceptServerToClientMessage(const NoInterceptor*, void*, void*, + CallSpineInterface*) {} + +template +inline void InterceptServerToClientMessage( + ServerMetadataHandle (Derived::Call::*fn)(const Message&), + typename Derived::Call* call, Derived*, CallSpineInterface* call_spine) { + GPR_DEBUG_ASSERT(fn == &Derived::Call::OnServerToClientMessage); + call_spine->server_to_client_messages().sender.InterceptAndMap( + [call, call_spine](MessageHandle msg) -> absl::optional { + auto return_md = call->OnServerToClientMessage(*msg); + if (return_md == nullptr) return std::move(msg); + return call_spine->Cancel(std::move(return_md)); + }); +} + +template +inline void InterceptServerToClientMessage( + ServerMetadataHandle (Derived::Call::*fn)(const Message&, Derived*), + typename Derived::Call* call, Derived* channel, + CallSpineInterface* call_spine) { + GPR_DEBUG_ASSERT(fn == &Derived::Call::OnServerToClientMessage); + call_spine->server_to_client_messages().sender.InterceptAndMap( + [call, call_spine, + channel](MessageHandle msg) -> absl::optional { + auto return_md = call->OnServerToClientMessage(*msg, channel); + if (return_md == nullptr) return std::move(msg); + return call_spine->Cancel(std::move(return_md)); + }); +} + +inline void InterceptServerTrailingMetadata(const NoInterceptor*, void*, void*, + CallSpineInterface*) {} + +template +inline void InterceptServerTrailingMetadata( + void (Derived::Call::*fn)(ServerMetadata&), typename Derived::Call* call, + Derived*, CallSpineInterface* call_spine) { + GPR_DEBUG_ASSERT(fn == &Derived::Call::OnServerTrailingMetadata); + call_spine->server_trailing_metadata().sender.InterceptAndMap( + [call](ServerMetadataHandle md) { + call->OnServerTrailingMetadata(*md); + return md; + }); +} + +template +inline void InterceptServerTrailingMetadata( + absl::Status (Derived::Call::*fn)(ServerMetadata&), + typename Derived::Call* call, Derived*, CallSpineInterface* call_spine) { + GPR_DEBUG_ASSERT(fn == &Derived::Call::OnServerTrailingMetadata); + call_spine->server_trailing_metadata().sender.InterceptAndMap( + [call](ServerMetadataHandle md) -> absl::optional { + auto status = call->OnServerTrailingMetadata(*md); + if (status.ok()) return std::move(md); + return ServerMetadataFromStatus(status); + }); +} + +inline void InterceptFinalize(const NoInterceptor*, void*) {} + +template +inline void InterceptFinalize(void (Call::*fn)(const grpc_call_final_info*), + Call* call) { + GPR_DEBUG_ASSERT(fn == &Call::OnFinalize); + GetContext()->Add( + [call](const grpc_call_final_info* final_info) { + call->OnFinalize(final_info); + }); +} + +template +absl::enable_if_t>::value, + FilterCallData*> +MakeFilterCall(Derived*) { + static FilterCallData call{nullptr}; + return &call; +} + +template +absl::enable_if_t>::value, + FilterCallData*> +MakeFilterCall(Derived* derived) { + return GetContext()->ManagedNew>(derived); +} + +} // namespace promise_filter_detail + +// Base class for promise-based channel filters. +// Eventually this machinery will move elsewhere (the interception logic will +// move directly into the channel stack, and so filters will just directly +// derive from `ChannelFilter`) +// +// Implements new-style call filters, and polyfills them into the previous +// scheme. +// +// Call filters: +// Derived types should declare a class `Call` with the following members: +// - OnClientInitialMetadata - $VALUE_TYPE = ClientMetadata +// - OnServerInitialMetadata - $VALUE_TYPE = ServerMetadata +// - OnServerToClientMessage - $VALUE_TYPE = Message +// - OnClientToServerMessage - $VALUE_TYPE = Message +// - OnServerTrailingMetadata - $VALUE_TYPE = ServerMetadata +// - OnFinalize - special, see below +// These members define an interception point for a particular event in +// the call lifecycle. +// The type of these members matters, and is selectable by the class +// author. For $INTERCEPTOR_NAME in the above list: +// - static const NoInterceptor $INTERCEPTOR_NAME: +// defines that this filter does not intercept this event. +// there is zero runtime cost added to handling that event by this filter. +// - void $INTERCEPTOR_NAME($VALUE_TYPE&): +// the filter intercepts this event, and can modify the value. +// it never fails. +// - absl::Status $INTERCEPTOR_NAME($VALUE_TYPE&): +// the filter intercepts this event, and can modify the value. +// it can fail, in which case the call will be aborted. +// - ServerMetadataHandle $INTERCEPTOR_NAME($VALUE_TYPE&) +// the filter intercepts this event, and can modify the value. +// the filter can return nullptr for success, or a metadata handle for +// failure (in which case the call will be aborted). +// useful for cases where the exact metadata returned needs to be customized. +// - void $INTERCEPTOR_NAME($VALUE_TYPE&, Derived*): +// the filter intercepts this event, and can modify the value. +// it can access the channel via the second argument. +// it never fails. +// - absl::Status $INTERCEPTOR_NAME($VALUE_TYPE&, Derived*): +// the filter intercepts this event, and can modify the value. +// it can access the channel via the second argument. +// it can fail, in which case the call will be aborted. +// - ServerMetadataHandle $INTERCEPTOR_NAME($VALUE_TYPE&, Derived*) +// the filter intercepts this event, and can modify the value. +// it can access the channel via the second argument. +// the filter can return nullptr for success, or a metadata handle for +// failure (in which case the call will be aborted). +// useful for cases where the exact metadata returned needs to be customized. +// Finally, OnFinalize can be added to intecept call finalization. +// It must have one of the signatures: +// - static const NoInterceptor OnFinalize: +// the filter does not intercept call finalization. +// - void OnFinalize(const grpc_call_final_info*): +// the filter intercepts call finalization. +template +class ImplementChannelFilter : public ChannelFilter { + public: + // Natively construct a v3 call. + void InitCall(CallSpineInterface* call_spine) { + typename Derived::Call* call = + GetContext() + ->ManagedNew>( + static_cast(this)); + promise_filter_detail::InterceptClientInitialMetadata( + &Derived::Call::OnClientInitialMetadata, call, + static_cast(this), call_spine); + promise_filter_detail::InterceptClientToServerMessage( + &Derived::Call::OnClientToServerMessage, call, + static_cast(this), call_spine); + promise_filter_detail::InterceptServerInitialMetadata( + &Derived::Call::OnServerInitialMetadata, call, + static_cast(this), call_spine); + promise_filter_detail::InterceptServerToClientMessage( + &Derived::Call::OnServerToClientMessage, call, + static_cast(this), call_spine); + promise_filter_detail::InterceptServerTrailingMetadata( + &Derived::Call::OnServerTrailingMetadata, call, + static_cast(this), call_spine); + promise_filter_detail::InterceptFinalize(&Derived::Call::OnFinalize, call); + } + + // Polyfill for the original promise scheme. + // Allows writing v3 filters that work with v2 stacks. + // (and consequently also v1 stacks since we can polyfill back to that too). + ArenaPromise MakeCallPromise( + CallArgs call_args, NextPromiseFactory next_promise_factory) final { + auto* call = promise_filter_detail::MakeFilterCall( + static_cast(this)); + promise_filter_detail::InterceptClientToServerMessage( + &Derived::Call::OnClientToServerMessage, call, call_args); + promise_filter_detail::InterceptServerInitialMetadata( + &Derived::Call::OnServerInitialMetadata, call, call_args); + promise_filter_detail::InterceptServerToClientMessage( + &Derived::Call::OnServerToClientMessage, call, call_args); + promise_filter_detail::InterceptFinalize( + &Derived::Call::OnFinalize, + static_cast(&call->call)); + return promise_filter_detail::MapResult( + &Derived::Call::OnServerTrailingMetadata, + promise_filter_detail::RaceAsyncCompletion< + promise_filter_detail::CallHasAsyncErrorInterceptor()>:: + Run(promise_filter_detail::RunCall( + &Derived::Call::OnClientInitialMetadata, + std::move(call_args), std::move(next_promise_factory), + call), + &call->error_latch), + call); + } +}; + // Designator for whether a filter is client side or server side. // Please don't use this outside calls to MakePromiseBasedFilter - it's // intended to be deleted once the promise conversion is complete. @@ -912,7 +1577,49 @@ struct ChannelFilterWithFlagsMethods { // ChannelArgs channel_args, ChannelFilter::Args filter_args); // }; template -absl::enable_if_t::value, grpc_channel_filter> +absl::enable_if_t::value && + !std::is_base_of, F>::value, + grpc_channel_filter> +MakePromiseBasedFilter(const char* name) { + using CallData = promise_filter_detail::CallData; + + return grpc_channel_filter{ + // start_transport_stream_op_batch + promise_filter_detail::BaseCallDataMethods::StartTransportStreamOpBatch, + // make_call_promise + promise_filter_detail::ChannelFilterMethods::MakeCallPromise, + nullptr, + // start_transport_op + promise_filter_detail::ChannelFilterMethods::StartTransportOp, + // sizeof_call_data + sizeof(CallData), + // init_call_elem + promise_filter_detail::CallDataFilterWithFlagsMethods< + CallData, kFlags>::InitCallElem, + // set_pollset_or_pollset_set + promise_filter_detail::BaseCallDataMethods::SetPollsetOrPollsetSet, + // destroy_call_elem + promise_filter_detail::CallDataFilterWithFlagsMethods< + CallData, kFlags>::DestroyCallElem, + // sizeof_channel_data + sizeof(F), + // init_channel_elem + promise_filter_detail::ChannelFilterWithFlagsMethods< + F, kFlags>::InitChannelElem, + // post_init_channel_elem + promise_filter_detail::ChannelFilterMethods::PostInitChannelElem, + // destroy_channel_elem + promise_filter_detail::ChannelFilterMethods::DestroyChannelElem, + // get_channel_info + promise_filter_detail::ChannelFilterMethods::GetChannelInfo, + // name + name, + }; +} + +template +absl::enable_if_t, F>::value, + grpc_channel_filter> MakePromiseBasedFilter(const char* name) { using CallData = promise_filter_detail::CallData; @@ -921,6 +1628,9 @@ MakePromiseBasedFilter(const char* name) { promise_filter_detail::BaseCallDataMethods::StartTransportStreamOpBatch, // make_call_promise promise_filter_detail::ChannelFilterMethods::MakeCallPromise, + [](grpc_channel_element* elem, CallSpineInterface* args) { + static_cast(elem->channel_data)->InitCall(args); + }, // start_transport_op promise_filter_detail::ChannelFilterMethods::StartTransportOp, // sizeof_call_data diff --git a/src/core/lib/channel/server_call_tracer_filter.cc b/src/core/lib/channel/server_call_tracer_filter.cc index 026216fd59e2c..c2450a97d4c94 100644 --- a/src/core/lib/channel/server_call_tracer_filter.cc +++ b/src/core/lib/channel/server_call_tracer_filter.cc @@ -42,19 +42,55 @@ namespace grpc_core { namespace { -// TODO(yashykt): This filter is not really needed. We should be able to move -// this to the connected filter. -class ServerCallTracerFilter : public ChannelFilter { +class ServerCallTracerFilter + : public ImplementChannelFilter { public: static const grpc_channel_filter kFilter; static absl::StatusOr Create( const ChannelArgs& /*args*/, ChannelFilter::Args /*filter_args*/); - ArenaPromise MakeCallPromise( - CallArgs call_args, NextPromiseFactory next_promise_factory) override; + class Call { + public: + void OnClientInitialMetadata(ClientMetadata& client_initial_metadata) { + auto* call_tracer = CallTracer(); + if (call_tracer == nullptr) return; + call_tracer->RecordReceivedInitialMetadata(&client_initial_metadata); + } + + void OnServerInitialMetadata(ServerMetadata& server_initial_metadata) { + auto* call_tracer = CallTracer(); + if (call_tracer == nullptr) return; + call_tracer->RecordSendInitialMetadata(&server_initial_metadata); + } + + void OnFinalize(const grpc_call_final_info* final_info) { + auto* call_tracer = CallTracer(); + if (call_tracer == nullptr) return; + call_tracer->RecordEnd(final_info); + } + + void OnServerTrailingMetadata(ServerMetadata& server_trailing_metadata) { + auto* call_tracer = CallTracer(); + if (call_tracer == nullptr) return; + call_tracer->RecordSendTrailingMetadata(&server_trailing_metadata); + } + + static const NoInterceptor OnClientToServerMessage; + static const NoInterceptor OnServerToClientMessage; + + private: + static ServerCallTracer* CallTracer() { + auto* call_context = GetContext(); + return static_cast( + call_context[GRPC_CONTEXT_CALL_TRACER].value); + } + }; }; +const NoInterceptor ServerCallTracerFilter::Call::OnClientToServerMessage; +const NoInterceptor ServerCallTracerFilter::Call::OnServerToClientMessage; + const grpc_channel_filter ServerCallTracerFilter::kFilter = MakePromiseBasedFilter( @@ -65,34 +101,6 @@ absl::StatusOr ServerCallTracerFilter::Create( return ServerCallTracerFilter(); } -ArenaPromise ServerCallTracerFilter::MakeCallPromise( - CallArgs call_args, NextPromiseFactory next_promise_factory) { - auto* call_context = GetContext(); - auto* call_tracer = static_cast( - call_context[GRPC_CONTEXT_CALL_TRACER].value); - if (call_tracer == nullptr) { - return next_promise_factory(std::move(call_args)); - } - call_tracer->RecordReceivedInitialMetadata( - call_args.client_initial_metadata.get()); - call_args.server_initial_metadata->InterceptAndMap( - [call_tracer](ServerMetadataHandle metadata) { - call_tracer->RecordSendInitialMetadata(metadata.get()); - return metadata; - }); - GetContext()->Add( - [call_tracer](const grpc_call_final_info* final_info) { - call_tracer->RecordEnd(final_info); - }); - return OnCancel( - Map(next_promise_factory(std::move(call_args)), - [call_tracer](ServerMetadataHandle md) { - call_tracer->RecordSendTrailingMetadata(md.get()); - return md; - }), - [call_tracer]() { call_tracer->RecordCancel(absl::CancelledError()); }); -} - } // namespace void RegisterServerCallTracerFilter(CoreConfiguration::Builder* builder) { diff --git a/src/core/lib/event_engine/ares_resolver.cc b/src/core/lib/event_engine/ares_resolver.cc index d88f0a9c2450b..70813eb649a7d 100644 --- a/src/core/lib/event_engine/ares_resolver.cc +++ b/src/core/lib/event_engine/ares_resolver.cc @@ -193,7 +193,7 @@ AresResolver::CreateAresResolver( AresResolver::AresResolver( std::unique_ptr polled_fd_factory, std::shared_ptr event_engine, ares_channel channel) - : grpc_core::InternallyRefCounted( + : RefCountedDNSResolverInterface( GRPC_TRACE_FLAG_ENABLED(grpc_trace_ares_resolver) ? "AresResolver" : nullptr), channel_(channel), @@ -230,8 +230,8 @@ void AresResolver::Orphan() { } void AresResolver::LookupHostname( - absl::string_view name, absl::string_view default_port, - EventEngine::DNSResolver::LookupHostnameCallback callback) { + EventEngine::DNSResolver::LookupHostnameCallback callback, + absl::string_view name, absl::string_view default_port) { absl::string_view host; absl::string_view port_string; if (!grpc_core::SplitHostPort(name, &host, &port_string)) { @@ -296,8 +296,8 @@ void AresResolver::LookupHostname( } void AresResolver::LookupSRV( - absl::string_view name, - EventEngine::DNSResolver::LookupSRVCallback callback) { + EventEngine::DNSResolver::LookupSRVCallback callback, + absl::string_view name) { absl::string_view host; absl::string_view port; if (!grpc_core::SplitHostPort(name, &host, &port)) { @@ -325,8 +325,8 @@ void AresResolver::LookupSRV( } void AresResolver::LookupTXT( - absl::string_view name, - EventEngine::DNSResolver::LookupTXTCallback callback) { + EventEngine::DNSResolver::LookupTXTCallback callback, + absl::string_view name) { absl::string_view host; absl::string_view port; if (!grpc_core::SplitHostPort(name, &host, &port)) { @@ -387,7 +387,8 @@ void AresResolver::CheckSocketsLocked() { event_engine_->Run( [self = Ref(DEBUG_LOCATION, "CheckSocketsLocked"), fd_node]() mutable { - self->OnReadable(fd_node, absl::OkStatus()); + static_cast(self.get()) + ->OnReadable(fd_node, absl::OkStatus()); }); } else { // Otherwise register with the poller for readable event. @@ -396,7 +397,8 @@ void AresResolver::CheckSocketsLocked() { fd_node->polled_fd->RegisterForOnReadableLocked( [self = Ref(DEBUG_LOCATION, "CheckSocketsLocked"), fd_node](absl::Status status) mutable { - self->OnReadable(fd_node, status); + static_cast(self.get()) + ->OnReadable(fd_node, status); }); } } @@ -410,7 +412,8 @@ void AresResolver::CheckSocketsLocked() { fd_node->polled_fd->RegisterForOnWriteableLocked( [self = Ref(DEBUG_LOCATION, "CheckSocketsLocked"), fd_node](absl::Status status) mutable { - self->OnWritable(fd_node, status); + static_cast(self.get()) + ->OnWritable(fd_node, status); }); } } @@ -453,7 +456,7 @@ void AresResolver::MaybeStartTimerLocked() { ares_backup_poll_alarm_handle_ = event_engine_->RunAfter( kAresBackupPollAlarmDuration, [self = Ref(DEBUG_LOCATION, "MaybeStartTimerLocked")]() { - self->OnAresBackupPollAlarm(); + static_cast(self.get())->OnAresBackupPollAlarm(); }); } diff --git a/src/core/lib/event_engine/ares_resolver.h b/src/core/lib/event_engine/ares_resolver.h index df394ca69dc20..a1837f2ccf066 100644 --- a/src/core/lib/event_engine/ares_resolver.h +++ b/src/core/lib/event_engine/ares_resolver.h @@ -39,6 +39,7 @@ #include #include "src/core/lib/event_engine/grpc_polled_fd.h" +#include "src/core/lib/event_engine/ref_counted_dns_resolver_interface.h" #include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/sync.h" @@ -54,7 +55,7 @@ extern grpc_core::TraceFlag grpc_trace_ares_resolver; } \ } while (0) -class AresResolver : public grpc_core::InternallyRefCounted { +class AresResolver : public RefCountedDNSResolverInterface { public: static absl::StatusOr> CreateAresResolver(absl::string_view dns_server, @@ -67,15 +68,13 @@ class AresResolver : public grpc_core::InternallyRefCounted { ~AresResolver() override; void Orphan() override ABSL_LOCKS_EXCLUDED(mutex_); - void LookupHostname(absl::string_view name, absl::string_view default_port, - EventEngine::DNSResolver::LookupHostnameCallback callback) - ABSL_LOCKS_EXCLUDED(mutex_); - void LookupSRV(absl::string_view name, - EventEngine::DNSResolver::LookupSRVCallback callback) - ABSL_LOCKS_EXCLUDED(mutex_); - void LookupTXT(absl::string_view name, - EventEngine::DNSResolver::LookupTXTCallback callback) - ABSL_LOCKS_EXCLUDED(mutex_); + void LookupHostname(EventEngine::DNSResolver::LookupHostnameCallback callback, + absl::string_view name, absl::string_view default_port) + ABSL_LOCKS_EXCLUDED(mutex_) override; + void LookupSRV(EventEngine::DNSResolver::LookupSRVCallback callback, + absl::string_view name) ABSL_LOCKS_EXCLUDED(mutex_) override; + void LookupTXT(EventEngine::DNSResolver::LookupTXTCallback callback, + absl::string_view name) ABSL_LOCKS_EXCLUDED(mutex_) override; private: // A FdNode saves (not owns) a live socket/fd which c-ares creates, and owns a diff --git a/src/core/lib/event_engine/memory_allocator.cc b/src/core/lib/event_engine/memory_allocator.cc deleted file mode 100644 index 69ece21c2944a..0000000000000 --- a/src/core/lib/event_engine/memory_allocator.cc +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2021 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include -#include - -#include -#include -#include - -#include -#include -#include - -#include "src/core/lib/slice/slice_refcount.h" - -namespace grpc_event_engine { -namespace experimental { - -namespace { - -// Reference count for a slice allocated by MemoryAllocator::MakeSlice. -// Takes care of releasing memory back when the slice is destroyed. -class SliceRefCount : public grpc_slice_refcount { - public: - SliceRefCount(std::shared_ptr allocator, - size_t size) - : grpc_slice_refcount(Destroy), - allocator_(std::move(allocator)), - size_(size) { - // Nothing to do here. - } - ~SliceRefCount() { allocator_->Release(size_); } - - private: - static void Destroy(grpc_slice_refcount* p) { - auto* rc = static_cast(p); - rc->~SliceRefCount(); - free(rc); - } - - std::shared_ptr allocator_; - size_t size_; -}; - -} // namespace - -grpc_slice MemoryAllocator::MakeSlice(MemoryRequest request) { - auto size = Reserve(request.Increase(sizeof(SliceRefCount))); - void* p = malloc(size); - new (p) SliceRefCount(allocator_, size); - grpc_slice slice; - slice.refcount = static_cast(p); - slice.data.refcounted.bytes = - static_cast(p) + sizeof(SliceRefCount); - slice.data.refcounted.length = size - sizeof(SliceRefCount); - return slice; -} - -} // namespace experimental -} // namespace grpc_event_engine diff --git a/src/core/lib/event_engine/posix_engine/native_dns_resolver.cc b/src/core/lib/event_engine/posix_engine/native_dns_resolver.cc new file mode 100644 index 0000000000000..79a9489d66189 --- /dev/null +++ b/src/core/lib/event_engine/posix_engine/native_dns_resolver.cc @@ -0,0 +1,131 @@ +// Copyright 2023 The gRPC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "src/core/lib/iomgr/port.h" + +#ifdef GRPC_POSIX_SOCKET_RESOLVE_ADDRESS + +#include +#include +#include + +#include +#include +#include +#include + +#include "absl/functional/any_invocable.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" + +#include "src/core/lib/event_engine/posix_engine/native_dns_resolver.h" +#include "src/core/lib/gpr/useful.h" +#include "src/core/lib/gprpp/host_port.h" + +namespace grpc_event_engine { +namespace experimental { +namespace { + +absl::StatusOr> +LookupHostnameBlocking(absl::string_view name, absl::string_view default_port) { + struct addrinfo hints; + struct addrinfo *result = nullptr, *resp; + std::string host; + std::string port; + // parse name, splitting it into host and port parts + grpc_core::SplitHostPort(name, &host, &port); + if (host.empty()) { + return absl::InvalidArgumentError(absl::StrCat("Unparseable name: ", name)); + } + if (port.empty()) { + if (default_port.empty()) { + return absl::InvalidArgumentError( + absl::StrFormat("No port in name %s or default_port argument", name)); + } + port = std::string(default_port); + } + // Call getaddrinfo + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; // ipv4 or ipv6 + hints.ai_socktype = SOCK_STREAM; // stream socket + hints.ai_flags = AI_PASSIVE; // for wildcard IP address + int s = getaddrinfo(host.c_str(), port.c_str(), &hints, &result); + if (s != 0) { + // Retry if well-known service name is recognized + const char* svc[][2] = {{"http", "80"}, {"https", "443"}}; + for (size_t i = 0; i < GPR_ARRAY_SIZE(svc); i++) { + if (port == svc[i][0]) { + s = getaddrinfo(host.c_str(), svc[i][1], &hints, &result); + break; + } + } + } + if (s != 0) { + return absl::UnknownError(absl::StrFormat( + "Address lookup failed for %s os_error: %s syscall: getaddrinfo", name, + gai_strerror(s))); + } + // Success path: fill in addrs + std::vector addresses; + for (resp = result; resp != nullptr; resp = resp->ai_next) { + addresses.emplace_back(resp->ai_addr, resp->ai_addrlen); + } + if (result) { + freeaddrinfo(result); + } + return addresses; +} + +} // namespace + +NativeDNSResolver::NativeDNSResolver(std::shared_ptr event_engine) + : event_engine_(std::move(event_engine)) {} + +void NativeDNSResolver::LookupHostname( + EventEngine::DNSResolver::LookupHostnameCallback on_resolved, + absl::string_view name, absl::string_view default_port) { + event_engine_->Run( + [name, default_port, on_resolved = std::move(on_resolved)]() mutable { + on_resolved(LookupHostnameBlocking(name, default_port)); + }); +} + +void NativeDNSResolver::LookupSRV( + EventEngine::DNSResolver::LookupSRVCallback on_resolved, + absl::string_view /* name */) { + // Not supported + event_engine_->Run([on_resolved = std::move(on_resolved)]() mutable { + on_resolved(absl::UnimplementedError( + "The Native resolver does not support looking up SRV records")); + }); +} + +void NativeDNSResolver::LookupTXT( + EventEngine::DNSResolver::LookupTXTCallback on_resolved, + absl::string_view /* name */) { + // Not supported + event_engine_->Run([on_resolved = std::move(on_resolved)]() mutable { + on_resolved(absl::UnimplementedError( + "The Native resolver does not support looking up TXT records")); + }); +} + +} // namespace experimental +} // namespace grpc_event_engine + +#endif // GRPC_POSIX_SOCKET_RESOLVE_ADDRESS diff --git a/src/core/lib/event_engine/posix_engine/native_dns_resolver.h b/src/core/lib/event_engine/posix_engine/native_dns_resolver.h new file mode 100644 index 0000000000000..411b090ab64e2 --- /dev/null +++ b/src/core/lib/event_engine/posix_engine/native_dns_resolver.h @@ -0,0 +1,61 @@ +// Copyright 2023 The gRPC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GRPC_SRC_CORE_LIB_EVENT_ENGINE_POSIX_ENGINE_NATIVE_DNS_RESOLVER_H +#define GRPC_SRC_CORE_LIB_EVENT_ENGINE_POSIX_ENGINE_NATIVE_DNS_RESOLVER_H + +#include + +#include + +#include "absl/strings/string_view.h" + +#include "src/core/lib/iomgr/port.h" + +#ifdef GRPC_POSIX_SOCKET_RESOLVE_ADDRESS + +#include + +#include "src/core/lib/event_engine/ref_counted_dns_resolver_interface.h" + +namespace grpc_event_engine { +namespace experimental { + +// An asynchronous DNS resolver which uses the native platform's getaddrinfo +// API. Only supports A/AAAA records. +class NativeDNSResolver : public RefCountedDNSResolverInterface { + public: + explicit NativeDNSResolver(std::shared_ptr event_engine); + + void LookupHostname( + EventEngine::DNSResolver::LookupHostnameCallback on_resolved, + absl::string_view name, absl::string_view default_port) override; + + void LookupSRV(EventEngine::DNSResolver::LookupSRVCallback on_resolved, + absl::string_view name) override; + + void LookupTXT(EventEngine::DNSResolver::LookupTXTCallback on_resolved, + absl::string_view name) override; + + void Orphan() override { delete this; } + + private: + std::shared_ptr event_engine_; +}; + +} // namespace experimental +} // namespace grpc_event_engine + +#endif // GRPC_POSIX_SOCKET_RESOLVE_ADDRESS +#endif // GRPC_SRC_CORE_LIB_EVENT_ENGINE_POSIX_ENGINE_NATIVE_DNS_RESOLVER_H diff --git a/src/core/lib/event_engine/posix_engine/posix_engine.cc b/src/core/lib/event_engine/posix_engine/posix_engine.cc index f7e9a035f6393..a5a4ee1b40fce 100644 --- a/src/core/lib/event_engine/posix_engine/posix_engine.cc +++ b/src/core/lib/event_engine/posix_engine/posix_engine.cc @@ -28,6 +28,7 @@ #include "absl/cleanup/cleanup.h" #include "absl/functional/any_invocable.h" #include "absl/status/status.h" +#include "absl/strings/match.h" #include "absl/strings/str_cat.h" #include @@ -36,12 +37,15 @@ #include #include +#include "src/core/lib/config/config_vars.h" #include "src/core/lib/debug/trace.h" +#include "src/core/lib/event_engine/ares_resolver.h" #include "src/core/lib/event_engine/forkable.h" #include "src/core/lib/event_engine/grpc_polled_fd.h" #include "src/core/lib/event_engine/poller.h" #include "src/core/lib/event_engine/posix.h" #include "src/core/lib/event_engine/posix_engine/grpc_polled_fd_posix.h" +#include "src/core/lib/event_engine/posix_engine/native_dns_resolver.h" #include "src/core/lib/event_engine/posix_engine/tcp_socket_utils.h" #include "src/core/lib/event_engine/posix_engine/timer.h" #include "src/core/lib/event_engine/tcp_socket_utils.h" @@ -90,6 +94,15 @@ class TimerForkCallbackMethods { static void PostforkChild() { g_timer_fork_manager->PostforkChild(); } }; +bool ShouldUseAresDnsResolver() { +#if GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_ARES_EV_DRIVER) + auto resolver_env = grpc_core::ConfigVars::Get().DnsResolver(); + return resolver_env.empty() || absl::EqualsIgnoreCase(resolver_env, "ares"); +#else // GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_ARES_EV_DRIVER) + return false; +#endif // GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_ARES_EV_DRIVER) +} + } // namespace #ifdef GRPC_POSIX_SOCKET_TCP @@ -517,49 +530,54 @@ EventEngine::TaskHandle PosixEventEngine::RunAfterInternal( return handle; } -#if GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_TCP) - PosixEventEngine::PosixDNSResolver::PosixDNSResolver( - grpc_core::OrphanablePtr ares_resolver) - : ares_resolver_(std::move(ares_resolver)) {} + grpc_core::OrphanablePtr dns_resolver) + : dns_resolver_(std::move(dns_resolver)) {} void PosixEventEngine::PosixDNSResolver::LookupHostname( LookupHostnameCallback on_resolve, absl::string_view name, absl::string_view default_port) { - ares_resolver_->LookupHostname(name, default_port, std::move(on_resolve)); + dns_resolver_->LookupHostname(std::move(on_resolve), name, default_port); } void PosixEventEngine::PosixDNSResolver::LookupSRV(LookupSRVCallback on_resolve, absl::string_view name) { - ares_resolver_->LookupSRV(name, std::move(on_resolve)); + dns_resolver_->LookupSRV(std::move(on_resolve), name); } void PosixEventEngine::PosixDNSResolver::LookupTXT(LookupTXTCallback on_resolve, absl::string_view name) { - ares_resolver_->LookupTXT(name, std::move(on_resolve)); + dns_resolver_->LookupTXT(std::move(on_resolve), name); } -#endif // GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_TCP) - absl::StatusOr> PosixEventEngine::GetDNSResolver( - const EventEngine::DNSResolver::ResolverOptions& options) { -#if GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_TCP) - auto ares_resolver = AresResolver::CreateAresResolver( - options.dns_server, - std::make_unique(poller_manager_->Poller()), - shared_from_this()); - if (!ares_resolver.ok()) { - return ares_resolver.status(); + GRPC_UNUSED const EventEngine::DNSResolver::ResolverOptions& options) { +#ifndef GRPC_POSIX_SOCKET_RESOLVE_ADDRESS + grpc_core::Crash("Unable to get DNS resolver for this platform."); +#else // GRPC_POSIX_SOCKET_RESOLVE_ADDRESS + // If c-ares is supported on the platform, build according to user's + // configuration. + if (ShouldUseAresDnsResolver()) { +#if GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_ARES_EV_DRIVER) + GRPC_EVENT_ENGINE_DNS_TRACE("PosixEventEngine:%p creating AresResolver", + this); + auto ares_resolver = AresResolver::CreateAresResolver( + options.dns_server, + std::make_unique(poller_manager_->Poller()), + shared_from_this()); + if (!ares_resolver.ok()) { + return ares_resolver.status(); + } + return std::make_unique( + std::move(*ares_resolver)); +#endif // GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_ARES_EV_DRIVER) } + GRPC_EVENT_ENGINE_DNS_TRACE("PosixEventEngine:%p creating NativeDNSResolver", + this); return std::make_unique( - std::move(*ares_resolver)); -#else // GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_TCP) - // TODO(yijiem): Implement a basic A/AAAA-only native resolver in - // PosixEventEngine. - (void)options; - grpc_core::Crash("unimplemented"); -#endif // GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_TCP) + grpc_core::MakeOrphanable(shared_from_this())); +#endif // GRPC_POSIX_SOCKET_RESOLVE_ADDRESS } bool PosixEventEngine::IsWorkerThread() { grpc_core::Crash("unimplemented"); } diff --git a/src/core/lib/event_engine/posix_engine/posix_engine.h b/src/core/lib/event_engine/posix_engine/posix_engine.h index 50810a53b0875..c025d46c6c895 100644 --- a/src/core/lib/event_engine/posix_engine/posix_engine.h +++ b/src/core/lib/event_engine/posix_engine/posix_engine.h @@ -34,11 +34,11 @@ #include #include -#include "src/core/lib/event_engine/ares_resolver.h" #include "src/core/lib/event_engine/handle_containers.h" #include "src/core/lib/event_engine/posix.h" #include "src/core/lib/event_engine/posix_engine/event_poller.h" #include "src/core/lib/event_engine/posix_engine/timer_manager.h" +#include "src/core/lib/event_engine/ref_counted_dns_resolver_interface.h" #include "src/core/lib/event_engine/thread_pool/thread_pool.h" #include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/sync.h" @@ -141,11 +141,8 @@ class PosixEventEngine final : public PosixEventEngineWithFdSupport, public: class PosixDNSResolver : public EventEngine::DNSResolver { public: - PosixDNSResolver() = delete; -#if GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_TCP) explicit PosixDNSResolver( - grpc_core::OrphanablePtr ares_resolver); -#endif // GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_TCP) + grpc_core::OrphanablePtr dns_resolver); void LookupHostname(LookupHostnameCallback on_resolve, absl::string_view name, absl::string_view default_port) override; @@ -154,10 +151,8 @@ class PosixEventEngine final : public PosixEventEngineWithFdSupport, void LookupTXT(LookupTXTCallback on_resolve, absl::string_view name) override; -#if GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_TCP) private: - grpc_core::OrphanablePtr ares_resolver_; -#endif // GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_TCP) + grpc_core::OrphanablePtr dns_resolver_; }; #ifdef GRPC_POSIX_SOCKET_TCP @@ -203,7 +198,7 @@ class PosixEventEngine final : public PosixEventEngineWithFdSupport, bool CancelConnect(ConnectionHandle handle) override; bool IsWorkerThread() override; absl::StatusOr> GetDNSResolver( - const DNSResolver::ResolverOptions& options) override; + GRPC_UNUSED const DNSResolver::ResolverOptions& options) override; void Run(Closure* closure) override; void Run(absl::AnyInvocable closure) override; // Caution!! The timer implementation cannot create any fds. See #20418. diff --git a/src/core/lib/event_engine/posix_engine/tcp_socket_utils.cc b/src/core/lib/event_engine/posix_engine/tcp_socket_utils.cc index c5b2277d6a753..76e1b0a57e199 100644 --- a/src/core/lib/event_engine/posix_engine/tcp_socket_utils.cc +++ b/src/core/lib/event_engine/posix_engine/tcp_socket_utils.cc @@ -25,6 +25,7 @@ #include "absl/types/optional.h" #include +#include #include #include "src/core/lib/gpr/useful.h" @@ -209,6 +210,13 @@ PosixTcpOptions TcpOptionsFromEndpointConfig(const EndpointConfig& config) { options.socket_mutator = grpc_socket_mutator_ref(static_cast(value)); } + value = + config.GetVoidPointer(GRPC_ARG_EVENT_ENGINE_USE_MEMORY_ALLOCATOR_FACTORY); + if (value != nullptr) { + options.memory_allocator_factory = + static_cast( + value); + } return options; } diff --git a/src/core/lib/event_engine/posix_engine/tcp_socket_utils.h b/src/core/lib/event_engine/posix_engine/tcp_socket_utils.h index 278e70ddbddbd..83b52be657c85 100644 --- a/src/core/lib/event_engine/posix_engine/tcp_socket_utils.h +++ b/src/core/lib/event_engine/posix_engine/tcp_socket_utils.h @@ -26,6 +26,7 @@ #include #include +#include #include #include @@ -75,6 +76,8 @@ struct PosixTcpOptions { int dscp = kDscpNotSet; grpc_core::RefCountedPtr resource_quota; struct grpc_socket_mutator* socket_mutator = nullptr; + grpc_event_engine::experimental::MemoryAllocatorFactory* + memory_allocator_factory = nullptr; PosixTcpOptions() = default; // Move ctor PosixTcpOptions(PosixTcpOptions&& other) noexcept { @@ -89,6 +92,8 @@ struct PosixTcpOptions { } socket_mutator = std::exchange(other.socket_mutator, nullptr); resource_quota = std::move(other.resource_quota); + memory_allocator_factory = + std::exchange(other.memory_allocator_factory, nullptr); CopyIntegerOptions(other); return *this; } @@ -98,6 +103,7 @@ struct PosixTcpOptions { socket_mutator = grpc_socket_mutator_ref(other.socket_mutator); } resource_quota = other.resource_quota; + memory_allocator_factory = other.memory_allocator_factory; CopyIntegerOptions(other); } // Copy assignment @@ -113,6 +119,7 @@ struct PosixTcpOptions { socket_mutator = grpc_socket_mutator_ref(other.socket_mutator); } resource_quota = other.resource_quota; + memory_allocator_factory = other.memory_allocator_factory; CopyIntegerOptions(other); return *this; } diff --git a/src/core/lib/event_engine/query_extensions.h b/src/core/lib/event_engine/query_extensions.h new file mode 100644 index 0000000000000..2ef15ccfdab6f --- /dev/null +++ b/src/core/lib/event_engine/query_extensions.h @@ -0,0 +1,70 @@ +// Copyright 2023 gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#ifndef GRPC_SRC_CORE_LIB_EVENT_ENGINE_QUERY_EXTENSIONS_H +#define GRPC_SRC_CORE_LIB_EVENT_ENGINE_QUERY_EXTENSIONS_H + +#include + +#include "absl/strings/string_view.h" + +#include + +namespace grpc_event_engine { +namespace experimental { + +namespace endpoint_detail { + +template +struct QueryExtensionRecursion; + +template +struct QueryExtensionRecursion { + static void* Query(absl::string_view id, Querying* p) { + if (id == E::EndpointExtensionName()) return static_cast(p); + return QueryExtensionRecursion::Query(id, p); + } +}; + +template +struct QueryExtensionRecursion { + static void* Query(absl::string_view, Querying*) { return nullptr; } +}; + +} // namespace endpoint_detail + +// A helper class to derive from some set of base classes and export +// QueryExtension for them all. +// Endpoint implementations which need to support different extensions just need +// to derive from ExtendedEndpoint class. +template +class ExtendedEndpoint : public EventEngine::Endpoint, public Exports... { + public: + void* QueryExtension(absl::string_view id) override { + return endpoint_detail::QueryExtensionRecursion::Query(id, + this); + } +}; + +/// A helper method which returns a valid pointer if the extension is supported +/// by the endpoint. +template +T* QueryExtension(EventEngine::Endpoint* endpoint) { + return static_cast(endpoint->QueryExtension(T::EndpointExtensionName())); +} + +} // namespace experimental +} // namespace grpc_event_engine + +#endif // GRPC_SRC_CORE_LIB_EVENT_ENGINE_QUERY_EXTENSIONS_H diff --git a/src/core/lib/event_engine/ref_counted_dns_resolver_interface.h b/src/core/lib/event_engine/ref_counted_dns_resolver_interface.h new file mode 100644 index 0000000000000..645788e9a833f --- /dev/null +++ b/src/core/lib/event_engine/ref_counted_dns_resolver_interface.h @@ -0,0 +1,55 @@ +// Copyright 2023 The gRPC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GRPC_SRC_CORE_LIB_EVENT_ENGINE_REF_COUNTED_DNS_RESOLVER_INTERFACE_H +#define GRPC_SRC_CORE_LIB_EVENT_ENGINE_REF_COUNTED_DNS_RESOLVER_INTERFACE_H + +#include + +#include + +#include "absl/strings/string_view.h" + +#include + +#include "src/core/lib/gprpp/orphanable.h" + +namespace grpc_event_engine { +namespace experimental { + +class RefCountedDNSResolverInterface + : public grpc_core::InternallyRefCounted { + public: + explicit RefCountedDNSResolverInterface(const char* trace = nullptr, + intptr_t initial_refcount = 1) + : grpc_core::InternallyRefCounted( + trace, initial_refcount) {} + + virtual void LookupHostname( + EventEngine::DNSResolver::LookupHostnameCallback on_resolved, + absl::string_view name, absl::string_view default_port) = 0; + + virtual void LookupSRV( + EventEngine::DNSResolver::LookupSRVCallback on_resolved, + absl::string_view name) = 0; + + virtual void LookupTXT( + EventEngine::DNSResolver::LookupTXTCallback on_resolved, + absl::string_view name) = 0; +}; + +} // namespace experimental +} // namespace grpc_event_engine + +#endif // GRPC_SRC_CORE_LIB_EVENT_ENGINE_REF_COUNTED_DNS_RESOLVER_INTERFACE_H diff --git a/src/core/lib/event_engine/windows/windows_engine.cc b/src/core/lib/event_engine/windows/windows_engine.cc index 24b5c8898e683..2512026703945 100644 --- a/src/core/lib/event_engine/windows/windows_engine.cc +++ b/src/core/lib/event_engine/windows/windows_engine.cc @@ -204,17 +204,17 @@ WindowsEventEngine::WindowsDNSResolver::WindowsDNSResolver( void WindowsEventEngine::WindowsDNSResolver::LookupHostname( LookupHostnameCallback on_resolve, absl::string_view name, absl::string_view default_port) { - ares_resolver_->LookupHostname(name, default_port, std::move(on_resolve)); + ares_resolver_->LookupHostname(std::move(on_resolve), name, default_port); } void WindowsEventEngine::WindowsDNSResolver::LookupSRV( LookupSRVCallback on_resolve, absl::string_view name) { - ares_resolver_->LookupSRV(name, std::move(on_resolve)); + ares_resolver_->LookupSRV(std::move(on_resolve), name); } void WindowsEventEngine::WindowsDNSResolver::LookupTXT( LookupTXTCallback on_resolve, absl::string_view name) { - ares_resolver_->LookupTXT(name, std::move(on_resolve)); + ares_resolver_->LookupTXT(std::move(on_resolve), name); } #endif // GRPC_ARES == 1 && defined(GRPC_WINDOWS_SOCKET_ARES_EV_DRIVER) diff --git a/src/core/lib/experiments/experiments.cc b/src/core/lib/experiments/experiments.cc index 1e8c852545dda..d44fdac9f7216 100644 --- a/src/core/lib/experiments/experiments.cc +++ b/src/core/lib/experiments/experiments.cc @@ -24,10 +24,6 @@ #if defined(GRPC_CFSTREAM) namespace { -const char* const description_block_excessive_requests_before_settings_ack = - "If set, block excessive requests before receiving SETTINGS ACK."; -const char* const - additional_constraints_block_excessive_requests_before_settings_ack = "{}"; const char* const description_call_status_override_on_cancellation = "Avoid overriding call status of successfully finished calls if it races " "with cancellation."; @@ -41,21 +37,6 @@ const char* const description_client_idleness = const char* const additional_constraints_client_idleness = "{}"; const char* const description_client_privacy = "If set, client privacy"; const char* const additional_constraints_client_privacy = "{}"; -const char* const description_combiner_offload_to_event_engine = - "Offload Combiner work onto the EventEngine instead of the Executor."; -const char* const additional_constraints_combiner_offload_to_event_engine = - "{}"; -const char* const description_chttp2_batch_requests = - "Cap the number of requests received by one transport read prior to " - "offload."; -const char* const additional_constraints_chttp2_batch_requests = "{}"; -const uint8_t required_experiments_chttp2_batch_requests[] = { - static_cast(grpc_core::kExperimentIdCombinerOffloadToEventEngine)}; -const char* const description_chttp2_offload_on_rst_stream = - "Offload work on RST_STREAM."; -const char* const additional_constraints_chttp2_offload_on_rst_stream = "{}"; -const uint8_t required_experiments_chttp2_offload_on_rst_stream[] = { - static_cast(grpc_core::kExperimentIdCombinerOffloadToEventEngine)}; const char* const description_event_engine_client = "Use EventEngine clients instead of iomgr's grpc_tcp_client"; const char* const additional_constraints_event_engine_client = "{}"; @@ -79,10 +60,6 @@ const char* const description_keepalive_server_fix = "Allows overriding keepalive_permit_without_calls for servers. Refer " "https://github.com/grpc/grpc/pull/33917 for more information."; const char* const additional_constraints_keepalive_server_fix = "{}"; -const char* const description_lazier_stream_updates = - "Allow streams to consume up to 50% of the incoming window before we force " - "send a flow control update."; -const char* const additional_constraints_lazier_stream_updates = "{}"; const char* const description_memory_pressure_controller = "New memory pressure controller"; const char* const additional_constraints_memory_pressure_controller = "{}"; @@ -112,10 +89,6 @@ const char* const additional_constraints_pending_queue_cap = "{}"; const char* const description_pick_first_happy_eyeballs = "Use Happy Eyeballs in pick_first."; const char* const additional_constraints_pick_first_happy_eyeballs = "{}"; -const char* const description_ping_on_rst_stream = - "Send a ping on receiving some RST_STREAM frames on the server (proportion " - "configurable via grpc.http2.ping_on_rst_stream_percent channel arg)."; -const char* const additional_constraints_ping_on_rst_stream = "{}"; const char* const description_promise_based_client_call = "If set, use the new gRPC promise based call code when it's appropriate " "(ie when all filters in a stack are promise based)"; @@ -127,8 +100,6 @@ const char* const description_promise_based_server_call = "If set, use the new gRPC promise based call code when it's appropriate " "(ie when all filters in a stack are promise based)"; const char* const additional_constraints_promise_based_server_call = "{}"; -const uint8_t required_experiments_promise_based_server_call[] = { - static_cast(grpc_core::kExperimentIdLazierStreamUpdates)}; const char* const description_red_max_concurrent_streams = "Perform random early rejection of requests that would exceed a newly " "reduced MAX_CONCURRENT_STREAMS but are allowed by the current."; @@ -158,20 +129,8 @@ const char* const description_schedule_cancellation_over_write = "Allow cancellation op to be scheduled over a write"; const char* const additional_constraints_schedule_cancellation_over_write = "{}"; -const char* const description_separate_ping_from_keepalive = - "Keep a different keepalive timeout (resolution is seeing data after " - "sending a ping) from a ping timeout (resolution is getting a ping ack " - "after sending a ping) The first can be short and determines liveness. The " - "second can be longer and determines protocol correctness."; -const char* const additional_constraints_separate_ping_from_keepalive = "{}"; const char* const description_server_privacy = "If set, server privacy"; const char* const additional_constraints_server_privacy = "{}"; -const char* const description_settings_timeout = - "If set, use the settings timeout to send settings frame to the peer."; -const char* const additional_constraints_settings_timeout = "{}"; -const char* const description_tarpit = - "If set, tarpit invalid requests for some amount of time"; -const char* const additional_constraints_tarpit = "{}"; const char* const description_tcp_frame_size_tuning = "If set, enables TCP to use RPC size estimation made by higher layers. TCP " "would not indicate completion of a read operation until a specified " @@ -188,10 +147,6 @@ const char* const description_unconstrained_max_quota_buffer_size = "Discard the cap on the max free pool size for one memory allocator"; const char* const additional_constraints_unconstrained_max_quota_buffer_size = "{}"; -const char* const description_uniquely_unowned = - "Ensure HPACK table takes a unique copy of data when parsing unknown " - "metadata"; -const char* const additional_constraints_uniquely_unowned = "{}"; const char* const description_work_serializer_clears_time_cache = "Have the work serializer clear the time cache when it dispatches work."; const char* const additional_constraints_work_serializer_clears_time_cache = @@ -223,10 +178,6 @@ const bool kDefaultForDebugOnly = true; namespace grpc_core { const ExperimentMetadata g_experiment_metadata[] = { - {"block_excessive_requests_before_settings_ack", - description_block_excessive_requests_before_settings_ack, - additional_constraints_block_excessive_requests_before_settings_ack, - nullptr, 0, true, true}, {"call_status_override_on_cancellation", description_call_status_override_on_cancellation, additional_constraints_call_status_override_on_cancellation, nullptr, 0, @@ -237,16 +188,6 @@ const ExperimentMetadata g_experiment_metadata[] = { additional_constraints_client_idleness, nullptr, 0, true, true}, {"client_privacy", description_client_privacy, additional_constraints_client_privacy, nullptr, 0, false, false}, - {"combiner_offload_to_event_engine", - description_combiner_offload_to_event_engine, - additional_constraints_combiner_offload_to_event_engine, nullptr, 0, true, - true}, - {"chttp2_batch_requests", description_chttp2_batch_requests, - additional_constraints_chttp2_batch_requests, - required_experiments_chttp2_batch_requests, 1, true, true}, - {"chttp2_offload_on_rst_stream", description_chttp2_offload_on_rst_stream, - additional_constraints_chttp2_offload_on_rst_stream, - required_experiments_chttp2_offload_on_rst_stream, 1, true, true}, {"event_engine_client", description_event_engine_client, additional_constraints_event_engine_client, nullptr, 0, false, true}, {"event_engine_dns", description_event_engine_dns, @@ -261,8 +202,6 @@ const ExperimentMetadata g_experiment_metadata[] = { additional_constraints_keepalive_fix, nullptr, 0, false, false}, {"keepalive_server_fix", description_keepalive_server_fix, additional_constraints_keepalive_server_fix, nullptr, 0, false, false}, - {"lazier_stream_updates", description_lazier_stream_updates, - additional_constraints_lazier_stream_updates, nullptr, 0, true, true}, {"memory_pressure_controller", description_memory_pressure_controller, additional_constraints_memory_pressure_controller, nullptr, 0, false, true}, @@ -278,8 +217,6 @@ const ExperimentMetadata g_experiment_metadata[] = { additional_constraints_pending_queue_cap, nullptr, 0, true, true}, {"pick_first_happy_eyeballs", description_pick_first_happy_eyeballs, additional_constraints_pick_first_happy_eyeballs, nullptr, 0, true, true}, - {"ping_on_rst_stream", description_ping_on_rst_stream, - additional_constraints_ping_on_rst_stream, nullptr, 0, true, true}, {"promise_based_client_call", description_promise_based_client_call, additional_constraints_promise_based_client_call, nullptr, 0, false, true}, {"promise_based_inproc_transport", @@ -287,8 +224,7 @@ const ExperimentMetadata g_experiment_metadata[] = { additional_constraints_promise_based_inproc_transport, nullptr, 0, false, false}, {"promise_based_server_call", description_promise_based_server_call, - additional_constraints_promise_based_server_call, - required_experiments_promise_based_server_call, 1, false, true}, + additional_constraints_promise_based_server_call, nullptr, 0, false, true}, {"red_max_concurrent_streams", description_red_max_concurrent_streams, additional_constraints_red_max_concurrent_streams, nullptr, 0, false, true}, @@ -311,15 +247,8 @@ const ExperimentMetadata g_experiment_metadata[] = { description_schedule_cancellation_over_write, additional_constraints_schedule_cancellation_over_write, nullptr, 0, false, true}, - {"separate_ping_from_keepalive", description_separate_ping_from_keepalive, - additional_constraints_separate_ping_from_keepalive, nullptr, 0, true, - true}, {"server_privacy", description_server_privacy, additional_constraints_server_privacy, nullptr, 0, false, false}, - {"settings_timeout", description_settings_timeout, - additional_constraints_settings_timeout, nullptr, 0, true, true}, - {"tarpit", description_tarpit, additional_constraints_tarpit, nullptr, 0, - true, true}, {"tcp_frame_size_tuning", description_tcp_frame_size_tuning, additional_constraints_tcp_frame_size_tuning, nullptr, 0, false, true}, {"tcp_rcv_lowat", description_tcp_rcv_lowat, @@ -330,8 +259,6 @@ const ExperimentMetadata g_experiment_metadata[] = { description_unconstrained_max_quota_buffer_size, additional_constraints_unconstrained_max_quota_buffer_size, nullptr, 0, false, true}, - {"uniquely_unowned", description_uniquely_unowned, - additional_constraints_uniquely_unowned, nullptr, 0, true, true}, {"work_serializer_clears_time_cache", description_work_serializer_clears_time_cache, additional_constraints_work_serializer_clears_time_cache, nullptr, 0, true, @@ -351,10 +278,6 @@ const ExperimentMetadata g_experiment_metadata[] = { #elif defined(GPR_WINDOWS) namespace { -const char* const description_block_excessive_requests_before_settings_ack = - "If set, block excessive requests before receiving SETTINGS ACK."; -const char* const - additional_constraints_block_excessive_requests_before_settings_ack = "{}"; const char* const description_call_status_override_on_cancellation = "Avoid overriding call status of successfully finished calls if it races " "with cancellation."; @@ -368,21 +291,6 @@ const char* const description_client_idleness = const char* const additional_constraints_client_idleness = "{}"; const char* const description_client_privacy = "If set, client privacy"; const char* const additional_constraints_client_privacy = "{}"; -const char* const description_combiner_offload_to_event_engine = - "Offload Combiner work onto the EventEngine instead of the Executor."; -const char* const additional_constraints_combiner_offload_to_event_engine = - "{}"; -const char* const description_chttp2_batch_requests = - "Cap the number of requests received by one transport read prior to " - "offload."; -const char* const additional_constraints_chttp2_batch_requests = "{}"; -const uint8_t required_experiments_chttp2_batch_requests[] = { - static_cast(grpc_core::kExperimentIdCombinerOffloadToEventEngine)}; -const char* const description_chttp2_offload_on_rst_stream = - "Offload work on RST_STREAM."; -const char* const additional_constraints_chttp2_offload_on_rst_stream = "{}"; -const uint8_t required_experiments_chttp2_offload_on_rst_stream[] = { - static_cast(grpc_core::kExperimentIdCombinerOffloadToEventEngine)}; const char* const description_event_engine_client = "Use EventEngine clients instead of iomgr's grpc_tcp_client"; const char* const additional_constraints_event_engine_client = "{}"; @@ -406,10 +314,6 @@ const char* const description_keepalive_server_fix = "Allows overriding keepalive_permit_without_calls for servers. Refer " "https://github.com/grpc/grpc/pull/33917 for more information."; const char* const additional_constraints_keepalive_server_fix = "{}"; -const char* const description_lazier_stream_updates = - "Allow streams to consume up to 50% of the incoming window before we force " - "send a flow control update."; -const char* const additional_constraints_lazier_stream_updates = "{}"; const char* const description_memory_pressure_controller = "New memory pressure controller"; const char* const additional_constraints_memory_pressure_controller = "{}"; @@ -439,10 +343,6 @@ const char* const additional_constraints_pending_queue_cap = "{}"; const char* const description_pick_first_happy_eyeballs = "Use Happy Eyeballs in pick_first."; const char* const additional_constraints_pick_first_happy_eyeballs = "{}"; -const char* const description_ping_on_rst_stream = - "Send a ping on receiving some RST_STREAM frames on the server (proportion " - "configurable via grpc.http2.ping_on_rst_stream_percent channel arg)."; -const char* const additional_constraints_ping_on_rst_stream = "{}"; const char* const description_promise_based_client_call = "If set, use the new gRPC promise based call code when it's appropriate " "(ie when all filters in a stack are promise based)"; @@ -454,8 +354,6 @@ const char* const description_promise_based_server_call = "If set, use the new gRPC promise based call code when it's appropriate " "(ie when all filters in a stack are promise based)"; const char* const additional_constraints_promise_based_server_call = "{}"; -const uint8_t required_experiments_promise_based_server_call[] = { - static_cast(grpc_core::kExperimentIdLazierStreamUpdates)}; const char* const description_red_max_concurrent_streams = "Perform random early rejection of requests that would exceed a newly " "reduced MAX_CONCURRENT_STREAMS but are allowed by the current."; @@ -485,20 +383,8 @@ const char* const description_schedule_cancellation_over_write = "Allow cancellation op to be scheduled over a write"; const char* const additional_constraints_schedule_cancellation_over_write = "{}"; -const char* const description_separate_ping_from_keepalive = - "Keep a different keepalive timeout (resolution is seeing data after " - "sending a ping) from a ping timeout (resolution is getting a ping ack " - "after sending a ping) The first can be short and determines liveness. The " - "second can be longer and determines protocol correctness."; -const char* const additional_constraints_separate_ping_from_keepalive = "{}"; const char* const description_server_privacy = "If set, server privacy"; const char* const additional_constraints_server_privacy = "{}"; -const char* const description_settings_timeout = - "If set, use the settings timeout to send settings frame to the peer."; -const char* const additional_constraints_settings_timeout = "{}"; -const char* const description_tarpit = - "If set, tarpit invalid requests for some amount of time"; -const char* const additional_constraints_tarpit = "{}"; const char* const description_tcp_frame_size_tuning = "If set, enables TCP to use RPC size estimation made by higher layers. TCP " "would not indicate completion of a read operation until a specified " @@ -515,10 +401,6 @@ const char* const description_unconstrained_max_quota_buffer_size = "Discard the cap on the max free pool size for one memory allocator"; const char* const additional_constraints_unconstrained_max_quota_buffer_size = "{}"; -const char* const description_uniquely_unowned = - "Ensure HPACK table takes a unique copy of data when parsing unknown " - "metadata"; -const char* const additional_constraints_uniquely_unowned = "{}"; const char* const description_work_serializer_clears_time_cache = "Have the work serializer clear the time cache when it dispatches work."; const char* const additional_constraints_work_serializer_clears_time_cache = @@ -550,10 +432,6 @@ const bool kDefaultForDebugOnly = true; namespace grpc_core { const ExperimentMetadata g_experiment_metadata[] = { - {"block_excessive_requests_before_settings_ack", - description_block_excessive_requests_before_settings_ack, - additional_constraints_block_excessive_requests_before_settings_ack, - nullptr, 0, true, true}, {"call_status_override_on_cancellation", description_call_status_override_on_cancellation, additional_constraints_call_status_override_on_cancellation, nullptr, 0, @@ -564,16 +442,6 @@ const ExperimentMetadata g_experiment_metadata[] = { additional_constraints_client_idleness, nullptr, 0, true, true}, {"client_privacy", description_client_privacy, additional_constraints_client_privacy, nullptr, 0, false, false}, - {"combiner_offload_to_event_engine", - description_combiner_offload_to_event_engine, - additional_constraints_combiner_offload_to_event_engine, nullptr, 0, true, - true}, - {"chttp2_batch_requests", description_chttp2_batch_requests, - additional_constraints_chttp2_batch_requests, - required_experiments_chttp2_batch_requests, 1, true, true}, - {"chttp2_offload_on_rst_stream", description_chttp2_offload_on_rst_stream, - additional_constraints_chttp2_offload_on_rst_stream, - required_experiments_chttp2_offload_on_rst_stream, 1, true, true}, {"event_engine_client", description_event_engine_client, additional_constraints_event_engine_client, nullptr, 0, false, true}, {"event_engine_dns", description_event_engine_dns, @@ -588,8 +456,6 @@ const ExperimentMetadata g_experiment_metadata[] = { additional_constraints_keepalive_fix, nullptr, 0, false, false}, {"keepalive_server_fix", description_keepalive_server_fix, additional_constraints_keepalive_server_fix, nullptr, 0, false, false}, - {"lazier_stream_updates", description_lazier_stream_updates, - additional_constraints_lazier_stream_updates, nullptr, 0, true, true}, {"memory_pressure_controller", description_memory_pressure_controller, additional_constraints_memory_pressure_controller, nullptr, 0, false, true}, @@ -605,8 +471,6 @@ const ExperimentMetadata g_experiment_metadata[] = { additional_constraints_pending_queue_cap, nullptr, 0, true, true}, {"pick_first_happy_eyeballs", description_pick_first_happy_eyeballs, additional_constraints_pick_first_happy_eyeballs, nullptr, 0, true, true}, - {"ping_on_rst_stream", description_ping_on_rst_stream, - additional_constraints_ping_on_rst_stream, nullptr, 0, true, true}, {"promise_based_client_call", description_promise_based_client_call, additional_constraints_promise_based_client_call, nullptr, 0, false, true}, {"promise_based_inproc_transport", @@ -614,8 +478,7 @@ const ExperimentMetadata g_experiment_metadata[] = { additional_constraints_promise_based_inproc_transport, nullptr, 0, false, false}, {"promise_based_server_call", description_promise_based_server_call, - additional_constraints_promise_based_server_call, - required_experiments_promise_based_server_call, 1, false, true}, + additional_constraints_promise_based_server_call, nullptr, 0, false, true}, {"red_max_concurrent_streams", description_red_max_concurrent_streams, additional_constraints_red_max_concurrent_streams, nullptr, 0, false, true}, @@ -638,15 +501,8 @@ const ExperimentMetadata g_experiment_metadata[] = { description_schedule_cancellation_over_write, additional_constraints_schedule_cancellation_over_write, nullptr, 0, false, true}, - {"separate_ping_from_keepalive", description_separate_ping_from_keepalive, - additional_constraints_separate_ping_from_keepalive, nullptr, 0, true, - true}, {"server_privacy", description_server_privacy, additional_constraints_server_privacy, nullptr, 0, false, false}, - {"settings_timeout", description_settings_timeout, - additional_constraints_settings_timeout, nullptr, 0, true, true}, - {"tarpit", description_tarpit, additional_constraints_tarpit, nullptr, 0, - true, true}, {"tcp_frame_size_tuning", description_tcp_frame_size_tuning, additional_constraints_tcp_frame_size_tuning, nullptr, 0, false, true}, {"tcp_rcv_lowat", description_tcp_rcv_lowat, @@ -657,8 +513,6 @@ const ExperimentMetadata g_experiment_metadata[] = { description_unconstrained_max_quota_buffer_size, additional_constraints_unconstrained_max_quota_buffer_size, nullptr, 0, false, true}, - {"uniquely_unowned", description_uniquely_unowned, - additional_constraints_uniquely_unowned, nullptr, 0, true, true}, {"work_serializer_clears_time_cache", description_work_serializer_clears_time_cache, additional_constraints_work_serializer_clears_time_cache, nullptr, 0, true, @@ -678,10 +532,6 @@ const ExperimentMetadata g_experiment_metadata[] = { #else namespace { -const char* const description_block_excessive_requests_before_settings_ack = - "If set, block excessive requests before receiving SETTINGS ACK."; -const char* const - additional_constraints_block_excessive_requests_before_settings_ack = "{}"; const char* const description_call_status_override_on_cancellation = "Avoid overriding call status of successfully finished calls if it races " "with cancellation."; @@ -695,21 +545,6 @@ const char* const description_client_idleness = const char* const additional_constraints_client_idleness = "{}"; const char* const description_client_privacy = "If set, client privacy"; const char* const additional_constraints_client_privacy = "{}"; -const char* const description_combiner_offload_to_event_engine = - "Offload Combiner work onto the EventEngine instead of the Executor."; -const char* const additional_constraints_combiner_offload_to_event_engine = - "{}"; -const char* const description_chttp2_batch_requests = - "Cap the number of requests received by one transport read prior to " - "offload."; -const char* const additional_constraints_chttp2_batch_requests = "{}"; -const uint8_t required_experiments_chttp2_batch_requests[] = { - static_cast(grpc_core::kExperimentIdCombinerOffloadToEventEngine)}; -const char* const description_chttp2_offload_on_rst_stream = - "Offload work on RST_STREAM."; -const char* const additional_constraints_chttp2_offload_on_rst_stream = "{}"; -const uint8_t required_experiments_chttp2_offload_on_rst_stream[] = { - static_cast(grpc_core::kExperimentIdCombinerOffloadToEventEngine)}; const char* const description_event_engine_client = "Use EventEngine clients instead of iomgr's grpc_tcp_client"; const char* const additional_constraints_event_engine_client = "{}"; @@ -733,10 +568,6 @@ const char* const description_keepalive_server_fix = "Allows overriding keepalive_permit_without_calls for servers. Refer " "https://github.com/grpc/grpc/pull/33917 for more information."; const char* const additional_constraints_keepalive_server_fix = "{}"; -const char* const description_lazier_stream_updates = - "Allow streams to consume up to 50% of the incoming window before we force " - "send a flow control update."; -const char* const additional_constraints_lazier_stream_updates = "{}"; const char* const description_memory_pressure_controller = "New memory pressure controller"; const char* const additional_constraints_memory_pressure_controller = "{}"; @@ -766,10 +597,6 @@ const char* const additional_constraints_pending_queue_cap = "{}"; const char* const description_pick_first_happy_eyeballs = "Use Happy Eyeballs in pick_first."; const char* const additional_constraints_pick_first_happy_eyeballs = "{}"; -const char* const description_ping_on_rst_stream = - "Send a ping on receiving some RST_STREAM frames on the server (proportion " - "configurable via grpc.http2.ping_on_rst_stream_percent channel arg)."; -const char* const additional_constraints_ping_on_rst_stream = "{}"; const char* const description_promise_based_client_call = "If set, use the new gRPC promise based call code when it's appropriate " "(ie when all filters in a stack are promise based)"; @@ -781,8 +608,6 @@ const char* const description_promise_based_server_call = "If set, use the new gRPC promise based call code when it's appropriate " "(ie when all filters in a stack are promise based)"; const char* const additional_constraints_promise_based_server_call = "{}"; -const uint8_t required_experiments_promise_based_server_call[] = { - static_cast(grpc_core::kExperimentIdLazierStreamUpdates)}; const char* const description_red_max_concurrent_streams = "Perform random early rejection of requests that would exceed a newly " "reduced MAX_CONCURRENT_STREAMS but are allowed by the current."; @@ -812,20 +637,8 @@ const char* const description_schedule_cancellation_over_write = "Allow cancellation op to be scheduled over a write"; const char* const additional_constraints_schedule_cancellation_over_write = "{}"; -const char* const description_separate_ping_from_keepalive = - "Keep a different keepalive timeout (resolution is seeing data after " - "sending a ping) from a ping timeout (resolution is getting a ping ack " - "after sending a ping) The first can be short and determines liveness. The " - "second can be longer and determines protocol correctness."; -const char* const additional_constraints_separate_ping_from_keepalive = "{}"; const char* const description_server_privacy = "If set, server privacy"; const char* const additional_constraints_server_privacy = "{}"; -const char* const description_settings_timeout = - "If set, use the settings timeout to send settings frame to the peer."; -const char* const additional_constraints_settings_timeout = "{}"; -const char* const description_tarpit = - "If set, tarpit invalid requests for some amount of time"; -const char* const additional_constraints_tarpit = "{}"; const char* const description_tcp_frame_size_tuning = "If set, enables TCP to use RPC size estimation made by higher layers. TCP " "would not indicate completion of a read operation until a specified " @@ -842,10 +655,6 @@ const char* const description_unconstrained_max_quota_buffer_size = "Discard the cap on the max free pool size for one memory allocator"; const char* const additional_constraints_unconstrained_max_quota_buffer_size = "{}"; -const char* const description_uniquely_unowned = - "Ensure HPACK table takes a unique copy of data when parsing unknown " - "metadata"; -const char* const additional_constraints_uniquely_unowned = "{}"; const char* const description_work_serializer_clears_time_cache = "Have the work serializer clear the time cache when it dispatches work."; const char* const additional_constraints_work_serializer_clears_time_cache = @@ -877,10 +686,6 @@ const bool kDefaultForDebugOnly = true; namespace grpc_core { const ExperimentMetadata g_experiment_metadata[] = { - {"block_excessive_requests_before_settings_ack", - description_block_excessive_requests_before_settings_ack, - additional_constraints_block_excessive_requests_before_settings_ack, - nullptr, 0, true, true}, {"call_status_override_on_cancellation", description_call_status_override_on_cancellation, additional_constraints_call_status_override_on_cancellation, nullptr, 0, @@ -891,16 +696,6 @@ const ExperimentMetadata g_experiment_metadata[] = { additional_constraints_client_idleness, nullptr, 0, true, true}, {"client_privacy", description_client_privacy, additional_constraints_client_privacy, nullptr, 0, false, false}, - {"combiner_offload_to_event_engine", - description_combiner_offload_to_event_engine, - additional_constraints_combiner_offload_to_event_engine, nullptr, 0, true, - true}, - {"chttp2_batch_requests", description_chttp2_batch_requests, - additional_constraints_chttp2_batch_requests, - required_experiments_chttp2_batch_requests, 1, true, true}, - {"chttp2_offload_on_rst_stream", description_chttp2_offload_on_rst_stream, - additional_constraints_chttp2_offload_on_rst_stream, - required_experiments_chttp2_offload_on_rst_stream, 1, true, true}, {"event_engine_client", description_event_engine_client, additional_constraints_event_engine_client, nullptr, 0, false, true}, {"event_engine_dns", description_event_engine_dns, @@ -915,8 +710,6 @@ const ExperimentMetadata g_experiment_metadata[] = { additional_constraints_keepalive_fix, nullptr, 0, false, false}, {"keepalive_server_fix", description_keepalive_server_fix, additional_constraints_keepalive_server_fix, nullptr, 0, false, false}, - {"lazier_stream_updates", description_lazier_stream_updates, - additional_constraints_lazier_stream_updates, nullptr, 0, true, true}, {"memory_pressure_controller", description_memory_pressure_controller, additional_constraints_memory_pressure_controller, nullptr, 0, false, true}, @@ -932,8 +725,6 @@ const ExperimentMetadata g_experiment_metadata[] = { additional_constraints_pending_queue_cap, nullptr, 0, true, true}, {"pick_first_happy_eyeballs", description_pick_first_happy_eyeballs, additional_constraints_pick_first_happy_eyeballs, nullptr, 0, true, true}, - {"ping_on_rst_stream", description_ping_on_rst_stream, - additional_constraints_ping_on_rst_stream, nullptr, 0, true, true}, {"promise_based_client_call", description_promise_based_client_call, additional_constraints_promise_based_client_call, nullptr, 0, false, true}, {"promise_based_inproc_transport", @@ -941,8 +732,7 @@ const ExperimentMetadata g_experiment_metadata[] = { additional_constraints_promise_based_inproc_transport, nullptr, 0, false, false}, {"promise_based_server_call", description_promise_based_server_call, - additional_constraints_promise_based_server_call, - required_experiments_promise_based_server_call, 1, false, true}, + additional_constraints_promise_based_server_call, nullptr, 0, false, true}, {"red_max_concurrent_streams", description_red_max_concurrent_streams, additional_constraints_red_max_concurrent_streams, nullptr, 0, false, true}, @@ -965,15 +755,8 @@ const ExperimentMetadata g_experiment_metadata[] = { description_schedule_cancellation_over_write, additional_constraints_schedule_cancellation_over_write, nullptr, 0, false, true}, - {"separate_ping_from_keepalive", description_separate_ping_from_keepalive, - additional_constraints_separate_ping_from_keepalive, nullptr, 0, true, - true}, {"server_privacy", description_server_privacy, additional_constraints_server_privacy, nullptr, 0, false, false}, - {"settings_timeout", description_settings_timeout, - additional_constraints_settings_timeout, nullptr, 0, true, true}, - {"tarpit", description_tarpit, additional_constraints_tarpit, nullptr, 0, - true, true}, {"tcp_frame_size_tuning", description_tcp_frame_size_tuning, additional_constraints_tcp_frame_size_tuning, nullptr, 0, false, true}, {"tcp_rcv_lowat", description_tcp_rcv_lowat, @@ -984,8 +767,6 @@ const ExperimentMetadata g_experiment_metadata[] = { description_unconstrained_max_quota_buffer_size, additional_constraints_unconstrained_max_quota_buffer_size, nullptr, 0, false, true}, - {"uniquely_unowned", description_uniquely_unowned, - additional_constraints_uniquely_unowned, nullptr, 0, true, true}, {"work_serializer_clears_time_cache", description_work_serializer_clears_time_cache, additional_constraints_work_serializer_clears_time_cache, nullptr, 0, true, diff --git a/src/core/lib/experiments/experiments.h b/src/core/lib/experiments/experiments.h index 60db6dfc1f36d..1b023557b33d0 100644 --- a/src/core/lib/experiments/experiments.h +++ b/src/core/lib/experiments/experiments.h @@ -57,8 +57,6 @@ namespace grpc_core { #ifdef GRPC_EXPERIMENTS_ARE_FINAL #if defined(GRPC_CFSTREAM) -#define GRPC_EXPERIMENT_IS_INCLUDED_BLOCK_EXCESSIVE_REQUESTS_BEFORE_SETTINGS_ACK -inline bool IsBlockExcessiveRequestsBeforeSettingsAckEnabled() { return true; } #ifndef NDEBUG #define GRPC_EXPERIMENT_IS_INCLUDED_CALL_STATUS_OVERRIDE_ON_CANCELLATION #endif @@ -73,12 +71,6 @@ inline bool IsCanaryClientPrivacyEnabled() { return false; } #define GRPC_EXPERIMENT_IS_INCLUDED_CLIENT_IDLENESS inline bool IsClientIdlenessEnabled() { return true; } inline bool IsClientPrivacyEnabled() { return false; } -#define GRPC_EXPERIMENT_IS_INCLUDED_COMBINER_OFFLOAD_TO_EVENT_ENGINE -inline bool IsCombinerOffloadToEventEngineEnabled() { return true; } -#define GRPC_EXPERIMENT_IS_INCLUDED_CHTTP2_BATCH_REQUESTS -inline bool IsChttp2BatchRequestsEnabled() { return true; } -#define GRPC_EXPERIMENT_IS_INCLUDED_CHTTP2_OFFLOAD_ON_RST_STREAM -inline bool IsChttp2OffloadOnRstStreamEnabled() { return true; } inline bool IsEventEngineClientEnabled() { return false; } inline bool IsEventEngineDnsEnabled() { return false; } inline bool IsEventEngineListenerEnabled() { return false; } @@ -87,8 +79,6 @@ inline bool IsFreeLargeAllocatorEnabled() { return false; } inline bool IsHttp2StatsFixEnabled() { return true; } inline bool IsKeepaliveFixEnabled() { return false; } inline bool IsKeepaliveServerFixEnabled() { return false; } -#define GRPC_EXPERIMENT_IS_INCLUDED_LAZIER_STREAM_UPDATES -inline bool IsLazierStreamUpdatesEnabled() { return true; } inline bool IsMemoryPressureControllerEnabled() { return false; } #define GRPC_EXPERIMENT_IS_INCLUDED_MONITORING_EXPERIMENT inline bool IsMonitoringExperimentEnabled() { return true; } @@ -100,8 +90,6 @@ inline bool IsPeerStateBasedFramingEnabled() { return false; } inline bool IsPendingQueueCapEnabled() { return true; } #define GRPC_EXPERIMENT_IS_INCLUDED_PICK_FIRST_HAPPY_EYEBALLS inline bool IsPickFirstHappyEyeballsEnabled() { return true; } -#define GRPC_EXPERIMENT_IS_INCLUDED_PING_ON_RST_STREAM -inline bool IsPingOnRstStreamEnabled() { return true; } inline bool IsPromiseBasedClientCallEnabled() { return false; } inline bool IsPromiseBasedInprocTransportEnabled() { return false; } inline bool IsPromiseBasedServerCallEnabled() { return false; } @@ -114,19 +102,11 @@ inline bool IsRfcMaxConcurrentStreamsEnabled() { return false; } inline bool IsRoundRobinDelegateToPickFirstEnabled() { return true; } inline bool IsRstpitEnabled() { return false; } inline bool IsScheduleCancellationOverWriteEnabled() { return false; } -#define GRPC_EXPERIMENT_IS_INCLUDED_SEPARATE_PING_FROM_KEEPALIVE -inline bool IsSeparatePingFromKeepaliveEnabled() { return true; } inline bool IsServerPrivacyEnabled() { return false; } -#define GRPC_EXPERIMENT_IS_INCLUDED_SETTINGS_TIMEOUT -inline bool IsSettingsTimeoutEnabled() { return true; } -#define GRPC_EXPERIMENT_IS_INCLUDED_TARPIT -inline bool IsTarpitEnabled() { return true; } inline bool IsTcpFrameSizeTuningEnabled() { return false; } inline bool IsTcpRcvLowatEnabled() { return false; } inline bool IsTraceRecordCallopsEnabled() { return false; } inline bool IsUnconstrainedMaxQuotaBufferSizeEnabled() { return false; } -#define GRPC_EXPERIMENT_IS_INCLUDED_UNIQUELY_UNOWNED -inline bool IsUniquelyUnownedEnabled() { return true; } #define GRPC_EXPERIMENT_IS_INCLUDED_WORK_SERIALIZER_CLEARS_TIME_CACHE inline bool IsWorkSerializerClearsTimeCacheEnabled() { return true; } inline bool IsWorkSerializerDispatchEnabled() { return false; } @@ -138,8 +118,6 @@ inline bool IsWriteSizeCapEnabled() { return true; } inline bool IsWrrDelegateToPickFirstEnabled() { return true; } #elif defined(GPR_WINDOWS) -#define GRPC_EXPERIMENT_IS_INCLUDED_BLOCK_EXCESSIVE_REQUESTS_BEFORE_SETTINGS_ACK -inline bool IsBlockExcessiveRequestsBeforeSettingsAckEnabled() { return true; } #ifndef NDEBUG #define GRPC_EXPERIMENT_IS_INCLUDED_CALL_STATUS_OVERRIDE_ON_CANCELLATION #endif @@ -154,12 +132,6 @@ inline bool IsCanaryClientPrivacyEnabled() { return false; } #define GRPC_EXPERIMENT_IS_INCLUDED_CLIENT_IDLENESS inline bool IsClientIdlenessEnabled() { return true; } inline bool IsClientPrivacyEnabled() { return false; } -#define GRPC_EXPERIMENT_IS_INCLUDED_COMBINER_OFFLOAD_TO_EVENT_ENGINE -inline bool IsCombinerOffloadToEventEngineEnabled() { return true; } -#define GRPC_EXPERIMENT_IS_INCLUDED_CHTTP2_BATCH_REQUESTS -inline bool IsChttp2BatchRequestsEnabled() { return true; } -#define GRPC_EXPERIMENT_IS_INCLUDED_CHTTP2_OFFLOAD_ON_RST_STREAM -inline bool IsChttp2OffloadOnRstStreamEnabled() { return true; } inline bool IsEventEngineClientEnabled() { return false; } inline bool IsEventEngineDnsEnabled() { return false; } #define GRPC_EXPERIMENT_IS_INCLUDED_EVENT_ENGINE_LISTENER @@ -169,8 +141,6 @@ inline bool IsFreeLargeAllocatorEnabled() { return false; } inline bool IsHttp2StatsFixEnabled() { return true; } inline bool IsKeepaliveFixEnabled() { return false; } inline bool IsKeepaliveServerFixEnabled() { return false; } -#define GRPC_EXPERIMENT_IS_INCLUDED_LAZIER_STREAM_UPDATES -inline bool IsLazierStreamUpdatesEnabled() { return true; } inline bool IsMemoryPressureControllerEnabled() { return false; } #define GRPC_EXPERIMENT_IS_INCLUDED_MONITORING_EXPERIMENT inline bool IsMonitoringExperimentEnabled() { return true; } @@ -182,8 +152,6 @@ inline bool IsPeerStateBasedFramingEnabled() { return false; } inline bool IsPendingQueueCapEnabled() { return true; } #define GRPC_EXPERIMENT_IS_INCLUDED_PICK_FIRST_HAPPY_EYEBALLS inline bool IsPickFirstHappyEyeballsEnabled() { return true; } -#define GRPC_EXPERIMENT_IS_INCLUDED_PING_ON_RST_STREAM -inline bool IsPingOnRstStreamEnabled() { return true; } inline bool IsPromiseBasedClientCallEnabled() { return false; } inline bool IsPromiseBasedInprocTransportEnabled() { return false; } inline bool IsPromiseBasedServerCallEnabled() { return false; } @@ -196,19 +164,11 @@ inline bool IsRfcMaxConcurrentStreamsEnabled() { return false; } inline bool IsRoundRobinDelegateToPickFirstEnabled() { return true; } inline bool IsRstpitEnabled() { return false; } inline bool IsScheduleCancellationOverWriteEnabled() { return false; } -#define GRPC_EXPERIMENT_IS_INCLUDED_SEPARATE_PING_FROM_KEEPALIVE -inline bool IsSeparatePingFromKeepaliveEnabled() { return true; } inline bool IsServerPrivacyEnabled() { return false; } -#define GRPC_EXPERIMENT_IS_INCLUDED_SETTINGS_TIMEOUT -inline bool IsSettingsTimeoutEnabled() { return true; } -#define GRPC_EXPERIMENT_IS_INCLUDED_TARPIT -inline bool IsTarpitEnabled() { return true; } inline bool IsTcpFrameSizeTuningEnabled() { return false; } inline bool IsTcpRcvLowatEnabled() { return false; } inline bool IsTraceRecordCallopsEnabled() { return false; } inline bool IsUnconstrainedMaxQuotaBufferSizeEnabled() { return false; } -#define GRPC_EXPERIMENT_IS_INCLUDED_UNIQUELY_UNOWNED -inline bool IsUniquelyUnownedEnabled() { return true; } #define GRPC_EXPERIMENT_IS_INCLUDED_WORK_SERIALIZER_CLEARS_TIME_CACHE inline bool IsWorkSerializerClearsTimeCacheEnabled() { return true; } inline bool IsWorkSerializerDispatchEnabled() { return false; } @@ -220,8 +180,6 @@ inline bool IsWriteSizeCapEnabled() { return true; } inline bool IsWrrDelegateToPickFirstEnabled() { return true; } #else -#define GRPC_EXPERIMENT_IS_INCLUDED_BLOCK_EXCESSIVE_REQUESTS_BEFORE_SETTINGS_ACK -inline bool IsBlockExcessiveRequestsBeforeSettingsAckEnabled() { return true; } #ifndef NDEBUG #define GRPC_EXPERIMENT_IS_INCLUDED_CALL_STATUS_OVERRIDE_ON_CANCELLATION #endif @@ -236,12 +194,6 @@ inline bool IsCanaryClientPrivacyEnabled() { return false; } #define GRPC_EXPERIMENT_IS_INCLUDED_CLIENT_IDLENESS inline bool IsClientIdlenessEnabled() { return true; } inline bool IsClientPrivacyEnabled() { return false; } -#define GRPC_EXPERIMENT_IS_INCLUDED_COMBINER_OFFLOAD_TO_EVENT_ENGINE -inline bool IsCombinerOffloadToEventEngineEnabled() { return true; } -#define GRPC_EXPERIMENT_IS_INCLUDED_CHTTP2_BATCH_REQUESTS -inline bool IsChttp2BatchRequestsEnabled() { return true; } -#define GRPC_EXPERIMENT_IS_INCLUDED_CHTTP2_OFFLOAD_ON_RST_STREAM -inline bool IsChttp2OffloadOnRstStreamEnabled() { return true; } inline bool IsEventEngineClientEnabled() { return false; } inline bool IsEventEngineDnsEnabled() { return false; } #define GRPC_EXPERIMENT_IS_INCLUDED_EVENT_ENGINE_LISTENER @@ -251,8 +203,6 @@ inline bool IsFreeLargeAllocatorEnabled() { return false; } inline bool IsHttp2StatsFixEnabled() { return true; } inline bool IsKeepaliveFixEnabled() { return false; } inline bool IsKeepaliveServerFixEnabled() { return false; } -#define GRPC_EXPERIMENT_IS_INCLUDED_LAZIER_STREAM_UPDATES -inline bool IsLazierStreamUpdatesEnabled() { return true; } inline bool IsMemoryPressureControllerEnabled() { return false; } #define GRPC_EXPERIMENT_IS_INCLUDED_MONITORING_EXPERIMENT inline bool IsMonitoringExperimentEnabled() { return true; } @@ -264,8 +214,6 @@ inline bool IsPeerStateBasedFramingEnabled() { return false; } inline bool IsPendingQueueCapEnabled() { return true; } #define GRPC_EXPERIMENT_IS_INCLUDED_PICK_FIRST_HAPPY_EYEBALLS inline bool IsPickFirstHappyEyeballsEnabled() { return true; } -#define GRPC_EXPERIMENT_IS_INCLUDED_PING_ON_RST_STREAM -inline bool IsPingOnRstStreamEnabled() { return true; } inline bool IsPromiseBasedClientCallEnabled() { return false; } inline bool IsPromiseBasedInprocTransportEnabled() { return false; } inline bool IsPromiseBasedServerCallEnabled() { return false; } @@ -278,19 +226,11 @@ inline bool IsRfcMaxConcurrentStreamsEnabled() { return false; } inline bool IsRoundRobinDelegateToPickFirstEnabled() { return true; } inline bool IsRstpitEnabled() { return false; } inline bool IsScheduleCancellationOverWriteEnabled() { return false; } -#define GRPC_EXPERIMENT_IS_INCLUDED_SEPARATE_PING_FROM_KEEPALIVE -inline bool IsSeparatePingFromKeepaliveEnabled() { return true; } inline bool IsServerPrivacyEnabled() { return false; } -#define GRPC_EXPERIMENT_IS_INCLUDED_SETTINGS_TIMEOUT -inline bool IsSettingsTimeoutEnabled() { return true; } -#define GRPC_EXPERIMENT_IS_INCLUDED_TARPIT -inline bool IsTarpitEnabled() { return true; } inline bool IsTcpFrameSizeTuningEnabled() { return false; } inline bool IsTcpRcvLowatEnabled() { return false; } inline bool IsTraceRecordCallopsEnabled() { return false; } inline bool IsUnconstrainedMaxQuotaBufferSizeEnabled() { return false; } -#define GRPC_EXPERIMENT_IS_INCLUDED_UNIQUELY_UNOWNED -inline bool IsUniquelyUnownedEnabled() { return true; } #define GRPC_EXPERIMENT_IS_INCLUDED_WORK_SERIALIZER_CLEARS_TIME_CACHE inline bool IsWorkSerializerClearsTimeCacheEnabled() { return true; } inline bool IsWorkSerializerDispatchEnabled() { return false; } @@ -304,14 +244,10 @@ inline bool IsWrrDelegateToPickFirstEnabled() { return true; } #else enum ExperimentIds { - kExperimentIdBlockExcessiveRequestsBeforeSettingsAck, kExperimentIdCallStatusOverrideOnCancellation, kExperimentIdCanaryClientPrivacy, kExperimentIdClientIdleness, kExperimentIdClientPrivacy, - kExperimentIdCombinerOffloadToEventEngine, - kExperimentIdChttp2BatchRequests, - kExperimentIdChttp2OffloadOnRstStream, kExperimentIdEventEngineClient, kExperimentIdEventEngineDns, kExperimentIdEventEngineListener, @@ -319,7 +255,6 @@ enum ExperimentIds { kExperimentIdHttp2StatsFix, kExperimentIdKeepaliveFix, kExperimentIdKeepaliveServerFix, - kExperimentIdLazierStreamUpdates, kExperimentIdMemoryPressureController, kExperimentIdMonitoringExperiment, kExperimentIdMultiping, @@ -327,7 +262,6 @@ enum ExperimentIds { kExperimentIdPeerStateBasedFraming, kExperimentIdPendingQueueCap, kExperimentIdPickFirstHappyEyeballs, - kExperimentIdPingOnRstStream, kExperimentIdPromiseBasedClientCall, kExperimentIdPromiseBasedInprocTransport, kExperimentIdPromiseBasedServerCall, @@ -338,15 +272,11 @@ enum ExperimentIds { kExperimentIdRoundRobinDelegateToPickFirst, kExperimentIdRstpit, kExperimentIdScheduleCancellationOverWrite, - kExperimentIdSeparatePingFromKeepalive, kExperimentIdServerPrivacy, - kExperimentIdSettingsTimeout, - kExperimentIdTarpit, kExperimentIdTcpFrameSizeTuning, kExperimentIdTcpRcvLowat, kExperimentIdTraceRecordCallops, kExperimentIdUnconstrainedMaxQuotaBufferSize, - kExperimentIdUniquelyUnowned, kExperimentIdWorkSerializerClearsTimeCache, kExperimentIdWorkSerializerDispatch, kExperimentIdWriteSizePolicy, @@ -354,11 +284,6 @@ enum ExperimentIds { kExperimentIdWrrDelegateToPickFirst, kNumExperiments }; -#define GRPC_EXPERIMENT_IS_INCLUDED_BLOCK_EXCESSIVE_REQUESTS_BEFORE_SETTINGS_ACK -inline bool IsBlockExcessiveRequestsBeforeSettingsAckEnabled() { - return IsExperimentEnabled( - kExperimentIdBlockExcessiveRequestsBeforeSettingsAck); -} #define GRPC_EXPERIMENT_IS_INCLUDED_CALL_STATUS_OVERRIDE_ON_CANCELLATION inline bool IsCallStatusOverrideOnCancellationEnabled() { return IsExperimentEnabled(kExperimentIdCallStatusOverrideOnCancellation); @@ -375,18 +300,6 @@ inline bool IsClientIdlenessEnabled() { inline bool IsClientPrivacyEnabled() { return IsExperimentEnabled(kExperimentIdClientPrivacy); } -#define GRPC_EXPERIMENT_IS_INCLUDED_COMBINER_OFFLOAD_TO_EVENT_ENGINE -inline bool IsCombinerOffloadToEventEngineEnabled() { - return IsExperimentEnabled(kExperimentIdCombinerOffloadToEventEngine); -} -#define GRPC_EXPERIMENT_IS_INCLUDED_CHTTP2_BATCH_REQUESTS -inline bool IsChttp2BatchRequestsEnabled() { - return IsExperimentEnabled(kExperimentIdChttp2BatchRequests); -} -#define GRPC_EXPERIMENT_IS_INCLUDED_CHTTP2_OFFLOAD_ON_RST_STREAM -inline bool IsChttp2OffloadOnRstStreamEnabled() { - return IsExperimentEnabled(kExperimentIdChttp2OffloadOnRstStream); -} #define GRPC_EXPERIMENT_IS_INCLUDED_EVENT_ENGINE_CLIENT inline bool IsEventEngineClientEnabled() { return IsExperimentEnabled(kExperimentIdEventEngineClient); @@ -415,10 +328,6 @@ inline bool IsKeepaliveFixEnabled() { inline bool IsKeepaliveServerFixEnabled() { return IsExperimentEnabled(kExperimentIdKeepaliveServerFix); } -#define GRPC_EXPERIMENT_IS_INCLUDED_LAZIER_STREAM_UPDATES -inline bool IsLazierStreamUpdatesEnabled() { - return IsExperimentEnabled(kExperimentIdLazierStreamUpdates); -} #define GRPC_EXPERIMENT_IS_INCLUDED_MEMORY_PRESSURE_CONTROLLER inline bool IsMemoryPressureControllerEnabled() { return IsExperimentEnabled(kExperimentIdMemoryPressureController); @@ -447,10 +356,6 @@ inline bool IsPendingQueueCapEnabled() { inline bool IsPickFirstHappyEyeballsEnabled() { return IsExperimentEnabled(kExperimentIdPickFirstHappyEyeballs); } -#define GRPC_EXPERIMENT_IS_INCLUDED_PING_ON_RST_STREAM -inline bool IsPingOnRstStreamEnabled() { - return IsExperimentEnabled(kExperimentIdPingOnRstStream); -} #define GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_CLIENT_CALL inline bool IsPromiseBasedClientCallEnabled() { return IsExperimentEnabled(kExperimentIdPromiseBasedClientCall); @@ -491,22 +396,10 @@ inline bool IsRstpitEnabled() { inline bool IsScheduleCancellationOverWriteEnabled() { return IsExperimentEnabled(kExperimentIdScheduleCancellationOverWrite); } -#define GRPC_EXPERIMENT_IS_INCLUDED_SEPARATE_PING_FROM_KEEPALIVE -inline bool IsSeparatePingFromKeepaliveEnabled() { - return IsExperimentEnabled(kExperimentIdSeparatePingFromKeepalive); -} #define GRPC_EXPERIMENT_IS_INCLUDED_SERVER_PRIVACY inline bool IsServerPrivacyEnabled() { return IsExperimentEnabled(kExperimentIdServerPrivacy); } -#define GRPC_EXPERIMENT_IS_INCLUDED_SETTINGS_TIMEOUT -inline bool IsSettingsTimeoutEnabled() { - return IsExperimentEnabled(kExperimentIdSettingsTimeout); -} -#define GRPC_EXPERIMENT_IS_INCLUDED_TARPIT -inline bool IsTarpitEnabled() { - return IsExperimentEnabled(kExperimentIdTarpit); -} #define GRPC_EXPERIMENT_IS_INCLUDED_TCP_FRAME_SIZE_TUNING inline bool IsTcpFrameSizeTuningEnabled() { return IsExperimentEnabled(kExperimentIdTcpFrameSizeTuning); @@ -523,10 +416,6 @@ inline bool IsTraceRecordCallopsEnabled() { inline bool IsUnconstrainedMaxQuotaBufferSizeEnabled() { return IsExperimentEnabled(kExperimentIdUnconstrainedMaxQuotaBufferSize); } -#define GRPC_EXPERIMENT_IS_INCLUDED_UNIQUELY_UNOWNED -inline bool IsUniquelyUnownedEnabled() { - return IsExperimentEnabled(kExperimentIdUniquelyUnowned); -} #define GRPC_EXPERIMENT_IS_INCLUDED_WORK_SERIALIZER_CLEARS_TIME_CACHE inline bool IsWorkSerializerClearsTimeCacheEnabled() { return IsExperimentEnabled(kExperimentIdWorkSerializerClearsTimeCache); diff --git a/src/core/lib/experiments/experiments.yaml b/src/core/lib/experiments/experiments.yaml index 60ed3d7eb2e10..e3b89572e4388 100644 --- a/src/core/lib/experiments/experiments.yaml +++ b/src/core/lib/experiments/experiments.yaml @@ -37,12 +37,6 @@ # This file only defines the experiments. Refer to rollouts.yaml for the rollout # state of each experiment. -- name: block_excessive_requests_before_settings_ack - description: - If set, block excessive requests before receiving SETTINGS ACK. - expiry: 2024/03/03 - owner: ctiller@google.com - test_tags: [bad_client_test] - name: call_status_override_on_cancellation description: Avoid overriding call status of successfully finished calls if it races with @@ -57,23 +51,9 @@ owner: alishananda@google.com test_tags: [] allow_in_fuzzing_config: false -- name: chttp2_batch_requests - description: - Cap the number of requests received by one transport read prior to offload. - expiry: 2024/03/03 - owner: ctiller@google.com - test_tags: ["cpp_end2end_test", "flow_control_test"] - requires: [combiner_offload_to_event_engine] -- name: chttp2_offload_on_rst_stream - description: - Offload work on RST_STREAM. - expiry: 2024/03/03 - owner: ctiller@google.com - test_tags: ["cpp_end2end_test", "flow_control_test"] - requires: [combiner_offload_to_event_engine] - name: client_idleness description: If enabled, client channel idleness is enabled by default. - expiry: 2023/12/15 + expiry: 2024/03/15 owner: roth@google.com test_tags: [] - name: client_privacy @@ -83,12 +63,6 @@ owner: alishananda@google.com test_tags: [] allow_in_fuzzing_config: false -- name: combiner_offload_to_event_engine - description: - Offload Combiner work onto the EventEngine instead of the Executor. - expiry: 2024/01/15 - owner: hork@google.com - test_tags: [] - name: event_engine_client description: Use EventEngine clients instead of iomgr's grpc_tcp_client expiry: 2024/01/21 @@ -133,13 +107,6 @@ owner: yashkt@google.com test_tags: [] allow_in_fuzzing_config: false -- name: lazier_stream_updates - description: - Allow streams to consume up to 50% of the incoming window before we - force send a flow control update. - expiry: 2024/01/23 - owner: ctiller@google.com - test_tags: [flow_control_test] - name: memory_pressure_controller description: New memory pressure controller expiry: 2024/05/05 @@ -185,16 +152,9 @@ - name: pick_first_happy_eyeballs description: Use Happy Eyeballs in pick_first. - expiry: 2023/12/15 + expiry: 2024/03/15 owner: roth@google.com test_tags: ["lb_unit_test", "cpp_lb_end2end_test", "xds_end2end_test"] -- name: ping_on_rst_stream - description: - Send a ping on receiving some RST_STREAM frames on the server - (proportion configurable via grpc.http2.ping_on_rst_stream_percent channel arg). - expiry: 2024/03/03 - owner: ctiller@google.com - test_tags: [] - name: promise_based_client_call description: If set, use the new gRPC promise based call code when it's appropriate @@ -216,7 +176,6 @@ expiry: 2024/06/14 owner: ctiller@google.com test_tags: ["core_end2end_test", "cpp_end2end_test", "xds_end2end_test", "logging_test"] - requires: [lazier_stream_updates] - name: red_max_concurrent_streams description: Perform random early rejection of requests that would exceed a newly reduced @@ -248,7 +207,7 @@ description: Change round_robin code to delegate to pick_first as per dualstack backend design. - expiry: 2023/12/15 + expiry: 2024/03/15 owner: roth@google.com test_tags: ["lb_unit_test", "cpp_lb_end2end_test", "xds_end2end_test"] - name: rstpit @@ -262,15 +221,6 @@ expiry: 2024/01/01 owner: vigneshbabu@google.com test_tags: [] -- name: separate_ping_from_keepalive - description: - Keep a different keepalive timeout (resolution is seeing data after sending a ping) - from a ping timeout (resolution is getting a ping ack after sending a ping) - The first can be short and determines liveness. - The second can be longer and determines protocol correctness. - expiry: 2024/03/03 - owner: ctiller@google.com - test_tags: [] - name: server_privacy description: If set, server privacy @@ -278,18 +228,6 @@ owner: alishananda@google.com test_tags: [] allow_in_fuzzing_config: false -- name: settings_timeout - description: - If set, use the settings timeout to send settings frame to the peer. - expiry: 2024/03/03 - owner: ctiller@google.com - test_tags: [] -- name: tarpit - description: - If set, tarpit invalid requests for some amount of time - expiry: 2024/03/03 - owner: ctiller@google.com - test_tags: [bad_client_test] - name: tcp_frame_size_tuning description: If set, enables TCP to use RPC size estimation made by higher layers. @@ -314,10 +252,6 @@ expiry: 2024/02/01 owner: ctiller@google.com test_tags: [resource_quota_test] -- name: uniquely_unowned - description: Ensure HPACK table takes a unique copy of data when parsing unknown metadata - expiry: 2024/03/03 - owner: ctiller@google.com - name: work_serializer_clears_time_cache description: Have the work serializer clear the time cache when it dispatches work. @@ -349,6 +283,6 @@ description: Change WRR code to delegate to pick_first as per dualstack backend design. - expiry: 2023/12/15 + expiry: 2024/03/15 owner: roth@google.com test_tags: ["lb_unit_test", "cpp_lb_end2end_test", "xds_end2end_test"] diff --git a/src/core/lib/experiments/rollouts.yaml b/src/core/lib/experiments/rollouts.yaml index 00b404c61de90..160a12e55a01a 100644 --- a/src/core/lib/experiments/rollouts.yaml +++ b/src/core/lib/experiments/rollouts.yaml @@ -40,22 +40,14 @@ # # Supported platforms: ios, windows, posix -- name: block_excessive_requests_before_settings_ack - default: true - name: call_status_override_on_cancellation default: debug - name: canary_client_privacy default: false -- name: chttp2_batch_requests - default: true -- name: chttp2_offload_on_rst_stream - default: true - name: client_idleness default: true - name: client_privacy default: false -- name: combiner_offload_to_event_engine - default: true - name: event_engine_client default: # not tested on iOS at all @@ -82,14 +74,10 @@ default: false - name: http2_stats_fix default: true -- name: jitter_max_idle - default: true - name: keepalive_fix default: false - name: keepalive_server_fix default: false -- name: lazier_stream_updates - default: true - name: memory_pressure_controller default: false - name: monitoring_experiment @@ -102,8 +90,6 @@ default: true - name: pick_first_happy_eyeballs default: true -- name: ping_on_rst_stream - default: true - name: promise_based_client_call default: false - name: promise_based_server_call @@ -122,14 +108,8 @@ default: false - name: schedule_cancellation_over_write default: false -- name: separate_ping_from_keepalive - default: true - name: server_privacy default: false -- name: settings_timeout - default: true -- name: tarpit - default: true - name: tcp_frame_size_tuning default: false - name: tcp_rcv_lowat @@ -138,8 +118,6 @@ default: false - name: unconstrained_max_quota_buffer_size default: false -- name: uniquely_unowned - default: true - name: work_serializer_clears_time_cache default: true - name: work_serializer_dispatch diff --git a/src/core/lib/gprpp/orphanable.h b/src/core/lib/gprpp/orphanable.h index 86319429e5c2d..5a77ad46b452c 100644 --- a/src/core/lib/gprpp/orphanable.h +++ b/src/core/lib/gprpp/orphanable.h @@ -108,6 +108,17 @@ class InternallyRefCounted : public Orphanable { } } + GRPC_MUST_USE_RESULT RefCountedPtr RefIfNonZero() { + return RefCountedPtr(refs_.RefIfNonZero() ? static_cast(this) + : nullptr); + } + GRPC_MUST_USE_RESULT RefCountedPtr RefIfNonZero( + const DebugLocation& location, const char* reason) { + return RefCountedPtr(refs_.RefIfNonZero(location, reason) + ? static_cast(this) + : nullptr); + } + private: void IncrementRefCount() { refs_.Ref(); } void IncrementRefCount(const DebugLocation& location, const char* reason) { diff --git a/src/core/lib/gprpp/ref_counted.h b/src/core/lib/gprpp/ref_counted.h index cdf692c5ce7df..5eaf5cda0f1fb 100644 --- a/src/core/lib/gprpp/ref_counted.h +++ b/src/core/lib/gprpp/ref_counted.h @@ -219,7 +219,7 @@ class NonPolymorphicRefCount { // Default behavior: Delete the object. struct UnrefDelete { template - void operator()(T* p) { + void operator()(T* p) const { delete p; } }; @@ -231,7 +231,7 @@ struct UnrefDelete { // later by identifying entries for which RefIfNonZero() returns null. struct UnrefNoDelete { template - void operator()(T* /*p*/) {} + void operator()(T* /*p*/) const {} }; // Call the object's dtor but do not delete it. This is useful for cases @@ -239,7 +239,7 @@ struct UnrefNoDelete { // arena). struct UnrefCallDtor { template - void operator()(T* p) { + void operator()(T* p) const { p->~T(); } }; @@ -279,32 +279,44 @@ class RefCounted : public Impl { // Note: Depending on the Impl used, this dtor can be implicitly virtual. ~RefCounted() = default; + // Ref() for mutable types. GRPC_MUST_USE_RESULT RefCountedPtr Ref() { IncrementRefCount(); return RefCountedPtr(static_cast(this)); } - GRPC_MUST_USE_RESULT RefCountedPtr Ref(const DebugLocation& location, const char* reason) { IncrementRefCount(location, reason); return RefCountedPtr(static_cast(this)); } + // Ref() for const types. + GRPC_MUST_USE_RESULT RefCountedPtr Ref() const { + IncrementRefCount(); + return RefCountedPtr(static_cast(this)); + } + GRPC_MUST_USE_RESULT RefCountedPtr Ref( + const DebugLocation& location, const char* reason) const { + IncrementRefCount(location, reason); + return RefCountedPtr(static_cast(this)); + } + // TODO(roth): Once all of our code is converted to C++ and can use // RefCountedPtr<> instead of manual ref-counting, make this method // private, since it will only be used by RefCountedPtr<>, which is a // friend of this class. - void Unref() { + void Unref() const { if (GPR_UNLIKELY(refs_.Unref())) { - unref_behavior_(static_cast(this)); + unref_behavior_(static_cast(this)); } } - void Unref(const DebugLocation& location, const char* reason) { + void Unref(const DebugLocation& location, const char* reason) const { if (GPR_UNLIKELY(refs_.Unref(location, reason))) { - unref_behavior_(static_cast(this)); + unref_behavior_(static_cast(this)); } } + // RefIfNonZero() for mutable types. GRPC_MUST_USE_RESULT RefCountedPtr RefIfNonZero() { return RefCountedPtr(refs_.RefIfNonZero() ? static_cast(this) : nullptr); @@ -316,6 +328,18 @@ class RefCounted : public Impl { : nullptr); } + // RefIfNonZero() for const types. + GRPC_MUST_USE_RESULT RefCountedPtr RefIfNonZero() const { + return RefCountedPtr( + refs_.RefIfNonZero() ? static_cast(this) : nullptr); + } + GRPC_MUST_USE_RESULT RefCountedPtr RefIfNonZero( + const DebugLocation& location, const char* reason) const { + return RefCountedPtr(refs_.RefIfNonZero(location, reason) + ? static_cast(this) + : nullptr); + } + // Not copyable nor movable. RefCounted(const RefCounted&) = delete; RefCounted& operator=(const RefCounted&) = delete; @@ -336,12 +360,13 @@ class RefCounted : public Impl { template friend class RefCountedPtr; - void IncrementRefCount() { refs_.Ref(); } - void IncrementRefCount(const DebugLocation& location, const char* reason) { + void IncrementRefCount() const { refs_.Ref(); } + void IncrementRefCount(const DebugLocation& location, + const char* reason) const { refs_.Ref(location, reason); } - RefCount refs_; + mutable RefCount refs_; GPR_NO_UNIQUE_ADDRESS UnrefBehavior unref_behavior_; }; diff --git a/src/core/lib/iomgr/combiner.cc b/src/core/lib/iomgr/combiner.cc index e29c6fcbe35b4..0d77bb579d4cb 100644 --- a/src/core/lib/iomgr/combiner.cc +++ b/src/core/lib/iomgr/combiner.cc @@ -51,19 +51,11 @@ static void combiner_finally_exec(grpc_core::Combiner* lock, grpc_closure* closure, grpc_error_handle error); -// TODO(ctiller): delete this when the combiner_offload_to_event_engine -// experiment is removed. -static void offload(void* arg, grpc_error_handle error); - grpc_core::Combiner* grpc_combiner_create( std::shared_ptr event_engine) { grpc_core::Combiner* lock = new grpc_core::Combiner(); - if (grpc_core::IsCombinerOffloadToEventEngineEnabled()) { - lock->event_engine = event_engine; - } else { - GRPC_CLOSURE_INIT(&lock->offload, offload, lock, nullptr); - } + lock->event_engine = event_engine; gpr_ref_init(&lock->refs, 1); gpr_atm_no_barrier_store(&lock->state, STATE_UNORPHANED); grpc_closure_list_init(&lock->final_list); @@ -173,27 +165,18 @@ static void move_next() { } } -static void offload(void* arg, grpc_error_handle /*error*/) { - grpc_core::Combiner* lock = static_cast(arg); - push_last_on_exec_ctx(lock); -} - static void queue_offload(grpc_core::Combiner* lock) { move_next(); // Make the combiner look uncontended by storing a non-null value here, so // that we don't immediately offload again. gpr_atm_no_barrier_store(&lock->initiating_exec_ctx_or_null, 1); GRPC_COMBINER_TRACE(gpr_log(GPR_INFO, "C:%p queue_offload", lock)); - if (grpc_core::IsCombinerOffloadToEventEngineEnabled()) { - lock->event_engine->Run([lock] { - grpc_core::ApplicationCallbackExecCtx callback_exec_ctx; - grpc_core::ExecCtx exec_ctx(0); - push_last_on_exec_ctx(lock); - exec_ctx.Flush(); - }); - } else { - grpc_core::Executor::Run(&lock->offload, absl::OkStatus()); - } + lock->event_engine->Run([lock] { + grpc_core::ApplicationCallbackExecCtx callback_exec_ctx; + grpc_core::ExecCtx exec_ctx(0); + push_last_on_exec_ctx(lock); + exec_ctx.Flush(); + }); } bool grpc_combiner_continue_exec_ctx() { @@ -215,33 +198,14 @@ bool grpc_combiner_continue_exec_ctx() { grpc_core::ExecCtx::Get()->IsReadyToFinish(), lock->time_to_execute_final_list)); - if (grpc_core::IsCombinerOffloadToEventEngineEnabled()) { - // offload only if both (1) the combiner is contended and has more than one - // closure to execute, and (2) the current execution context needs to finish - // as soon as possible - if (contended && grpc_core::ExecCtx::Get()->IsReadyToFinish()) { - // this execution context wants to move on: schedule remaining work to be - // picked up on the executor - queue_offload(lock); - return true; - } - } else { - // TODO(ctiller): delete this when the combiner_offload_to_event_engine - // experiment is removed. - - // offload only if all the following conditions are true: - // 1. the combiner is contended and has more than one closure to execute - // 2. the current execution context needs to finish as soon as possible - // 3. the current thread is not a worker for any background poller - // 4. the DEFAULT executor is threaded - if (contended && grpc_core::ExecCtx::Get()->IsReadyToFinish() && - !grpc_iomgr_platform_is_any_background_poller_thread() && - grpc_core::Executor::IsThreadedDefault()) { - // this execution context wants to move on: schedule remaining work to be - // picked up on the executor - queue_offload(lock); - return true; - } + // offload only if both (1) the combiner is contended and has more than one + // closure to execute, and (2) the current execution context needs to finish + // as soon as possible + if (contended && grpc_core::ExecCtx::Get()->IsReadyToFinish()) { + // this execution context wants to move on: schedule remaining work to be + // picked up on the executor + queue_offload(lock); + return true; } if (!lock->time_to_execute_final_list || diff --git a/src/core/lib/iomgr/event_engine_shims/endpoint.cc b/src/core/lib/iomgr/event_engine_shims/endpoint.cc index 341fe1e577655..b1e8fdf890490 100644 --- a/src/core/lib/iomgr/event_engine_shims/endpoint.cc +++ b/src/core/lib/iomgr/event_engine_shims/endpoint.cc @@ -69,6 +69,8 @@ class EventEngineEndpointWrapper { explicit EventEngineEndpointWrapper( std::unique_ptr endpoint); + EventEngine::Endpoint* endpoint() { return endpoint_.get(); } + int Fd() { grpc_core::MutexLock lock(&mu_); return fd_; @@ -428,6 +430,17 @@ bool grpc_is_event_engine_endpoint(grpc_endpoint* ep) { return ep->vtable == &grpc_event_engine_endpoint_vtable; } +EventEngine::Endpoint* grpc_get_wrapped_event_engine_endpoint( + grpc_endpoint* ep) { + if (!grpc_is_event_engine_endpoint(ep)) { + return nullptr; + } + auto* eeep = + reinterpret_cast( + ep); + return eeep->wrapper->endpoint(); +} + void grpc_event_engine_endpoint_destroy_and_release_fd( grpc_endpoint* ep, int* fd, grpc_closure* on_release_fd) { auto* eeep = diff --git a/src/core/lib/iomgr/event_engine_shims/endpoint.h b/src/core/lib/iomgr/event_engine_shims/endpoint.h index bc018f1e4d71d..efd57c6ea6d2b 100644 --- a/src/core/lib/iomgr/event_engine_shims/endpoint.h +++ b/src/core/lib/iomgr/event_engine_shims/endpoint.h @@ -31,6 +31,11 @@ grpc_endpoint* grpc_event_engine_endpoint_create( /// Returns true if the passed endpoint is an event engine shim endpoint. bool grpc_is_event_engine_endpoint(grpc_endpoint* ep); +/// Returns the wrapped event engine endpoint if the given grpc_endpoint is an +/// event engine shim endpoint. Otherwise it returns nullptr. +EventEngine::Endpoint* grpc_get_wrapped_event_engine_endpoint( + grpc_endpoint* ep); + /// Destroys the passed in event engine shim endpoint and schedules the /// asynchronous execution of the on_release_fd callback. The int pointer fd is /// set to the underlying endpoint's file descriptor. diff --git a/src/core/lib/promise/detail/promise_like.h b/src/core/lib/promise/detail/promise_like.h index 486653856e35e..4bec366164294 100644 --- a/src/core/lib/promise/detail/promise_like.h +++ b/src/core/lib/promise/detail/promise_like.h @@ -68,6 +68,10 @@ class PromiseLike { private: GPR_NO_UNIQUE_ADDRESS F f_; + static_assert(!std::is_void::type>::value, + "PromiseLike cannot be used with a function that returns void " + "- return Empty{} instead"); + public: // NOLINTNEXTLINE - internal detail that drastically simplifies calling code. PromiseLike(F&& f) : f_(std::forward(f)) {} diff --git a/src/core/lib/promise/status_flag.h b/src/core/lib/promise/status_flag.h index 063cb76a079e6..54019d387408a 100644 --- a/src/core/lib/promise/status_flag.h +++ b/src/core/lib/promise/status_flag.h @@ -25,11 +25,31 @@ namespace grpc_core { +struct Failure {}; +struct Success {}; + +inline bool IsStatusOk(Failure) { return false; } +inline bool IsStatusOk(Success) { return true; } + +template <> +struct StatusCastImpl { + static absl::Status Cast(Success) { return absl::OkStatus(); } +}; + +template <> +struct StatusCastImpl { + static absl::Status Cast(Success) { return absl::OkStatus(); } +}; + // A boolean representing whether an operation succeeded (true) or failed // (false). class StatusFlag { public: explicit StatusFlag(bool value) : value_(value) {} + // NOLINTNEXTLINE(google-explicit-constructor) + StatusFlag(Failure) : value_(false) {} + // NOLINTNEXTLINE(google-explicit-constructor) + StatusFlag(Success) : value_(true) {} bool ok() const { return value_; } @@ -46,14 +66,21 @@ struct StatusCastImpl { } }; -struct Failure {}; +template <> +struct StatusCastImpl { + static absl::Status Cast(StatusFlag flag) { + return flag.ok() ? absl::OkStatus() : absl::CancelledError(); + } +}; // A value if an operation was successful, or a failure flag if not. template class ValueOrFailure { public: - explicit ValueOrFailure(T value) : value_(std::move(value)) {} - explicit ValueOrFailure(Failure) {} + // NOLINTNEXTLINE(google-explicit-constructor) + ValueOrFailure(T value) : value_(std::move(value)) {} + // NOLINTNEXTLINE(google-explicit-constructor) + ValueOrFailure(Failure) {} static ValueOrFailure FromOptional(absl::optional value) { return ValueOrFailure{std::move(value)}; @@ -75,6 +102,11 @@ inline bool IsStatusOk(const ValueOrFailure& value) { return value.ok(); } +template +inline T TakeValue(ValueOrFailure&& value) { + return std::move(value.value()); +} + template struct StatusCastImpl> { static absl::Status Cast(const ValueOrFailure flag) { diff --git a/src/core/lib/resource_quota/memory_quota.cc b/src/core/lib/resource_quota/memory_quota.cc index 6335a54c484fd..0e63e45b4df6e 100644 --- a/src/core/lib/resource_quota/memory_quota.cc +++ b/src/core/lib/resource_quota/memory_quota.cc @@ -20,11 +20,19 @@ #include #include +#include +#include +#include +#include #include +#include #include "absl/status/status.h" #include "absl/strings/str_cat.h" +#include +#include + #include "src/core/lib/debug/trace.h" #include "src/core/lib/gpr/useful.h" #include "src/core/lib/gprpp/mpscq.h" @@ -34,6 +42,7 @@ #include "src/core/lib/promise/race.h" #include "src/core/lib/promise/seq.h" #include "src/core/lib/resource_quota/trace.h" +#include "src/core/lib/slice/slice_refcount.h" namespace grpc_core { @@ -90,6 +99,39 @@ class MemoryQuotaTracker { Mutex mu_; std::vector> quotas_ ABSL_GUARDED_BY(mu_); }; + +// Reference count for a slice allocated by MemoryAllocator::MakeSlice. +// Takes care of releasing memory back when the slice is destroyed. +class SliceRefCount : public grpc_slice_refcount { + public: + SliceRefCount( + std::shared_ptr< + grpc_event_engine::experimental::internal::MemoryAllocatorImpl> + allocator, + size_t size) + : grpc_slice_refcount(Destroy), + allocator_(std::move(allocator)), + size_(size) { + // Nothing to do here. + } + ~SliceRefCount() { + allocator_->Release(size_); + allocator_.reset(); + } + + private: + static void Destroy(grpc_slice_refcount* p) { + auto* rc = static_cast(p); + rc->~SliceRefCount(); + free(rc); + } + + std::shared_ptr< + grpc_event_engine::experimental::internal::MemoryAllocatorImpl> + allocator_; + size_t size_; +}; + } // namespace // @@ -337,6 +379,18 @@ void GrpcMemoryAllocatorImpl::Replenish() { free_bytes_.fetch_add(amount, std::memory_order_acq_rel); } +grpc_slice GrpcMemoryAllocatorImpl::MakeSlice(MemoryRequest request) { + auto size = Reserve(request.Increase(sizeof(SliceRefCount))); + void* p = malloc(size); + new (p) SliceRefCount(shared_from_this(), size); + grpc_slice slice; + slice.refcount = static_cast(p); + slice.data.refcounted.bytes = + static_cast(p) + sizeof(SliceRefCount); + slice.data.refcounted.length = size - sizeof(SliceRefCount); + return slice; +} + // // BasicMemoryQuota // diff --git a/src/core/lib/resource_quota/memory_quota.h b/src/core/lib/resource_quota/memory_quota.h index a76f2594d206c..4f38b8dd32d3d 100644 --- a/src/core/lib/resource_quota/memory_quota.h +++ b/src/core/lib/resource_quota/memory_quota.h @@ -400,6 +400,12 @@ class GrpcMemoryAllocatorImpl final : public EventEngineMemoryAllocatorImpl { // Returns the number of bytes reserved. size_t Reserve(MemoryRequest request) override; + /// Allocate a slice, using MemoryRequest to size the number of returned + /// bytes. For a variable length request, check the returned slice length to + /// verify how much memory was allocated. Takes care of reserving memory for + /// any relevant control structures also. + grpc_slice MakeSlice(MemoryRequest request) override; + // Release some bytes that were previously reserved. void Release(size_t n) override { // Add the released memory to our free bytes counter... if this increases diff --git a/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.h b/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.h index 22ccf7ebd74a8..d9f2527a33cfc 100644 --- a/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.h +++ b/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.h @@ -39,7 +39,6 @@ #include "src/core/lib/gprpp/sync.h" #include "src/core/lib/gprpp/thd.h" #include "src/core/lib/gprpp/unique_type_name.h" -#include "src/core/lib/iomgr/iomgr_fwd.h" #include "src/core/lib/security/credentials/tls/grpc_tls_certificate_distributor.h" #include "src/core/lib/security/security_connector/ssl_utils.h" @@ -55,8 +54,6 @@ struct grpc_tls_certificate_provider : public grpc_core::RefCounted { public: - virtual grpc_pollset_set* interested_parties() const { return nullptr; } - virtual grpc_core::RefCountedPtr distributor() const = 0; diff --git a/src/core/lib/surface/call_trace.cc b/src/core/lib/surface/call_trace.cc index 163418f5083fb..d9b544ee1f9d6 100644 --- a/src/core/lib/surface/call_trace.cc +++ b/src/core/lib/surface/call_trace.cc @@ -77,7 +77,8 @@ const grpc_channel_filter* PromiseTracingFilterFor( return r; }; }, - grpc_channel_next_op, /* sizeof_call_data: */ 0, + /* init_call: */ nullptr, grpc_channel_next_op, + /* sizeof_call_data: */ 0, // init_call_elem: [](grpc_call_element*, const grpc_call_element_args*) { return absl::OkStatus(); diff --git a/src/core/lib/surface/server.cc b/src/core/lib/surface/server.cc index 2402f45e9da37..b2d1be8fea52b 100644 --- a/src/core/lib/surface/server.cc +++ b/src/core/lib/surface/server.cc @@ -757,6 +757,7 @@ class ChannelBroadcaster { const grpc_channel_filter Server::kServerTopFilter = { Server::CallData::StartTransportStreamOpBatch, Server::ChannelData::MakeCallPromise, + /* init_call: */ nullptr, grpc_channel_next_op, sizeof(Server::CallData), Server::CallData::InitCallElement, @@ -838,9 +839,9 @@ void Server::Start() { if (unregistered_request_matcher_ == nullptr) { unregistered_request_matcher_ = make_real_request_matcher(); } - for (std::unique_ptr& rm : registered_methods_) { - if (rm->matcher == nullptr) { - rm->matcher = make_real_request_matcher(); + for (auto& rm : registered_methods_) { + if (rm.second->matcher == nullptr) { + rm.second->matcher = make_real_request_matcher(); } } { @@ -927,20 +928,11 @@ void Server::RegisterCompletionQueue(grpc_completion_queue* cq) { cqs_.push_back(cq); } -namespace { - -bool streq(const std::string& a, const char* b) { - return (a.empty() && b == nullptr) || - ((b != nullptr) && !strcmp(a.c_str(), b)); -} - -} // namespace - Server::RegisteredMethod* Server::RegisterMethod( const char* method, const char* host, grpc_server_register_method_payload_handling payload_handling, uint32_t flags) { - if (IsRegisteredMethodsMapEnabled() && started_) { + if (started_) { Crash("Attempting to register method after server started"); } @@ -949,21 +941,21 @@ Server::RegisteredMethod* Server::RegisterMethod( "grpc_server_register_method method string cannot be NULL"); return nullptr; } - for (std::unique_ptr& m : registered_methods_) { - if (streq(m->method, method) && streq(m->host, host)) { - gpr_log(GPR_ERROR, "duplicate registration for %s@%s", method, - host ? host : "*"); - return nullptr; - } + auto key = std::make_pair(host ? host : "", method); + if (registered_methods_.find(key) != registered_methods_.end()) { + gpr_log(GPR_ERROR, "duplicate registration for %s@%s", method, + host ? host : "*"); + return nullptr; } if (flags != 0) { gpr_log(GPR_ERROR, "grpc_server_register_method invalid flags 0x%08x", flags); return nullptr; } - registered_methods_.emplace_back(std::make_unique( - method, host, payload_handling, flags)); - return registered_methods_.back().get(); + auto it = registered_methods_.emplace( + key, std::make_unique(method, host, payload_handling, + flags)); + return it.first->second.get(); } void Server::DoneRequestEvent(void* req, grpc_cq_completion* /*c*/) { @@ -1014,9 +1006,9 @@ void Server::KillPendingWorkLocked(grpc_error_handle error) { if (started_) { unregistered_request_matcher_->KillRequests(error); unregistered_request_matcher_->ZombifyPending(); - for (std::unique_ptr& rm : registered_methods_) { - rm->matcher->KillRequests(error); - rm->matcher->ZombifyPending(); + for (auto& rm : registered_methods_) { + rm.second->matcher->KillRequests(error); + rm.second->matcher->ZombifyPending(); } } } @@ -1251,7 +1243,6 @@ class Server::ChannelData::ConnectivityWatcher // Server::ChannelData::~ChannelData() { - old_registered_methods_.reset(); if (server_ != nullptr) { if (server_->channelz_node_ != nullptr && channelz_socket_uuid_ != 0) { server_->channelz_node_->RemoveChildSocket(channelz_socket_uuid_); @@ -1275,50 +1266,6 @@ void Server::ChannelData::InitTransport(RefCountedPtr server, channel_ = channel; cq_idx_ = cq_idx; channelz_socket_uuid_ = channelz_socket_uuid; - // Build a lookup table phrased in terms of mdstr's in this channels context - // to quickly find registered methods. - size_t num_registered_methods = server_->registered_methods_.size(); - if (!IsRegisteredMethodsMapEnabled() && num_registered_methods > 0) { - uint32_t max_probes = 0; - size_t slots = 2 * num_registered_methods; - old_registered_methods_ = - std::make_unique>(slots); - for (std::unique_ptr& rm : server_->registered_methods_) { - Slice host; - Slice method = Slice::FromExternalString(rm->method); - const bool has_host = !rm->host.empty(); - if (has_host) { - host = Slice::FromExternalString(rm->host); - } - uint32_t hash = MixHash32(has_host ? host.Hash() : 0, method.Hash()); - uint32_t probes = 0; - for (probes = 0; (*old_registered_methods_)[(hash + probes) % slots] - .server_registered_method != nullptr; - probes++) { - } - if (probes > max_probes) max_probes = probes; - ChannelRegisteredMethod* crm = - &(*old_registered_methods_)[(hash + probes) % slots]; - crm->server_registered_method = rm.get(); - crm->flags = rm->flags; - crm->has_host = has_host; - if (has_host) { - crm->host = std::move(host); - } - crm->method = std::move(method); - } - GPR_ASSERT(slots <= UINT32_MAX); - registered_method_max_probes_ = max_probes; - } else if (IsRegisteredMethodsMapEnabled()) { - for (std::unique_ptr& rm : server_->registered_methods_) { - auto key = std::make_pair(!rm->host.empty() ? rm->host : "", rm->method); - registered_methods_.emplace( - key, std::make_unique( - rm.get(), rm->flags, /*has_host=*/!rm->host.empty(), - Slice::FromExternalString(rm->method), - Slice::FromExternalString(rm->host))); - } - } // Publish channel. { MutexLock lock(&server_->mu_global_); @@ -1330,7 +1277,10 @@ void Server::ChannelData::InitTransport(RefCountedPtr server, op->set_accept_stream = true; op->set_accept_stream_fn = AcceptStream; if (IsRegisteredMethodLookupInTransportEnabled()) { - op->set_registered_method_matcher_fn = SetRegisteredMethodOnMetadata; + op->set_registered_method_matcher_fn = [](void* arg, + ClientMetadata* metadata) { + static_cast(arg)->SetRegisteredMethodOnMetadata(*metadata); + }; } // op->set_registered_method_matcher_fn = Registered op->set_accept_stream_user_data = this; @@ -1341,75 +1291,41 @@ void Server::ChannelData::InitTransport(RefCountedPtr server, transport->PerformOp(op); } -Server::ChannelRegisteredMethod* Server::ChannelData::GetRegisteredMethod( - const grpc_slice& host, const grpc_slice& path) { - if (old_registered_methods_ == nullptr) return nullptr; - // TODO(ctiller): unify these two searches - // check for an exact match with host - uint32_t hash = MixHash32(grpc_slice_hash(host), grpc_slice_hash(path)); - for (size_t i = 0; i <= registered_method_max_probes_; i++) { - ChannelRegisteredMethod* rm = &( - *old_registered_methods_)[(hash + i) % old_registered_methods_->size()]; - if (rm->server_registered_method == nullptr) break; - if (!rm->has_host) continue; - if (rm->host != host) continue; - if (rm->method != path) continue; - return rm; - } - // check for a wildcard method definition (no host set) - hash = MixHash32(0, grpc_slice_hash(path)); - for (size_t i = 0; i <= registered_method_max_probes_; i++) { - ChannelRegisteredMethod* rm = &( - *old_registered_methods_)[(hash + i) % old_registered_methods_->size()]; - if (rm->server_registered_method == nullptr) break; - if (rm->has_host) continue; - if (rm->method != path) continue; - return rm; - } - return nullptr; -} - -Server::ChannelRegisteredMethod* Server::ChannelData::GetRegisteredMethod( +Server::RegisteredMethod* Server::ChannelData::GetRegisteredMethod( const absl::string_view& host, const absl::string_view& path) { - if (registered_methods_.empty()) return nullptr; + if (server_->registered_methods_.empty()) return nullptr; // check for an exact match with host - auto it = registered_methods_.find(std::make_pair(host, path)); - if (it != registered_methods_.end()) { + auto it = server_->registered_methods_.find(std::make_pair(host, path)); + if (it != server_->registered_methods_.end()) { return it->second.get(); } // check for wildcard method definition (no host set) - it = registered_methods_.find(std::make_pair("", path)); - if (it != registered_methods_.end()) { + it = server_->registered_methods_.find(std::make_pair("", path)); + if (it != server_->registered_methods_.end()) { return it->second.get(); } return nullptr; } void Server::ChannelData::SetRegisteredMethodOnMetadata( - void* arg, ServerMetadata* metadata) { - auto* chand = static_cast(arg); - auto* authority = metadata->get_pointer(HttpAuthorityMetadata()); + ClientMetadata& metadata) { + auto* authority = metadata.get_pointer(HttpAuthorityMetadata()); if (authority == nullptr) { - authority = metadata->get_pointer(HostMetadata()); + authority = metadata.get_pointer(HostMetadata()); if (authority == nullptr) { // Authority not being set is an RPC error. return; } } - auto* path = metadata->get_pointer(HttpPathMetadata()); + auto* path = metadata.get_pointer(HttpPathMetadata()); if (path == nullptr) { // Path not being set would result in an RPC error. return; } - ChannelRegisteredMethod* method; - if (!IsRegisteredMethodsMapEnabled()) { - method = chand->GetRegisteredMethod(authority->c_slice(), path->c_slice()); - } else { - method = chand->GetRegisteredMethod(authority->as_string_view(), - path->as_string_view()); - } + RegisteredMethod* method = + GetRegisteredMethod(authority->as_string_view(), path->as_string_view()); // insert in metadata - metadata->Set(GrpcRegisteredMethod(), method); + metadata.Set(GrpcRegisteredMethod(), method); } void Server::ChannelData::AcceptStream(void* arg, Transport* /*transport*/, @@ -1478,24 +1394,20 @@ ArenaPromise Server::ChannelData::MakeCallPromise( Timestamp deadline = GetContext()->deadline(); // Find request matcher. RequestMatcherInterface* matcher; - ChannelRegisteredMethod* rm = nullptr; + RegisteredMethod* rm = nullptr; if (IsRegisteredMethodLookupInTransportEnabled()) { - rm = static_cast( + rm = static_cast( call_args.client_initial_metadata->get(GrpcRegisteredMethod()) .value_or(nullptr)); } else { - if (!IsRegisteredMethodsMapEnabled()) { - rm = chand->GetRegisteredMethod(host_ptr->c_slice(), path->c_slice()); - } else { - rm = chand->GetRegisteredMethod(host_ptr->as_string_view(), - path->as_string_view()); - } + rm = chand->GetRegisteredMethod(host_ptr->as_string_view(), + path->as_string_view()); } ArenaPromise>> maybe_read_first_message([] { return NextResult(); }); if (rm != nullptr) { - matcher = rm->server_registered_method->matcher.get(); - switch (rm->server_registered_method->payload_handling) { + matcher = rm->matcher.get(); + switch (rm->payload_handling) { case GRPC_SRM_PAYLOAD_NONE: break; case GRPC_SRM_PAYLOAD_READ_INITIAL_BYTE_BUFFER: @@ -1749,22 +1661,18 @@ void Server::CallData::StartNewRpc(grpc_call_element* elem) { grpc_server_register_method_payload_handling payload_handling = GRPC_SRM_PAYLOAD_NONE; if (path_.has_value() && host_.has_value()) { - ChannelRegisteredMethod* rm; + RegisteredMethod* rm; if (IsRegisteredMethodLookupInTransportEnabled()) { - rm = static_cast( + rm = static_cast( recv_initial_metadata_->get(GrpcRegisteredMethod()) .value_or(nullptr)); } else { - if (!IsRegisteredMethodsMapEnabled()) { - rm = chand->GetRegisteredMethod(host_->c_slice(), path_->c_slice()); - } else { - rm = chand->GetRegisteredMethod(host_->as_string_view(), - path_->as_string_view()); - } + rm = chand->GetRegisteredMethod(host_->as_string_view(), + path_->as_string_view()); } if (rm != nullptr) { - matcher_ = rm->server_registered_method->matcher.get(); - payload_handling = rm->server_registered_method->payload_handling; + matcher_ = rm->matcher.get(); + payload_handling = rm->payload_handling; } } // Start recv_message op if needed. diff --git a/src/core/lib/surface/server.h b/src/core/lib/surface/server.h index f5999bcda30ed..226fb9fc1329a 100644 --- a/src/core/lib/surface/server.h +++ b/src/core/lib/surface/server.h @@ -211,26 +211,6 @@ class Server : public InternallyRefCounted, private: struct RequestedCall; - struct ChannelRegisteredMethod { - ChannelRegisteredMethod() = default; - ChannelRegisteredMethod(RegisteredMethod* server_registered_method_arg, - uint32_t flags_arg, bool has_host_arg, - Slice method_arg, Slice host_arg) - : server_registered_method(server_registered_method_arg), - flags(flags_arg), - has_host(has_host_arg), - method(std::move(method_arg)), - host(std::move(host_arg)) {} - - ~ChannelRegisteredMethod() = default; - - RegisteredMethod* server_registered_method = nullptr; - uint32_t flags; - bool has_host; - Slice method; - Slice host; - }; - class RequestMatcherInterface; class RealRequestMatcherFilterStack; class RealRequestMatcherPromises; @@ -251,11 +231,8 @@ class Server : public InternallyRefCounted, Channel* channel() const { return channel_.get(); } size_t cq_idx() const { return cq_idx_; } - ChannelRegisteredMethod* GetRegisteredMethod(const grpc_slice& host, - const grpc_slice& path); - - ChannelRegisteredMethod* GetRegisteredMethod(const absl::string_view& host, - const absl::string_view& path); + RegisteredMethod* GetRegisteredMethod(const absl::string_view& host, + const absl::string_view& path); // Filter vtable functions. static grpc_error_handle InitChannelElement( grpc_channel_element* elem, grpc_channel_element_args* args); @@ -268,43 +245,18 @@ class Server : public InternallyRefCounted, static void AcceptStream(void* arg, Transport* /*transport*/, const void* transport_server_data); - static void SetRegisteredMethodOnMetadata(void* arg, - ServerMetadata* metadata); + void SetRegisteredMethodOnMetadata(ClientMetadata& metadata); void Destroy() ABSL_EXCLUSIVE_LOCKS_REQUIRED(server_->mu_global_); static void FinishDestroy(void* arg, grpc_error_handle error); - struct StringViewStringViewPairHash - : absl::flat_hash_set< - std::pair>::hasher { - using is_transparent = void; - }; - - struct StringViewStringViewPairEq - : std::equal_to> { - using is_transparent = void; - }; - RefCountedPtr server_; RefCountedPtr channel_; // The index into Server::cqs_ of the CQ used as a starting point for // where to publish new incoming calls. size_t cq_idx_; absl::optional::iterator> list_position_; - // A hash-table of the methods and hosts of the registered methods. - // TODO(vjpai): Convert this to an STL map type as opposed to a direct - // bucket implementation. (Consider performance impact, hash function to - // use, etc.) - std::unique_ptr> - old_registered_methods_; - // Map of registered methods. - absl::flat_hash_map /*host, method*/, - std::unique_ptr, - StringViewStringViewPairHash, - StringViewStringViewPairEq> - registered_methods_; - uint32_t registered_method_max_probes_; grpc_closure finish_destroy_channel_closure_; intptr_t channelz_socket_uuid_; }; @@ -413,6 +365,17 @@ class Server : public InternallyRefCounted, grpc_cq_completion completion; }; + struct StringViewStringViewPairHash + : absl::flat_hash_set< + std::pair>::hasher { + using is_transparent = void; + }; + + struct StringViewStringViewPairEq + : std::equal_to> { + using is_transparent = void; + }; + static void ListenerDestroyDone(void* arg, grpc_error_handle error); static void DoneShutdownEvent(void* server, @@ -498,7 +461,11 @@ class Server : public InternallyRefCounted, bool starting_ ABSL_GUARDED_BY(mu_global_) = false; CondVar starting_cv_; - std::vector> registered_methods_; + // Map of registered methods. + absl::flat_hash_map /*host, method*/, + std::unique_ptr, + StringViewStringViewPairHash, StringViewStringViewPairEq> + registered_methods_; // Request matcher for unregistered methods. std::unique_ptr unregistered_request_matcher_; diff --git a/src/core/lib/transport/metadata_batch.h b/src/core/lib/transport/metadata_batch.h index 0a81bd22a0ab2..6bb12289d5aed 100644 --- a/src/core/lib/transport/metadata_batch.h +++ b/src/core/lib/transport/metadata_batch.h @@ -649,9 +649,8 @@ class ParseHelper { return ParsedMetadata( typename ParsedMetadata::FromSlicePair{}, Slice::FromCopiedString(key), - IsUniquelyUnownedEnabled() && will_keep_past_request_lifetime_ - ? value_.TakeUniquelyOwned() - : std::move(value_), + will_keep_past_request_lifetime_ ? value_.TakeUniquelyOwned() + : std::move(value_), transport_size_); } diff --git a/src/core/lib/transport/transport.h b/src/core/lib/transport/transport.h index 6f451a9ae4be4..f34692c8ddc0a 100644 --- a/src/core/lib/transport/transport.h +++ b/src/core/lib/transport/transport.h @@ -55,7 +55,10 @@ #include "src/core/lib/promise/context.h" #include "src/core/lib/promise/detail/status.h" #include "src/core/lib/promise/latch.h" +#include "src/core/lib/promise/party.h" #include "src/core/lib/promise/pipe.h" +#include "src/core/lib/promise/race.h" +#include "src/core/lib/promise/status_flag.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/transport/connectivity_state.h" @@ -228,6 +231,254 @@ struct CallArgs { using NextPromiseFactory = std::function(CallArgs)>; +// The common middle part of a call - a reference is held by each of +// CallInitiator and CallHandler - which provide interfaces that are appropriate +// for each side of a call. +// The spine will ultimately host the pipes, filters, and context for one part +// of a call: ie top-half client channel, sub channel call, server call. +// TODO(ctiller): eventually drop this when we don't need to reference into +// legacy promise calls anymore +class CallSpineInterface { + public: + virtual ~CallSpineInterface() = default; + virtual Pipe& client_initial_metadata() = 0; + virtual Pipe& server_initial_metadata() = 0; + virtual Pipe& client_to_server_messages() = 0; + virtual Pipe& server_to_client_messages() = 0; + virtual Pipe& server_trailing_metadata() = 0; + virtual Latch& cancel_latch() = 0; + virtual Party& party() = 0; + virtual void IncrementRefCount() = 0; + virtual void Unref() = 0; + + // Cancel the call with the given metadata. + // Regarding the `MUST_USE_RESULT absl::nullopt_t`: + // Most cancellation calls right now happen in pipe interceptors; + // there `nullopt` indicates terminate processing of this pipe and close with + // error. + // It's convenient then to have the Cancel operation (setting the latch to + // terminate the call) be the last thing that occurs in a pipe interceptor, + // and this construction supports that (and has helped the author not write + // some bugs). + GRPC_MUST_USE_RESULT absl::nullopt_t Cancel(ServerMetadataHandle metadata) { + GPR_DEBUG_ASSERT(Activity::current() == &party()); + auto& c = cancel_latch(); + if (c.is_set()) return absl::nullopt; + c.Set(std::move(metadata)); + return absl::nullopt; + } + + auto WaitForCancel() { + GPR_DEBUG_ASSERT(Activity::current() == &party()); + return cancel_latch().Wait(); + } + + // Wrap a promise so that if it returns failure it automatically cancels + // the rest of the call. + // The resulting (returned) promise will resolve to Empty. + template + auto CancelIfFails(Promise promise) { + GPR_DEBUG_ASSERT(Activity::current() == &party()); + using P = promise_detail::PromiseLike; + using ResultType = typename P::Result; + return Map(std::move(promise), [this](ResultType r) { + if (!IsStatusOk(r)) { + std::ignore = Cancel(StatusCast(std::move(r))); + } + return Empty{}; + }); + } + + // Spawn a promise that returns Empty{} and save some boilerplate handling + // that detail. + template + void SpawnInfallible(absl::string_view name, PromiseFactory promise_factory) { + party().Spawn(name, std::move(promise_factory), [](Empty) {}); + } + + // Spawn a promise that returns some status-like type; if the status + // represents failure automatically cancel the rest of the call. + template + void SpawnGuarded(absl::string_view name, PromiseFactory promise_factory) { + using FactoryType = + promise_detail::OncePromiseFactory; + using PromiseType = typename FactoryType::Promise; + using ResultType = typename PromiseType::Result; + static_assert( + std::is_same()))>::value, + "SpawnGuarded promise must return a status-like object"); + party().Spawn(name, std::move(promise_factory), [this](ResultType r) { + if (!IsStatusOk(r)) { + std::ignore = Cancel(StatusCast(std::move(r))); + } + }); + } +}; + +class CallSpine final : public CallSpineInterface { + public: + Pipe& client_initial_metadata() override { + return client_initial_metadata_; + } + Pipe& server_initial_metadata() override { + return server_initial_metadata_; + } + Pipe& client_to_server_messages() override { + return client_to_server_messages_; + } + Pipe& server_to_client_messages() override { + return server_to_client_messages_; + } + Pipe& server_trailing_metadata() override { + return server_trailing_metadata_; + } + Latch& cancel_latch() override { return cancel_latch_; } + Party& party() override { Crash("unimplemented"); } + void IncrementRefCount() override { Crash("unimplemented"); } + void Unref() override { Crash("unimplemented"); } + + private: + // Initial metadata from client to server + Pipe client_initial_metadata_; + // Initial metadata from server to client + Pipe server_initial_metadata_; + // Messages travelling from the application to the transport. + Pipe client_to_server_messages_; + // Messages travelling from the transport to the application. + Pipe server_to_client_messages_; + // Trailing metadata from server to client + Pipe server_trailing_metadata_; + // Latch that can be set to terminate the call + Latch cancel_latch_; +}; + +class CallInitiator { + public: + explicit CallInitiator(RefCountedPtr spine) + : spine_(std::move(spine)) {} + + auto PushClientInitialMetadata(ClientMetadataHandle md) { + GPR_DEBUG_ASSERT(Activity::current() == &spine_->party()); + return Map(spine_->client_initial_metadata().sender.Push(std::move(md)), + [](bool ok) { return StatusFlag(ok); }); + } + + auto PullServerInitialMetadata() { + GPR_DEBUG_ASSERT(Activity::current() == &spine_->party()); + return Map(spine_->server_initial_metadata().receiver.Next(), + [](NextResult md) + -> ValueOrFailure { + if (!md.has_value()) return Failure{}; + return std::move(*md); + }); + } + + auto PullServerTrailingMetadata() { + GPR_DEBUG_ASSERT(Activity::current() == &spine_->party()); + return Race(spine_->WaitForCancel(), + Map(spine_->server_trailing_metadata().receiver.Next(), + [spine = spine_](NextResult md) + -> ServerMetadataHandle { + GPR_ASSERT(md.has_value()); + return std::move(*md); + })); + } + + auto PullMessage() { + GPR_DEBUG_ASSERT(Activity::current() == &spine_->party()); + return spine_->server_to_client_messages().receiver.Next(); + } + + auto PushMessage(MessageHandle message) { + GPR_DEBUG_ASSERT(Activity::current() == &spine_->party()); + return spine_->client_to_server_messages().sender.Push(std::move(message)); + } + + template + auto CancelIfFails(Promise promise) { + return spine_->CancelIfFails(std::move(promise)); + } + + template + void SpawnGuarded(absl::string_view name, PromiseFactory promise_factory) { + spine_->SpawnGuarded(name, std::move(promise_factory)); + } + + template + void SpawnInfallible(absl::string_view name, PromiseFactory promise_factory) { + spine_->SpawnInfallible(name, std::move(promise_factory)); + } + + private: + const RefCountedPtr spine_; +}; + +class CallHandler { + public: + explicit CallHandler(RefCountedPtr spine) + : spine_(std::move(spine)) {} + + auto PullClientInitialMetadata() { + GPR_DEBUG_ASSERT(Activity::current() == &spine_->party()); + return Map(spine_->client_initial_metadata().receiver.Next(), + [](NextResult md) + -> ValueOrFailure { + if (!md.has_value()) return Failure{}; + return std::move(*md); + }); + } + + auto PushServerInitialMetadata(ClientMetadataHandle md) { + GPR_DEBUG_ASSERT(Activity::current() == &spine_->party()); + return Map(spine_->server_initial_metadata().sender.Push(std::move(md)), + [](bool ok) { return StatusFlag(ok); }); + } + + auto PushServerTrailingMetadata(ClientMetadataHandle md) { + GPR_DEBUG_ASSERT(Activity::current() == &spine_->party()); + return Map(spine_->server_initial_metadata().sender.Push(std::move(md)), + [](bool ok) { return StatusFlag(ok); }); + } + + auto PullMessage() { + GPR_DEBUG_ASSERT(Activity::current() == &spine_->party()); + return spine_->client_to_server_messages().receiver.Next(); + } + + auto PushMessage(MessageHandle message) { + GPR_DEBUG_ASSERT(Activity::current() == &spine_->party()); + return spine_->server_to_client_messages().sender.Push(std::move(message)); + } + + template + auto CancelIfFails(Promise promise) { + return spine_->CancelIfFails(std::move(promise)); + } + + template + void SpawnGuarded(absl::string_view name, PromiseFactory promise_factory) { + spine_->SpawnGuarded(name, std::move(promise_factory)); + } + + template + void SpawnInfallible(absl::string_view name, PromiseFactory promise_factory) { + spine_->SpawnInfallible(name, std::move(promise_factory)); + } + + private: + const RefCountedPtr spine_; +}; + +template +auto OutgoingMessages(CallHalf& h) { + struct Wrapper { + CallHalf& h; + auto Next() { return h.PullMessage(); } + }; + return Wrapper{h}; +} + } // namespace grpc_core // forward declarations @@ -633,9 +884,7 @@ class FilterStackTransport { class ClientTransport { public: - // Create a promise to execute one client call. - virtual ArenaPromise MakeCallPromise( - CallArgs call_args) = 0; + virtual void StartCall(CallHandler call_handler) = 0; protected: ~ClientTransport() = default; @@ -643,10 +892,14 @@ class ClientTransport { class ServerTransport { public: - // Register the factory function for the filter stack part of a call - // promise. - void SetCallPromiseFactory( - absl::AnyInvocable(CallArgs) const>); + // AcceptFunction takes initial metadata for a new call and returns a + // CallInitiator object for it, for the transport to use to communicate with + // the CallHandler object passed to the application. + using AcceptFunction = + absl::AnyInvocable(ClientMetadata&) const>; + + // Called once slightly after transport setup to register the accept function. + virtual void SetAcceptFunction(AcceptFunction accept_function) = 0; protected: ~ServerTransport() = default; diff --git a/src/cpp/server/secure_server_credentials.cc b/src/cpp/server/secure_server_credentials.cc index 9ec5d24031f6e..fd198898d2f8a 100644 --- a/src/cpp/server/secure_server_credentials.cc +++ b/src/cpp/server/secure_server_credentials.cc @@ -36,15 +36,15 @@ namespace grpc { -void AuthMetadataProcessorAyncWrapper::Destroy(void* wrapper) { - auto* w = static_cast(wrapper); +void AuthMetadataProcessorAsyncWrapper::Destroy(void* wrapper) { + auto* w = static_cast(wrapper); delete w; } -void AuthMetadataProcessorAyncWrapper::Process( +void AuthMetadataProcessorAsyncWrapper::Process( void* wrapper, grpc_auth_context* context, const grpc_metadata* md, size_t num_md, grpc_process_auth_metadata_done_cb cb, void* user_data) { - auto* w = static_cast(wrapper); + auto* w = static_cast(wrapper); if (!w->processor_) { // Early exit. cb(user_data, nullptr, 0, nullptr, 0, GRPC_STATUS_OK, nullptr); @@ -52,8 +52,8 @@ void AuthMetadataProcessorAyncWrapper::Process( } if (w->processor_->IsBlocking()) { w->thread_pool_->Add([w, context, md, num_md, cb, user_data] { - w->AuthMetadataProcessorAyncWrapper::InvokeProcessor(context, md, num_md, - cb, user_data); + w->AuthMetadataProcessorAsyncWrapper::InvokeProcessor(context, md, num_md, + cb, user_data); }); } else { // invoke directly. @@ -61,7 +61,7 @@ void AuthMetadataProcessorAyncWrapper::Process( } } -void AuthMetadataProcessorAyncWrapper::InvokeProcessor( +void AuthMetadataProcessorAsyncWrapper::InvokeProcessor( grpc_auth_context* context, const grpc_metadata* md, size_t num_md, grpc_process_auth_metadata_done_cb cb, void* user_data) { AuthMetadataProcessor::InputMetadata metadata; @@ -104,10 +104,10 @@ int SecureServerCredentials::AddPortToServer(const std::string& addr, void SecureServerCredentials::SetAuthMetadataProcessor( const std::shared_ptr& processor) { - auto* wrapper = new grpc::AuthMetadataProcessorAyncWrapper(processor); + auto* wrapper = new grpc::AuthMetadataProcessorAsyncWrapper(processor); grpc_server_credentials_set_auth_metadata_processor( - creds_, {grpc::AuthMetadataProcessorAyncWrapper::Process, - grpc::AuthMetadataProcessorAyncWrapper::Destroy, wrapper}); + creds_, {grpc::AuthMetadataProcessorAsyncWrapper::Process, + grpc::AuthMetadataProcessorAsyncWrapper::Destroy, wrapper}); } std::shared_ptr SslServerCredentials( diff --git a/src/cpp/server/secure_server_credentials.h b/src/cpp/server/secure_server_credentials.h index ca90a25f4f245..a62e51f2c24e3 100644 --- a/src/cpp/server/secure_server_credentials.h +++ b/src/cpp/server/secure_server_credentials.h @@ -35,7 +35,7 @@ namespace grpc { class SecureServerCredentials; -class AuthMetadataProcessorAyncWrapper final { +class AuthMetadataProcessorAsyncWrapper final { public: static void Destroy(void* wrapper); @@ -43,7 +43,7 @@ class AuthMetadataProcessorAyncWrapper final { const grpc_metadata* md, size_t num_md, grpc_process_auth_metadata_done_cb cb, void* user_data); - explicit AuthMetadataProcessorAyncWrapper( + explicit AuthMetadataProcessorAsyncWrapper( const std::shared_ptr& processor) : processor_(processor) { if (processor && processor->IsBlocking()) { @@ -78,7 +78,7 @@ class SecureServerCredentials final : public ServerCredentials { SecureServerCredentials* AsSecureServerCredentials() override { return this; } grpc_server_credentials* creds_; - std::unique_ptr processor_; + std::unique_ptr processor_; }; } // namespace grpc diff --git a/src/objective-c/PrivacyInfo.xcprivacy b/src/objective-c/PrivacyInfo.xcprivacy new file mode 100644 index 0000000000000..276f7610da6df --- /dev/null +++ b/src/objective-c/PrivacyInfo.xcprivacy @@ -0,0 +1,23 @@ + + + + + NSPrivacyTracking + + NSPrivacyCollectedDataTypes + + NSPrivacyTrackingDomains + + NSPrivacyAccessedAPITypes + + + NSPrivacyAccessedAPIType + NSPrivacyAccessedAPICategoryFileTimestamp + NSPrivacyAccessedAPITypeReasons + + C617.1 + + + + + diff --git a/src/python/grpcio/grpc/_server.py b/src/python/grpcio/grpc/_server.py index 595056136b766..4ea1d5e459784 100644 --- a/src/python/grpcio/grpc/_server.py +++ b/src/python/grpcio/grpc/_server.py @@ -579,7 +579,7 @@ def _call_behavior( exception.__traceback__, ) ) - traceback.print_exc() + traceback.print_exc() _LOGGER.exception(details) _abort( state, diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 1d2fdd5bacc37..d6304bccbdca8 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -508,12 +508,12 @@ 'src/core/lib/event_engine/default_event_engine_factory.cc', 'src/core/lib/event_engine/event_engine.cc', 'src/core/lib/event_engine/forkable.cc', - 'src/core/lib/event_engine/memory_allocator.cc', 'src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc', 'src/core/lib/event_engine/posix_engine/ev_poll_posix.cc', 'src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc', 'src/core/lib/event_engine/posix_engine/internal_errqueue.cc', 'src/core/lib/event_engine/posix_engine/lockfree_event.cc', + 'src/core/lib/event_engine/posix_engine/native_dns_resolver.cc', 'src/core/lib/event_engine/posix_engine/posix_endpoint.cc', 'src/core/lib/event_engine/posix_engine/posix_engine.cc', 'src/core/lib/event_engine/posix_engine/posix_engine_listener.cc', diff --git a/templates/gRPC.podspec.template b/templates/gRPC.podspec.template index 9efb28643a8e4..c1f739ce85e84 100644 --- a/templates/gRPC.podspec.template +++ b/templates/gRPC.podspec.template @@ -34,6 +34,8 @@ :tag => "v#{version}", } + s.resource = 'src/objective-c/PrivacyInfo.xcprivacy' + name = 'GRPCClient' s.module_name = name s.header_dir = name diff --git a/templates/tools/dockerfile/grpc_iwyu/Dockerfile.template b/templates/tools/dockerfile/grpc_iwyu/Dockerfile.template deleted file mode 100644 index ecb4deef2839b..0000000000000 --- a/templates/tools/dockerfile/grpc_iwyu/Dockerfile.template +++ /dev/null @@ -1,30 +0,0 @@ -%YAML 1.2 ---- | - # Copyright 2021 gRPC authors. - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - - # Docker file for running IWYU. - # Updated: 2022-11-03 - - FROM silkeh/clang:16-bullseye - - # Install prerequisites for the iwyu script - RUN apt-get update && apt-get install -y python3 jq git cmake python zlib1g-dev libtinfo-dev libclang-16-dev && apt-get clean - ADD iwyu.sh / - - # When running locally, we'll be impersonating the current user, so we need - # to make the script runnable by everyone. - RUN chmod a+rx /iwyu.sh - - CMD ["echo 'Run with tools/distrib/iwyu.sh'"] diff --git a/test/core/bad_client/tests/initial_settings_frame.cc b/test/core/bad_client/tests/initial_settings_frame.cc index 4f0af52b01621..384d1662ab24c 100644 --- a/test/core/bad_client/tests/initial_settings_frame.cc +++ b/test/core/bad_client/tests/initial_settings_frame.cc @@ -200,12 +200,10 @@ int main(int argc, char** argv) { GRPC_BAD_CLIENT_DISCONNECT); // too many requests before the settings ack is sent should be cancelled - if (grpc_core::IsBlockExcessiveRequestsBeforeSettingsAckEnabled()) { - GRPC_RUN_BAD_CLIENT_TEST(single_request_verifier, nullptr, - PFX_STR ZERO_SETTING_HDR FOOBAR_0 FOOBAR_2 - SETTING_ACK RST_STREAM_1 RST_STREAM_3 FOOBAR_1, - GRPC_BAD_CLIENT_MAX_CONCURRENT_REQUESTS_OF_ONE); - } + GRPC_RUN_BAD_CLIENT_TEST(single_request_verifier, nullptr, + PFX_STR ZERO_SETTING_HDR FOOBAR_0 FOOBAR_2 + SETTING_ACK RST_STREAM_1 RST_STREAM_3 FOOBAR_1, + GRPC_BAD_CLIENT_MAX_CONCURRENT_REQUESTS_OF_ONE); grpc_shutdown(); return 0; diff --git a/test/core/channel/channel_args_test.cc b/test/core/channel/channel_args_test.cc index 10a05d35e2637..fd035ccc12d46 100644 --- a/test/core/channel/channel_args_test.cc +++ b/test/core/channel/channel_args_test.cc @@ -209,6 +209,37 @@ TEST(ChannelArgsTest, GetNonOwningEventEngine) { ASSERT_EQ(p.use_count(), 2); } +struct MutableValue : public RefCounted { + static constexpr absl::string_view ChannelArgName() { + return "grpc.test.mutable_value"; + } + static int ChannelArgsCompare(const MutableValue* a, const MutableValue* b) { + return a->i - b->i; + } + int i = 42; +}; + +struct ConstValue : public RefCounted { + static constexpr absl::string_view ChannelArgName() { + return "grpc.test.const_value"; + } + static constexpr bool ChannelArgUseConstPtr() { return true; }; + static int ChannelArgsCompare(const ConstValue* a, const ConstValue* b) { + return a->i - b->i; + } + int i = 42; +}; + +TEST(ChannelArgsTest, SetObjectRespectsMutabilityConstraints) { + auto m = MakeRefCounted(); + auto c = MakeRefCounted(); + auto args = ChannelArgs().SetObject(m).SetObject(c); + RefCountedPtr m1 = args.GetObjectRef(); + RefCountedPtr c1 = args.GetObjectRef(); + EXPECT_EQ(m1.get(), m.get()); + EXPECT_EQ(c1.get(), c.get()); +} + } // namespace grpc_core TEST(GrpcChannelArgsTest, Create) { diff --git a/test/core/channel/channel_stack_builder_test.cc b/test/core/channel/channel_stack_builder_test.cc index 4e253b832d0f6..65d4466eff9bd 100644 --- a/test/core/channel/channel_stack_builder_test.cc +++ b/test/core/channel/channel_stack_builder_test.cc @@ -62,8 +62,8 @@ const grpc_channel_filter* FilterNamed(const char* name) { ->emplace( name, new grpc_channel_filter{ - grpc_call_next_op, nullptr, grpc_channel_next_op, 0, CallInitFunc, - grpc_call_stack_ignore_set_pollset_or_pollset_set, + grpc_call_next_op, nullptr, nullptr, grpc_channel_next_op, 0, + CallInitFunc, grpc_call_stack_ignore_set_pollset_or_pollset_set, CallDestroyFunc, 0, ChannelInitFunc, [](grpc_channel_stack*, grpc_channel_element*) {}, ChannelDestroyFunc, grpc_channel_next_get_info, name}) diff --git a/test/core/channel/channel_stack_test.cc b/test/core/channel/channel_stack_test.cc index 7ed34f60b6419..4054e6a8844f2 100644 --- a/test/core/channel/channel_stack_test.cc +++ b/test/core/channel/channel_stack_test.cc @@ -83,6 +83,7 @@ TEST(ChannelStackTest, CreateChannelStack) { const grpc_channel_filter filter = { call_func, nullptr, + nullptr, channel_func, sizeof(int), call_init_func, diff --git a/test/core/client_channel/resolvers/dns_resolver_cooldown_test.cc b/test/core/client_channel/resolvers/dns_resolver_cooldown_test.cc index fc663837a8bf1..08f7e161a8fb0 100644 --- a/test/core/client_channel/resolvers/dns_resolver_cooldown_test.cc +++ b/test/core/client_channel/resolvers/dns_resolver_cooldown_test.cc @@ -42,6 +42,7 @@ #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/config/core_configuration.h" #include "src/core/lib/event_engine/default_event_engine.h" +#include "src/core/lib/experiments/experiments.h" #include "src/core/lib/gprpp/debug_location.h" #include "src/core/lib/gprpp/no_destruct.h" #include "src/core/lib/gprpp/notification.h" @@ -409,6 +410,14 @@ static void test_cooldown() { } TEST(DnsResolverCooldownTest, MainTest) { + // TODO(yijiem): This test tests the cooldown behavior of the PollingResolver + // interface. To do that, it overrides the grpc_dns_lookup_hostname_ares + // function and overrides the iomgr's g_dns_resolver system. We would need to + // rewrite this test for EventEngine using a custom EE DNSResolver or adding + // to the resolver_fuzzer. + if (grpc_core::IsEventEngineDnsEnabled()) { + GTEST_SKIP() << "Not with event engine dns"; + } grpc_init(); auto work_serializer = std::make_shared( diff --git a/test/core/end2end/end2end_test_corpus/negative_deadline/5769288995635200 b/test/core/end2end/end2end_test_corpus/negative_deadline/5769288995635200 new file mode 100644 index 0000000000000..bb65fceedf8ac --- /dev/null +++ b/test/core/end2end/end2end_test_corpus/negative_deadline/5769288995635200 @@ -0,0 +1,11 @@ +test_id: 3 +event_engine_actions { + run_delay: 9851624184873214 + run_delay: 1 + run_delay: 1 + run_delay: 0 + run_delay: 53876069761024 +} +config_vars { + experiments: 280384054960896 +} diff --git a/test/core/end2end/tests/filter_causes_close.cc b/test/core/end2end/tests/filter_causes_close.cc index 751507495de05..91b917ced3b74 100644 --- a/test/core/end2end/tests/filter_causes_close.cc +++ b/test/core/end2end/tests/filter_causes_close.cc @@ -102,6 +102,7 @@ const grpc_channel_filter test_filter = { return Immediate(ServerMetadataFromStatus( absl::PermissionDeniedError("Failure that's not preventable."))); }, + nullptr, grpc_channel_next_op, sizeof(call_data), init_call_elem, diff --git a/test/core/end2end/tests/filter_context.cc b/test/core/end2end/tests/filter_context.cc index 8156384fbd64d..1da07e814e965 100644 --- a/test/core/end2end/tests/filter_context.cc +++ b/test/core/end2end/tests/filter_context.cc @@ -82,7 +82,7 @@ grpc_error_handle init_channel_elem(grpc_channel_element* /*elem*/, void destroy_channel_elem(grpc_channel_element* /*elem*/) {} const grpc_channel_filter test_filter = { - start_transport_stream_op_batch, nullptr, grpc_channel_next_op, + start_transport_stream_op_batch, nullptr, nullptr, grpc_channel_next_op, sizeof(call_data), init_call_elem, grpc_call_stack_ignore_set_pollset_or_pollset_set, destroy_call_elem, 0, init_channel_elem, grpc_channel_stack_no_post_init, destroy_channel_elem, diff --git a/test/core/end2end/tests/filter_init_fails.cc b/test/core/end2end/tests/filter_init_fails.cc index a640bebc5f0e7..a160b38b60af6 100644 --- a/test/core/end2end/tests/filter_init_fails.cc +++ b/test/core/end2end/tests/filter_init_fails.cc @@ -80,7 +80,7 @@ const grpc_channel_filter test_filter = { return Immediate(ServerMetadataFromStatus( absl::PermissionDeniedError("access denied"))); }, - grpc_channel_next_op, 0, init_call_elem, + nullptr, grpc_channel_next_op, 0, init_call_elem, grpc_call_stack_ignore_set_pollset_or_pollset_set, destroy_call_elem, 0, init_channel_elem, grpc_channel_stack_no_post_init, destroy_channel_elem, grpc_channel_next_get_info, diff --git a/test/core/end2end/tests/retry_cancel_with_multiple_send_batches.cc b/test/core/end2end/tests/retry_cancel_with_multiple_send_batches.cc index 85948b208c508..22de5660e0553 100644 --- a/test/core/end2end/tests/retry_cancel_with_multiple_send_batches.cc +++ b/test/core/end2end/tests/retry_cancel_with_multiple_send_batches.cc @@ -159,6 +159,7 @@ class FailSendOpsFilter { grpc_channel_filter FailSendOpsFilter::kFilterVtable = { CallData::StartTransportStreamOpBatch, nullptr, + nullptr, grpc_channel_next_op, sizeof(CallData), CallData::Init, diff --git a/test/core/end2end/tests/retry_recv_message_replay.cc b/test/core/end2end/tests/retry_recv_message_replay.cc index 74e5f98bfc74c..befb2391ddea6 100644 --- a/test/core/end2end/tests/retry_recv_message_replay.cc +++ b/test/core/end2end/tests/retry_recv_message_replay.cc @@ -109,6 +109,7 @@ class FailFirstSendOpFilter { grpc_channel_filter FailFirstSendOpFilter::kFilterVtable = { CallData::StartTransportStreamOpBatch, nullptr, + nullptr, grpc_channel_next_op, sizeof(CallData), CallData::Init, diff --git a/test/core/end2end/tests/retry_recv_trailing_metadata_error.cc b/test/core/end2end/tests/retry_recv_trailing_metadata_error.cc index 8a277ff4b6dc4..7a43114ec3a2e 100644 --- a/test/core/end2end/tests/retry_recv_trailing_metadata_error.cc +++ b/test/core/end2end/tests/retry_recv_trailing_metadata_error.cc @@ -104,6 +104,7 @@ class InjectStatusFilter { grpc_channel_filter InjectStatusFilter::kFilterVtable = { CallData::StartTransportStreamOpBatch, nullptr, + nullptr, grpc_channel_next_op, sizeof(CallData), CallData::Init, diff --git a/test/core/end2end/tests/retry_send_op_fails.cc b/test/core/end2end/tests/retry_send_op_fails.cc index 3bb2715dc5ae4..2fdc38186944b 100644 --- a/test/core/end2end/tests/retry_send_op_fails.cc +++ b/test/core/end2end/tests/retry_send_op_fails.cc @@ -110,6 +110,7 @@ class FailFirstCallFilter { grpc_channel_filter FailFirstCallFilter::kFilterVtable = { CallData::StartTransportStreamOpBatch, nullptr, + nullptr, grpc_channel_next_op, sizeof(CallData), CallData::Init, diff --git a/test/core/end2end/tests/retry_transparent_goaway.cc b/test/core/end2end/tests/retry_transparent_goaway.cc index 98d3f744d18df..091a800f210ca 100644 --- a/test/core/end2end/tests/retry_transparent_goaway.cc +++ b/test/core/end2end/tests/retry_transparent_goaway.cc @@ -116,6 +116,7 @@ class FailFirstCallFilter { grpc_channel_filter FailFirstCallFilter::kFilterVtable = { CallData::StartTransportStreamOpBatch, nullptr, + nullptr, grpc_channel_next_op, sizeof(CallData), CallData::Init, diff --git a/test/core/end2end/tests/retry_transparent_max_concurrent_streams.cc b/test/core/end2end/tests/retry_transparent_max_concurrent_streams.cc index ceb374042475c..663fd6f610597 100644 --- a/test/core/end2end/tests/retry_transparent_max_concurrent_streams.cc +++ b/test/core/end2end/tests/retry_transparent_max_concurrent_streams.cc @@ -104,6 +104,8 @@ CORE_END2END_TEST(RetryHttp2Test, RetryTransparentMaxConcurrentStreams) { EXPECT_EQ(server_status.status(), GRPC_STATUS_OK); EXPECT_EQ(server_status.message(), "xyz"); // Destroy server and then restart it. + // TODO(hork): hack to solve PosixEventEngine Listener's async shutdown issue. + absl::SleepFor(absl::Milliseconds(250)); InitServer(server_args); // Server should get the second call. auto s2 = RequestCall(201); diff --git a/test/core/end2end/tests/retry_transparent_not_sent_on_wire.cc b/test/core/end2end/tests/retry_transparent_not_sent_on_wire.cc index 87542064ad4b5..05bf6c1ac1210 100644 --- a/test/core/end2end/tests/retry_transparent_not_sent_on_wire.cc +++ b/test/core/end2end/tests/retry_transparent_not_sent_on_wire.cc @@ -115,6 +115,7 @@ class FailFirstTenCallsFilter { grpc_channel_filter FailFirstTenCallsFilter::kFilterVtable = { CallData::StartTransportStreamOpBatch, nullptr, + nullptr, grpc_channel_next_op, sizeof(CallData), CallData::Init, diff --git a/test/core/event_engine/BUILD b/test/core/event_engine/BUILD index 13543244c14f4..1cdf7d24cd398 100644 --- a/test/core/event_engine/BUILD +++ b/test/core/event_engine/BUILD @@ -232,3 +232,16 @@ grpc_cc_library( "//src/core:time", ], ) + +grpc_cc_test( + name = "query_extensions_test", + srcs = ["query_extensions_test.cc"], + external_deps = ["gtest"], + language = "C++", + uses_event_engine = False, + uses_polling = False, + deps = [ + "//:gpr_platform", + "//src/core:event_engine_query_extensions", + ], +) diff --git a/test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.h b/test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.h index 5f83f44c9ea8b..78e36a65d2a9e 100644 --- a/test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.h +++ b/test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.h @@ -48,7 +48,9 @@ namespace experimental { // EventEngine implementation to be used by fuzzers. // It's only allowed to have one FuzzingEventEngine instantiated at a time. -class FuzzingEventEngine : public EventEngine { +class FuzzingEventEngine + : public EventEngine, + public std::enable_shared_from_this { public: struct Options { Duration max_delay_run_after = std::chrono::seconds(30); diff --git a/test/core/event_engine/query_extensions_test.cc b/test/core/event_engine/query_extensions_test.cc new file mode 100644 index 0000000000000..712a496f38ce2 --- /dev/null +++ b/test/core/event_engine/query_extensions_test.cc @@ -0,0 +1,95 @@ +// Copyright 2023 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include + +#include "src/core/lib/event_engine/query_extensions.h" + +#include + +#include "absl/functional/any_invocable.h" +#include "absl/status/status.h" +#include "gtest/gtest.h" + +#include +#include + +#include "src/core/lib/gprpp/crash.h" + +namespace grpc_event_engine { +namespace experimental { +namespace { + +template +class TestExtension { + public: + TestExtension() = default; + ~TestExtension() = default; + + static std::string EndpointExtensionName() { + return "grpc.test.test_extension" + std::to_string(i); + } + + int GetValue() const { return val_; } + + private: + int val_ = i; +}; + +class ExtendedTestEndpoint + : public ExtendedEndpoint, TestExtension<1>, + TestExtension<2>> { + public: + ExtendedTestEndpoint() = default; + ~ExtendedTestEndpoint() override = default; + bool Read(absl::AnyInvocable /*on_read*/, + SliceBuffer* /*buffer*/, const ReadArgs* /*args*/) override { + grpc_core::Crash("Not implemented"); + }; + bool Write(absl::AnyInvocable /*on_writable*/, + SliceBuffer* /*data*/, const WriteArgs* /*args*/) override { + grpc_core::Crash("Not implemented"); + } + /// Returns an address in the format described in DNSResolver. The returned + /// values are expected to remain valid for the life of the Endpoint. + const EventEngine::ResolvedAddress& GetPeerAddress() const override { + grpc_core::Crash("Not implemented"); + } + const EventEngine::ResolvedAddress& GetLocalAddress() const override { + grpc_core::Crash("Not implemented"); + }; +}; + +TEST(QueryExtensionsTest, EndpointSupportsMultipleExtensions) { + ExtendedTestEndpoint endpoint; + TestExtension<0>* extension_0 = QueryExtension>(&endpoint); + TestExtension<1>* extension_1 = QueryExtension>(&endpoint); + TestExtension<2>* extension_2 = QueryExtension>(&endpoint); + + EXPECT_NE(extension_0, nullptr); + EXPECT_NE(extension_1, nullptr); + EXPECT_NE(extension_2, nullptr); + + EXPECT_EQ(extension_0->GetValue(), 0); + EXPECT_EQ(extension_1->GetValue(), 1); + EXPECT_EQ(extension_2->GetValue(), 2); +} +} // namespace + +} // namespace experimental +} // namespace grpc_event_engine + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/test/core/event_engine/test_suite/BUILD b/test/core/event_engine/test_suite/BUILD index 63c37eb70ac6f..7abf2c2bf9b09 100644 --- a/test/core/event_engine/test_suite/BUILD +++ b/test/core/event_engine/test_suite/BUILD @@ -56,6 +56,24 @@ grpc_cc_test( ], ) +grpc_cc_test( + name = "posix_event_engine_native_dns_test", + srcs = ["posix_event_engine_native_dns_test.cc"], + tags = [ + "no_mac", + "no_windows", + ], + uses_event_engine = True, + uses_polling = True, + deps = [ + "//:config_vars", + "//src/core:posix_event_engine", + "//test/core/event_engine:event_engine_test_utils", + "//test/core/event_engine/test_suite/posix:oracle_event_engine_posix", + "//test/core/event_engine/test_suite/tests:dns", + ], +) + grpc_cc_test( name = "thready_posix_event_engine_test", srcs = ["thready_posix_event_engine_test.cc"], diff --git a/test/core/event_engine/test_suite/posix_event_engine_native_dns_test.cc b/test/core/event_engine/test_suite/posix_event_engine_native_dns_test.cc new file mode 100644 index 0000000000000..0f4ff26720bd5 --- /dev/null +++ b/test/core/event_engine/test_suite/posix_event_engine_native_dns_test.cc @@ -0,0 +1,51 @@ +// Copyright 2023 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include +#include +#include + +#include + +#include + +#include "src/core/lib/config/config_vars.h" +#include "src/core/lib/event_engine/posix_engine/posix_engine.h" +#include "test/core/event_engine/test_suite/event_engine_test_framework.h" +#include "test/core/event_engine/test_suite/posix/oracle_event_engine_posix.h" +#include "test/core/event_engine/test_suite/tests/dns_test.h" +#include "test/core/util/test_config.h" + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + grpc::testing::TestEnvironment env(&argc, argv); + SetEventEngineFactories( + []() { + return std::make_unique< + grpc_event_engine::experimental::PosixEventEngine>(); + }, + []() { + return std::make_unique< + grpc_event_engine::experimental::PosixOracleEventEngine>(); + }); + // TODO(ctiller): EventEngine temporarily needs grpc to be initialized first + // until we clear out the iomgr shutdown code. + grpc_core::ConfigVars::Overrides overrides; + overrides.dns_resolver = "native"; + grpc_core::ConfigVars::SetOverrides(overrides); + grpc_event_engine::experimental::InitDNSTests(); + grpc_init(); + int r = RUN_ALL_TESTS(); + grpc_shutdown(); + return r; +} diff --git a/test/core/event_engine/test_suite/tests/dns_test.cc b/test/core/event_engine/test_suite/tests/dns_test.cc index 4e585b59a7d6f..587b7094f975e 100644 --- a/test/core/event_engine/test_suite/tests/dns_test.cc +++ b/test/core/event_engine/test_suite/tests/dns_test.cc @@ -35,6 +35,7 @@ #include +#include "src/core/lib/config/config_vars.h" #include "src/core/lib/event_engine/tcp_socket_utils.h" #include "src/core/lib/gprpp/crash.h" // IWYU pragma: keep #include "src/core/lib/gprpp/notification.h" @@ -96,6 +97,15 @@ MATCHER(StatusCodeEq, "") { return std::get<0>(arg).code() == std::get<1>(arg); } +#define SKIP_TEST_FOR_NATIVE_DNS_RESOLVER() \ + do { \ + if (grpc_core::ConfigVars::Get().DnsResolver() == "native") { \ + GTEST_SKIP() \ + << "This test specifies a target DNS server which the native " \ + "DNS resolver does not support."; \ + } \ + } while (0) + } // namespace class EventEngineDNSTest : public EventEngineTest { @@ -227,6 +237,7 @@ EventEngineDNSTest::DNSServer EventEngineDNSTest::dns_server_; #ifndef GRPC_IOS_EVENT_ENGINE_CLIENT TEST_F(EventEngineDNSTest, QueryNXHostname) { + SKIP_TEST_FOR_NATIVE_DNS_RESOLVER(); auto dns_resolver = CreateDefaultDNSResolver(); dns_resolver->LookupHostname( [this](auto result) { @@ -242,6 +253,7 @@ TEST_F(EventEngineDNSTest, QueryNXHostname) { } TEST_F(EventEngineDNSTest, QueryWithIPLiteral) { + SKIP_TEST_FOR_NATIVE_DNS_RESOLVER(); auto dns_resolver = CreateDefaultDNSResolver(); dns_resolver->LookupHostname( [this](auto result) { @@ -257,6 +269,7 @@ TEST_F(EventEngineDNSTest, QueryWithIPLiteral) { } TEST_F(EventEngineDNSTest, QueryARecord) { + SKIP_TEST_FOR_NATIVE_DNS_RESOLVER(); auto dns_resolver = CreateDefaultDNSResolver(); dns_resolver->LookupHostname( [this](auto result) { @@ -274,6 +287,7 @@ TEST_F(EventEngineDNSTest, QueryARecord) { } TEST_F(EventEngineDNSTest, QueryAAAARecord) { + SKIP_TEST_FOR_NATIVE_DNS_RESOLVER(); auto dns_resolver = CreateDefaultDNSResolver(); dns_resolver->LookupHostname( [this](auto result) { @@ -294,6 +308,7 @@ TEST_F(EventEngineDNSTest, QueryAAAARecord) { } TEST_F(EventEngineDNSTest, TestAddressSorting) { + SKIP_TEST_FOR_NATIVE_DNS_RESOLVER(); auto dns_resolver = CreateDefaultDNSResolver(); dns_resolver->LookupHostname( [this](auto result) { @@ -311,6 +326,7 @@ TEST_F(EventEngineDNSTest, TestAddressSorting) { } TEST_F(EventEngineDNSTest, QuerySRVRecord) { + SKIP_TEST_FOR_NATIVE_DNS_RESOLVER(); const SRVRecord kExpectedRecords[] = { {/*host=*/"ipv4-only-multi-target.dns-test.event-engine", /*port=*/1234, /*priority=*/0, /*weight=*/0}, @@ -329,6 +345,7 @@ TEST_F(EventEngineDNSTest, QuerySRVRecord) { } TEST_F(EventEngineDNSTest, QuerySRVRecordWithLocalhost) { + SKIP_TEST_FOR_NATIVE_DNS_RESOLVER(); auto dns_resolver = CreateDefaultDNSResolver(); dns_resolver->LookupSRV( [this](auto result) { @@ -341,6 +358,7 @@ TEST_F(EventEngineDNSTest, QuerySRVRecordWithLocalhost) { } TEST_F(EventEngineDNSTest, QueryTXTRecord) { + SKIP_TEST_FOR_NATIVE_DNS_RESOLVER(); // clang-format off const std::string kExpectedRecord = "grpc_config=[{" @@ -370,6 +388,7 @@ TEST_F(EventEngineDNSTest, QueryTXTRecord) { } TEST_F(EventEngineDNSTest, QueryTXTRecordWithLocalhost) { + SKIP_TEST_FOR_NATIVE_DNS_RESOLVER(); auto dns_resolver = CreateDefaultDNSResolver(); dns_resolver->LookupTXT( [this](auto result) { @@ -382,6 +401,7 @@ TEST_F(EventEngineDNSTest, QueryTXTRecordWithLocalhost) { } TEST_F(EventEngineDNSTest, TestCancelActiveDNSQuery) { + SKIP_TEST_FOR_NATIVE_DNS_RESOLVER(); const std::string name = "dont-care-since-wont-be-resolved.test.com:1234"; auto dns_resolver = CreateDNSResolverWithNonResponsiveServer(); dns_resolver->LookupHostname( diff --git a/test/core/ext/filters/event_engine_client_channel_resolver/resolver_fuzzer.cc b/test/core/ext/filters/event_engine_client_channel_resolver/resolver_fuzzer.cc index 483ad6b0e4ac7..97ed6eaa71c65 100644 --- a/test/core/ext/filters/event_engine_client_channel_resolver/resolver_fuzzer.cc +++ b/test/core/ext/filters/event_engine_client_channel_resolver/resolver_fuzzer.cc @@ -71,7 +71,8 @@ absl::Status ErrorToAbslStatus( } class FuzzingResolverEventEngine - : public grpc_event_engine::experimental::AbortingEventEngine { + : public grpc_event_engine::experimental::AbortingEventEngine, + public std::enable_shared_from_this { public: explicit FuzzingResolverEventEngine( const event_engine_client_channel_resolver::Msg& msg, diff --git a/test/core/gprpp/orphanable_test.cc b/test/core/gprpp/orphanable_test.cc index e5b741466614e..cea10b748f778 100644 --- a/test/core/gprpp/orphanable_test.cc +++ b/test/core/gprpp/orphanable_test.cc @@ -103,6 +103,32 @@ TEST(OrphanablePtr, InternallyRefCountedWithTracing) { baz->FinishWork(); } +class Qux : public InternallyRefCounted { + public: + Qux() : Qux(0) {} + explicit Qux(int value) : InternallyRefCounted("Qux"), value_(value) {} + ~Qux() override { self_ref_ = RefIfNonZero(DEBUG_LOCATION, "extra_work"); } + void Orphan() override { Unref(); } + int value() const { return value_; } + + void StartWork() { self_ref_ = RefIfNonZero(DEBUG_LOCATION, "work"); } + void FinishWork() { + // This is a little ugly, but it makes the logged ref and unref match up. + self_ref_.release(); + Unref(DEBUG_LOCATION, "work"); + } + + private: + int value_; + RefCountedPtr self_ref_; +}; + +TEST(OrphanablePtr, InternallyRefCountedIfNonZero) { + auto qux = MakeOrphanable(); + qux->StartWork(); + qux->FinishWork(); +} + } // namespace } // namespace testing } // namespace grpc_core diff --git a/test/core/gprpp/ref_counted_test.cc b/test/core/gprpp/ref_counted_test.cc index 4d8761ecb1db0..7c28cddc6f645 100644 --- a/test/core/gprpp/ref_counted_test.cc +++ b/test/core/gprpp/ref_counted_test.cc @@ -53,6 +53,17 @@ TEST(RefCounted, ExtraRef) { foo->Unref(); } +TEST(RefCounted, Const) { + const Foo* foo = new Foo(); + RefCountedPtr foop = foo->Ref(); + foop.release(); + foop = foo->RefIfNonZero(); + foop.release(); + foo->Unref(); + foo->Unref(); + foo->Unref(); +} + class Value : public RefCounted { public: Value(int value, std::set>* registry) : value_(value) { diff --git a/test/core/promise/status_flag_test.cc b/test/core/promise/status_flag_test.cc index 799a08917e722..dcde3641f049c 100644 --- a/test/core/promise/status_flag_test.cc +++ b/test/core/promise/status_flag_test.cc @@ -33,6 +33,8 @@ TEST(StatusFlagTest, Basics) { EXPECT_EQ(ValueOrFailure(42).value(), 42); EXPECT_EQ(StatusCast>(ValueOrFailure(42)).value(), 42); + EXPECT_TRUE(IsStatusOk(Success{})); + EXPECT_FALSE(IsStatusOk(Failure{})); } } // namespace grpc_core diff --git a/test/core/surface/channel_init_test.cc b/test/core/surface/channel_init_test.cc index 22fdad11da97b..bab56a38d3a6e 100644 --- a/test/core/surface/channel_init_test.cc +++ b/test/core/surface/channel_init_test.cc @@ -35,9 +35,9 @@ const grpc_channel_filter* FilterNamed(const char* name) { if (it != filters->end()) return it->second; return filters ->emplace(name, - new grpc_channel_filter{nullptr, nullptr, nullptr, 0, nullptr, - nullptr, nullptr, 0, nullptr, nullptr, - nullptr, nullptr, name}) + new grpc_channel_filter{nullptr, nullptr, nullptr, nullptr, 0, + nullptr, nullptr, nullptr, 0, nullptr, + nullptr, nullptr, nullptr, name}) .first->second; } diff --git a/test/core/transport/chttp2/streams_not_seen_test.cc b/test/core/transport/chttp2/streams_not_seen_test.cc index e38160d4b41ba..420686ea6e616 100644 --- a/test/core/transport/chttp2/streams_not_seen_test.cc +++ b/test/core/transport/chttp2/streams_not_seen_test.cc @@ -199,6 +199,7 @@ class TrailingMetadataRecordingFilter { grpc_channel_filter TrailingMetadataRecordingFilter::kFilterVtable = { CallData::StartTransportStreamOpBatch, nullptr, + nullptr, grpc_channel_next_op, sizeof(CallData), CallData::Init, diff --git a/test/core/xds/xds_audit_logger_registry_test.cc b/test/core/xds/xds_audit_logger_registry_test.cc index abfc73801e674..7249b1c8b1bfb 100644 --- a/test/core/xds/xds_audit_logger_registry_test.cc +++ b/test/core/xds/xds_audit_logger_registry_test.cc @@ -65,10 +65,10 @@ absl::StatusOr ConvertAuditLoggerConfig( const AuditLoggerConfigProto& config) { std::string serialized_config = config.SerializeAsString(); upb::Arena arena; - upb::SymbolTable symtab; - XdsResourceType::DecodeContext context = {nullptr, - GrpcXdsBootstrap::GrpcXdsServer(), - nullptr, symtab.ptr(), arena.ptr()}; + upb::DefPool def_pool; + XdsResourceType::DecodeContext context = { + nullptr, GrpcXdsBootstrap::GrpcXdsServer(), nullptr, def_pool.ptr(), + arena.ptr()}; auto* upb_config = envoy_config_rbac_v3_RBAC_AuditLoggingOptions_AuditLoggerConfig_parse( serialized_config.data(), serialized_config.size(), arena.ptr()); diff --git a/test/core/xds/xds_channel_stack_modifier_test.cc b/test/core/xds/xds_channel_stack_modifier_test.cc index ab6c78242f054..ffd0b9f450d44 100644 --- a/test/core/xds/xds_channel_stack_modifier_test.cc +++ b/test/core/xds/xds_channel_stack_modifier_test.cc @@ -95,11 +95,11 @@ TEST(XdsChannelStackModifierTest, XdsHttpFiltersInsertion) { grpc_init(); // Add 2 test filters to XdsChannelStackModifier const grpc_channel_filter test_filter_1 = { - nullptr, nullptr, nullptr, 0, nullptr, nullptr, nullptr, - 0, nullptr, nullptr, nullptr, nullptr, kTestFilter1}; + nullptr, nullptr, nullptr, nullptr, 0, nullptr, nullptr, + nullptr, 0, nullptr, nullptr, nullptr, nullptr, kTestFilter1}; const grpc_channel_filter test_filter_2 = { - nullptr, nullptr, nullptr, 0, nullptr, nullptr, nullptr, - 0, nullptr, nullptr, nullptr, nullptr, kTestFilter2}; + nullptr, nullptr, nullptr, nullptr, 0, nullptr, nullptr, + nullptr, 0, nullptr, nullptr, nullptr, nullptr, kTestFilter2}; auto channel_stack_modifier = MakeRefCounted( std::vector{&test_filter_1, &test_filter_2}); grpc_arg arg = channel_stack_modifier->MakeChannelArg(); diff --git a/test/core/xds/xds_lb_policy_registry_test.cc b/test/core/xds/xds_lb_policy_registry_test.cc index b6b10dddd3cd4..d139d6005e3d1 100644 --- a/test/core/xds/xds_lb_policy_registry_test.cc +++ b/test/core/xds/xds_lb_policy_registry_test.cc @@ -74,10 +74,10 @@ absl::StatusOr ConvertXdsPolicy( const LoadBalancingPolicyProto& policy) { std::string serialized_policy = policy.SerializeAsString(); upb::Arena arena; - upb::SymbolTable symtab; - XdsResourceType::DecodeContext context = {nullptr, - GrpcXdsBootstrap::GrpcXdsServer(), - nullptr, symtab.ptr(), arena.ptr()}; + upb::DefPool def_pool; + XdsResourceType::DecodeContext context = { + nullptr, GrpcXdsBootstrap::GrpcXdsServer(), nullptr, def_pool.ptr(), + arena.ptr()}; auto* upb_policy = envoy_config_cluster_v3_LoadBalancingPolicy_parse( serialized_policy.data(), serialized_policy.size(), arena.ptr()); ValidationErrors errors; diff --git a/test/distrib/cpp/run_distrib_test_cmake_for_dll.bat b/test/distrib/cpp/run_distrib_test_cmake_for_dll.bat index 2adfaf141419f..887c20dcd74a9 100644 --- a/test/distrib/cpp/run_distrib_test_cmake_for_dll.bat +++ b/test/distrib/cpp/run_distrib_test_cmake_for_dll.bat @@ -78,6 +78,11 @@ popd @rem folders, like the following command trying to imitate. git submodule foreach bash -c "cd $toplevel; rm -rf $name" +@rem TODO(dawidcha): Remove this once this DLL test can pass { +echo Skipped! +exit /b 0 +@rem TODO(dawidcha): Remove this once this DLL test can pass } + @rem Install gRPC @rem NOTE(jtattermusch): The -DProtobuf_USE_STATIC_LIBS=ON is necessary on cmake3.16+ @rem since by default "find_package(Protobuf ...)" uses the cmake's builtin diff --git a/tools/buildgen/generate_projects.sh b/tools/buildgen/generate_projects.sh index 9ddd588bb37a9..e7905a40c9552 100755 --- a/tools/buildgen/generate_projects.sh +++ b/tools/buildgen/generate_projects.sh @@ -28,8 +28,6 @@ fi cd `dirname $0`/../.. -tools/distrib/fix_build_deps.py - echo "Generating build_autogenerated.yaml from bazel BUILD file" rm -f build_autogenerated.yaml python3 tools/buildgen/extract_metadata_from_bazel_xml.py diff --git a/tools/distrib/fix_build_deps.py b/tools/distrib/fix_build_deps.py deleted file mode 100755 index 0c014ec2b76ea..0000000000000 --- a/tools/distrib/fix_build_deps.py +++ /dev/null @@ -1,684 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2022 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import collections -from doctest import SKIP -import multiprocessing -import os -import re -import sys - -import run_buildozer - -# find our home -ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), "../..")) -os.chdir(ROOT) - -vendors = collections.defaultdict(list) -scores = collections.defaultdict(int) -avoidness = collections.defaultdict(int) -consumes = {} -no_update = set() -buildozer_commands = [] -original_deps = {} -original_external_deps = {} -skip_headers = collections.defaultdict(set) - -# TODO(ctiller): ideally we wouldn't hardcode a bunch of paths here. -# We can likely parse out BUILD files from dependencies to generate this index. -EXTERNAL_DEPS = { - "absl/algorithm/container.h": "absl/algorithm:container", - "absl/base/attributes.h": "absl/base:core_headers", - "absl/base/call_once.h": "absl/base", - # TODO(ctiller) remove this - "absl/base/internal/endian.h": "absl/base:endian", - "absl/base/thread_annotations.h": "absl/base:core_headers", - "absl/container/flat_hash_map.h": "absl/container:flat_hash_map", - "absl/container/flat_hash_set.h": "absl/container:flat_hash_set", - "absl/container/inlined_vector.h": "absl/container:inlined_vector", - "absl/cleanup/cleanup.h": "absl/cleanup", - "absl/debugging/failure_signal_handler.h": ( - "absl/debugging:failure_signal_handler" - ), - "absl/debugging/stacktrace.h": "absl/debugging:stacktrace", - "absl/debugging/symbolize.h": "absl/debugging:symbolize", - "absl/flags/flag.h": "absl/flags:flag", - "absl/flags/marshalling.h": "absl/flags:marshalling", - "absl/flags/parse.h": "absl/flags:parse", - "absl/functional/any_invocable.h": "absl/functional:any_invocable", - "absl/functional/bind_front.h": "absl/functional:bind_front", - "absl/functional/function_ref.h": "absl/functional:function_ref", - "absl/hash/hash.h": "absl/hash", - "absl/memory/memory.h": "absl/memory", - "absl/meta/type_traits.h": "absl/meta:type_traits", - "absl/numeric/int128.h": "absl/numeric:int128", - "absl/random/random.h": "absl/random", - "absl/random/bit_gen_ref.h": "absl/random:bit_gen_ref", - "absl/random/mocking_bit_gen.h": "absl/random:mocking_bit_gen", - "absl/random/distributions.h": "absl/random:distributions", - "absl/random/uniform_int_distribution.h": "absl/random:distributions", - "absl/status/status.h": "absl/status", - "absl/status/statusor.h": "absl/status:statusor", - "absl/strings/ascii.h": "absl/strings", - "absl/strings/cord.h": "absl/strings:cord", - "absl/strings/escaping.h": "absl/strings", - "absl/strings/match.h": "absl/strings", - "absl/strings/numbers.h": "absl/strings", - "absl/strings/str_cat.h": "absl/strings", - "absl/strings/str_format.h": "absl/strings:str_format", - "absl/strings/str_join.h": "absl/strings", - "absl/strings/str_replace.h": "absl/strings", - "absl/strings/str_split.h": "absl/strings", - "absl/strings/string_view.h": "absl/strings", - "absl/strings/strip.h": "absl/strings", - "absl/strings/substitute.h": "absl/strings", - "absl/synchronization/mutex.h": "absl/synchronization", - "absl/synchronization/notification.h": "absl/synchronization", - "absl/time/clock.h": "absl/time", - "absl/time/time.h": "absl/time", - "absl/types/optional.h": "absl/types:optional", - "absl/types/span.h": "absl/types:span", - "absl/types/variant.h": "absl/types:variant", - "absl/utility/utility.h": "absl/utility", - "address_sorting/address_sorting.h": "address_sorting", - "google/cloud/opentelemetry/resource_detector.h": "google_cloud_cpp:opentelemetry", - "opentelemetry/common/attribute_value.h": "otel/api", - "opentelemetry/common/key_value_iterable.h": "otel/api", - "opentelemetry/nostd/function_ref.h": "otel/api", - "opentelemetry/nostd/string_view.h": "otel/api", - "opentelemetry/context/context.h": "otel/api", - "opentelemetry/metrics/meter.h": "otel/api", - "opentelemetry/metrics/meter_provider.h": "otel/api", - "opentelemetry/metrics/provider.h": "otel/api", - "opentelemetry/metrics/sync_instruments.h": "otel/api", - "opentelemetry/nostd/shared_ptr.h": "otel/api", - "opentelemetry/nostd/unique_ptr.h": "otel/api", - "opentelemetry/sdk/metrics/meter_provider.h": "otel/sdk/src/metrics", - "opentelemetry/sdk/common/attribute_utils.h": "otel/sdk:headers", - "opentelemetry/sdk/resource/resource.h": "otel/sdk:headers", - "opentelemetry/sdk/resource/resource_detector.h": "otel/sdk:headers", - "opentelemetry/sdk/resource/semantic_conventions.h": "otel/sdk:headers", - "ares.h": "cares", - "fuzztest/fuzztest.h": ["fuzztest", "fuzztest_main"], - "google/api/monitored_resource.pb.h": ( - "google/api:monitored_resource_cc_proto" - ), - "google/devtools/cloudtrace/v2/tracing.grpc.pb.h": ( - "googleapis_trace_grpc_service" - ), - "google/logging/v2/logging.grpc.pb.h": "googleapis_logging_grpc_service", - "google/logging/v2/logging.pb.h": "googleapis_logging_cc_proto", - "google/logging/v2/log_entry.pb.h": "googleapis_logging_cc_proto", - "google/monitoring/v3/metric_service.grpc.pb.h": ( - "googleapis_monitoring_grpc_service" - ), - "gmock/gmock.h": "gtest", - "gtest/gtest.h": "gtest", - "opencensus/exporters/stats/stackdriver/stackdriver_exporter.h": ( - "opencensus-stats-stackdriver_exporter" - ), - "opencensus/exporters/trace/stackdriver/stackdriver_exporter.h": ( - "opencensus-trace-stackdriver_exporter" - ), - "opencensus/trace/context_util.h": "opencensus-trace-context_util", - "opencensus/trace/propagation/grpc_trace_bin.h": ( - "opencensus-trace-propagation" - ), - "opencensus/tags/context_util.h": "opencensus-tags-context_util", - "opencensus/trace/span_context.h": "opencensus-trace-span_context", - "openssl/base.h": "libssl", - "openssl/bio.h": "libssl", - "openssl/bn.h": "libcrypto", - "openssl/buffer.h": "libcrypto", - "openssl/crypto.h": "libcrypto", - "openssl/digest.h": "libssl", - "openssl/engine.h": "libcrypto", - "openssl/err.h": "libcrypto", - "openssl/evp.h": "libcrypto", - "openssl/hmac.h": "libcrypto", - "openssl/mem.h": "libcrypto", - "openssl/param_build.h": "libcrypto", - "openssl/pem.h": "libcrypto", - "openssl/rsa.h": "libcrypto", - "openssl/sha.h": "libcrypto", - "openssl/ssl.h": "libssl", - "openssl/tls1.h": "libssl", - "openssl/x509.h": "libcrypto", - "openssl/x509v3.h": "libcrypto", - "re2/re2.h": "re2", - "upb/base/string_view.h": "upb_base_lib", - "upb/collections/map.h": "upb_collections_lib", - "upb/reflection/def.h": "upb_reflection", - "upb/json/encode.h": "upb_json_lib", - "upb/mem/arena.h": "upb_mem_lib", - "upb/text/encode.h": "upb_textformat_lib", - "upb/reflection/def.hpp": "upb_reflection", - "upb/upb.h": "upb_amalgamation_lib", - "upb/upb.hpp": "upb_lib", - "xxhash.h": "xxhash", - "zlib.h": "madler_zlib", -} - -INTERNAL_DEPS = { - "test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.h": ( - "//test/core/event_engine/fuzzing_event_engine" - ), - "test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.pb.h": "//test/core/event_engine/fuzzing_event_engine:fuzzing_event_engine_proto", - "test/core/experiments/test_experiments.h": "//test/core/experiments:test_experiments_lib", - "google/api/expr/v1alpha1/syntax.upb.h": "google_api_expr_v1alpha1_syntax_upb", - "google/rpc/status.upb.h": "google_rpc_status_upb", - "google/protobuf/any.upb.h": "protobuf_any_upb", - "google/protobuf/duration.upb.h": "protobuf_duration_upb", - "google/protobuf/struct.upb.h": "protobuf_struct_upb", - "google/protobuf/timestamp.upb.h": "protobuf_timestamp_upb", - "google/protobuf/wrappers.upb.h": "protobuf_wrappers_upb", - "grpc/status.h": "grpc_public_hdrs", - "src/proto/grpc/channelz/channelz.grpc.pb.h": ( - "//src/proto/grpc/channelz:channelz_proto" - ), - "src/proto/grpc/core/stats.pb.h": "//src/proto/grpc/core:stats_proto", - "src/proto/grpc/health/v1/health.upb.h": "grpc_health_upb", - "src/proto/grpc/lb/v1/load_reporter.grpc.pb.h": ( - "//src/proto/grpc/lb/v1:load_reporter_proto" - ), - "src/proto/grpc/lb/v1/load_balancer.upb.h": "grpc_lb_upb", - "src/proto/grpc/reflection/v1alpha/reflection.grpc.pb.h": ( - "//src/proto/grpc/reflection/v1alpha:reflection_proto" - ), - "src/proto/grpc/gcp/transport_security_common.upb.h": "alts_upb", - "src/proto/grpc/gcp/handshaker.upb.h": "alts_upb", - "src/proto/grpc/gcp/altscontext.upb.h": "alts_upb", - "src/proto/grpc/lookup/v1/rls.upb.h": "rls_upb", - "src/proto/grpc/lookup/v1/rls_config.upb.h": "rls_config_upb", - "src/proto/grpc/lookup/v1/rls_config.upbdefs.h": "rls_config_upbdefs", - "src/proto/grpc/testing/xds/v3/csds.grpc.pb.h": ( - "//src/proto/grpc/testing/xds/v3:csds_proto" - ), - "xds/data/orca/v3/orca_load_report.upb.h": "xds_orca_upb", - "xds/service/orca/v3/orca.upb.h": "xds_orca_service_upb", - "xds/type/v3/typed_struct.upb.h": "xds_type_upb", -} - - -class FakeSelects: - def config_setting_group(self, **kwargs): - pass - - -num_cc_libraries = 0 -num_opted_out_cc_libraries = 0 -parsing_path = None - - -# Convert the source or header target to a relative path. -def _get_filename(name, parsing_path): - filename = "%s%s" % ( - ( - parsing_path + "/" - if (parsing_path and not name.startswith("//")) - else "" - ), - name, - ) - filename = filename.replace("//:", "") - filename = filename.replace("//src/core:", "src/core/") - filename = filename.replace( - "//src/cpp/ext/filters/census:", "src/cpp/ext/filters/census/" - ) - return filename - - -def grpc_cc_library( - name, - hdrs=[], - public_hdrs=[], - srcs=[], - select_deps=None, - tags=[], - deps=[], - external_deps=[], - proto=None, - **kwargs, -): - global args - global num_cc_libraries - global num_opted_out_cc_libraries - global parsing_path - assert parsing_path is not None - name = "//%s:%s" % (parsing_path, name) - num_cc_libraries += 1 - if select_deps or "nofixdeps" in tags: - if args.whats_left and not select_deps and "nofixdeps" not in tags: - num_opted_out_cc_libraries += 1 - print("Not opted in: {}".format(name)) - no_update.add(name) - scores[name] = len(public_hdrs + hdrs) - # avoid_dep is the internal way of saying prefer something else - # we add grpc_avoid_dep to allow internal grpc-only stuff to avoid each - # other, whilst not biasing dependent projects - if "avoid_dep" in tags or "grpc_avoid_dep" in tags: - avoidness[name] += 10 - if proto: - proto_hdr = "%s%s" % ( - (parsing_path + "/" if parsing_path else ""), - proto.replace(".proto", ".pb.h"), - ) - skip_headers[name].add(proto_hdr) - - for hdr in hdrs + public_hdrs: - vendors[_get_filename(hdr, parsing_path)].append(name) - inc = set() - original_deps[name] = frozenset(deps) - original_external_deps[name] = frozenset(external_deps) - for src in hdrs + public_hdrs + srcs: - for line in open(_get_filename(src, parsing_path)): - m = re.search(r"^#include <(.*)>", line) - if m: - inc.add(m.group(1)) - m = re.search(r'^#include "(.*)"', line) - if m: - inc.add(m.group(1)) - consumes[name] = list(inc) - - -def grpc_proto_library(name, srcs, **kwargs): - global parsing_path - assert parsing_path is not None - name = "//%s:%s" % (parsing_path, name) - for src in srcs: - proto_hdr = src.replace(".proto", ".pb.h") - vendors[_get_filename(proto_hdr, parsing_path)].append(name) - - -def buildozer(cmd, target): - buildozer_commands.append("%s|%s" % (cmd, target)) - - -def buildozer_set_list(name, values, target, via=""): - if not values: - buildozer("remove %s" % name, target) - return - adjust = via if via else name - buildozer( - "set %s %s" % (adjust, " ".join('"%s"' % s for s in values)), target - ) - if via: - buildozer("remove %s" % name, target) - buildozer("rename %s %s" % (via, name), target) - - -def score_edit_distance(proposed, existing): - """Score a proposed change primarily by edit distance""" - sum = 0 - for p in proposed: - if p not in existing: - sum += 1 - for e in existing: - if e not in proposed: - sum += 1 - return sum - - -def total_score(proposal): - return sum(scores[dep] for dep in proposal) - - -def total_avoidness(proposal): - return sum(avoidness[dep] for dep in proposal) - - -def score_list_size(proposed, existing): - """Score a proposed change primarily by number of dependencies""" - return len(proposed) - - -def score_best(proposed, existing): - """Score a proposed change primarily by dependency score""" - return 0 - - -SCORERS = { - "edit_distance": score_edit_distance, - "list_size": score_list_size, - "best": score_best, -} - -parser = argparse.ArgumentParser(description="Fix build dependencies") -parser.add_argument( - "targets", nargs="*", default=[], help="targets to fix (empty => all)" -) -parser.add_argument( - "--score", - type=str, - default="edit_distance", - help="scoring function to use: one of " + ", ".join(SCORERS.keys()), -) -parser.add_argument( - "--whats_left", - action="store_true", - default=False, - help="show what is left to opt in", -) -parser.add_argument( - "--explain", - action="store_true", - default=False, - help="try to explain some decisions", -) -parser.add_argument( - "--why", - type=str, - default=None, - help="with --explain, target why a given dependency is needed", -) -args = parser.parse_args() - -for dirname in [ - "", - "src/core", - "src/cpp/ext/gcp", - "src/cpp/ext/csm", - "src/cpp/ext/otel", - "test/core/backoff", - "test/core/experiments", - "test/core/uri", - "test/core/util", - "test/core/end2end", - "test/core/event_engine", - "test/core/filters", - "test/core/promise", - "test/core/resource_quota", - "test/core/transport/chaotic_good", - "fuzztest", - "fuzztest/core/channel", - "fuzztest/core/transport/chttp2", -]: - parsing_path = dirname - exec( - open("%sBUILD" % (dirname + "/" if dirname else ""), "r").read(), - { - "load": lambda filename, *args: None, - "licenses": lambda licenses: None, - "package": lambda **kwargs: None, - "exports_files": lambda files, visibility=None: None, - "bool_flag": lambda **kwargs: None, - "config_setting": lambda **kwargs: None, - "selects": FakeSelects(), - "python_config_settings": lambda **kwargs: None, - "grpc_cc_binary": grpc_cc_library, - "grpc_cc_library": grpc_cc_library, - "grpc_cc_test": grpc_cc_library, - "grpc_core_end2end_test": lambda **kwargs: None, - "grpc_fuzzer": grpc_cc_library, - "grpc_fuzz_test": grpc_cc_library, - "grpc_proto_fuzzer": grpc_cc_library, - "grpc_proto_library": grpc_proto_library, - "select": lambda d: d["//conditions:default"], - "glob": lambda files: None, - "grpc_end2end_tests": lambda: None, - "grpc_upb_proto_library": lambda name, **kwargs: None, - "grpc_upb_proto_reflection_library": lambda name, **kwargs: None, - "grpc_generate_one_off_targets": lambda: None, - "grpc_generate_one_off_internal_targets": lambda: None, - "grpc_package": lambda **kwargs: None, - "filegroup": lambda name, **kwargs: None, - "sh_library": lambda name, **kwargs: None, - }, - {}, - ) - parsing_path = None - -if args.whats_left: - print( - "{}/{} libraries are opted in".format( - num_cc_libraries - num_opted_out_cc_libraries, num_cc_libraries - ) - ) - - -def make_relative_path(dep, lib): - if lib is None: - return dep - lib_path = lib[: lib.rfind(":") + 1] - if dep.startswith(lib_path): - return dep[len(lib_path) :] - return dep - - -if args.whats_left: - print( - "{}/{} libraries are opted in".format( - num_cc_libraries - num_opted_out_cc_libraries, num_cc_libraries - ) - ) - - -# Keeps track of all possible sets of dependencies that could satify the -# problem. (models the list monad in Haskell!) -class Choices: - def __init__(self, library, substitutions): - self.library = library - self.to_add = [] - self.to_remove = [] - self.substitutions = substitutions - - def add_one_of(self, choices, trigger): - if not choices: - return - choices = sum( - [self.apply_substitutions(choice) for choice in choices], [] - ) - if args.explain and (args.why is None or args.why in choices): - print( - "{}: Adding one of {} for {}".format( - self.library, choices, trigger - ) - ) - self.to_add.append( - tuple( - make_relative_path(choice, self.library) for choice in choices - ) - ) - - def add(self, choice, trigger): - self.add_one_of([choice], trigger) - - def remove(self, remove): - for remove in self.apply_substitutions(remove): - self.to_remove.append(make_relative_path(remove, self.library)) - - def apply_substitutions(self, dep): - if dep in self.substitutions: - return self.substitutions[dep] - return [dep] - - def best(self, scorer): - choices = set() - choices.add(frozenset()) - - for add in sorted(set(self.to_add), key=lambda x: (len(x), x)): - new_choices = set() - for append_choice in add: - for choice in choices: - new_choices.add(choice.union([append_choice])) - choices = new_choices - for remove in sorted(set(self.to_remove)): - new_choices = set() - for choice in choices: - new_choices.add(choice.difference([remove])) - choices = new_choices - - best = None - - def final_scorer(x): - return (total_avoidness(x), scorer(x), total_score(x)) - - for choice in choices: - if best is None or final_scorer(choice) < final_scorer(best): - best = choice - return best - - -def make_library(library): - error = False - hdrs = sorted(consumes[library]) - # we need a little trickery here since grpc_base has channel.cc, which calls grpc_init - # which is in grpc, which is illegal but hard to change - # once EventEngine lands we can clean this up - deps = Choices( - library, - {"//:grpc_base": ["//:grpc", "//:grpc_unsecure"]} - if library.startswith("//test/") - else {}, - ) - external_deps = Choices(None, {}) - for hdr in hdrs: - if hdr in skip_headers[library]: - continue - - if hdr == "systemd/sd-daemon.h": - continue - - if hdr == "src/core/lib/profiling/stap_probes.h": - continue - - if hdr.startswith("src/libfuzzer/"): - continue - - if hdr == "grpc/grpc.h" and library.startswith("//test:"): - # not the root build including grpc.h ==> //:grpc - deps.add_one_of(["//:grpc", "//:grpc_unsecure"], hdr) - continue - - if hdr in INTERNAL_DEPS: - dep = INTERNAL_DEPS[hdr] - if isinstance(dep, list): - for d in dep: - deps.add(d, hdr) - else: - if not ("//" in dep): - dep = "//:" + dep - deps.add(dep, hdr) - continue - - if hdr in vendors: - deps.add_one_of(vendors[hdr], hdr) - continue - - if "include/" + hdr in vendors: - deps.add_one_of(vendors["include/" + hdr], hdr) - continue - - if "." not in hdr: - # assume a c++ system include - continue - - if hdr in EXTERNAL_DEPS: - if isinstance(EXTERNAL_DEPS[hdr], list): - for dep in EXTERNAL_DEPS[hdr]: - external_deps.add(dep, hdr) - else: - external_deps.add(EXTERNAL_DEPS[hdr], hdr) - continue - - if hdr.startswith("opencensus/"): - trail = hdr[len("opencensus/") :] - trail = trail[: trail.find("/")] - external_deps.add("opencensus-" + trail, hdr) - continue - - if hdr.startswith("envoy/"): - path, file = os.path.split(hdr) - file = file.split(".") - path = path.split("/") - dep = "_".join(path[:-1] + [file[1]]) - deps.add(dep, hdr) - continue - - if hdr.startswith("google/protobuf/") and not hdr.endswith(".upb.h"): - external_deps.add("protobuf_headers", hdr) - continue - - if "/" not in hdr: - # assume a system include - continue - - is_sys_include = False - for sys_path in [ - "sys", - "arpa", - "gperftools", - "netinet", - "linux", - "android", - "mach", - "net", - "CoreFoundation", - ]: - if hdr.startswith(sys_path + "/"): - is_sys_include = True - break - if is_sys_include: - # assume a system include - continue - - print( - "# ERROR: can't categorize header: %s used by %s" % (hdr, library) - ) - error = True - - deps.remove(library) - - deps = sorted( - deps.best(lambda x: SCORERS[args.score](x, original_deps[library])) - ) - external_deps = sorted( - external_deps.best( - lambda x: SCORERS[args.score](x, original_external_deps[library]) - ) - ) - - return (library, error, deps, external_deps) - - -def main() -> None: - update_libraries = [] - for library in sorted(consumes.keys()): - if library in no_update: - continue - if args.targets and library not in args.targets: - continue - update_libraries.append(library) - with multiprocessing.Pool(processes=multiprocessing.cpu_count()) as p: - updated_libraries = p.map(make_library, update_libraries, 1) - - error = False - for library, lib_error, deps, external_deps in updated_libraries: - if lib_error: - error = True - continue - buildozer_set_list("external_deps", external_deps, library, via="deps") - buildozer_set_list("deps", deps, library) - - run_buildozer.run_buildozer(buildozer_commands) - - if error: - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/tools/distrib/iwyu.sh b/tools/distrib/iwyu.sh deleted file mode 100755 index 9a00fde2b4d38..0000000000000 --- a/tools/distrib/iwyu.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -# Copyright 2021 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -echo "NOTE: to automagically apply fixes, invoke with --fix" - -set -ex - -# change to root directory -cd $(dirname $0)/../.. -REPO_ROOT=$(pwd) - -# grep targets with manual tag, which is not included in a result of bazel build using ... -# let's get a list of them using query command and pass it to gen_compilation_database.py -export MANUAL_TARGETS=$(bazel query 'attr("tags", "manual", tests(//test/cpp/...))' | grep -v _on_ios) - -# generate a clang compilation database for all C/C++ sources in the repo. -tools/distrib/gen_compilation_database.py \ - --include_headers \ - --ignore_system_headers \ - --dedup_targets \ - "//:*" \ - "//src/core/..." \ - "//src/cpp/ext/csm/..." \ - "//src/cpp/ext/gcp/..." \ - "//src/cpp/ext/otel/..." \ - "//src/compiler/..." \ - "//test/core/..." \ - "//test/cpp/..." \ - "//fuzztest/..." \ - $MANUAL_TARGETS - -# run iwyu against the checked out codebase -# when modifying the checked-out files, the current user will be impersonated -# so that the updated files don't end up being owned by "root". -if [ "$IWYU_SKIP_DOCKER" == "" ] -then - # build iwyu docker image - docker build -t grpc_iwyu tools/dockerfile/grpc_iwyu - - docker run \ - -e TEST="$TEST" \ - -e CHANGED_FILES="$CHANGED_FILES" \ - -e IWYU_ROOT="/local-code" \ - --rm=true \ - -v "${REPO_ROOT}":/local-code \ - -v "${HOME/.cache/bazel}":"${HOME/.cache/bazel}" \ - --user "$(id -u):$(id -g)" \ - -t grpc_iwyu /iwyu.sh "$@" -else - IWYU_ROOT="${REPO_ROOT}" tools/dockerfile/grpc_iwyu/iwyu.sh -fi diff --git a/tools/distrib/iwyu_mappings.imp b/tools/distrib/iwyu_mappings.imp deleted file mode 100644 index 3f993a9aee5fd..0000000000000 --- a/tools/distrib/iwyu_mappings.imp +++ /dev/null @@ -1,9 +0,0 @@ -[ - { include: ["", "private", "", "public"] }, - { include: ["", "public", "\"src/core/lib/iomgr/sockaddr.h\"", "public"]}, - { include: ["", "public", "\"src/core/lib/iomgr/sockaddr.h\"", "public"]}, - { include: ["", "private", "", "public"] }, - { include: ["", "private", "", "public"] }, - # workaround: https://github.com/include-what-you-use/include-what-you-use/issues/908 - { symbol: ["std::max", "private", "", "public" ] }, -] diff --git a/tools/dockerfile/grpc_iwyu/Dockerfile b/tools/dockerfile/grpc_iwyu/Dockerfile deleted file mode 100644 index 01b9a2d3a4aee..0000000000000 --- a/tools/dockerfile/grpc_iwyu/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2021 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Docker file for running IWYU. -# Updated: 2022-11-03 - -FROM silkeh/clang:16-bullseye - -# Install prerequisites for the iwyu script -RUN apt-get update && apt-get install -y python3 jq git cmake python zlib1g-dev libtinfo-dev libclang-16-dev && apt-get clean -ADD iwyu.sh / - -# When running locally, we'll be impersonating the current user, so we need -# to make the script runnable by everyone. -RUN chmod a+rx /iwyu.sh - -CMD ["echo 'Run with tools/distrib/iwyu.sh'"] diff --git a/tools/dockerfile/grpc_iwyu/iwyu.sh b/tools/dockerfile/grpc_iwyu/iwyu.sh deleted file mode 100755 index 96f5ba7459207..0000000000000 --- a/tools/dockerfile/grpc_iwyu/iwyu.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/bin/sh -# Copyright 2017 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -x - -cd ${IWYU_ROOT} - -export PATH=${PATH}:${IWYU_ROOT}/iwyu_build/bin - -# number of CPUs available -CPU_COUNT=`nproc` - -rm -rf iwyu || true -git clone https://github.com/include-what-you-use/include-what-you-use.git iwyu - -############################################################################### -# -# BEWARE! BEWARE! BEWARE! BEWARE! BEWARE! BEWARE! BEWARE! BEWARE! -# -# Changing the version of iwyu can bring along subtle changes. -# You *must* test the new version of iwyu: -# 1. run it on the entire codebase before submitting -# 2. UPLOAD A CHANGE THAT SHOULD BE BROKEN AFTER SUBMISSION OF THIS CHANGE -# ensure that the broken change is caught by the new version of iwyu -# -# BEWARE! BEWARE! BEWARE! BEWARE! BEWARE! BEWARE! BEWARE! BEWARE! -# -############################################################################### - -# latest commit on the clang_16 branch -cd ${IWYU_ROOT}/iwyu -git checkout 7301b1fc88e5e16d8df73aecea55037d9c0a371b -if [ $? -ne 0 ]; then - echo "Failed to checkout iwyu commit" - exit 1 -fi -mkdir -p ${IWYU_ROOT}/iwyu_build -cd ${IWYU_ROOT}/iwyu_build -cmake -G "Unix Makefiles" -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_ROOT_DIR=/usr/lib/llvm-16 ${IWYU_ROOT}/iwyu -if [ $? -ne 0 ]; then - echo "Failed to cmake iwyu" - exit 1 -fi -make -j $CPU_COUNT -if [ $? -ne 0 ]; then - echo "Failed to make iwyu" - exit 1 -fi -cd ${IWYU_ROOT} - -cat compile_commands.json \ - | sed "s/ -DNDEBUG//g" \ - | sed "s/ -std=c\\+\\+14/ -std=c++17/g" \ - | sed "s,\"file\": \",\"file\": \"${IWYU_ROOT}/,g" \ - > compile_commands_for_iwyu.json - -export ENABLED_MODULES=' - src/core/ext - src/core/lib - src/cpp - src/python/grpcio_observability - test/core - fuzztest -' - -export DISABLED_MODULES=' - src/core/lib/gpr - src/core/lib/iomgr - src/core/ext/transport/binder - test/core/alts - test/core/iomgr - test/core/security - test/core/tsi - test/core/transport/binder -' - -export INCLUSION_REGEX=`echo $ENABLED_MODULES | sed 's/ /|/g' | sed 's,\\(.*\\),^(\\1)/,g'` -export EXCLUSION_REGEX=`echo $DISABLED_MODULES | sed 's/ /|/g' | sed 's,\\(.*\\),^(\\1)/,g'` - -# figure out which files to include -cat compile_commands.json | jq -r '.[].file' \ - | grep -E $INCLUSION_REGEX \ - | grep -v -E "/upb-gen/|/upbdefs-gen/" \ - | grep -v -E $EXCLUSION_REGEX \ - | grep -v src/core/lib/security/credentials/tls/grpc_tls_credentials_options.h \ - | grep -v test/core/end2end/end2end_tests.cc \ - | sort \ - > iwyu_files0.txt - -cat iwyu_files0.txt \ - | xargs -d '\n' ls -1df 2> /dev/null \ - > iwyu_files.txt \ - || true - -echo '#!/bin/sh -${IWYU_ROOT}/iwyu/iwyu_tool.py -p compile_commands_for_iwyu.json $1 \ - -- -Xiwyu --no_fwd_decls \ - -Xiwyu --update_comments \ - -Xiwyu --mapping_file=${IWYU_ROOT}/tools/distrib/iwyu_mappings.imp \ - | grep -v -E "port_platform.h" \ - | grep -v -E "repeated_ptr_field.h" \ - | grep -v -E "repeated_field.h" \ - | grep -v -E "^(- )?namespace " \ - > iwyu/iwyu.`echo $1 | sha1sum`.out -' > iwyu/run_iwyu_on.sh -chmod +x iwyu/run_iwyu_on.sh - -# run iwyu, filtering out changes to port_platform.h -xargs -n 1 -P $CPU_COUNT -a iwyu_files.txt ${IWYU_ROOT}/iwyu/run_iwyu_on.sh - -cat iwyu/iwyu.*.out > iwyu.out - -# apply the suggested changes -${IWYU_ROOT}/iwyu/fix_includes.py \ - --nocomments \ - --nosafe_headers \ - --ignore_re='^(include/.*|src/core/lib/security/credentials/tls/grpc_tls_credentials_options\.h|external/.*)' \ - < iwyu.out \ - | grep 'IWYU edited 0 files on your behalf' - -if [ $? -ne 0 ] -then - echo "Iwyu edited some files. Here is the diff of files edited by iwyu:" - git --no-pager diff - # Exit with a non zero error code to ensure sanity checks fail accordingly. - exit 1 -fi diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 5905f239563cf..21666ae9e51b2 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -2265,7 +2265,6 @@ src/core/lib/event_engine/forkable.cc \ src/core/lib/event_engine/forkable.h \ src/core/lib/event_engine/grpc_polled_fd.h \ src/core/lib/event_engine/handle_containers.h \ -src/core/lib/event_engine/memory_allocator.cc \ src/core/lib/event_engine/memory_allocator_factory.h \ src/core/lib/event_engine/nameser.h \ src/core/lib/event_engine/poller.h \ @@ -2282,6 +2281,8 @@ src/core/lib/event_engine/posix_engine/internal_errqueue.cc \ src/core/lib/event_engine/posix_engine/internal_errqueue.h \ src/core/lib/event_engine/posix_engine/lockfree_event.cc \ src/core/lib/event_engine/posix_engine/lockfree_event.h \ +src/core/lib/event_engine/posix_engine/native_dns_resolver.cc \ +src/core/lib/event_engine/posix_engine/native_dns_resolver.h \ src/core/lib/event_engine/posix_engine/posix_endpoint.cc \ src/core/lib/event_engine/posix_engine/posix_endpoint.h \ src/core/lib/event_engine/posix_engine/posix_engine.cc \ @@ -2308,6 +2309,7 @@ src/core/lib/event_engine/posix_engine/wakeup_fd_pipe.h \ src/core/lib/event_engine/posix_engine/wakeup_fd_posix.h \ src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.cc \ src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.h \ +src/core/lib/event_engine/ref_counted_dns_resolver_interface.h \ src/core/lib/event_engine/resolved_address.cc \ src/core/lib/event_engine/resolved_address_internal.h \ src/core/lib/event_engine/shim.cc \ @@ -2657,6 +2659,7 @@ src/core/lib/promise/race.h \ src/core/lib/promise/seq.h \ src/core/lib/promise/sleep.cc \ src/core/lib/promise/sleep.h \ +src/core/lib/promise/status_flag.h \ src/core/lib/promise/trace.cc \ src/core/lib/promise/trace.h \ src/core/lib/promise/try_seq.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index aa35ecb50b374..b7b1ae5bf255a 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -2041,7 +2041,6 @@ src/core/lib/event_engine/forkable.cc \ src/core/lib/event_engine/forkable.h \ src/core/lib/event_engine/grpc_polled_fd.h \ src/core/lib/event_engine/handle_containers.h \ -src/core/lib/event_engine/memory_allocator.cc \ src/core/lib/event_engine/memory_allocator_factory.h \ src/core/lib/event_engine/nameser.h \ src/core/lib/event_engine/poller.h \ @@ -2058,6 +2057,8 @@ src/core/lib/event_engine/posix_engine/internal_errqueue.cc \ src/core/lib/event_engine/posix_engine/internal_errqueue.h \ src/core/lib/event_engine/posix_engine/lockfree_event.cc \ src/core/lib/event_engine/posix_engine/lockfree_event.h \ +src/core/lib/event_engine/posix_engine/native_dns_resolver.cc \ +src/core/lib/event_engine/posix_engine/native_dns_resolver.h \ src/core/lib/event_engine/posix_engine/posix_endpoint.cc \ src/core/lib/event_engine/posix_engine/posix_endpoint.h \ src/core/lib/event_engine/posix_engine/posix_engine.cc \ @@ -2084,6 +2085,7 @@ src/core/lib/event_engine/posix_engine/wakeup_fd_pipe.h \ src/core/lib/event_engine/posix_engine/wakeup_fd_posix.h \ src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.cc \ src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.h \ +src/core/lib/event_engine/ref_counted_dns_resolver_interface.h \ src/core/lib/event_engine/resolved_address.cc \ src/core/lib/event_engine/resolved_address_internal.h \ src/core/lib/event_engine/shim.cc \ @@ -2436,6 +2438,7 @@ src/core/lib/promise/race.h \ src/core/lib/promise/seq.h \ src/core/lib/promise/sleep.cc \ src/core/lib/promise/sleep.h \ +src/core/lib/promise/status_flag.h \ src/core/lib/promise/trace.cc \ src/core/lib/promise/trace.h \ src/core/lib/promise/try_seq.h \ diff --git a/tools/internal_ci/linux/grpc_e2e_performance_gke.sh b/tools/internal_ci/linux/grpc_e2e_performance_gke.sh index 15951cccc756e..a8da8a9fae6a0 100755 --- a/tools/internal_ci/linux/grpc_e2e_performance_gke.sh +++ b/tools/internal_ci/linux/grpc_e2e_performance_gke.sh @@ -131,9 +131,9 @@ configLangArgs32core+=( -l c++ ) runnerLangArgs+=( -l "cxx:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}" ) # dotnet -configLangArgs8core+=( -l dotnet ) -configLangArgs32core+=( -l dotnet ) -runnerLangArgs+=( -l "dotnet:${GRPC_DOTNET_REPO}:${GRPC_DOTNET_COMMIT}" ) +# configLangArgs8core+=( -l dotnet ) +# configLangArgs32core+=( -l dotnet ) +# runnerLangArgs+=( -l "dotnet:${GRPC_DOTNET_REPO}:${GRPC_DOTNET_COMMIT}" ) # # go configLangArgs8core+=( -l go ) diff --git a/tools/internal_ci/linux/grpc_e2e_performance_gke_experiment.sh b/tools/internal_ci/linux/grpc_e2e_performance_gke_experiment.sh index f62d9920b72a4..c3b99af80ebb6 100755 --- a/tools/internal_ci/linux/grpc_e2e_performance_gke_experiment.sh +++ b/tools/internal_ci/linux/grpc_e2e_performance_gke_experiment.sh @@ -125,9 +125,9 @@ configLangArgs32core+=( -l c++ ) runnerLangArgs+=( -l "cxx:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}" ) # dotnet -configLangArgs8core+=( -l dotnet ) -configLangArgs32core+=( -l dotnet ) -runnerLangArgs+=( -l "dotnet:${GRPC_DOTNET_REPO}:${GRPC_DOTNET_COMMIT}" ) +# configLangArgs8core+=( -l dotnet ) +# configLangArgs32core+=( -l dotnet ) +# runnerLangArgs+=( -l "dotnet:${GRPC_DOTNET_REPO}:${GRPC_DOTNET_COMMIT}" ) # # go configLangArgs8core+=( -l go ) diff --git a/tools/internal_ci/linux/grpc_iwyu.cfg b/tools/internal_ci/linux/grpc_iwyu.cfg deleted file mode 100644 index 33fbbd68bad00..0000000000000 --- a/tools/internal_ci/linux/grpc_iwyu.cfg +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2017 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Config file for the internal CI (in protobuf text format) - -# Location of the continuous shell script in repository. -build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" -timeout_mins: 60 -action { - define_artifacts { - regex: "**/*sponge_log.*" - regex: "github/grpc/reports/**" - } -} - -env_vars { - key: "RUN_TESTS_FLAGS" - value: "-f basictests linux iwyu --inner_jobs 16 -j 1 --internal_ci --bq_result_table aggregate_results" -} diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py index 110e29ce2f1aa..836485bdb6949 100644 --- a/tools/interop_matrix/client_matrix.py +++ b/tools/interop_matrix/client_matrix.py @@ -430,7 +430,7 @@ def __init__(self, patch=[], runtimes=[], testcases_file=None): ("v1.56.0", ReleaseInfo()), ("v1.57.2", ReleaseInfo()), ("v1.58.0", ReleaseInfo()), - ("v1.59.0", ReleaseInfo()), + ("v1.59.1", ReleaseInfo()), ] ), "python": OrderedDict( diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 35bdddc0eb0b4..6b2a8024173cf 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -6881,6 +6881,26 @@ ], "uses_polling": true }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "posix_event_engine_native_dns_test", + "platforms": [ + "linux", + "posix" + ], + "uses_polling": true + }, { "args": [], "benchmark": false, @@ -7189,6 +7209,30 @@ ], "uses_polling": true }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "query_extensions_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": false + }, { "args": [], "benchmark": false, diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 808bc0a30bf84..064dbf3328d3c 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -21,8 +21,8 @@ import itertools import json import os -import pipes import re +import shlex import sys import time @@ -121,7 +121,7 @@ def create_scenario_jobspec( if bq_result_table: cmd += 'BQ_RESULT_TABLE="%s" ' % bq_result_table cmd += "tools/run_tests/performance/run_qps_driver.sh " - cmd += "--scenarios_json=%s " % pipes.quote( + cmd += "--scenarios_json=%s " % shlex.quote( json.dumps({"scenarios": [scenario_json]}) ) cmd += "--scenario_result_file=scenario_result.json " @@ -135,7 +135,7 @@ def create_scenario_jobspec( user_at_host = "%s@%s" % (_REMOTE_HOST_USERNAME, remote_host) cmd = 'ssh %s "cd ~/performance_workspace/grpc/ && "%s' % ( user_at_host, - pipes.quote(cmd), + shlex.quote(cmd), ) return jobset.JobSpec( @@ -157,7 +157,7 @@ def create_quit_jobspec(workers, remote_host=None): user_at_host = "%s@%s" % (_REMOTE_HOST_USERNAME, remote_host) cmd = 'ssh %s "cd ~/performance_workspace/grpc/ && "%s' % ( user_at_host, - pipes.quote(cmd), + shlex.quote(cmd), ) return jobset.JobSpec( @@ -192,7 +192,7 @@ def create_netperf_jobspec( user_at_host = "%s@%s" % (_REMOTE_HOST_USERNAME, client_host) cmd = 'ssh %s "cd ~/performance_workspace/grpc/ && "%s' % ( user_at_host, - pipes.quote(cmd), + shlex.quote(cmd), ) return jobset.JobSpec( diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 5bd0175781002..ea1b1b0205269 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -26,10 +26,10 @@ import multiprocessing import os import os.path -import pipes import platform import random import re +import shlex import socket import subprocess import sys @@ -479,7 +479,7 @@ def test_specs(self): cmdline = [binary] + target["args"] shortname = target.get( "shortname", - " ".join(pipes.quote(arg) for arg in cmdline), + " ".join(shlex.quote(arg) for arg in cmdline), ) shortname += shortname_ext out.append( @@ -1317,7 +1317,6 @@ def __str__(self): "objc": ObjCLanguage(), "sanity": Sanity("sanity_tests.yaml"), "clang-tidy": Sanity("clang_tidy_tests.yaml"), - "iwyu": Sanity("iwyu_tests.yaml"), } _MSBUILD_CONFIG = { diff --git a/tools/run_tests/sanity/iwyu_tests.yaml b/tools/run_tests/sanity/iwyu_tests.yaml deleted file mode 100644 index 2c9c7836d7fbd..0000000000000 --- a/tools/run_tests/sanity/iwyu_tests.yaml +++ /dev/null @@ -1,2 +0,0 @@ -- script: tools/distrib/iwyu.sh - cpu_cost: 1000 diff --git a/tools/run_tests/sanity/sanity_tests.yaml b/tools/run_tests/sanity/sanity_tests.yaml index 439891f75bf06..6c4494b00c284 100644 --- a/tools/run_tests/sanity/sanity_tests.yaml +++ b/tools/run_tests/sanity/sanity_tests.yaml @@ -6,7 +6,6 @@ - script: tools/run_tests/sanity/check_buildifier.sh - script: tools/run_tests/sanity/check_cache_mk.sh - script: tools/run_tests/sanity/check_deprecated_grpc++.py -- script: tools/run_tests/sanity/check_do_not_submit.sh - script: tools/run_tests/sanity/check_illegal_terms.sh - script: tools/run_tests/sanity/check_port_platform.py - script: tools/run_tests/sanity/check_include_style.py diff --git a/tools/run_tests/xds_k8s_test_driver/bin/run_ping_pong.py b/tools/run_tests/xds_k8s_test_driver/bin/run_ping_pong.py index c321e3899077e..d811a16a378f3 100755 --- a/tools/run_tests/xds_k8s_test_driver/bin/run_ping_pong.py +++ b/tools/run_tests/xds_k8s_test_driver/bin/run_ping_pong.py @@ -79,6 +79,7 @@ def get_client_rpc_stats( def run_ping_pong(test_client: _XdsTestClient, num_rpcs: int): + test_client.wait_for_active_xds_channel() test_client.wait_for_server_channel_ready() lb_stats = get_client_rpc_stats(test_client, num_rpcs) for backend, rpcs_count in lb_stats.rpcs_by_peer.items(): diff --git a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/api.py b/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/api.py index 05bd08ab6f2a6..33e48d760092a 100644 --- a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/api.py +++ b/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/api.py @@ -501,7 +501,7 @@ def _delete_resource( return True except _HttpError as error: if error.resp and error.resp.status == 404: - logger.info("%s not deleted since it does not exist", full_name) + logger.debug("%s not deleted since it doesn't exist", full_name) else: logger.warning("Failed to delete %s, %r", full_name, error) return False diff --git a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/compute.py b/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/compute.py index 93ed60d25fc23..b06a1007b4ef1 100644 --- a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/compute.py +++ b/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/compute.py @@ -565,8 +565,8 @@ def _delete_resource( return True except googleapiclient.errors.HttpError as error: if error.resp and error.resp.status == 404: - logger.info( - 'Resource %s "%s" not deleted since it does not exist', + logger.debug( + "Resource %s %s not deleted since it doesn't exist", resource_type, resource_name, ) diff --git a/tools/run_tests/xds_k8s_test_driver/framework/test_app/runners/k8s/k8s_base_runner.py b/tools/run_tests/xds_k8s_test_driver/framework/test_app/runners/k8s/k8s_base_runner.py index da17961d5f443..3df9bd19a253e 100644 --- a/tools/run_tests/xds_k8s_test_driver/framework/test_app/runners/k8s/k8s_base_runner.py +++ b/tools/run_tests/xds_k8s_test_driver/framework/test_app/runners/k8s/k8s_base_runner.py @@ -582,20 +582,32 @@ def _delete_gamma_route(self, name, wait_for_deletion=True): logger.info("Deleting HTTPRoute %s", name) try: self.k8s_namespace.delete_gamma_route(name) - except (retryers.RetryError, k8s.NotFound) as e: - logger.info("HTTPRoute %s deletion failed: %s", name, e) + except k8s.NotFound: + logger.debug( + "HTTPRoute %s not deleted since it doesn't exist", name + ) + return + except retryers.RetryError as e: + logger.warning("HTTPRoute %s deletion failed: %s", name, e) return if wait_for_deletion: self.k8s_namespace.wait_for_get_gamma_route_deleted(name) - logger.debug("HTTPRoute %s deleted", name) + logger.info("HTTPRoute %s deleted", name) def _delete_session_affinity_policy(self, name, wait_for_deletion=True): logger.info("Deleting GCPSessionAffinityPolicy %s", name) try: self.k8s_namespace.delete_session_affinity_policy(name) - except (retryers.RetryError, k8s.NotFound) as e: - logger.info( + except k8s.NotFound: + logger.debug( + "GCPSessionAffinityPolicy %s not deleted since it" + " doesn't exist", + name, + ) + return + except retryers.RetryError as e: + logger.warning( "GCPSessionAffinityPolicy %s deletion failed: %s", name, e ) return @@ -604,14 +616,21 @@ def _delete_session_affinity_policy(self, name, wait_for_deletion=True): self.k8s_namespace.wait_for_get_session_affinity_policy_deleted( name ) - logger.debug("GCPSessionAffinityPolicy %s deleted", name) + logger.info("GCPSessionAffinityPolicy %s deleted", name) def _delete_session_affinity_filter(self, name, wait_for_deletion=True): logger.info("Deleting GCPSessionAffinityFilter %s", name) try: self.k8s_namespace.delete_session_affinity_filter(name) - except (retryers.RetryError, k8s.NotFound) as e: - logger.info( + except k8s.NotFound: + logger.debug( + "GCPSessionAffinityFilter %s not deleted since it" + " doesn't exist", + name, + ) + return + except retryers.RetryError as e: + logger.warning( "GCPSessionAffinityFilter %s deletion failed: %s", name, e ) return @@ -620,71 +639,95 @@ def _delete_session_affinity_filter(self, name, wait_for_deletion=True): self.k8s_namespace.wait_for_get_session_affinity_filter_deleted( name ) - logger.debug("GCPSessionAffinityFilter %s deleted", name) + logger.info("GCPSessionAffinityFilter %s deleted", name) def _delete_backend_policy(self, name, wait_for_deletion=True): logger.info("Deleting GCPBackendPolicy %s", name) try: self.k8s_namespace.delete_backend_policy(name) - except (retryers.RetryError, k8s.NotFound) as e: - logger.info("GGCPBackendPolicy %s deletion failed: %s", name, e) + except k8s.NotFound: + logger.debug( + "GGCPBackendPolicy %s not deleted since it doesn't exist", name + ) + return + except retryers.RetryError as e: + logger.warning("GGCPBackendPolicy %s deletion failed: %s", name, e) return if wait_for_deletion: self.k8s_namespace.wait_for_get_backend_policy_deleted(name) - logger.debug("GCPBackendPolicy %s deleted", name) + logger.info("GCPBackendPolicy %s deleted", name) def _delete_deployment(self, name, wait_for_deletion=True): logger.info("Deleting deployment %s", name) self.stop_pod_dependencies() try: self.k8s_namespace.delete_deployment(name) - except (retryers.RetryError, k8s.NotFound) as e: - logger.info("Deployment %s deletion failed: %s", name, e) + except k8s.NotFound: + logger.debug( + "Deployment %s not deleted since it doesn't exist", name + ) + return + except retryers.RetryError as e: + logger.warning("Deployment %s deletion failed: %s", name, e) return if wait_for_deletion: self.k8s_namespace.wait_for_deployment_deleted(name) - logger.debug("Deployment %s deleted", name) + logger.info("Deployment %s deleted", name) def _delete_service(self, name, wait_for_deletion=True): logger.info("Deleting service %s", name) try: self.k8s_namespace.delete_service(name) - except (retryers.RetryError, k8s.NotFound) as e: - logger.info("Service %s deletion failed: %s", name, e) + except k8s.NotFound: + logger.debug("Service %s not deleted since it doesn't exist", name) + return + except retryers.RetryError as e: + logger.warning("Service %s deletion failed: %s", name, e) return if wait_for_deletion: self.k8s_namespace.wait_for_service_deleted(name) - logger.debug("Service %s deleted", name) + logger.info("Service %s deleted", name) def _delete_service_account(self, name, wait_for_deletion=True): logger.info("Deleting service account %s", name) try: self.k8s_namespace.delete_service_account(name) - except (retryers.RetryError, k8s.NotFound) as e: - logger.info("Service account %s deletion failed: %s", name, e) + except k8s.NotFound: + logger.debug( + "Service account %s not deleted since it doesn't exist", name + ) + return + except retryers.RetryError as e: + logger.warning("Service account %s deletion failed: %s", name, e) return if wait_for_deletion: self.k8s_namespace.wait_for_service_account_deleted(name) - logger.debug("Service account %s deleted", name) + logger.info("Service account %s deleted", name) def delete_namespace(self, wait_for_deletion=True): logger.info("Deleting namespace %s", self.k8s_namespace.name) try: self.k8s_namespace.delete() - except (retryers.RetryError, k8s.NotFound) as e: - logger.info( + except k8s.NotFound: + logger.debug( + "Namespace %s not deleted since it doesn't exist", + self.k8s_namespace.name, + ) + return + except retryers.RetryError as e: + logger.warning( "Namespace %s deletion failed: %s", self.k8s_namespace.name, e ) return if wait_for_deletion: self.k8s_namespace.wait_for_namespace_deleted() - logger.debug("Namespace %s deleted", self.k8s_namespace.name) + logger.info("Namespace %s deleted", self.k8s_namespace.name) def _wait_deployment_with_available_replicas(self, name, count=1, **kwargs): logger.info(