diff --git a/mobile/library/common/engine_common.cc b/mobile/library/common/engine_common.cc index 61875a82f0e2..9295be831cee 100644 --- a/mobile/library/common/engine_common.cc +++ b/mobile/library/common/engine_common.cc @@ -54,6 +54,12 @@ void registerMobileProtoDescriptors() { namespace Envoy { +class ServerLite : public Server::InstanceBase { +public: + using Server::InstanceBase::InstanceBase; + void maybeCreateHeapShrinker() override {} +}; + EngineCommon::EngineCommon(std::unique_ptr&& options) : options_(std::move(options)) { @@ -69,9 +75,8 @@ EngineCommon::EngineCommon(std::unique_ptr&& options) ThreadLocal::Instance& tls, Thread::ThreadFactory& thread_factory, Filesystem::Instance& file_system, std::unique_ptr process_context, Buffer::WatermarkFactorySharedPtr watermark_factory) { - // TODO(alyssawilk) use InstanceLite not InstanceImpl. auto local_address = Network::Utility::getLocalAddress(options.localAddressIpVersion()); - auto server = std::make_unique( + auto server = std::make_unique( init_manager, options, time_system, hooks, restarter, store, access_log_lock, std::move(random_generator), tls, thread_factory, file_system, std::move(process_context), watermark_factory); diff --git a/source/exe/BUILD b/source/exe/BUILD index 853dd6be295a..f236980d398a 100644 --- a/source/exe/BUILD +++ b/source/exe/BUILD @@ -42,7 +42,7 @@ envoy_cc_library( "//source/server:drain_manager_lib", "//source/server:listener_hooks_lib", "//source/server:options_lib", - "//source/server:server_lib", + "//source/server:server_base_lib", ] + select({ "//bazel:windows_x86_64": envoy_all_extensions(WINDOWS_SKIP_TARGETS), "//bazel:linux_ppc": envoy_all_extensions(PPC_SKIP_TARGETS), @@ -144,7 +144,7 @@ envoy_cc_library( "//source/server:drain_manager_lib", "//source/server:listener_hooks_lib", "//source/server:options_lib", - "//source/server:server_lib", + "//source/server:server_base_lib", ] + envoy_all_core_extensions() + # TODO(rojkov): drop io_uring dependency when it's fully integrated. select({ @@ -175,6 +175,7 @@ envoy_cc_library( "//source/extensions/listener_managers/validation_listener_manager:validation_listener_manager_lib", "//source/server:hot_restart_lib", "//source/server:hot_restart_nop_lib", + "//source/server:server_lib", "//source/server/config_validation:server_lib", ] + envoy_select_signal_trace([ "//source/common/signal:sigaction_lib", diff --git a/source/exe/main_common.cc b/source/exe/main_common.cc index d1b3d1270e1e..5005fa457efc 100644 --- a/source/exe/main_common.cc +++ b/source/exe/main_common.cc @@ -16,9 +16,9 @@ #include "source/server/config_validation/server.h" #include "source/server/drain_manager_impl.h" #include "source/server/hot_restart_nop_impl.h" +#include "source/server/instance_impl.h" #include "source/server/listener_hooks.h" #include "source/server/options_impl.h" -#include "source/server/server.h" #include "absl/debugging/symbolize.h" #include "absl/strings/str_split.h" diff --git a/source/server/BUILD b/source/server/BUILD index 3a4cf9c2261f..fe2c30191b76 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -376,7 +376,7 @@ envoy_cc_library( ) envoy_cc_library( - name = "server_lib", + name = "server_base_lib", srcs = ["server.cc"], hdrs = ["server.h"], external_deps = [ @@ -424,7 +424,6 @@ envoy_cc_library( "//source/common/http:headers_lib", "//source/common/init:manager_lib", "//source/common/local_info:local_info_lib", - "//source/common/memory:heap_shrinker_lib", "//source/common/memory:stats_lib", "//source/common/protobuf:utility_lib", "//source/common/quic:quic_stat_names_lib", @@ -444,6 +443,16 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "server_lib", + srcs = ["instance_impl.cc"], + hdrs = ["instance_impl.h"], + deps = [ + ":server_base_lib", + "//source/common/memory:heap_shrinker_lib", + ], +) + envoy_cc_library( name = "ssl_context_manager_lib", srcs = ["ssl_context_manager.cc"], diff --git a/source/server/instance_impl.cc b/source/server/instance_impl.cc new file mode 100644 index 000000000000..67a54231d86f --- /dev/null +++ b/source/server/instance_impl.cc @@ -0,0 +1,11 @@ +#include "source/server/instance_impl.h" + +namespace Envoy { +namespace Server { +void InstanceImpl::maybeCreateHeapShrinker() { + heap_shrinker_ = + std::make_unique(dispatcher(), overloadManager(), *stats().rootScope()); +} + +} // namespace Server +} // namespace Envoy diff --git a/source/server/instance_impl.h b/source/server/instance_impl.h new file mode 100644 index 000000000000..6a91adfec2fb --- /dev/null +++ b/source/server/instance_impl.h @@ -0,0 +1,22 @@ +#pragma once + +#include "source/common/memory/heap_shrinker.h" +#include "source/server/server.h" + +namespace Envoy { +namespace Server { + +// The production server instance, which creates all of the required components. +class InstanceImpl : public InstanceBase { +public: + using InstanceBase::InstanceBase; + +protected: + void maybeCreateHeapShrinker() override; + +private: + std::unique_ptr heap_shrinker_; +}; + +} // namespace Server +} // namespace Envoy diff --git a/source/server/server.cc b/source/server/server.cc index 575b29716ffc..43b1e81fa8ab 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -74,7 +74,7 @@ std::unique_ptr getHandler(Event::Dispatcher& dispatcher) { } // namespace -InstanceImpl::InstanceImpl(Init::Manager& init_manager, const Options& options, +InstanceBase::InstanceBase(Init::Manager& init_manager, const Options& options, Event::TimeSystem& time_system, ListenerHooks& hooks, HotRestart& restarter, Stats::StoreRoot& store, Thread::BasicLockable& access_log_lock, @@ -106,14 +106,14 @@ InstanceImpl::InstanceImpl(Init::Manager& init_manager, const Options& options, hooks_(hooks), quic_stat_names_(store.symbolTable()), server_contexts_(*this), enable_reuse_port_default_(true), stats_flush_in_progress_(false) {} -InstanceImpl::~InstanceImpl() { +InstanceBase::~InstanceBase() { terminate(); // Stop logging to file before all the AccessLogManager and its dependencies are // destructed to avoid crashing at shutdown. file_logger_.reset(); - // Destruct the ListenerManager explicitly, before InstanceImpl's local init_manager_ is + // Destruct the ListenerManager explicitly, before InstanceBase's local init_manager_ is // destructed. // // The ListenerManager's DestinationPortsMap contains FilterChainSharedPtrs. There is a rare race @@ -138,17 +138,17 @@ InstanceImpl::~InstanceImpl() { #endif } -Upstream::ClusterManager& InstanceImpl::clusterManager() { +Upstream::ClusterManager& InstanceBase::clusterManager() { ASSERT(config_.clusterManager() != nullptr); return *config_.clusterManager(); } -const Upstream::ClusterManager& InstanceImpl::clusterManager() const { +const Upstream::ClusterManager& InstanceBase::clusterManager() const { ASSERT(config_.clusterManager() != nullptr); return *config_.clusterManager(); } -void InstanceImpl::drainListeners(OptRef options) { +void InstanceBase::drainListeners(OptRef options) { ENVOY_LOG(info, "closing and draining listeners"); listener_manager_->stopListeners(ListenerManager::StopListenersType::All, options.has_value() ? *options @@ -156,7 +156,7 @@ void InstanceImpl::drainListeners(OptRefstartDrainSequence([] {}); } -void InstanceImpl::failHealthcheck(bool fail) { +void InstanceBase::failHealthcheck(bool fail) { live_.store(!fail); server_stats_->live_.set(live_.load()); } @@ -228,7 +228,7 @@ void InstanceUtil::flushMetricsToSinks(const std::list& sinks, S } } -void InstanceImpl::flushStats() { +void InstanceBase::flushStats() { if (stats_flush_in_progress_) { ENVOY_LOG(debug, "skipping stats flush as flush is already in progress"); server_stats_->dropped_stat_flushes_.inc(); @@ -251,7 +251,7 @@ void InstanceImpl::flushStats() { } } -void InstanceImpl::updateServerStats() { +void InstanceBase::updateServerStats() { // mergeParentStatsIfAny() does nothing and returns a struct of 0s if there is no parent. HotRestart::ServerStatsFromParent parent_stats = restarter_.mergeParentStatsIfAny(stats_store_); @@ -278,7 +278,7 @@ void InstanceImpl::updateServerStats() { stats_store_.symbolTable().getRecentLookups([](absl::string_view, uint64_t) {})); } -void InstanceImpl::flushStatsInternal() { +void InstanceBase::flushStatsInternal() { updateServerStats(); auto& stats_config = config_.statsConfig(); InstanceUtil::flushMetricsToSinks(stats_config.sinks(), stats_store_, clusterManager(), @@ -291,9 +291,9 @@ void InstanceImpl::flushStatsInternal() { stats_flush_in_progress_ = false; } -bool InstanceImpl::healthCheckFailed() { return !live_.load(); } +bool InstanceBase::healthCheckFailed() { return !live_.load(); } -ProcessContextOptRef InstanceImpl::processContext() { +ProcessContextOptRef InstanceBase::processContext() { if (process_context_ == nullptr) { return absl::nullopt; } @@ -385,7 +385,7 @@ void InstanceUtil::loadBootstrapConfig(envoy::config::bootstrap::v3::Bootstrap& MessageUtil::validate(bootstrap, validation_visitor); } -void InstanceImpl::initialize(Network::Address::InstanceConstSharedPtr local_address, +void InstanceBase::initialize(Network::Address::InstanceConstSharedPtr local_address, ComponentFactory& component_factory) { std::function set_up_logger = [&] { TRY_ASSERT_MAIN_THREAD { @@ -424,7 +424,7 @@ void InstanceImpl::initialize(Network::Address::InstanceConstSharedPtr local_add }); } -void InstanceImpl::initializeOrThrow(Network::Address::InstanceConstSharedPtr local_address, +void InstanceBase::initializeOrThrow(Network::Address::InstanceConstSharedPtr local_address, ComponentFactory& component_factory) { ENVOY_LOG(info, "initializing epoch {} (base id={}, hot restart version={})", options_.restartEpoch(), restarter_.baseId(), restarter_.version()); @@ -524,7 +524,7 @@ void InstanceImpl::initializeOrThrow(Network::Address::InstanceConstSharedPtr lo server_stats_->initialization_time_ms_, timeSource()); server_stats_->concurrency_.set(options_.concurrency()); server_stats_->hot_restart_epoch_.set(options_.restartEpoch()); - InstanceImpl::failHealthcheck(false); + InstanceBase::failHealthcheck(false); // Check if bootstrap has server version override set, if yes, we should use that as // 'server.version' stat. @@ -610,8 +610,7 @@ void InstanceImpl::initializeOrThrow(Network::Address::InstanceConstSharedPtr lo *dispatcher_, *stats_store_.rootScope(), thread_local_, bootstrap_.overload_manager(), messageValidationContext().staticValidationVisitor(), *api_, options_); - heap_shrinker_ = std::make_unique(*dispatcher_, *overload_manager_, - *stats_store_.rootScope()); + maybeCreateHeapShrinker(); for (const auto& bootstrap_extension : bootstrap_.bootstrap_extensions()) { auto& factory = Config::Utility::getAndCheckFactory( @@ -735,7 +734,7 @@ void InstanceImpl::initializeOrThrow(Network::Address::InstanceConstSharedPtr lo // Now the configuration gets parsed. The configuration may start setting // thread local data per above. See MainImpl::initialize() for why ConfigImpl - // is constructed as part of the InstanceImpl and then populated once + // is constructed as part of the InstanceBase and then populated once // cluster_manager_factory_ is available. config_.initialize(bootstrap_, *this, *cluster_manager_factory_); @@ -784,12 +783,12 @@ void InstanceImpl::initializeOrThrow(Network::Address::InstanceConstSharedPtr lo *stats_store_.rootScope(), config_.workerWatchdogConfig(), *api_, "workers"); } -void InstanceImpl::onClusterManagerPrimaryInitializationComplete() { +void InstanceBase::onClusterManagerPrimaryInitializationComplete() { // If RTDS was not configured the `onRuntimeReady` callback is immediately invoked. runtime().startRtdsSubscriptions([this]() { onRuntimeReady(); }); } -void InstanceImpl::onRuntimeReady() { +void InstanceBase::onRuntimeReady() { // Begin initializing secondary clusters after RTDS configuration has been applied. // Initializing can throw exceptions, so catch these. TRY_ASSERT_MAIN_THREAD { clusterManager().initializeSecondaryClusters(bootstrap_); } @@ -820,7 +819,7 @@ void InstanceImpl::onRuntimeReady() { } } -void InstanceImpl::startWorkers() { +void InstanceBase::startWorkers() { // The callback will be called after workers are started. listener_manager_->startWorkers(*worker_guard_dog_, [this]() { if (isShutdown()) { @@ -850,7 +849,7 @@ Runtime::LoaderPtr InstanceUtil::createRuntime(Instance& server, server.messageValidationContext().dynamicValidationVisitor(), server.api()); } -void InstanceImpl::loadServerFlags(const absl::optional& flags_path) { +void InstanceBase::loadServerFlags(const absl::optional& flags_path) { if (!flags_path) { return; } @@ -858,7 +857,7 @@ void InstanceImpl::loadServerFlags(const absl::optional& flags_path ENVOY_LOG(info, "server flags path: {}", flags_path.value()); if (api_->fileSystem().fileExists(flags_path.value() + "/drain")) { ENVOY_LOG(info, "starting server in drain mode"); - InstanceImpl::failHealthcheck(true); + InstanceBase::failHealthcheck(true); } } @@ -938,7 +937,7 @@ RunHelper::RunHelper(Instance& instance, const Options& options, Event::Dispatch }); } -void InstanceImpl::run() { +void InstanceBase::run() { // RunHelper exists primarily to facilitate testing of how we respond to early shutdown during // startup (see RunHelperTest in server_test.cc). const auto run_helper = RunHelper(*this, options_, *dispatcher_, clusterManager(), @@ -960,7 +959,7 @@ void InstanceImpl::run() { terminate(); } -void InstanceImpl::terminate() { +void InstanceBase::terminate() { if (terminated_) { return; } @@ -1008,16 +1007,16 @@ void InstanceImpl::terminate() { FatalErrorHandler::clearFatalActionsOnTerminate(); } -Runtime::Loader& InstanceImpl::runtime() { return *runtime_; } +Runtime::Loader& InstanceBase::runtime() { return *runtime_; } -void InstanceImpl::shutdown() { +void InstanceBase::shutdown() { ENVOY_LOG(info, "shutting down server instance"); shutdown_ = true; restarter_.sendParentTerminateRequest(); notifyCallbacksForStage(Stage::ShutdownExit, [this] { dispatcher_->exit(); }); } -void InstanceImpl::shutdownAdmin() { +void InstanceBase::shutdownAdmin() { ENVOY_LOG(warn, "shutting down admin due to child startup"); stat_flush_timer_.reset(); handler_->stopListeners(); @@ -1030,21 +1029,21 @@ void InstanceImpl::shutdownAdmin() { restarter_.sendParentTerminateRequest(); } -ServerLifecycleNotifier::HandlePtr InstanceImpl::registerCallback(Stage stage, +ServerLifecycleNotifier::HandlePtr InstanceBase::registerCallback(Stage stage, StageCallback callback) { auto& callbacks = stage_callbacks_[stage]; return std::make_unique>(callbacks, callback); } ServerLifecycleNotifier::HandlePtr -InstanceImpl::registerCallback(Stage stage, StageCallbackWithCompletion callback) { +InstanceBase::registerCallback(Stage stage, StageCallbackWithCompletion callback) { ASSERT(stage == Stage::ShutdownExit); auto& callbacks = stage_completable_callbacks_[stage]; return std::make_unique>(callbacks, callback); } -void InstanceImpl::notifyCallbacksForStage(Stage stage, std::function completion_cb) { +void InstanceBase::notifyCallbacksForStage(Stage stage, std::function completion_cb) { ASSERT_IS_MAIN_OR_TEST_THREAD(); const auto it = stage_callbacks_.find(stage); if (it != stage_callbacks_.end()) { @@ -1073,7 +1072,7 @@ void InstanceImpl::notifyCallbacksForStage(Stage stage, std::function co } } -ProtobufTypes::MessagePtr InstanceImpl::dumpBootstrapConfig() { +ProtobufTypes::MessagePtr InstanceBase::dumpBootstrapConfig() { auto config_dump = std::make_unique(); config_dump->mutable_bootstrap()->MergeFrom(bootstrap_); TimestampUtil::systemClockToTimestamp(bootstrap_config_update_time_, @@ -1081,7 +1080,7 @@ ProtobufTypes::MessagePtr InstanceImpl::dumpBootstrapConfig() { return config_dump; } -Network::DnsResolverSharedPtr InstanceImpl::getOrCreateDnsResolver() { +Network::DnsResolverSharedPtr InstanceBase::getOrCreateDnsResolver() { if (!dns_resolver_) { envoy::config::core::v3::TypedExtensionConfig typed_dns_resolver_config; Network::DnsResolverFactory& dns_resolver_factory = @@ -1092,7 +1091,7 @@ Network::DnsResolverSharedPtr InstanceImpl::getOrCreateDnsResolver() { return dns_resolver_; } -bool InstanceImpl::enableReusePortDefault() { return enable_reuse_port_default_; } +bool InstanceBase::enableReusePortDefault() { return enable_reuse_port_default_; } } // namespace Server } // namespace Envoy diff --git a/source/server/server.h b/source/server/server.h index 859eb5d4789b..fdafe2dfbef7 100644 --- a/source/server/server.h +++ b/source/server/server.h @@ -32,7 +32,6 @@ #include "source/common/grpc/context_impl.h" #include "source/common/http/context_impl.h" #include "source/common/init/manager_impl.h" -#include "source/common/memory/heap_shrinker.h" #include "source/common/protobuf/message_validator_impl.h" #include "source/common/quic/quic_stat_names.h" #include "source/common/router/context_impl.h" @@ -149,7 +148,7 @@ class InstanceUtil : Logger::Loggable { }; /** - * This is a helper used by InstanceImpl::run() on the stack. It's broken out to make testing + * This is a helper used by InstanceBase::run() on the stack. It's broken out to make testing * easier. */ class RunHelper : Logger::Loggable { @@ -223,16 +222,17 @@ class ServerFactoryContextImpl : public Configuration::ServerFactoryContext, }; /** - * This is the actual full standalone server which stitches together various common components. + * This is the base class for the standalone server which stitches together various common + * components. Some components are optional (so PURE) and can be created or not by subclasses. */ -class InstanceImpl final : Logger::Loggable, - public Instance, - public ServerLifecycleNotifier { +class InstanceBase : Logger::Loggable, + public Instance, + public ServerLifecycleNotifier { public: /** * @throw EnvoyException if initialization fails. */ - InstanceImpl(Init::Manager& init_manager, const Options& options, Event::TimeSystem& time_system, + InstanceBase(Init::Manager& init_manager, const Options& options, Event::TimeSystem& time_system, ListenerHooks& hooks, HotRestart& restarter, Stats::StoreRoot& store, Thread::BasicLockable& access_log_lock, Random::RandomGeneratorPtr&& random_generator, ThreadLocal::Instance& tls, @@ -243,7 +243,9 @@ class InstanceImpl final : Logger::Loggable, // initialize the server. This must be called before run(). void initialize(Network::Address::InstanceConstSharedPtr local_address, ComponentFactory& component_factory); - ~InstanceImpl() override; + ~InstanceBase() override; + + virtual void maybeCreateHeapShrinker() PURE; void run() override; @@ -396,7 +398,6 @@ class InstanceImpl final : Logger::Loggable, Http::ContextImpl http_context_; Router::ContextImpl router_context_; std::unique_ptr process_context_; - std::unique_ptr heap_shrinker_; // initialization_time is a histogram for tracking the initialization time across hot restarts // whenever we have support for histogram merge across hot restarts. Stats::TimespanPtr initialization_timer_; diff --git a/test/integration/server.cc b/test/integration/server.cc index 4995765c9481..91f491595a7c 100644 --- a/test/integration/server.cc +++ b/test/integration/server.cc @@ -12,6 +12,7 @@ #include "source/common/stats/thread_local_store.h" #include "source/common/thread_local/thread_local_impl.h" #include "source/server/hot_restart_nop_impl.h" +#include "source/server/instance_impl.h" #include "source/server/options_impl.h" #include "source/server/process_context_impl.h" diff --git a/test/server/server_fuzz_test.cc b/test/server/server_fuzz_test.cc index 93e269c00786..c2f2ee9a22c6 100644 --- a/test/server/server_fuzz_test.cc +++ b/test/server/server_fuzz_test.cc @@ -6,8 +6,8 @@ #include "source/common/common/random_generator.h" #include "source/common/network/address_impl.h" #include "source/common/thread_local/thread_local_impl.h" +#include "source/server/instance_impl.h" #include "source/server/listener_hooks.h" -#include "source/server/server.h" #include "test/fuzz/fuzz_runner.h" #include "test/integration/server.h" diff --git a/test/server/server_test.cc b/test/server/server_test.cc index 8e8bca32f2f0..699798a34e52 100644 --- a/test/server/server_test.cc +++ b/test/server/server_test.cc @@ -15,8 +15,8 @@ #include "source/common/protobuf/protobuf.h" #include "source/common/thread_local/thread_local_impl.h" #include "source/common/version/version.h" +#include "source/server/instance_impl.h" #include "source/server/process_context_impl.h" -#include "source/server/server.h" #include "test/common/config/dummy_config.pb.h" #include "test/common/stats/stat_test_utility.h"