Merge pull request rabbitmq#9875 from rabbitmq/logger-exchange-wait-f…

…or-init Logger exchange: fix race condition during initialisation
SimonUnge · Nov 9, 2023 · 09a95a5 · 09a95a5
2 parents ac09b6e + 55f724a
commit 09a95a5
Show file tree

Hide file tree

Showing 3 changed files with 43 additions and 3 deletions.
diff --git a/deps/rabbit/src/rabbit.erl b/deps/rabbit/src/rabbit.erl
@@ -1079,6 +1079,7 @@ stop(State) ->
         [] -> rabbit_prelaunch:set_stop_reason(normal);
         _  -> rabbit_prelaunch:set_stop_reason(State)
     end,
+    rabbit_db:clear_init_finished(),
     rabbit_boot_state:set(stopped),
     ok.
 

diff --git a/deps/rabbit/src/rabbit_db.erl b/deps/rabbit/src/rabbit_db.erl
@@ -14,13 +14,17 @@
 
 -include_lib("rabbit_common/include/logging.hrl").
 
+-define(PT_KEY_INIT_FINISHED, {?MODULE, node(), initialisation_finished}).
+
 -export([init/0,
          reset/0,
          force_reset/0,
          force_load_on_next_boot/0,
          is_virgin_node/0, is_virgin_node/1,
          dir/0,
-         ensure_dir_exists/0]).
+         ensure_dir_exists/0,
+         is_init_finished/0,
+         clear_init_finished/0]).
 
 %% Exported to be used by various rabbit_db_* modules
 -export([
@@ -62,6 +66,7 @@ init() ->
                "DB: initialization successeful",
                #{domain => ?RMQLOG_DOMAIN_DB}),
 
+            init_finished(),
             post_init(IsVirgin),
 
             ok;
@@ -102,6 +107,24 @@ init_using_khepri() ->
                #{domain => ?RMQLOG_DOMAIN_DB})
     end.
 
+init_finished() ->
+    %% Used during initialisation by rabbit_logger_exchange_h.erl
+    %% If an exchange logger is configured, it needs to declare the
+    %% exchange. For this, it requires the metadata store to be
+    %% initialised. The initialisation happens on a rabbit boot step,
+    %% after the second phase of the prelaunch where the logger is
+    %% configured.
+    %% Using this persistent term the logger exchange can delay
+    %% declaring the exchange until the metadata store is ready.
+    persistent_term:put(?PT_KEY_INIT_FINISHED, true).
+
+is_init_finished() ->
+    persistent_term:get(?PT_KEY_INIT_FINISHED, false).
+
+clear_init_finished() ->
+    _ = persistent_term:erase(?PT_KEY_INIT_FINISHED),
+    ok.
+
 -spec reset() -> Ret when
       Ret :: ok.
 %% @doc Resets the database and the node.

diff --git a/deps/rabbit/src/rabbit_logger_exchange_h.erl b/deps/rabbit/src/rabbit_logger_exchange_h.erl
@@ -124,11 +124,27 @@ start_setup_proc(#{config := InternalConfig} = Config) ->
     {ok, DefaultVHost} = application:get_env(rabbit, default_vhost),
     Exchange = rabbit_misc:r(DefaultVHost, exchange, ?LOG_EXCH_NAME),
     InternalConfig1 = InternalConfig#{exchange => Exchange},
-
-    Pid = spawn(fun() -> setup_proc(Config#{config => InternalConfig1}) end),
+    Pid = spawn(fun() ->
+                        wait_for_initial_pass(60),
+                        setup_proc(Config#{config => InternalConfig1})
+                end),
     InternalConfig2 = InternalConfig1#{setup_proc => Pid},
     Config#{config => InternalConfig2}.
 
+%% Declaring an exchange requires the metadata store to be ready
+%% which happens on a boot step after the second phase of the prelaunch.
+%% This function waits for the store initialisation.
+wait_for_initial_pass(0) ->
+    ok;
+wait_for_initial_pass(N) ->
+    case rabbit_db:is_init_finished() of
+        false ->
+            timer:sleep(1000),
+            wait_for_initial_pass(N - 1);
+        true ->
+            ok
+    end.
+
 setup_proc(
   #{config := #{exchange := #resource{name = Name,
                                       virtual_host = VHost}}} = Config) ->