Merge pull request #4450 from esl/components/dynamic_domains

Components/dynamic domains

big_tests/dynamic_domains.config

@@ -110,6 +110,11 @@
         {server, <<"domain.example.com">>},
         {host, <<"localhost">>},
         {password, <<"nicniema">>}]},
+    {astrid, [
+        {username, <<"astrid">>},
+        {server, <<"sogndal">>},
+        {host, <<"localhost">>},
+        {password, <<"doctor">>}]},
     {geralt, [
         {username, <<"geralt">>},
         {server, <<"domain.example.com">>},

big_tests/dynamic_domains.spec

@@ -107,6 +107,7 @@
 %% to minimise impact on other tests
 {suites, "tests", auth_methods_for_c2s_SUITE}.
 {suites, "tests", cluster_commands_SUITE}.
+{suites, "tests", component_SUITE}.
 {suites, "tests", dynamic_domains_SUITE}.
 {suites, "tests", graphql_server_SUITE}.
 {suites, "tests", last_SUITE}.

big_tests/tests/component_helper.erl

@@ -6,11 +6,10 @@
          disconnect_component/2,
          disconnect_components/2,
          connect_component_subdomain/1,
-         get_components/1, get_components/2, get_components/3, 
-         spec/2,
-         common/1,
-         common/2,
-         name/1
+         get_components/1,
+         second_port/1,
+         spec/1,
+         host/1
         ]).
 
 -export([component_start_stream/2,
@@ -128,33 +127,34 @@ component_start_stream_subdomain(Conn = #client{props = Props}, []) ->
 connect_component_subdomain(Component) ->
     connect_component(Component, component_start_stream_subdomain).
 
-spec(component_on_2, Config) ->
-    [{component, <<"yet_another_service">>}] ++ common(Config, mim2_service_port());
-spec(component_duplicate, Config) ->
-    [{component, <<"another_service">>}] ++ common(Config, mim2_service_port());
-spec(hidden_component, Config) ->
-    [{component, <<"hidden_component">>}] ++ common(Config, hidden_service_port());
-spec(kicking_component, Config) ->
-    [{component, <<"kicking_component">>}] ++ common(Config, kicking_service_port());
-spec(Other, Config) ->
-    [name(Other) | proplists:get_value(Other, Config, [])].
-
-common(Config) ->
-    common(Config, service_port()).
+host(CompSpec) ->
+    proplists:get_value(component, CompSpec).
+
+second_port(Spec) ->
+    lists:keyreplace(port, 1, Spec, {port, mim2_service_port()}).
+
+spec(vjud_component) ->
+    [{component, <<"vjud">>} | common(mim2_service_port())];
+spec(component_on_2) ->
+    [{component, <<"yet_another_service">>} | common(mim2_service_port())];
+spec(component_on_2) ->
+    [{component, <<"yet_another_service">>} | common(mim2_service_port())];
+spec(hidden_component) ->
+    [{component, <<"hidden_component">>} | common(hidden_service_port())];
+spec(kicking_component) ->
+    [{component, <<"kicking_component">>} | common(kicking_service_port())];
+spec(Other) ->
+    Prefix = integer_to_binary(erlang:unique_integer([monotonic, positive])),
+    Name = <<Prefix/binary, "_", (atom_to_binary(Other))/binary>>,
+    [{component, Name} |  common(service_port())].
 
 service_port() ->
     ct:get_config({hosts, mim, service_port}).
 
 get_components(Config) ->
-    Opts = common(Config),
-    get_components(Opts, Config).
-
-get_components(Opts, Config) ->
+    Opts = common(service_port()),
     Components = [component1, component2, vjud_component],
-    get_components(Opts, Components, Config).
-
-get_components(Opts, Components, Config) ->
-    [ {C, Opts ++ spec(C, Config)} || C <- Components ] ++ Config.
+    [ {C, Opts ++ spec(C)} || C <- Components ] ++ Config.
 
 kicking_service_port() ->
     ct:get_config({hosts, mim, kicking_service_port}).
@@ -165,17 +165,8 @@ hidden_service_port() ->
 mim2_service_port() ->
     ct:get_config({hosts, mim2, service_port}).
 
-common(_Config, Port) ->
+common(Port) ->
     [{server, ct:get_config({hosts, mim, domain})},
-     {host, ct:get_config({hosts, mim, domain})},
+     {host, <<"localhost">>},
      {password, <<"secret">>},
      {port, Port}].
-
-name(component1) ->
-    {component, <<"test_service">>};
-name(component2) ->
-    {component, <<"another_service">>};
-name(vjud_component) ->
-    {component, <<"vjud">>};
-name(kicking_component) ->
-    {component, <<"kicking_component">>}.

big_tests/tests/disco_and_caps_SUITE.erl

@@ -32,8 +32,7 @@ caps_test_cases() ->
      user_can_query_server_caps_via_disco].
 
 extra_feature_test_cases() ->
-    [user_can_query_extra_domains,
-     user_can_query_server_info].
+    [user_can_query_server_info].
 
 init_per_suite(C) ->
     instrument_helper:start(instrument_helper:declared_events(mod_disco)),
@@ -155,15 +154,6 @@ user_cannot_query_friend_resources_with_unknown_node(Config) ->
         escalus:assert(is_stanza_from, [BobJid], Stanza)
     end).
 
-user_can_query_extra_domains(Config) ->
-    escalus:fresh_story(Config, [{alice, 1}], fun(Alice) ->
-        Server = escalus_client:server(Alice),
-        escalus:send(Alice, escalus_stanza:service_discovery(Server)),
-        Stanza = escalus:wait_for_stanza(Alice),
-        escalus:assert(has_service, [extra_domain()], Stanza),
-        escalus:assert(is_stanza_from, [domain()], Stanza)
-    end).
-
 user_can_query_server_features(Config) ->
     escalus:fresh_story(Config, [{alice, 1}], fun(Alice) ->
         Server = escalus_client:server(Alice),
@@ -209,8 +199,7 @@ required_modules(disco_with_extra_features) ->
     [{mod_disco, mod_config(mod_disco, extra_disco_opts())}].
 
 extra_disco_opts() ->
-    #{extra_domains => [extra_domain()],
-      server_info => [server_info(abuse, #{}),
+    #{server_info => [server_info(abuse, #{}),
                       server_info(admin, #{modules => [mod_disco]}),
                       server_info(sales, #{modules => [mod_pubsub]})]}.
 
@@ -219,9 +208,6 @@ get_form_fields(Stanza) ->
                                {element_with_ns, <<"x">>, ?NS_DATA_FORMS},
                                {element, <<"field">>}]).
 
-extra_domain() ->
-    <<"eXtra.example.com">>.
-
 server_info(Type, Extra) ->
     maps:merge(#{name => name(Type), urls => urls(Type)}, Extra).
 

big_tests/tests/mod_global_distrib_SUITE.erl

@@ -644,7 +644,7 @@ test_components_in_different_regions(_Config) ->
 %% Ordinary user is not able to discover the hidden component from GD
 test_hidden_component_disco_in_different_region(Config) ->
     %% Hidden component from component_SUITE connects to mim1/europe_node1
-    HiddenComponentConfig = component_helper:spec(hidden_component, Config),
+    HiddenComponentConfig = component_helper:spec(hidden_component),
     {_HiddenComp, HiddenAddr, _} = component_helper:connect_component(HiddenComponentConfig),
 
     escalus:fresh_story(

big_tests/tests/service_mongoose_system_metrics_SUITE.erl

@@ -18,13 +18,7 @@
     instance_id = <<>>,
     app_secret = <<>>}).
 
--import(distributed_helper, [mim/0, mim2/0, mim3/0, rpc/4,
-                             require_rpc_nodes/1
-                            ]).
-
--import(component_helper, [connect_component/1,
-                           disconnect_component/2,
-                           get_components/1]).
+-import(distributed_helper, [mim/0, mim2/0, mim3/0, rpc/4, require_rpc_nodes/1]).
 
 -import(domain_helper, [host_type/0]).
 -import(config_parser_helper, [mod_config/2, config/2]).
@@ -120,9 +114,8 @@ init_per_testcase(system_metrics_are_reported_to_configurable_google_analytics,
     Config;
 init_per_testcase(xmpp_components_are_reported, Config) ->
     create_events_collection(),
-    Config1 = get_components(Config),
     enable_system_metrics(mim()),
-    Config1;
+    Config;
 init_per_testcase(xmpp_stanzas_counts_are_reported = CN, Config) ->
     create_events_collection(),
     enable_system_metrics(mim()),
@@ -248,12 +241,12 @@ mongoose_version_is_reported(_Config) ->
 cluster_uptime_is_reported(_Config) ->
     wait_helper:wait_until(fun is_cluster_uptime_reported/0, true).
 
-xmpp_components_are_reported(Config) ->
-    CompOpts = ?config(component1, Config),
-    {Component, Addr, _} = connect_component(CompOpts),
+xmpp_components_are_reported(_Config) ->
+    CompOpts = component_helper:spec(component1),
+    {Component, Addr, _} = component_helper:connect_component(CompOpts),
     wait_helper:wait_until(fun are_xmpp_components_reported/0, true),
     wait_helper:wait_until(fun more_than_one_component_is_reported/0, true),
-    disconnect_component(Component, Addr).
+    component_helper:disconnect_component(Component, Addr).
 
 api_are_reported(_Config) ->
     wait_helper:wait_until(fun is_api_reported/0, true).

doc/developers-guide/Stanza-routing.md

@@ -33,8 +33,8 @@ The default behaviour is the following:
 
 * `mongoose_router_global`: runs a global `filter_packet` hook.
 * `mongoose_router_localdomain`: if there is a local route registered for the destination domain (i.e. there is an entry in the `mongoose_router` ETS table), routes the stanza to it. When the recipient's domain is checked for the first time, the corresponding route is not registered yet, because the routes are added lazily - see `mongoose_router_dynamic_domains`.
-* `mongoose_router_external_localnode`: if there is an external component registered for the destination domain on the current node, routes the stanza to it. Such components are stored in the Mnesia table `external_component`, which is not replicated in the cluster.
-* `mongoose_router_external`: if there is an external component registered for the destination domain on any node in the cluster, routes the stanza to it. Such components are stored in the Mnesia table `external_component_global`, which is replicated among all cluster nodes.
+* `mongoose_router_external_localnode`: if there is an external component registered for the destination domain on the current node, routes the stanza to it.
+* `mongoose_router_external`: if there is an external component registered for the destination domain on any node in the cluster, routes the stanza to it.
 * `mongoose_router_dynamic_domains`: if the recipient's domain is hosted by the local server, a route is added for it, and the stanza is routed locally.
 * `ejabberd_s2s`: tries to find or establish a connection to another server and send the stanza there.
 

doc/listeners/listen-components.md

@@ -4,9 +4,11 @@ Interface for external services acting as XMPP components ([XEP-0114: Jabber Com
 
 According to [XEP-0114: Jabber Component Protocol](http://xmpp.org/extensions/xep-0114.html) the component's hostname should be given in the <stream:stream> element.
 
-!!! warning
-    This interface does not support [dynamic domains](../configuration/general.md#generalhost_types).
-    Do not use them both at the same time.
+!!! Note
+    The component might register _any_ domain, which might not necessarily be a static nor a dynamic domain, nor subdomain, recognised by the MongooseIM router. For routing to work, the modules `mongoose_router_external_localnode` and `mongoose_router_external` must be enabled in the [`general.routing_modules`](../configuration/general.md#generalrouting_modules) section.
+
+    Note the order of the routing modules: if a component is meant to supplant a domain served regularly by the MongooseIM server, the external routers should be ordered with higher priority.
+
 
 ## Configuration options
 

doc/modules/mod_disco.md

@@ -10,14 +10,6 @@ Implements [XEP-0030: Service Discovery](http://xmpp.org/extensions/xep-0030.htm
 Strategy to handle incoming stanzas. For details, please refer to
 [IQ processing policies](../configuration/Modules.md#iq-processing-policies).
 
-### `modules.mod_disco.extra_domains`
-* **Syntax:** array of strings, valid domain names
-* **Default:** no extra domains
-* **Example:** `extra_domains = ["custom_domain"]`
-
-Adds domains that are not registered with other means to a local item announcement (response to `http://jabber.org/protocol/disco#items` IQ get).
-Please note that `mod_disco` doesn't verify these domains, so if no handlers are registered later for them, a client will receive a `service-unavailable` error for every stanza sent to one of these hosts.
-
 ### `modules.mod_disco.server_info`
 * **Syntax:** array of tables described below
 * **Default:** no additional server info
@@ -50,7 +42,6 @@ will still receive full disco results.
 ```toml
 [modules.mod_disco]
   iqdisc.type = "one_queue"
-  extra_domains = ["some_domain", "another_domain"]
   server_info = [
     {name = "abuse-address", urls = ["[email protected]"]},
     {name = "friendly-spirits", urls = ["spirit1@localhost", "spirit2@localhost"], modules = ["mod_muc", "mod_disco"]}

src/component/mongoose_component_connection.erl

@@ -90,7 +90,6 @@ handle_event(internal, {connect, {SocketModule, SocketOpts}}, connect,
     {next_state, wait_for_stream, StateData1, state_timeout(LOpts)};
 handle_event(internal, #xmlstreamstart{name = Name, attrs = Attrs}, wait_for_stream, StateData) ->
     StreamStart = #xmlel{name = Name, attrs = Attrs},
-    execute_element_event(component_element_in, StreamStart, StateData),
     handle_stream_start(StateData, StreamStart);
 handle_event(internal, #xmlel{name = <<"handshake">>} = El, wait_for_handshake, StateData) ->
     execute_element_event(component_element_in, El, StateData),
@@ -176,10 +175,10 @@ handle_socket_packet(StateData = #component_data{parser = Parser}, Packet) ->
     end.
 
 -spec handle_socket_elements(data(), [exml_stream:element()], non_neg_integer()) -> fsm_res().
-handle_socket_elements(StateData = #component_data{shaper = Shaper}, Elements, Size) ->
+handle_socket_elements(StateData = #component_data{lserver = LServer, shaper = Shaper}, Elements, Size) ->
     {NewShaper, Pause} = mongoose_shaper:update(Shaper, Size),
     [mongoose_instrument:execute(
-       component_xmpp_element_size_in, #{}, #{byte_size => exml:xml_size(El)})
+       component_xmpp_element_size_in, #{}, #{byte_size => exml:xml_size(El), lserver => LServer})
      || El <- Elements],
     NewStateData = StateData#component_data{shaper = NewShaper},
     MaybePauseTimeout = maybe_pause(NewStateData, Pause),
@@ -212,6 +211,7 @@ handle_stream_start(S0, StreamStart) ->
             IsSubdomain = <<"true">> =:= exml_query:attr(StreamStart, <<"is_subdomain">>, <<>>),
             S1 = S0#component_data{lserver = LServer, is_subdomain = IsSubdomain},
             send_header(S1),
+            execute_element_event(component_element_in, StreamStart, S1),
             {next_state, wait_for_handshake, S1, state_timeout(S1)};
         {?NS_COMPONENT_ACCEPT, error} ->
             stream_start_error(S0, mongoose_xmpp_errors:host_unknown());
@@ -285,24 +285,6 @@ handle_stream_established(StateData, #xmlel{name = Name} = El) ->
     end,
     keep_state_and_data.
 
--spec register_routes(data()) -> any().
-register_routes(StateData) ->
-    #component_data{listener_opts = #{hidden_components := AreHidden}} = StateData,
-    Routes = get_routes(StateData),
-    Handler = mongoose_packet_handler:new(mongoose_component, #{pid => self()}),
-    mongoose_component:register_components(Routes, node(), Handler, AreHidden).
-
--spec get_routes(data()) -> [jid:lserver()].
-get_routes(#component_data{lserver = Subdomain, is_subdomain = true}) ->
-    Hosts = mongoose_config:get_opt(hosts),
-    component_routes(Subdomain, Hosts);
-get_routes(#component_data{lserver = Host}) ->
-    [Host].
-
--spec component_routes(binary(), [jid:lserver()]) -> [jid:lserver()].
-component_routes(Subdomain, Hosts) ->
-    [<<Subdomain/binary, ".", Host/binary>> || Host <- Hosts].
-
 -spec try_register_routes(data(), retries()) -> fsm_res().
 try_register_routes(StateData, Retries) ->
     case register_routes(StateData) of
@@ -320,6 +302,26 @@ try_register_routes(StateData, Retries) ->
             handle_registration_conflict(ConflictBehaviour, RoutesInfo, StateData, Retries)
     end.
 
+-spec register_routes(data()) -> any().
+register_routes(StateData) ->
+    #component_data{listener_opts = #{hidden_components := AreHidden}} = StateData,
+    Routes = get_routes(StateData),
+    Handler = mongoose_packet_handler:new(mongoose_component, #{pid => self()}),
+    mongoose_component:register_components(Routes, node(), Handler, AreHidden).
+
+-spec get_routes(data()) -> [jid:lserver()].
+get_routes(#component_data{lserver = Subdomain, is_subdomain = true}) ->
+    StaticDomains = mongoose_domain_api:get_all_static(),
+    DynamicDomains = mongoose_domain_api:get_all_dynamic(),
+    [ component_route(Subdomain, Domain) || {Domain, _} <- StaticDomains ]
+    ++ [ component_route(Subdomain, Domain) || {Domain, _} <- DynamicDomains ];
+get_routes(#component_data{lserver = Host}) ->
+    [Host].
+
+-spec component_route(binary(), jid:lserver()) -> jid:lserver().
+component_route(Subdomain, Host) ->
+    <<Subdomain/binary, ".", Host/binary>>.
+
 handle_registration_conflict(kick_old, RoutesInfo, StateData, Retries) when Retries > 0 ->
     %% see lookup_routes
     Pids = lists:usort(routes_info_to_pids(RoutesInfo)),
@@ -424,11 +426,11 @@ close_parser(#component_data{parser = Parser}) ->
 -spec send_xml(data(), exml_stream:element() | [exml_stream:element()]) -> maybe_ok().
 send_xml(Data, XmlElement) when is_tuple(XmlElement) ->
     send_xml(Data, [XmlElement]);
-send_xml(#component_data{socket = Socket} = StateData, XmlElements) when is_list(XmlElements) ->
+send_xml(#component_data{lserver = LServer, socket = Socket} = StateData, XmlElements) when is_list(XmlElements) ->
     [ begin
           execute_element_event(component_element_out, El, StateData),
           mongoose_instrument:execute(
-            component_xmpp_element_size_out, #{}, #{byte_size => exml:xml_size(El)})
+            component_xmpp_element_size_out, #{}, #{byte_size => exml:xml_size(El), lserver => LServer})
       end || El <- XmlElements],
     mongoose_component_socket:send_xml(Socket, XmlElements).
 

src/domain/mongoose_domain_api.erl

@@ -41,9 +41,6 @@
 %% For testing
 -export([get_all_dynamic/0]).
 
--ignore_xref([get_all_static/0]).
--ignore_xref([get_all_dynamic/0]).
-
 -type status() :: enabled | disabled | deleting.
 -type domain() :: jid:lserver().
 -type host_type() :: mongooseim:host_type().