From 72ed73c3233bde9c0ecc05af739f467a2eed5900 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Mon, 4 Dec 2023 02:26:00 +0100 Subject: [PATCH 01/10] feat: Declarative metrics and autorates --- src/behaviors/emqttb_behavior_conn.erl | 12 +- src/behaviors/emqttb_behavior_pub.erl | 48 ++---- src/behaviors/emqttb_behavior_sub.erl | 13 +- src/conf/emqttb_conf.erl | 33 ++-- src/conf/emqttb_conf_model.erl | 2 +- src/conf/emqttb_mt_scenario.erl | 12 +- src/framework/emqttb_app.erl | 5 +- src/framework/emqttb_autorate.erl | 122 +++++++++++++-- src/framework/emqttb_group.erl | 6 +- src/framework/emqttb_scenario.erl | 15 +- src/metrics/emqttb_metrics.erl | 148 +++++++++++++++--- src/restapi/emqttb_http_scenario_conf.erl | 4 +- .../emqttb_scenario_persistent_session.erl | 38 +++-- src/scenarios/emqttb_scenario_pub.erl | 33 ++-- src/scenarios/emqttb_scenario_pubsub_fwd.erl | 27 ++-- test/emqttb_worker_SUITE.erl | 12 +- 16 files changed, 388 insertions(+), 142 deletions(-) diff --git a/src/behaviors/emqttb_behavior_conn.erl b/src/behaviors/emqttb_behavior_conn.erl index 4c34534..a3b6106 100644 --- a/src/behaviors/emqttb_behavior_conn.erl +++ b/src/behaviors/emqttb_behavior_conn.erl @@ -20,12 +20,18 @@ %% behavior callbacks: -export([init_per_group/2, init/1, handle_message/3, terminate/2]). --export_type([]). +-export_type([prototype/0, config/0]). %%================================================================================ %% Type declarations %%================================================================================ +-type config() :: #{ expiry => non_neg_integer() + , clean_start => boolean() + }. + +-type prototype() :: {?MODULE, config()}. + %%================================================================================ %% behavior callbacks %%================================================================================ @@ -33,13 +39,11 @@ init_per_group(_Group, Opts) -> Expiry = maps:get(expiry, Opts, 0), CleanStart = maps:get(clean_start, Opts, true), - HostShift = maps:get(host_shift, Opts, 0), - HostSelection = maps:get(host_selection, Opts, random), #{ expiry => Expiry , clean_start => CleanStart }. -init(Opts0 = #{clean_start := CleanStart, expiry := Expiry}) -> +init(#{clean_start := CleanStart, expiry := Expiry}) -> Props = case Expiry of undefined -> #{}; _ -> #{'Session-Expiry-Interval' => Expiry} diff --git a/src/behaviors/emqttb_behavior_pub.erl b/src/behaviors/emqttb_behavior_pub.erl index 8092ae1..772d5c6 100644 --- a/src/behaviors/emqttb_behavior_pub.erl +++ b/src/behaviors/emqttb_behavior_pub.erl @@ -23,7 +23,7 @@ %% behavior callbacks: -export([init_per_group/2, init/1, handle_message/3, terminate/2]). --export_type([]). +-export_type([prototype/0, config/0]). -import(emqttb_worker, [send_after/2, send_after_rand/2, repeat/2, my_group/0, my_id/0, my_clientid/0, my_cfg/1, connect/2]). @@ -34,15 +34,20 @@ %% Type declarations %%================================================================================ --type config() :: #{ topic := binary() - , pubinterval := counters:counters_ref() - , qos := emqttb:qos() - , retain := boolean() - , set_latency := lee:key() - , msg_size := non_neg_integer() - , metadata => boolean() +-type config() :: #{ topic := binary() + , n_published := lee:model_key() + , pubinterval := lee:model_key() + , msg_size := non_neg_integer() + , qos := emqttb:qos() + , set_latency := lee:key() + , retain => boolean() + , metadata => boolean() + , host_shift => integer() + , host_selection => random | round_robin }. +-type prototype() :: {?MODULE, config()}. + %%================================================================================ %% API %%================================================================================ @@ -61,32 +66,25 @@ parse_metadata(<>) -> init_per_group(Group, #{ topic := Topic + , n_published := NPublishedMetricKey , pubinterval := PubInterval - , pub_autorate := AutorateConf , msg_size := MsgSize , qos := QoS - , retain := Retain , set_latency := SetLatencyKey } = Conf) when is_binary(Topic), is_integer(MsgSize), is_list(SetLatencyKey) -> AddMetadata = maps:get(metadata, Conf, false), - PubCnt = emqttb_metrics:new_counter(?CNT_PUB_MESSAGES(Group), - [ {help, <<"Number of published messages">>} - , {labels, [group]} - ]), + PubCnt = emqttb_metrics:from_model(NPublishedMetricKey), emqttb_worker:new_opstat(Group, ?AVG_PUB_TIME), - {auto, PubRate} = emqttb_autorate:ensure(#{ id => my_autorate(Group) - , error => fun() -> error_fun(SetLatencyKey, Group) end - , init_val => PubInterval - , conf_root => AutorateConf - }), + {auto, PubRate} = emqttb_autorate:from_model(PubInterval), MetadataSize = case AddMetadata of true -> (32 + 32 + 64) div 8; false -> 0 end, HostShift = maps:get(host_shift, Conf, 0), HostSelection = maps:get(host_selection, Conf, random), + Retain = maps:get(retain, Conf, false), #{ topic => Topic , message => message(max(0, MsgSize - MetadataSize)) , pub_opts => [{qos, QoS}, {retain, Retain}] @@ -135,18 +133,6 @@ terminate(_Shared, Conn) -> %% Internal functions %%================================================================================ -error_fun(SetLatencyKey, Group) -> - SetLatency = ?CFG(SetLatencyKey), - AvgWindow = 250, - %% Note that dependency of latency on publish interval is inverse: - %% lesser interval -> messages are sent more often -> more load -> more latency - %% - %% So the control must be reversed, and error is the negative of what one usually - %% expects: - %% Current - Target instead of Target - Current. - (emqttb_metrics:get_rolling_average(?GROUP_OP_TIME(Group, ?AVG_PUB_TIME), AvgWindow) - - erlang:convert_time_unit(SetLatency, millisecond, microsecond)). - my_autorate(Group) -> list_to_atom(atom_to_list(Group) ++ ".pub.rate"). diff --git a/src/behaviors/emqttb_behavior_sub.erl b/src/behaviors/emqttb_behavior_sub.erl index 1a48ab0..2a20bee 100644 --- a/src/behaviors/emqttb_behavior_sub.erl +++ b/src/behaviors/emqttb_behavior_sub.erl @@ -20,12 +20,23 @@ %% behavior callbacks: -export([init_per_group/2, init/1, handle_message/3, terminate/2]). --export_type([]). +-export_type([prototype/0, config/0]). %%================================================================================ %% Type declarations %%================================================================================ +-type config() :: #{ topic := binary() + , qos := 0..2 + , clean_start => boolean() + , expiry => non_neg_integer() | undefined + , host_shift => integer() + , host_selection => _ + , parse_metadata => boolean() + }. + +-type prototype() :: {?MODULE, config()}. + -define(CNT_SUB_MESSAGES(GRP), {emqttb_received_messages, GRP}). -define(CNT_SUB_LATENCY(GRP), {emqttb_e2e_latency, GRP}). -define(AVG_SUB_TIME, subscribe). diff --git a/src/conf/emqttb_conf.erl b/src/conf/emqttb_conf.erl index 4fd0d72..a5b0901 100644 --- a/src/conf/emqttb_conf.erl +++ b/src/conf/emqttb_conf.erl @@ -16,7 +16,7 @@ -module(emqttb_conf). %% API: --export([load_conf/0, get/1, list_keys/1, reload/0, patch/1]). +-export([load_model/0, load_conf/0, get/1, list_keys/1, reload/0, patch/1]). -export([compile_model/1]). -export_type([]). @@ -33,22 +33,11 @@ %% API funcions %%================================================================================ -load_conf() -> +load_model() -> case compile_model(emqttb_conf_model:model()) of {ok, Model} -> - Storage = lee_storage:new(lee_persistent_term_storage, ?CONF_STORE), persistent_term:put(?MODEL_STORE, Model), - case lee:init_config(Model, Storage) of - {ok, _Data, _Warnings} -> - maybe_load_repeat(), - maybe_load_conf_file(), - maybe_dump_conf(), - ok; - {error, Errors, _Warnings} -> - [logger:critical(E) || E <- Errors], - emqttb:setfail("invalid configuration"), - emqttb:terminate() - end; + ok; {error, Errors} -> logger:critical("Configuration model is invalid!"), [logger:critical(E) || E <- Errors], @@ -56,6 +45,20 @@ load_conf() -> emqttb:terminate() end. +load_conf() -> + Storage = lee_storage:new(lee_persistent_term_storage, ?CONF_STORE), + case lee:init_config(?MYMODEL, Storage) of + {ok, _Data, _Warnings} -> + maybe_load_repeat(), + maybe_load_conf_file(), + maybe_dump_conf(), + ok; + {error, Errors, _Warnings} -> + [logger:critical(E) || E <- Errors], + emqttb:setfail("invalid configuration"), + emqttb:terminate() + end. + compile_model(Model) -> MTs = metamodel(), lee_model:compile(MTs, [Model]). @@ -134,6 +137,8 @@ metamodel() -> , file => "/etc/emqttb/emqttb.conf" }) , lee_metatype:create(emqttb_mt_scenario) + , lee_metatype:create(emqttb_metrics) + , lee_metatype:create(emqttb_autorate) ]. cli_args_getter() -> diff --git a/src/conf/emqttb_conf_model.erl b/src/conf/emqttb_conf_model.erl index df6927a..93a70ac 100644 --- a/src/conf/emqttb_conf_model.erl +++ b/src/conf/emqttb_conf_model.erl @@ -246,7 +246,7 @@ model() -> }, emqttb_worker:model()} , autorate => - {[map, cli_action, default_instance], + {[map, cli_action], #{ cli_operand => "a" , key_elements => [[id]] }, diff --git a/src/conf/emqttb_mt_scenario.erl b/src/conf/emqttb_mt_scenario.erl index 47e3753..ce1e450 100644 --- a/src/conf/emqttb_mt_scenario.erl +++ b/src/conf/emqttb_mt_scenario.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -20,7 +20,7 @@ -behavior(lee_metatype). %% behavior callbacks: --export([names/1, post_patch/5]). +-export([names/1]). -include("emqttb.hrl"). @@ -31,7 +31,7 @@ names(_) -> [scenario]. -post_patch(scenario, _, _, _, {set, [?SK(Scenario)], _}) -> - emqttb_scenario:run(Scenario); -post_patch(scenario, _, _, _, {rm, [?SK(Scenario)]}) -> - emqttb_scenario:stop(Scenario). +%% post_patch(scenario, _, _, _, {set, [?SK(Scenario)], _}) -> +%% emqttb_scenario:run(Scenario); +%% post_patch(scenario, _, _, _, {rm, [?SK(Scenario)]}) -> +%% emqttb_scenario:stop(Scenario). diff --git a/src/framework/emqttb_app.erl b/src/framework/emqttb_app.erl index 6e70e9e..cb10137 100644 --- a/src/framework/emqttb_app.erl +++ b/src/framework/emqttb_app.erl @@ -13,8 +13,11 @@ -include("emqttb.hrl"). start(_StartType, _StartArgs) -> - Sup = emqttb_sup:start_link(), + emqttb_conf:load_model(), emqttb_conf:load_conf(), + Sup = emqttb_sup:start_link(), + emqttb_autorate:create_autorates(), + emqttb_scenario:run_scenarios(), CLIArgs = application:get_env(?APP, cli_args, []), emqttb_grafana:annotate(["Start emqttb " | lists:join($ , CLIArgs)]), post_init(), diff --git a/src/framework/emqttb_autorate.erl b/src/framework/emqttb_autorate.erl index d895ac9..20c7f0c 100644 --- a/src/framework/emqttb_autorate.erl +++ b/src/framework/emqttb_autorate.erl @@ -14,6 +14,9 @@ %% limitations under the License. %%-------------------------------------------------------------------- -module(emqttb_autorate). + +-behavior(gen_server). +-behavior(lee_metatype). %% This module uses velocity PI controller to automatically adjust %% rate of something to minimize error function. %% @@ -21,13 +24,15 @@ %% https://controlguru.com/integral-reset-windup-jacketing-logic-and-the-velocity-pi-form/ %% API: --export([ensure/1, get_counter/1, reset/2, info/0]). +-export([ensure/1, get_counter/1, reset/2, info/0, create_autorates/0]). -%% behavior callbacks: +%% gen_server callbacks: -export([init/1, handle_call/3, handle_cast/2, handle_info/2]). +%% lee_metatype callbacks: +-export([names/1, metaparams/1, meta_validate_node/4, meta_validate/2]). %% internal exports: --export([start_link/1, model/0]). +-export([start_link/1, model/0, from_model/1]). -export_type([config/0, scram_fun/0, info/0]). @@ -49,8 +54,10 @@ %% `false' if everything is normal. -type scram_fun() :: fun((_IsOverride :: boolean()) -> {true, integer()} | false). +-type id() :: atom(). + -type config() :: - #{ id := atom() + #{ id := id() , conf_root := atom() , error := fun(() -> number()) , scram => scram_fun() @@ -67,6 +74,9 @@ , sum := number() }. +-define(id(ID), {n, l, {?MODULE, ID}}). +-define(via(ID), {via, gproc, ?id(ID)}). + %%================================================================================ %% API funcions %%================================================================================ @@ -76,14 +86,19 @@ ensure(Conf) -> {ok, Pid} = emqttb_autorate_sup:ensure(Conf#{parent => self()}), {auto, get_counter(Pid)}. --spec get_counter(atom() | pid()) -> counters:counters_ref(). -get_counter(Id) -> - gen_server:call(Id, get_counter). +-spec get_counter(lee:key() | id() | pid()) -> counters:counters_ref(). +get_counter(Pid) when is_pid(Pid) -> + gen_server:call(Pid, get_counter); +get_counter(Id) when is_atom(Id) -> + gen_server:call(?via(Id), get_counter); +get_counter(Key) -> + #mnode{metaparams = #{autorate_id := Id}} = lee_model:get(Key, ?MYMODEL), + gen_server:call(?via(Id), get_counter). %% Set the current value to the specified value -spec reset(atom() | pid(), integer()) -> ok. reset(Id, Val) -> - gen_server:call(Id, {reset, Val}). + gen_server:call(?via(Id), {reset, Val}). -spec info() -> [info()]. info() -> @@ -100,7 +115,7 @@ info() -> %%================================================================================ start_link(Conf = #{id := Id}) -> - gen_server:start_link({local, Id}, ?MODULE, Conf, []). + gen_server:start_link(?via(Id), ?MODULE, Conf, []). model() -> #{ id => @@ -110,6 +125,14 @@ model() -> , cli_operand => "autorate" , cli_short => $a }} + , set_point => + {[value, cli_param], + #{ oneliner => "Value that the autorate will try to approach" + , type => number() + , default => 0 + , cli_operand => "setpoint" + , cli_short => $s + }} , min => {[value, cli_param], #{ oneliner => "Minimum value of the controlled parameter" @@ -159,7 +182,62 @@ model() -> }. %%================================================================================ -%% behavior callbacks +%% lee_metatype callbacks and helpers +%%================================================================================ + +names(_) -> + [autorate]. + +metaparams(autorate) -> + %% TBD: make it possible to configure alternative targets. + [ {mandatory, autorate_id, atom()} + , {mandatory, process_variable, lee:model_key()} + , {optional, error_coeff, number()} + ]. + + +meta_validate_node(autorate, Model, _Key, #mnode{metaparams = #{process_variable := ProcVarKey}}) -> + Error = {["process_variable parameter must point at a node of `metric' metatype"], []}, + try lee_model:get(ProcVarKey, Model) of + #mnode{metatypes = MTs} -> + case lists:member(metric, MTs) of + true -> {[], []}; + false -> Error + end + catch + _:_ -> Error + end. + +meta_validate(autorate, Model) -> + Ids = [begin + #mnode{metaparams = #{autorate_id := Id}} = lee_model:get(Key, Model), + Id + end || Key <- lee_model:get_metatype_index(autorate, Model)], + case length(lists:usort(Ids)) =:= length(Ids) of + false -> {["Autorate IDs must be unique"], [], []}; + true -> {[], [], []} + end. + +create_autorates() -> + [begin + #mnode{metaparams = MPs} = lee_model:get(Key, ?MYMODEL), + Id = ?m_attr(autorate, autorate_id, MPs), + emqttb_conf:patch([{set, [autorate, {Id}], []}]), + {ok, _} = emqttb_autorate_sup:ensure(#{ id => Id + , error => make_error_fun(Key) + , init_val => ?CFG(Key) + , conf_root => Id + }) + end || Key <- lee_model:get_metatype_index(autorate, ?MYMODEL)], %% TODO: wrong + ok. + + +-spec from_model(lee:key()) -> {auto, counter:counters_ref()}. +from_model(ModelKey) -> + {auto, get_counter(ModelKey)}. + +%%================================================================================ +%% gen_server callbacks %%================================================================================ -record(s, @@ -285,3 +363,27 @@ clamp(Val, _, _) -> my_cfg(ConfRoot, Key) -> ?CFG([autorate, {ConfRoot} | Key]). + +-spec make_error_fun(lee:key()) -> fun(() -> number()). +make_error_fun(Key) -> + ModelKey = lee_model:get_model_key(Key), + #mnode{metaparams = MP} = lee_model:get(ModelKey, ?MYMODEL), + ProcessVarKey = ?m_attr(autorate, process_variable, MP), + Id = ?m_attr(autorate, autorate_id, MP), + Coeff = ?m_attr(autorate, error_coeff, MP, 1), + MetricKey = emqttb_metrics:from_model(ProcessVarKey), + %% + #mnode{metaparams = PVarMPs} = lee_model:get(ProcessVarKey, ?MYMODEL), + case ?m_attr(metric, metric_type, PVarMPs) of + counter -> + fun() -> + SetPoint = my_cfg(Id, [set_point]), + Coeff * (SetPoint - emqttb_metrics:get_counter(MetricKey)) + end; + rolling_average -> + fun() -> + AvgWindow = 250, + SetPoint = my_cfg(Id, [set_point]), + Coeff * (SetPoint - emqttb_metrics:get_rolling_average(MetricKey, AvgWindow)) + end + end. diff --git a/src/framework/emqttb_group.erl b/src/framework/emqttb_group.erl index ce75537..aba613b 100644 --- a/src/framework/emqttb_group.erl +++ b/src/framework/emqttb_group.erl @@ -35,10 +35,14 @@ %% Type declarations %%================================================================================ +-type prototype() :: emqttb_behavior_pub:prototype() + | emqttb_behavior_sub:prototype() + | emqttb_behavior_conn:prototype(). + -type group_config() :: #{ id := atom() , client_config := atom() - , behavior := {module(), map()} + , behavior := prototype() , parent => pid() , autorate => atom() , start_n => integer() diff --git a/src/framework/emqttb_scenario.erl b/src/framework/emqttb_scenario.erl index dee99ca..edb16ed 100644 --- a/src/framework/emqttb_scenario.erl +++ b/src/framework/emqttb_scenario.erl @@ -25,7 +25,7 @@ -export([init/1, handle_call/3, handle_cast/2, handle_continue/2, terminate/2]). %% internal exports: --export([start_link/1]). +-export([start_link/1, run_scenarios/0]). -export_type([result/0]). @@ -158,6 +158,17 @@ info() -> start_link(Module) -> gen_server:start_link({local, Module}, ?MODULE, [Module], []). +-spec run_scenarios() -> ok. +run_scenarios() -> + lists:foreach( + fun([scenarios, Name]) -> + case lee:list(?MYCONF, [?SK(Name)]) of + [] -> ok; + [_] -> emqttb_scenario:run(Name) + end + end, + lee_model:get_metatype_index(scenario, ?MYMODEL)). + %%================================================================================ %% gen_server callbacks %%================================================================================ @@ -184,7 +195,7 @@ handle_call(_, _, S) -> handle_cast(_, S) -> {notrepy, S}. -terminate(_, State) -> +terminate(_, _State) -> persistent_term:erase(?SCENARIO_GROUP_LEADER(group_leader())). %%================================================================================ diff --git a/src/metrics/emqttb_metrics.erl b/src/metrics/emqttb_metrics.erl index 96ff99f..72fe34a 100644 --- a/src/metrics/emqttb_metrics.erl +++ b/src/metrics/emqttb_metrics.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -16,9 +16,11 @@ -module(emqttb_metrics). -behavior(gen_server). +-behavior(lee_metatype). %% API: --export([new_counter/2, counter_inc/2, counter_dec/2, get_counter/1, +-export([from_model/1, opstat/2, + new_counter/2, counter_inc/2, counter_dec/2, get_counter/1, new_gauge/2, gauge_set/2, gauge_ref/1, new_rolling_average/2, rolling_average_observe/2, get_rolling_average/1, get_rolling_average/2 @@ -27,6 +29,9 @@ %% gen_server callbacks: -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2]). +%% lee_metatype callbacks: +-export([names/1, metaparams/1, meta_validate/2]). + %% internal exports: -export([start_link/0]). @@ -34,11 +39,18 @@ -compile({inline, [counter_inc/2, counter_dec/2]}). +-include("emqttb.hrl"). +-include("../framework/emqttb_internal.hrl"). +-include_lib("typerefl/include/types.hrl"). +-include_lib("lee/include/lee.hrl"). + %%================================================================================ %% Type declarations %%================================================================================ --type key() :: {atom(), atom() | list() | tuple()}. +-type metric_id() :: {atom(), atom() | list() | tuple()}. + +-reflect_type([metric_id/0]). -define(SERVER, ?MODULE). @@ -57,58 +69,83 @@ start_link() -> gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). +%% Declarative API: + +-spec from_model(lee:model_key()) -> metric_id(). +from_model(ModelKey) -> + #mnode{metaparams = MPs} = lee_model:get(ModelKey, ?MYMODEL), + ?m_attr(metric, id, MPs). + +-spec opstat(atom(), atom()) -> lee:namespace(). +opstat(Group, Operation) -> + #{ avg_time => + {[metric], + #{ oneliner => "Average time of the operation" + , metric_type => rolling_average + , id => ?GROUP_OP_TIME(Group, Operation) + , labels => [group, operation] + }} + , pending => + {[metric], + #{ oneliner => "Number of pending operations" + , metric_type => counter + , id => ?GROUP_N_PENDING(Group, Operation) + , labels => [group, operation] + }} + }. + %% Simple counters and gauges: --spec new_counter(key(), list()) -> key(). +-spec new_counter(metric_id(), list()) -> metric_id(). new_counter(Key, PrometheusParams) -> - ok = gen_server:call(?SERVER, {new_counter, Key, PrometheusParams, [write_concurrency]}), + ok = gen_server:call(?SERVER, {new_counter, Key, PrometheusParams}), Key. --spec new_gauge(key(), list()) -> key(). +-spec new_gauge(metric_id(), list()) -> metric_id(). new_gauge(Key, PrometheusParams) -> - ok = gen_server:call(?SERVER, {new_counter, Key, PrometheusParams, []}), + ok = gen_server:call(?SERVER, {new_counter, Key, PrometheusParams}), Key. --spec counter_inc(key(), integer()) -> ok. +-spec counter_inc(metric_id(), integer()) -> ok. counter_inc(Key, Delta) -> counters:add(persistent_term:get(?C(Key)), 1, Delta). --spec counter_dec(key(), integer()) -> ok. +-spec counter_dec(metric_id(), integer()) -> ok. counter_dec(Key, Delta) -> counters:sub(persistent_term:get(?C(Key)), 1, Delta). --spec get_counter(key()) -> integer(). +-spec get_counter(metric_id()) -> integer(). get_counter(Key) -> counters:get(persistent_term:get(?C(Key)), 1). --spec gauge_set(key(), integer()) -> ok. +-spec gauge_set(metric_id(), integer()) -> ok. gauge_set(Key, Value) -> counters:put(persistent_term:get(?C(Key)), 1, Value). %% Get the raw counter reference for fast access: --spec gauge_ref(key()) -> counters:counters_ref(). +-spec gauge_ref(metric_id()) -> counters:counters_ref(). gauge_ref(Key) -> persistent_term:get(?C(Key)). %% Rolling average --spec new_rolling_average(key(), list()) -> key(). +-spec new_rolling_average(metric_id(), list()) -> metric_id(). new_rolling_average(Key, PrometheusParams) -> ok = gen_server:call(?SERVER, {new_rolling_average, Key, PrometheusParams}), Key. %% Fast update of rolling average of a value: --spec rolling_average_observe(key(), integer()) -> ok. +-spec rolling_average_observe(metric_id(), integer()) -> ok. rolling_average_observe(Key, Val) -> Cnt = persistent_term:get(?RA(Key)), counters:add(Cnt, 1, Val), %% Update the sum counters:add(Cnt, 2, 1). %% Update the number of samples --spec get_rolling_average(key()) -> integer(). +-spec get_rolling_average(metric_id()) -> integer(). get_rolling_average(Key) -> get_rolling_average(Key, ?DEFAULT_WINDOW). --spec get_rolling_average(key(), non_neg_integer()) -> integer(). +-spec get_rolling_average(metric_id(), non_neg_integer()) -> integer(). get_rolling_average(Key, Window) -> case gen_server:call(?SERVER, {get_rolling_average, Key, Window}) of undefined -> @@ -117,6 +154,31 @@ get_rolling_average(Key, Window) -> Val end. +%%================================================================================ +%% lee_metatype callbacks +%%================================================================================ + +names(_) -> + [metric]. + +metaparams(metric) -> + [ {mandatory, metric_type, typerefl:union([counter, gauge, rolling_average])} + , {mandatory, id, metric_id()} + , {mandatory, oneliner, string()} + , {mandatory, labels, [atom()]} + , {optional, unit, string()} + ]. + +meta_validate(metric, Model) -> + Ids = [begin + #mnode{metaparams = #{id := Id}} = lee_model:get(Key, Model), + Id + end || Key <- lee_model:get_metatype_index(metric, Model)], + case length(lists:usort(Ids)) =:= length(Ids) of + false -> {["Metric IDs must be unique"], [], []}; + true -> {[], [], []} + end. + %%================================================================================ %% gen_server callbacks %%================================================================================ @@ -133,21 +195,47 @@ get_rolling_average(Key, Window) -> %% Rolling average's state: -record(r, - { key :: key() + { key :: metric_id() , datapoints :: [#p{}] }). -record(s, - { counters = [] :: [key()] + { counters = [] :: [metric_id()] , rolling_average = [] :: [#r{}] }). init([]) -> + %% Create metrics defined in the model + State = lists:foldl( + fun(ModelKey, S) -> + #mnode{metaparams = MPs} = lee_model:get(ModelKey, ?MYMODEL), + PrometheusParams0 = case MPs of + #{unit := Unit, id := MetricId, labels := Labels} -> + [{unit, Unit}]; + #{id := MetricId, labels := Labels} -> + [] + end, + PrometheusParams = [ {help, list_to_binary(?m_attr(metric, oneliner, MPs))} + , {labels, Labels} + | PrometheusParams0 + ], + case ?m_attr(metric, metric_type, MPs) of + counter -> + mk_counter(S, MetricId, PrometheusParams); + gauge -> + mk_counter(S, MetricId, PrometheusParams); + rolling_average -> + mk_rolling_average(S, MetricId, PrometheusParams) + end + end, + #s{}, + lee_model:get_metatype_index(metric, ?MYMODEL)), + %% Start timer: self() ! tick, - {ok, #s{}}. + {ok, State}. -handle_call({new_counter, Key, PrometheusParams, Options}, _From, S) -> - {reply, ok, mk_counter(S, Key, PrometheusParams, Options)}; +handle_call({new_counter, Key, PrometheusParams}, _From, S) -> + {reply, ok, mk_counter(S, Key, PrometheusParams)}; handle_call({new_rolling_average, Key, PrometheusParams}, _From, S) -> {reply, ok, mk_rolling_average(S, Key, PrometheusParams)}; handle_call({get_rolling_average, Key, Window}, _From, S) -> @@ -194,7 +282,7 @@ do_get_rolling_average(#r{key = Key, datapoints = DP}, Window) -> (Sum - Sum0) div (N - N0) end. -look_back(StartTime, [A]) -> +look_back(_StartTime, [A]) -> A; look_back(StartTime, [A|Rest]) -> if A#p.time =< StartTime -> @@ -219,16 +307,24 @@ update_prometheus(#s{counters = CC, rolling_average = RR}) -> lists:foreach( fun(K = {Tag, Label}) -> LabelList = prom_labels(Label), - ok = prometheus_gauge:set(Tag, LabelList, get_counter(K)) + prometheus_gauge_set(Tag, LabelList, get_counter(K)) end, CC), lists:foreach( fun(R = #r{key = {Tag, Label}}) -> LabelList = prom_labels(Label), - ok = prometheus_gauge:set(Tag, LabelList, do_get_rolling_average(R, ?DEFAULT_WINDOW)) + ok = prometheus_gauge_set(Tag, LabelList, do_get_rolling_average(R, ?DEFAULT_WINDOW)) end, RR). +prometheus_gauge_set(Tag, LabelList, Val) -> + try + prometheus_gauge:set(Tag, LabelList, Val) + catch + EC:Err -> + logger:error(#{EC => Err, tag => Tag, labels => LabelList}) + end. + mk_rolling_average(S = #s{rolling_average = RR}, Key = {Tag, _Labels}, PrometheusParams) -> case lists:keyfind(Key, #r.key, RR) of false -> @@ -249,11 +345,11 @@ mk_rolling_average(S = #s{rolling_average = RR}, Key = {Tag, _Labels}, Prometheu S end. -mk_counter(S = #s{counters = CC}, Key = {Tag, _Labels}, PrometheusParams, Options) -> +mk_counter(S = #s{counters = CC}, Key = {Tag, _Labels}, PrometheusParams) -> case lists:member(Key, CC) of false -> prometheus_gauge:declare([{name, Tag}|PrometheusParams]), - Cnt = counters:new(1, Options), + Cnt = counters:new(1, [write_concurrency]), persistent_term:put(?C(Key), Cnt), S#s{counters = [Key|CC]}; true -> diff --git a/src/restapi/emqttb_http_scenario_conf.erl b/src/restapi/emqttb_http_scenario_conf.erl index 99d93c3..5c19f60 100644 --- a/src/restapi/emqttb_http_scenario_conf.erl +++ b/src/restapi/emqttb_http_scenario_conf.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -36,7 +36,7 @@ init(_Transport, _Req, []) -> allowed_methods(Req , State) -> {[<<"GET">>, <<"PUT">>], Req, State}. -resource_exists(Req = #{bindings := #{scenario := Scenario, key := Key}}, State) -> +resource_exists(Req = #{bindings := #{scenario := Scenario, key := _Key}}, State) -> Enabled = [atom_to_binary(I:name()) || I <- emqttb_scenario:list_enabled_scenarios()], Exists = lists:member(Scenario, Enabled), {Exists, Req, State}. diff --git a/src/scenarios/emqttb_scenario_persistent_session.erl b/src/scenarios/emqttb_scenario_persistent_session.erl index 43da955..4d2fe8f 100644 --- a/src/scenarios/emqttb_scenario_persistent_session.erl +++ b/src/scenarios/emqttb_scenario_persistent_session.erl @@ -63,7 +63,7 @@ { produced = 0 , consumed = 0 , to_consume = 0 - , pubinterval :: non_neg_integer() + , pubinterval :: non_neg_integer() | undefined }). %%================================================================================ @@ -87,12 +87,15 @@ model() -> , default => 256 }} , pubinterval => - {[value, cli_param], + {[value, cli_param, autorate], #{ oneliner => "Message publishing interval (microsecond)" , type => emqttb:duration_us() , default_ref => [interval] , cli_operand => "pubinterval" , cli_short => $i + , autorate_id => 'persistent_session/pubinterval' + , process_variable => [?SK(persistent_session), pub, pub_latency, pending] + , error_coeff => -1 }} , n => {[value, cli_param], @@ -109,14 +112,6 @@ model() -> , default_str => "100ms" , cli_operand => "publatency" }} - , pub_autorate => - {[value, cli_param, pointer], - #{ oneliner => "ID of the autorate config used to tune publish interval" - , type => atom() - , default => default - , cli_operand => "pubautorate" - , target_node => [autorate] - }} , topic_suffix => {[value, cli_param], #{ oneliner => "Suffix of the topic to publish to" @@ -133,6 +128,18 @@ model() -> , cli_operand => "pubtime" , cli_short => $T }} + %% Metrics: + , n_published => + {[metric], + #{ oneliner => "Total number of published messages" + , id => {emqttb_published_messages, persistent_session} + , metric_type => counter + , labels => [scenario] + }} + , pub_latency => + emqttb_metrics:opstat('persistent_session/pub', 'publish') + , conn_latency => + emqttb_metrics:opstat('persistent_session/pub', 'connect') } , sub => #{ qos => @@ -207,7 +214,7 @@ run() -> NCons = try emqttb_metrics:get_counter(?CNT_SUB_MESSAGES(?SUB_GROUP)) catch _:_ -> 0 end, - S = #s{produced = NProd, consumed = NCons, pubinterval = my_conf([pub, pubinterval])}, + S = #s{produced = NProd, consumed = NCons}, do_run(S, 0). %%================================================================================ @@ -250,7 +257,8 @@ publish_stage(S = #s{produced = NPub0, pubinterval = PubInterval}) -> TopicPrefix = topic_prefix(), TopicSuffix = my_conf([pub, topic_suffix]), PubOpts = #{ topic => <> - , pubinterval => PubInterval + , pubinterval => my_conf_key([pub, pubinterval]) + , n_published => my_conf_key([pub, n_published]) , msg_size => my_conf([pub, msg_size]) , qos => my_conf([pub, qos]) , set_latency => my_conf_key([pub, set_pub_latency]) @@ -261,13 +269,13 @@ publish_stage(S = #s{produced = NPub0, pubinterval = PubInterval}) -> , behavior => {emqttb_behavior_pub, PubOpts} }), Interval = my_conf([conninterval]), - {ok, N} = emqttb_group:set_target(?PUB_GROUP, my_conf([pub, n]), Interval), + {ok, _} = emqttb_group:set_target(?PUB_GROUP, my_conf([pub, n]), Interval), PubTime = my_conf([pub, pub_time]), timer:sleep(PubTime), - PubIntervalCref = emqttb_autorate:get_counter(emqttb_behavior_pub:my_autorate(?PUB_GROUP)), + PubIntervalCref = emqttb_autorate:get_counter('persistent_session/pubinterval'), PubInterval2 = counters:get(PubIntervalCref, 1), emqttb_group:stop(?PUB_GROUP), - NPub = emqttb_metrics:get_counter(?CNT_PUB_MESSAGES(?PUB_GROUP)), + NPub = emqttb_metrics:get_counter(emqttb_metrics:from_model(my_conf_key([pub, n_published]))), %% TODO: it doesn't take ramp up/down into account: prometheus_summary:observe(?PUB_THROUGHPUT, (NPub - NPub0) * timer:seconds(1) div PubTime), S#s{ produced = NPub diff --git a/src/scenarios/emqttb_scenario_pub.erl b/src/scenarios/emqttb_scenario_pub.erl index 6f35195..144f11d 100644 --- a/src/scenarios/emqttb_scenario_pub.erl +++ b/src/scenarios/emqttb_scenario_pub.erl @@ -74,20 +74,25 @@ model() -> , default => 256 }} , conninterval => - {[value, cli_param], + {[value, cli_param, autorate], #{ oneliner => "Client connection interval (microsecond)" , type => emqttb:duration_us() , default_ref => [interval] , cli_operand => "conninterval" , cli_short => $I + , autorate_id => 'pub/conn' + , process_variable => [?SK(pub), conn_latency, pending] }} , pubinterval => - {[value, cli_param], + {[value, cli_param, autorate], #{ oneliner => "Message publishing interval (microsecond)" , type => emqttb:duration_us() , default_ref => [interval] , cli_operand => "pubinterval" , cli_short => $i + , autorate_id => 'pub/pubinterval' + , process_variable => [?SK(pub), pub_latency, pending] + , error_coeff => -1 }} , set_pub_latency => {[value, cli_param], @@ -113,14 +118,6 @@ model() -> , cli_short => $g , target_node => [groups] }} - , pub_autorate => - {[value, cli_param, pointer], - #{ oneliner => "ID of the autorate config used to tune publish interval" - , type => atom() - , default => default - , cli_operand => "pubautorate" - , target_node => [autorate] - }} , metadata => {[value, cli_param], #{ oneliner => "Add metadata to the messages" @@ -135,12 +132,24 @@ model() -> , default => 0 , cli_operand => "start-n" }} + %% Metrics: + , n_published => + {[metric], + #{ oneliner => "Total number of published messages" + , metric_type => counter + , id => {emqttb_published_messages, pub} + , labels => [scenario] + }} + , pub_latency => + emqttb_metrics:opstat('pub', 'publish') + , conn_latency => + emqttb_metrics:opstat('pub', 'connect') }. run() -> PubOpts = #{ topic => my_conf([topic]) - , pubinterval => my_conf([pubinterval]) - , pub_autorate => my_conf([pub_autorate]) + , n_published => my_conf_key([n_published]) + , pubinterval => my_conf_key([pubinterval]) , msg_size => my_conf([msg_size]) , qos => my_conf([qos]) , retain => my_conf([retain]) diff --git a/src/scenarios/emqttb_scenario_pubsub_fwd.erl b/src/scenarios/emqttb_scenario_pubsub_fwd.erl index ed46b5c..a5f15a5 100644 --- a/src/scenarios/emqttb_scenario_pubsub_fwd.erl +++ b/src/scenarios/emqttb_scenario_pubsub_fwd.erl @@ -68,12 +68,15 @@ model() -> , default => 256 }} , pubinterval => - {[value, cli_param], + {[value, cli_param, autorate], #{ oneliner => "Message publishing interval (microsecond)" , type => emqttb:duration_us() , default_ref => [interval] , cli_operand => "pubinterval" , cli_short => $i + , autorate_id => 'pubsub_fwd/pubinterval' + , process_variable => [?SK(pubsub_fwd), pub, pub_latency, pending] + , error_coeff => -1 }} , set_pub_latency => {[value, cli_param], @@ -82,14 +85,18 @@ model() -> , default => 100 , cli_operand => "publatency" }} - , pub_autorate => - {[value, cli_param, pointer], - #{ oneliner => "ID of the autorate config used to tune publish interval" - , type => atom() - , default => default - , cli_operand => "pubautorate" - , target_node => [autorate] + %% Metrics: + , n_published => + {[metric], + #{ oneliner => "Total number of published messages" + , id => {emqttb_published_messages, pubsub_fwd} + , metric_type => counter + , labels => [scenario] }} + , pub_latency => + emqttb_metrics:opstat('pubsub_fwd/pub', 'publish') + , conn_latency => + emqttb_metrics:opstat('pubsub_fdw/pub', 'connect') } , sub => #{ qos => @@ -196,8 +203,8 @@ publish_stage() -> false -> 0 end, PubOpts = #{ topic => <> - , pubinterval => my_conf([pub, pubinterval]) - , pub_autorate => my_conf([pub, pub_autorate]) + , n_published => my_conf_key([pub, n_published]) + , pubinterval => my_conf_key([pub, pubinterval]) , msg_size => my_conf([pub, msg_size]) , qos => my_conf([pub, qos]) , set_latency => my_conf_key([pub, set_pub_latency]) diff --git a/test/emqttb_worker_SUITE.erl b/test/emqttb_worker_SUITE.erl index 9471958..8ea203d 100644 --- a/test/emqttb_worker_SUITE.erl +++ b/test/emqttb_worker_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -38,7 +38,7 @@ end_per_testcase(_, _Config) -> snabbkaffe:stop(), ok. -t_group(Config) -> +t_group(_Config) -> NClients = 10, Group = test_group, ?check_trace( @@ -70,16 +70,16 @@ t_group(Config) -> end} ]). -t_error_in_init(Config) -> +t_error_in_init(_Config) -> error_scenario(#{init => error}). -t_error_in_handle_msg(Config) -> +t_error_in_handle_msg(_Config) -> error_scenario(#{handle_message => error}). -t_invalid_return(Config) -> +t_invalid_return(_Config) -> error_scenario(#{handle_message => invalid_return}). -t_error_in_terminate(Config) -> +t_error_in_terminate(_Config) -> error_scenario(#{terminate => error}). error_scenario(Config) -> From 8f03da94eefb975e9e1c1d23280e8079b399fd7c Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Thu, 7 Dec 2023 22:01:11 +0100 Subject: [PATCH 02/10] feat: Declarative autoscales --- src/behaviors/emqttb_behavior_conn.erl | 30 ++-- src/behaviors/emqttb_behavior_pub.erl | 49 ++++--- src/behaviors/emqttb_behavior_sub.erl | 78 +++++++---- src/framework/emqttb_autorate.erl | 18 +-- src/framework/emqttb_group.erl | 130 ++++++++++-------- src/framework/emqttb_scenario.erl | 2 + src/framework/emqttb_worker.erl | 49 ++----- src/metrics/emqttb_metrics.erl | 27 +++- src/metrics/emqttb_pushgw.erl | 4 +- src/scenarios/emqttb_scenario_conn.erl | 14 +- .../emqttb_scenario_persistent_session.erl | 56 +++----- src/scenarios/emqttb_scenario_pub.erl | 41 ++---- src/scenarios/emqttb_scenario_pubsub_fwd.erl | 38 ++--- src/scenarios/emqttb_scenario_sub.erl | 17 +-- .../emqttb_scenario_sub_flapping.erl | 23 ++-- test/escript_SUITE.erl | 17 ++- 16 files changed, 321 insertions(+), 272 deletions(-) diff --git a/src/behaviors/emqttb_behavior_conn.erl b/src/behaviors/emqttb_behavior_conn.erl index a3b6106..5063968 100644 --- a/src/behaviors/emqttb_behavior_conn.erl +++ b/src/behaviors/emqttb_behavior_conn.erl @@ -17,6 +17,9 @@ -behavior(emqttb_worker). +%% API: +-export([model/1]). + %% behavior callbacks: -export([init_per_group/2, init/1, handle_message/3, terminate/2]). @@ -28,27 +31,38 @@ -type config() :: #{ expiry => non_neg_integer() , clean_start => boolean() + , metrics := lee:model_key() }. -type prototype() :: {?MODULE, config()}. %%================================================================================ -%% behavior callbacks +%% API %%================================================================================ -init_per_group(_Group, Opts) -> - Expiry = maps:get(expiry, Opts, 0), - CleanStart = maps:get(clean_start, Opts, true), - #{ expiry => Expiry - , clean_start => CleanStart +model(GroupId) -> + #{ conn_latency => + emqttb_metrics:opstat(GroupId, connect) }. -init(#{clean_start := CleanStart, expiry := Expiry}) -> +%%================================================================================ +%% behavior callbacks +%%================================================================================ + +init_per_group(_Group, Opts = #{metrics := MetricsKey}) -> + Defaults = #{ expiry => 0 + , clean_start => true + }, + Config = maps:merge(Defaults, Opts), + Config#{ conn_opstat => emqttb_metrics:opstat_from_model(MetricsKey ++ [conn_latency]) + }. + +init(#{clean_start := CleanStart, expiry := Expiry, conn_opstat := ConnOpstat}) -> Props = case Expiry of undefined -> #{}; _ -> #{'Session-Expiry-Interval' => Expiry} end, - {ok, Conn} = emqttb_worker:connect(Props, [{clean_start, CleanStart}], [], []), + {ok, Conn} = emqttb_worker:connect(ConnOpstat, Props, [{clean_start, CleanStart}], [], []), Conn. handle_message(_, Conn, _) -> diff --git a/src/behaviors/emqttb_behavior_pub.erl b/src/behaviors/emqttb_behavior_pub.erl index 772d5c6..1856641 100644 --- a/src/behaviors/emqttb_behavior_pub.erl +++ b/src/behaviors/emqttb_behavior_pub.erl @@ -18,7 +18,7 @@ -behavior(emqttb_worker). %% API --export([parse_metadata/1, my_autorate/1]). +-export([parse_metadata/1, model/1]). %% behavior callbacks: -export([init_per_group/2, init/1, handle_message/3, terminate/2]). @@ -35,11 +35,10 @@ %%================================================================================ -type config() :: #{ topic := binary() - , n_published := lee:model_key() , pubinterval := lee:model_key() , msg_size := non_neg_integer() + , metrics := lee:model_key() , qos := emqttb:qos() - , set_latency := lee:key() , retain => boolean() , metadata => boolean() , host_shift => integer() @@ -52,6 +51,21 @@ %% API %%================================================================================ +-spec model(atom()) -> lee:namespace(). +model(Group) -> + #{ conn_latency => + emqttb_metrics:opstat(Group, connect) + , pub_latency => + emqttb_metrics:opstat(Group, publish) + , n_published => + {[metric], + #{ oneliner => "Total number of messages published by the group" + , id => {emqttb_published_messages, Group} + , labels => [group] + , metric_type => counter + }} + }. + -spec parse_metadata(Msg) -> {ID, SeqNo, TS} when Msg :: binary(), ID :: integer(), @@ -66,18 +80,14 @@ parse_metadata(<>) -> init_per_group(Group, #{ topic := Topic - , n_published := NPublishedMetricKey , pubinterval := PubInterval , msg_size := MsgSize , qos := QoS - , set_latency := SetLatencyKey + , metrics := MetricsKey } = Conf) when is_binary(Topic), - is_integer(MsgSize), - is_list(SetLatencyKey) -> + is_integer(MsgSize) -> AddMetadata = maps:get(metadata, Conf, false), - PubCnt = emqttb_metrics:from_model(NPublishedMetricKey), - emqttb_worker:new_opstat(Group, ?AVG_PUB_TIME), - {auto, PubRate} = emqttb_autorate:from_model(PubInterval), + PubRate = emqttb_autorate:get_counter(emqttb_autorate:from_model(PubInterval)), MetadataSize = case AddMetadata of true -> (32 + 32 + 64) div 8; false -> 0 @@ -88,27 +98,31 @@ init_per_group(Group, #{ topic => Topic , message => message(max(0, MsgSize - MetadataSize)) , pub_opts => [{qos, QoS}, {retain, Retain}] - , pub_counter => PubCnt , pubinterval => PubRate , metadata => AddMetadata , host_shift => HostShift , host_selection => HostSelection + , pub_opstat => emqttb_metrics:opstat_from_model(MetricsKey ++ [pub_latency]) + , conn_opstat => emqttb_metrics:opstat_from_model(MetricsKey ++ [conn_latency]) + , pub_counter => emqttb_metrics:from_model(MetricsKey ++ [n_published]) }. -init(PubOpts = #{pubinterval := I}) -> +init(PubOpts = #{pubinterval := I, conn_opstat := ConnOpstat}) -> rand:seed(default), {SleepTime, N} = emqttb:get_duration_and_repeats(I), send_after_rand(SleepTime, {publish, N}), HostShift = maps:get(host_shift, PubOpts, 0), HostSelection = maps:get(host_selection, PubOpts, random), - {ok, Conn} = emqttb_worker:connect(#{ host_shift => HostShift + {ok, Conn} = emqttb_worker:connect(ConnOpstat, + #{ host_shift => HostShift , host_selection => HostSelection }), Conn. handle_message(Shared, Conn, {publish, N1}) -> #{ topic := TP, pubinterval := I, message := Msg0, pub_opts := PubOpts - , pub_counter := Cnt + , pub_counter := PubCounter + , pub_opstat := PubOpstat , metadata := AddMetadata } = Shared, {SleepTime, N2} = emqttb:get_duration_and_repeats(I), @@ -119,8 +133,8 @@ handle_message(Shared, Conn, {publish, N1}) -> end, T = emqttb_worker:format_topic(TP), repeat(N1, fun() -> - emqttb_worker:call_with_counter(?AVG_PUB_TIME, emqtt, publish, [Conn, T, Msg, PubOpts]), - emqttb_metrics:counter_inc(Cnt, 1) + emqttb_metrics:call_with_counter(PubOpstat, emqtt, publish, [Conn, T, Msg, PubOpts]), + emqttb_metrics:counter_inc(PubCounter, 1) end), {ok, Conn}; handle_message(_, Conn, _) -> @@ -133,9 +147,6 @@ terminate(_Shared, Conn) -> %% Internal functions %%================================================================================ -my_autorate(Group) -> - list_to_atom(atom_to_list(Group) ++ ".pub.rate"). - message(Size) -> list_to_binary([$A || _ <- lists:seq(1, Size)]). diff --git a/src/behaviors/emqttb_behavior_sub.erl b/src/behaviors/emqttb_behavior_sub.erl index 2a20bee..7b6e8e4 100644 --- a/src/behaviors/emqttb_behavior_sub.erl +++ b/src/behaviors/emqttb_behavior_sub.erl @@ -17,6 +17,9 @@ -behavior(emqttb_worker). +%% API: +-export([model/1]). + %% behavior callbacks: -export([init_per_group/2, init/1, handle_message/3, terminate/2]). @@ -28,6 +31,7 @@ -type config() :: #{ topic := binary() , qos := 0..2 + , metrics := lee:model_key() , clean_start => boolean() , expiry => non_neg_integer() | undefined , host_shift => integer() @@ -37,55 +41,81 @@ -type prototype() :: {?MODULE, config()}. --define(CNT_SUB_MESSAGES(GRP), {emqttb_received_messages, GRP}). --define(CNT_SUB_LATENCY(GRP), {emqttb_e2e_latency, GRP}). --define(AVG_SUB_TIME, subscribe). +%%================================================================================ +%% API +%%================================================================================ + +-spec model(atom()) -> lee:namespace(). +model(GroupId) -> + #{ n_received => + {[metric], + #{ oneliner => "Total number of received messages" + , id => {emqttb_received_messages, GroupId} + , metric_type => counter + , labels => [group] + }} + , conn_latency => + emqttb_metrics:opstat(GroupId, 'connect') + , sub_latency => + emqttb_metrics:opstat(GroupId, 'subscribe') + , e2e_latency => + {[metric], + #{ oneliner => "End-to-end latency" + , id => {emqttb_e2e_latency, GroupId} + , metric_type => rolling_average + , labels => [group] + , unit => "microsecond" + }} + }. %%================================================================================ %% behavior callbacks %%================================================================================ -init_per_group(Group, - #{ topic := Topic - , qos := _QoS +init_per_group(_Group, + #{ topic := Topic + , qos := _QoS + , metrics := MetricsModelKey } = Opts) when is_binary(Topic) -> - SubCnt = emqttb_metrics:new_counter(?CNT_SUB_MESSAGES(Group), - [ {help, <<"Number of received messages">>} - , {labels, [group]} - ]), - LatCnt = emqttb_metrics:new_rolling_average(?CNT_SUB_LATENCY(Group), - [ {help, <<"End-to-end latency">>} - , {labels, [group]} - ]), - emqttb_worker:new_opstat(Group, ?AVG_SUB_TIME), Defaults = #{ expiry => 0 , clean_start => true , host_shift => 0 , host_selection => random , parse_metadata => false }, - maps:merge(Defaults, Opts #{sub_counter => SubCnt, latency_counter => LatCnt}). + Conf = maps:merge(Defaults, Opts), + Conf#{ conn_opstat => emqttb_metrics:opstat_from_model(MetricsModelKey ++ [conn_latency]) + , sub_opstat => emqttb_metrics:opstat_from_model(MetricsModelKey ++ [sub_latency]) + , e2e_latency => emqttb_metrics:from_model(MetricsModelKey ++ [e2e_latency]) + , sub_counter => emqttb_metrics:from_model(MetricsModelKey ++ [n_received]) + }. -init(SubOpts0 = #{topic := T, qos := QoS, expiry := Expiry, clean_start := CleanStart}) -> +init(SubOpts0 = #{ topic := T + , qos := QoS + , expiry := Expiry + , clean_start := CleanStart + , conn_opstat := ConnOpstat + , sub_opstat := SubOpstat + }) -> SubOpts = maps:with([host_shift, host_selection], SubOpts0), Props = case Expiry of undefined -> SubOpts#{}; _ -> SubOpts#{'Session-Expiry-Interval' => Expiry} end, - {ok, Conn} = emqttb_worker:connect(Props, [{clean_start, CleanStart}], [], []), - emqttb_worker:call_with_counter(?AVG_SUB_TIME, emqtt, subscribe, [Conn, emqttb_worker:format_topic(T), QoS]), + {ok, Conn} = emqttb_worker:connect(ConnOpstat, Props, [{clean_start, CleanStart}], [], []), + emqttb_metrics:call_with_counter(SubOpstat, emqtt, subscribe, [Conn, emqttb_worker:format_topic(T), QoS]), Conn. -handle_message(#{sub_counter := Cnt, parse_metadata := ParseMetadata}, +handle_message(#{ parse_metadata := ParseMetadata, sub_counter := SubCnt, e2e_latency := E2ELatency}, Conn, - {publish, #{client_pid := Pid, payload := Payload}}) when - Pid =:= Conn -> - emqttb_metrics:counter_inc(Cnt, 1), + {publish, #{client_pid := Pid, payload := Payload}} + ) when Pid =:= Conn -> + emqttb_metrics:counter_inc(SubCnt, 1), case ParseMetadata of true -> {_Id, _SeqNo, TS} = emqttb_behavior_pub:parse_metadata(Payload), Dt = os:system_time(microsecond) - TS, - emqttb_metrics:rolling_average_observe(?CNT_SUB_LATENCY(emqttb_worker:my_group()), Dt); + emqttb_metrics:rolling_average_observe(E2ELatency, Dt); false -> ok end, diff --git a/src/framework/emqttb_autorate.erl b/src/framework/emqttb_autorate.erl index 20c7f0c..66c2093 100644 --- a/src/framework/emqttb_autorate.erl +++ b/src/framework/emqttb_autorate.erl @@ -54,10 +54,8 @@ %% `false' if everything is normal. -type scram_fun() :: fun((_IsOverride :: boolean()) -> {true, integer()} | false). --type id() :: atom(). - -type config() :: - #{ id := id() + #{ id := emqttb:autorate() , conf_root := atom() , error := fun(() -> number()) , scram => scram_fun() @@ -86,7 +84,7 @@ ensure(Conf) -> {ok, Pid} = emqttb_autorate_sup:ensure(Conf#{parent => self()}), {auto, get_counter(Pid)}. --spec get_counter(lee:key() | id() | pid()) -> counters:counters_ref(). +-spec get_counter(lee:key() | emqttb:autorate() | pid()) -> counters:counters_ref(). get_counter(Pid) when is_pid(Pid) -> gen_server:call(Pid, get_counter); get_counter(Id) when is_atom(Id) -> @@ -96,7 +94,10 @@ get_counter(Key) -> gen_server:call(?via(Id), get_counter). %% Set the current value to the specified value --spec reset(atom() | pid(), integer()) -> ok. +-spec reset(atom() | pid() | lee:ley(), integer()) -> ok. +reset(Key, Val) when is_list(Key) -> + #mnode{metaparams = #{autorate_id := Id}} = lee_model:get(Key, ?MYMODEL), + reset(Id, Val); reset(Id, Val) -> gen_server:call(?via(Id), {reset, Val}). @@ -232,9 +233,10 @@ create_autorates() -> ok. --spec from_model(lee:key()) -> {auto, counter:counters_ref()}. -from_model(ModelKey) -> - {auto, get_counter(ModelKey)}. +-spec from_model(lee:key()) -> atom(). +from_model(Key) -> + #mnode{metaparams = #{autorate_id := Id}} = lee_model:get(Key, ?MYMODEL), + Id. %%================================================================================ %% gen_server callbacks diff --git a/src/framework/emqttb_group.erl b/src/framework/emqttb_group.erl index aba613b..a46cde0 100644 --- a/src/framework/emqttb_group.erl +++ b/src/framework/emqttb_group.erl @@ -18,7 +18,8 @@ -behavior(gen_server). %% API: --export([ensure/1, stop/1, set_target/3, set_target_async/3, broadcast/2, report_dead_id/2, info/0]). +-export([ensure/1, stop/1, set_target/2, set_target/3, set_target_async/3, broadcast/2, report_dead_id/2, info/0, + conninterval_model/2]). %% gen_server callbacks: -export([init/1, handle_call/3, handle_cast/2, terminate/2, handle_info/2]). @@ -43,8 +44,8 @@ #{ id := atom() , client_config := atom() , behavior := prototype() + , conn_interval := atom() , parent => pid() - , autorate => atom() , start_n => integer() }. @@ -55,6 +56,15 @@ %% API funcions %%================================================================================ +-spec conninterval_model(emqttb:group_id(), lee:model_key()) -> map(). +conninterval_model(Group, AutoscaleMetric) -> + #{ oneliner => "Client connection interval" + , autorate_id => Group + , type => emqttb:duration_us() + , error_coeff => -1 + , process_variable => AutoscaleMetric + }. + -spec ensure(group_config()) -> ok. ensure(Conf) -> emqttb_group_sup:ensure(Conf#{parent => self()}). @@ -64,6 +74,11 @@ stop(ID) -> logger:info("Stopping group ~p", [ID]), emqttb_group_sup:stop(ID). +-spec set_target(emqttb:group(), NClients) -> NClients + when NClients :: emqttb:n_cycles(). +set_target(Group, NClients) -> + set_target(Group, NClients, undefined). + %% @doc Autoscale the group to the target number of workers. Returns %% value when the target or a ratelimit has been reached, or error %% when the new target has been set. @@ -129,6 +144,7 @@ info() -> , scaling :: #r{} | undefined , target :: non_neg_integer() | undefined , interval :: counters:counters_ref() + , autorate :: atom() , scale_timer :: reference() | undefined , tick_timer :: reference() , next_id = 0 :: non_neg_integer() @@ -142,11 +158,16 @@ init([Conf]) -> #{ id := ID , client_config := ConfID , behavior := {Behavior, BehSettings} + , conn_interval := ConnIntervalAutorateId } = Conf, ?tp(info, "Starting worker group", #{ id => ID , group_conf => ConfID }), + emqttb_metrics:new_counter(?GROUP_N_WORKERS(ID), + [ {help, <<"Number of workers in the group">>} + , {labels, [group]} + ]), ets:new(dead_id_pool(ID), [ordered_set, public, named_table]), StartN = maps:get(start_n, Conf, 0), persistent_term:put(?GROUP_LEADER_TO_GROUP_ID(self()), ID), @@ -154,14 +175,14 @@ init([Conf]) -> persistent_term:put(?GROUP_CONF_ID(self()), ConfID), BehSharedState = emqttb_worker:init_per_group(Behavior, ID, BehSettings), persistent_term:put(?GROUP_BEHAVIOR_SHARED_STATE(self()), BehSharedState), - declare_metrics(ID), - {auto, Autorate} = create_autorate(ID, ConfID), + Autorate = emqttb_autorate:get_counter(ConnIntervalAutorateId), S = #s{ id = ID , behavior = Behavior , conf_prefix = [groups, ConfID] , tick_timer = set_tick_timer() , parent_ref = maybe_monitor_parent(Conf) , interval = Autorate + , autorate = ConnIntervalAutorateId , next_id = StartN }, {ok, S}. @@ -235,7 +256,7 @@ wait_group_stop([MRef|Rest]) -> do_set_target(Target, InitInterval, OnComplete, S = #s{ scaling = Scaling , id = ID - , interval = Interval + , autorate = Autorate }) -> N = n_clients(S), Direction = if Target > N -> up; @@ -246,11 +267,11 @@ do_set_target(Target, InitInterval, OnComplete, S = #s{ scaling = Scaling case Direction of stay -> OnComplete({ok, N}), - S#s{scaling = undefined, target = Target, interval = Interval}; + S#s{scaling = undefined, target = Target}; _ -> case InitInterval of _ when is_integer(InitInterval) -> - emqttb_autorate:reset(my_autorate(ID), InitInterval); + emqttb_autorate:reset(Autorate, InitInterval); undefined -> ok end, @@ -369,57 +390,50 @@ maybe_monitor_parent(#{parent := Pid}) -> maybe_monitor_parent(_) -> undefined. -declare_metrics(ID) -> - emqttb_metrics:new_counter(?GROUP_N_WORKERS(ID), - [ {help, <<"Number of workers in the group">>} - , {labels, [group]} - ]), - emqttb_worker:new_opstat(ID, connect). - -create_autorate(GroupID, ConfID) -> - ID = my_autorate(GroupID), - DefaultConf = #{ [id] => ID - }, - emqttb_conf:patch(lee_lib:make_nested_patch(?MYMODEL, [autorate], DefaultConf)), - AutorateConf = #{ id => ID - , conf_root => ID - , error => fun() -> autoscale_error(GroupID) end - , scram => fun(Meltdown) -> autoscale_scram(GroupID, ConfID, Meltdown) end - }, - emqttb_autorate:ensure(AutorateConf). - -autoscale_scram(Group, ConfID, Meltdown) -> - MaxPending = ?CFG([groups, {ConfID}, scram, threshold]), - Hysteresis = ?CFG([groups, {ConfID}, scram, hysteresis]), - Override = ?CFG([groups, {ConfID}, scram, override]), - Pending = emqttb_metrics:get_counter(?GROUP_N_PENDING(Group, connect)), - if Meltdown andalso Pending >= (MaxPending * Hysteresis / 100) -> - {true, Override}; - Pending >= MaxPending -> - logger:warning("SCRAM is activated for group ~p. Unacked connections: ~p. Connection interval is dropped to ~p us.", - [Group, Pending, Override]), - {true, Override}; - Meltdown -> - logger:warning("SCRAM is deactivated for group ~p. Unacked connections: ~p. Connection rate is restored to normal value.", - [Group, Pending]), - false; - true -> - false - end. - -autoscale_error(Group) -> - %% Note that dependency of number of pending operations on connect - %% interval is inverse: lesser interval -> clients connect more - %% often -> more load -> more pending operations - %% - %% So the control must be reversed, and error is the negative of what one usually - %% expects: - %% Current - Target instead of Target - Current. - Target = ?CFG([groups, {Group}, target_conn_pending]), - emqttb_metrics:get_counter(?GROUP_N_PENDING(Group, connect)) - Target. - -my_autorate(Group) -> - list_to_atom(atom_to_list(Group) ++ "_autoscale"). +%% create_autorate(GroupID, ConfID) -> +%% ID = my_autorate(GroupID), +%% DefaultConf = #{ [id] => ID +%% }, +%% emqttb_conf:patch(lee_lib:make_nested_patch(?MYMODEL, [autorate], DefaultConf)), +%% AutorateConf = #{ id => ID +%% , conf_root => ID +%% , error => autoscale_error(Metric, GroupID) +%% , scram => fun(Meltdown) -> autoscale_scram(GroupID, ConfID, Meltdown) end +%% }, +%% emqttb_autorate:ensure(AutorateConf). + +%% autoscale_scram(Group, ConfID, Meltdown) -> +%% MaxPending = ?CFG([groups, {ConfID}, scram, threshold]), +%% Hysteresis = ?CFG([groups, {ConfID}, scram, hysteresis]), +%% Override = ?CFG([groups, {ConfID}, scram, override]), +%% Pending = emqttb_metrics:get_counter(?GROUP_N_PENDING(Group, connect)), +%% if Meltdown andalso Pending >= (MaxPending * Hysteresis / 100) -> +%% {true, Override}; +%% Pending >= MaxPending -> +%% logger:warning("SCRAM is activated for group ~p. Unacked connections: ~p. Connection interval is dropped to ~p us.", +%% [Group, Pending, Override]), +%% {true, Override}; +%% Meltdown -> +%% logger:warning("SCRAM is deactivated for group ~p. Unacked connections: ~p. Connection rate is restored to normal value.", +%% [Group, Pending]), +%% false; +%% true -> +%% false +%% end. + +%% autoscale_error(MetricKey, Group) -> +%% %% Note that dependency of number of pending operations on connect +%% %% interval is inverse: lesser interval -> clients connect more +%% %% often -> more load -> more pending operations +%% %% +%% %% So the control must be reversed, and error is the negative of what one usually +%% %% expects: +%% %% Current - Target instead of Target - Current. +%% Metric = emqttb_metrics:from_model(MetricKey), +%% fun() -> +%% Target = ?CFG([groups, {Group}, target_conn_pending]), +%% emqttb_metrics:get_counter(?GROUP_N_PENDING(Group, connect)) - Target +%% end. dead_id_pool(Group) -> Group. diff --git a/src/framework/emqttb_scenario.erl b/src/framework/emqttb_scenario.erl index edb16ed..cfc5ae0 100644 --- a/src/framework/emqttb_scenario.erl +++ b/src/framework/emqttb_scenario.erl @@ -48,6 +48,8 @@ -callback model() -> lee:lee_module(). +%% -callback name() -> atom(). + %%================================================================================ %% API functions %%================================================================================ diff --git a/src/framework/emqttb_worker.erl b/src/framework/emqttb_worker.erl index c722673..a6bb582 100644 --- a/src/framework/emqttb_worker.erl +++ b/src/framework/emqttb_worker.erl @@ -18,9 +18,9 @@ %% Worker API: -export([my_group/0, my_id/0, my_clientid/0, my_hostname/0, my_cfg/1, send_after/2, send_after_rand/2, repeat/2, - connect/1, connect/4, - format_topic/1, - new_opstat/2, call_with_counter/4]). + connect/2, connect/5, + format_topic/1 + ]). %% internal exports: -export([entrypoint/3, loop/1, start/3, init_per_group/3, model/0]). @@ -134,12 +134,12 @@ my_hostname() -> %% MQTT %%-------------------------------------------------------------------------------- --spec connect(map()) -> gen_statem:start_ret(). -connect(Properties) -> - connect(Properties, [], [], []). +-spec connect(emqttb_metrics:metric_ref(), map()) -> gen_statem:start_ret(). +connect(ConnOpstat, Properties) -> + connect(ConnOpstat, Properties, [], [], []). --spec connect(map(), [emqtt:option()], [gen_tcp:option()], [ssl:option()]) -> gen_statem:start_ret(). -connect(Properties0, CustomOptions, CustomTcpOptions, CustomSslOptions) -> +-spec connect(emqttb_metrics:metric_ref(), map(), [emqtt:option()], [gen_tcp:option()], [ssl:option()]) -> gen_statem:start_ret(). +connect(ConnOpstat, Properties0, CustomOptions, CustomTcpOptions, CustomSslOptions) -> HostShift = maps:get(host_shift, Properties0, 0), HostSelection = maps:get(host_selection, Properties0, random), Properties = maps:without([host_shift, host_selection], Properties0), @@ -164,40 +164,9 @@ connect(Properties0, CustomOptions, CustomTcpOptions, CustomSslOptions) -> ], {ok, Client} = emqtt:start_link(CustomOptions ++ Options), ConnectFun = connect_fun(), - {ok, _Properties} = call_with_counter(connect, emqtt, ConnectFun, [Client]), + {ok, _Properties} = emqttb_metrics:call_with_counter(ConnOpstat, emqtt, ConnectFun, [Client]), {ok, Client}. -%%-------------------------------------------------------------------------------- -%% Instrumentation -%%-------------------------------------------------------------------------------- - --spec call_with_counter(atom(), module(), atom(), list()) -> _. -call_with_counter(Operation, Mod, Fun, Args) -> - Grp = my_group(), - emqttb_metrics:counter_inc(?GROUP_N_PENDING(Grp, Operation), 1), - T0 = os:system_time(microsecond), - try apply(Mod, Fun, Args) - catch - EC:Err -> - EC(Err) - after - T = os:system_time(microsecond), - emqttb_metrics:counter_dec(?GROUP_N_PENDING(Grp, Operation), 1), - emqttb_metrics:rolling_average_observe(?GROUP_OP_TIME(Grp, Operation), T - T0) - end. - --spec new_opstat(emqttb:group(), atom()) -> ok. -new_opstat(Group, Operation) -> - emqttb_metrics:new_rolling_average(?GROUP_OP_TIME(Group, Operation), - [ {help, <<"Average run time of an operation (microseconds)">>} - , {labels, [group, operation]} - ]), - emqttb_metrics:new_counter(?GROUP_N_PENDING(Group, Operation), - [ {help, <<"Number of pending operations">>} - , {labels, [group, operation]} - ]), - ok. - %%================================================================================ %% Internal exports %%================================================================================ diff --git a/src/metrics/emqttb_metrics.erl b/src/metrics/emqttb_metrics.erl index 72fe34a..ab41010 100644 --- a/src/metrics/emqttb_metrics.erl +++ b/src/metrics/emqttb_metrics.erl @@ -19,7 +19,7 @@ -behavior(lee_metatype). %% API: --export([from_model/1, opstat/2, +-export([from_model/1, opstat_from_model/1, opstat/2, call_with_counter/4, new_counter/2, counter_inc/2, counter_dec/2, get_counter/1, new_gauge/2, gauge_set/2, gauge_ref/1, new_rolling_average/2, rolling_average_observe/2, @@ -35,7 +35,7 @@ %% internal exports: -export([start_link/0]). --export_type([]). +-export_type([metric_ref/0]). -compile({inline, [counter_inc/2, counter_dec/2]}). @@ -62,6 +62,8 @@ -define(MAX_WINDOW_SEC, 30). % Keep about 30 seconds of history -define(DEFAULT_WINDOW, 5000). +-opaque metric_ref() :: tuple(). + %%================================================================================ %% API funcions %%================================================================================ @@ -76,6 +78,11 @@ from_model(ModelKey) -> #mnode{metaparams = MPs} = lee_model:get(ModelKey, ?MYMODEL), ?m_attr(metric, id, MPs). +%% TODO: +-spec opstat_from_model(lee:model_key()) -> {metric_id(), metric_id()}. +opstat_from_model(Key) -> + {from_model(Key ++ [avg_time]), from_model(Key ++ [pending])}. + -spec opstat(atom(), atom()) -> lee:namespace(). opstat(Group, Operation) -> #{ avg_time => @@ -84,6 +91,7 @@ opstat(Group, Operation) -> , metric_type => rolling_average , id => ?GROUP_OP_TIME(Group, Operation) , labels => [group, operation] + , unit => "ms" }} , pending => {[metric], @@ -94,6 +102,21 @@ opstat(Group, Operation) -> }} }. +-spec call_with_counter(metric_ref(), module(), atom(), list()) -> _. +call_with_counter({AvgTime, NPending}, Mod, Fun, Args) -> + emqttb_metrics:counter_inc(NPending, 1), + T0 = os:system_time(microsecond), + try apply(Mod, Fun, Args) + catch + EC:Err -> + EC(Err) + after + T = os:system_time(microsecond), + emqttb_metrics:counter_dec(NPending, 1), + emqttb_metrics:rolling_average_observe(AvgTime, T - T0) + end. + + %% Simple counters and gauges: -spec new_counter(metric_id(), list()) -> metric_id(). diff --git a/src/metrics/emqttb_pushgw.erl b/src/metrics/emqttb_pushgw.erl index 9da197a..091b83b 100644 --- a/src/metrics/emqttb_pushgw.erl +++ b/src/metrics/emqttb_pushgw.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -65,5 +65,5 @@ do_collect() -> Data = prometheus_text_format:format(), Headers = [], Options = [], - {ok, Code, _RespHeaders, ClientRef} = hackney:post(Url, Headers, Data, Options), + {ok, _Code, _RespHeaders, ClientRef} = hackney:post(Url, Headers, Data, Options), hackney:skip_body(ClientRef). diff --git a/src/scenarios/emqttb_scenario_conn.erl b/src/scenarios/emqttb_scenario_conn.erl index ca4a4a1..04bd7d7 100644 --- a/src/scenarios/emqttb_scenario_conn.erl +++ b/src/scenarios/emqttb_scenario_conn.erl @@ -31,7 +31,7 @@ -include("emqttb.hrl"). -include_lib("typerefl/include/types.hrl"). --import(emqttb_scenario, [complete/1, loiter/0, my_conf/1, set_stage/2, set_stage/1]). +-import(emqttb_scenario, [complete/1, loiter/0, my_conf/1, my_conf_key/1, set_stage/2, set_stage/1]). %%================================================================================ %% Type declarations @@ -45,10 +45,9 @@ model() -> #{ conninterval => - {[value, cli_param], - #{ oneliner => "Client connection interval" - , type => emqttb:duration_us() - , default_ref => [interval] + {[value, cli_param, autorate], + (emqttb_group:conninterval_model('conn/conn', [?SK(conn), metrics, conn_latency, pending])) + #{ default_ref => [interval] , cli_operand => "conninterval" , cli_short => $I }} @@ -84,16 +83,21 @@ model() -> , cli_operand => "clean-start" , cli_short => $C }} + %% Metrics: + , metrics => + emqttb_behavior_conn:model('conn/conn') }. run() -> GroupId = ?GROUP, Opts = #{ expiry => my_conf([expiry]) , clean_start => my_conf([clean_start]) + , metrics => my_conf_key([metrics]) }, emqttb_group:ensure(#{ id => GroupId , client_config => my_conf([group]) , behavior => {emqttb_behavior_conn, Opts} + , conn_interval => emqttb_autorate:from_model(my_conf_key([conninterval])) }), Interval = my_conf([conninterval]), set_stage(ramp_up), diff --git a/src/scenarios/emqttb_scenario_persistent_session.erl b/src/scenarios/emqttb_scenario_persistent_session.erl index 4d2fe8f..7970bbc 100644 --- a/src/scenarios/emqttb_scenario_persistent_session.erl +++ b/src/scenarios/emqttb_scenario_persistent_session.erl @@ -94,7 +94,7 @@ model() -> , cli_operand => "pubinterval" , cli_short => $i , autorate_id => 'persistent_session/pubinterval' - , process_variable => [?SK(persistent_session), pub, pub_latency, pending] + , process_variable => [?SK(persistent_session), pub, metrics, pub_latency, pending] , error_coeff => -1 }} , n => @@ -105,13 +105,6 @@ model() -> , cli_operand => "num-publishers" , cli_short => $P }} - , set_pub_latency => - {[value, cli_param], - #{ oneliner => "Try to keep publishing time at this value (ms)" - , type => emqttb:duration_ms() - , default_str => "100ms" - , cli_operand => "publatency" - }} , topic_suffix => {[value, cli_param], #{ oneliner => "Suffix of the topic to publish to" @@ -128,18 +121,8 @@ model() -> , cli_operand => "pubtime" , cli_short => $T }} - %% Metrics: - , n_published => - {[metric], - #{ oneliner => "Total number of published messages" - , id => {emqttb_published_messages, persistent_session} - , metric_type => counter - , labels => [scenario] - }} - , pub_latency => - emqttb_metrics:opstat('persistent_session/pub', 'publish') - , conn_latency => - emqttb_metrics:opstat('persistent_session/pub', 'connect') + , metrics => + emqttb_behavior_pub:model('persistent_session/pub') } , sub => #{ qos => @@ -164,14 +147,16 @@ model() -> , default => 16#FFFFFFFF , cli_operand => "expiry" }} + , metrics => + emqttb_behavior_sub:model('persistent_session/sub') } , conninterval => - {[value, cli_param], - #{ oneliner => "Client connection interval (microsecond)" - , type => emqttb:duration_us() - , default => 0 - , cli_operand => "conninterval" + {[value, cli_param, autorate], + (emqttb_group:conninterval_model('persistent_session/pub', + [?SK(persistent_session), pub, metrics, conn_latency, pending])) + #{ cli_operand => "conninterval" , cli_short => $I + , default_str => "10ms" }} , group => {[value, cli_param], @@ -239,18 +224,19 @@ consume_stage(Cycle, S) -> , qos => my_conf([sub, qos]) , expiry => my_conf([sub, expiry]) , clean_start => Cycle =:= 0 + , metrics => my_conf_key([sub, metrics]) }, emqttb_group:ensure(#{ id => ?SUB_GROUP , client_config => my_conf([group]) , behavior => {emqttb_behavior_sub, SubOpts} + , conn_interval => emqttb_autorate:from_model(my_conf_key([conninterval])) }), N = my_conf([sub, n]), - Interval = my_conf([conninterval]), - {ok, N} = emqttb_group:set_target(?SUB_GROUP, N, Interval), + {ok, N} = emqttb_group:set_target(?SUB_GROUP, N), wait_consume_all(N, S), emqttb_group:stop(?SUB_GROUP), S#s{ to_consume = 0 - , consumed = emqttb_metrics:get_counter(?CNT_SUB_MESSAGES(?SUB_GROUP)) + , consumed = total_consumed_messages() }. publish_stage(S = #s{produced = NPub0, pubinterval = PubInterval}) -> @@ -258,24 +244,23 @@ publish_stage(S = #s{produced = NPub0, pubinterval = PubInterval}) -> TopicSuffix = my_conf([pub, topic_suffix]), PubOpts = #{ topic => <> , pubinterval => my_conf_key([pub, pubinterval]) - , n_published => my_conf_key([pub, n_published]) , msg_size => my_conf([pub, msg_size]) , qos => my_conf([pub, qos]) - , set_latency => my_conf_key([pub, set_pub_latency]) + , metrics => my_conf_key([pub, metrics]) , metadata => true }, emqttb_group:ensure(#{ id => ?PUB_GROUP , client_config => my_conf([group]) , behavior => {emqttb_behavior_pub, PubOpts} + , conn_interval => emqttb_autorate:from_model(my_conf_key([conninterval])) }), - Interval = my_conf([conninterval]), - {ok, _} = emqttb_group:set_target(?PUB_GROUP, my_conf([pub, n]), Interval), + {ok, _} = emqttb_group:set_target(?PUB_GROUP, my_conf([pub, n])), PubTime = my_conf([pub, pub_time]), timer:sleep(PubTime), PubIntervalCref = emqttb_autorate:get_counter('persistent_session/pubinterval'), PubInterval2 = counters:get(PubIntervalCref, 1), emqttb_group:stop(?PUB_GROUP), - NPub = emqttb_metrics:get_counter(emqttb_metrics:from_model(my_conf_key([pub, n_published]))), + NPub = emqttb_metrics:get_counter(emqttb_metrics:from_model(my_conf_key([pub, metrics, n_published]))), %% TODO: it doesn't take ramp up/down into account: prometheus_summary:observe(?PUB_THROUGHPUT, (NPub - NPub0) * timer:seconds(1) div PubTime), S#s{ produced = NPub @@ -284,7 +269,7 @@ publish_stage(S = #s{produced = NPub0, pubinterval = PubInterval}) -> }. wait_consume_all(Nsubs, #s{to_consume = Nmsgs, consumed = Consumed}) -> - LastConsumedMessages = emqttb_metrics:get_counter(?CNT_SUB_MESSAGES(?SUB_GROUP)), + LastConsumedMessages = total_consumed_messages(), do_consume(Consumed + Nsubs * Nmsgs, LastConsumedMessages, max_checks_without_progress()). do_consume(_, _, 0) -> @@ -311,7 +296,8 @@ topic_prefix() -> <<"pers_session/">>. total_consumed_messages() -> - emqttb_metrics:get_counter(?CNT_SUB_MESSAGES(?SUB_GROUP)). + CntrId = emqttb_metrics:from_model(my_conf_key([sub, metrics, n_received])), + emqttb_metrics:get_counter(CntrId). max_checks_without_progress() -> my_conf([max_stuck_time]) div ?CHECK_INTERVAL_MS. diff --git a/src/scenarios/emqttb_scenario_pub.erl b/src/scenarios/emqttb_scenario_pub.erl index 144f11d..0a314a9 100644 --- a/src/scenarios/emqttb_scenario_pub.erl +++ b/src/scenarios/emqttb_scenario_pub.erl @@ -81,7 +81,7 @@ model() -> , cli_operand => "conninterval" , cli_short => $I , autorate_id => 'pub/conn' - , process_variable => [?SK(pub), conn_latency, pending] + , process_variable => [?SK(pub), metrics, conn_latency, pending] }} , pubinterval => {[value, cli_param, autorate], @@ -91,16 +91,9 @@ model() -> , cli_operand => "pubinterval" , cli_short => $i , autorate_id => 'pub/pubinterval' - , process_variable => [?SK(pub), pub_latency, pending] + , process_variable => [?SK(pub), metrics, pub_latency, pending] , error_coeff => -1 }} - , set_pub_latency => - {[value, cli_param], - #{ oneliner => "Try to keep publishing time at this value (ms)" - , type => emqttb:duration_ms() - , default => 100 - , cli_operand => "publatency" - }} , n_clients => {[value, cli_param], #{ oneliner => "Number of clients" @@ -132,34 +125,24 @@ model() -> , default => 0 , cli_operand => "start-n" }} - %% Metrics: - , n_published => - {[metric], - #{ oneliner => "Total number of published messages" - , metric_type => counter - , id => {emqttb_published_messages, pub} - , labels => [scenario] - }} - , pub_latency => - emqttb_metrics:opstat('pub', 'publish') - , conn_latency => - emqttb_metrics:opstat('pub', 'connect') + , metrics => + emqttb_behavior_pub:model('pub/pub') }. run() -> - PubOpts = #{ topic => my_conf([topic]) - , n_published => my_conf_key([n_published]) - , pubinterval => my_conf_key([pubinterval]) - , msg_size => my_conf([msg_size]) - , qos => my_conf([qos]) - , retain => my_conf([retain]) - , set_latency => my_conf_key([set_pub_latency]) - , metadata => my_conf([metadata]) + PubOpts = #{ topic => my_conf([topic]) + , pubinterval => my_conf_key([pubinterval]) + , msg_size => my_conf([msg_size]) + , qos => my_conf([qos]) + , retain => my_conf([retain]) + , metadata => my_conf([metadata]) + , metrics => my_conf_key([metrics]) }, emqttb_group:ensure(#{ id => ?GROUP , client_config => my_conf([group]) , behavior => {emqttb_behavior_pub, PubOpts} , start_n => my_conf([start_n]) + , conn_interval => emqttb_autorate:from_model(my_conf_key([conninterval])) }), Interval = my_conf([conninterval]), set_stage(ramp_up), diff --git a/src/scenarios/emqttb_scenario_pubsub_fwd.erl b/src/scenarios/emqttb_scenario_pubsub_fwd.erl index a5f15a5..d2dc2d4 100644 --- a/src/scenarios/emqttb_scenario_pubsub_fwd.erl +++ b/src/scenarios/emqttb_scenario_pubsub_fwd.erl @@ -75,16 +75,9 @@ model() -> , cli_operand => "pubinterval" , cli_short => $i , autorate_id => 'pubsub_fwd/pubinterval' - , process_variable => [?SK(pubsub_fwd), pub, pub_latency, pending] + , process_variable => [?SK(pubsub_fwd), pub, metrics, pub_latency, pending] , error_coeff => -1 }} - , set_pub_latency => - {[value, cli_param], - #{ oneliner => "Try to keep publishing time at this value (ms)" - , type => emqttb:duration_ms() - , default => 100 - , cli_operand => "publatency" - }} %% Metrics: , n_published => {[metric], @@ -93,10 +86,8 @@ model() -> , metric_type => counter , labels => [scenario] }} - , pub_latency => - emqttb_metrics:opstat('pubsub_fwd/pub', 'publish') - , conn_latency => - emqttb_metrics:opstat('pubsub_fdw/pub', 'connect') + , metrics => + emqttb_behavior_pub:model('pubsub_fwd/pub') } , sub => #{ qos => @@ -106,14 +97,15 @@ model() -> , default => 1 , cli_operand => "sub-qos" }} + , metrics => + emqttb_behavior_sub:model('pubsub_fwd/sub') } , conninterval => - {[value, cli_param], - #{ oneliner => "Client connection interval (microsecond)" - , type => emqttb:duration_us() - , default => 0 - , cli_operand => "conninterval" + {[value, cli_param, autorate], + (emqttb_group:conninterval_model('pubsub_fwd/pub', [?SK(pubsub_fwd), pub, metrics, conn_latency, pending])) + #{ cli_operand => "conninterval" , cli_short => $I + , default_str => "10ms" }} , group => {[value, cli_param], @@ -180,15 +172,16 @@ subscribe_stage() -> , host_shift => 0 , host_selection => HostSelection , parse_metadata => true + , metrics => my_conf_key([sub, metrics]) }, emqttb_group:ensure(#{ id => ?SUB_GROUP , client_config => my_conf([group]) , behavior => {emqttb_behavior_sub, SubOpts} , start_n => my_conf([start_n]) + , conn_interval => emqttb_autorate:from_model(my_conf_key([conninterval])) }), N = my_conf([num_clients]) div 2, - Interval = my_conf([conninterval]), - {ok, _} = emqttb_group:set_target(?SUB_GROUP, N, Interval), + {ok, _} = emqttb_group:set_target(?SUB_GROUP, N), ok. publish_stage() -> @@ -203,23 +196,22 @@ publish_stage() -> false -> 0 end, PubOpts = #{ topic => <> - , n_published => my_conf_key([pub, n_published]) , pubinterval => my_conf_key([pub, pubinterval]) , msg_size => my_conf([pub, msg_size]) , qos => my_conf([pub, qos]) - , set_latency => my_conf_key([pub, set_pub_latency]) , metadata => true , host_shift => HostShift , host_selection => HostSelection + , metrics => my_conf_key([pub, metrics]) }, emqttb_group:ensure(#{ id => ?PUB_GROUP , client_config => my_conf([group]) , behavior => {emqttb_behavior_pub, PubOpts} , start_n => my_conf([start_n]) + , conn_interval => emqttb_autorate:from_model(my_conf_key([conninterval])) }), N = my_conf([num_clients]) div 2, - Interval = my_conf([conninterval]), - {ok, _} = emqttb_group:set_target(?PUB_GROUP, N, Interval), + {ok, _} = emqttb_group:set_target(?PUB_GROUP, N), ok. topic_prefix() -> diff --git a/src/scenarios/emqttb_scenario_sub.erl b/src/scenarios/emqttb_scenario_sub.erl index f887115..aac62e2 100644 --- a/src/scenarios/emqttb_scenario_sub.erl +++ b/src/scenarios/emqttb_scenario_sub.erl @@ -17,7 +17,6 @@ -behavior(emqttb_scenario). - %% behavior callbacks: -export([ model/0 , run/0 @@ -31,7 +30,7 @@ -include("emqttb.hrl"). -include_lib("typerefl/include/types.hrl"). --import(emqttb_scenario, [complete/1, loiter/0, my_conf/1, set_stage/2, set_stage/1]). +-import(emqttb_scenario, [complete/1, loiter/0, my_conf/1, my_conf_key/1, set_stage/2, set_stage/1]). %%================================================================================ %% Type declarations @@ -52,10 +51,9 @@ model() -> , cli_short => $t }} , conninterval => - {[value, cli_param], - #{ oneliner => "Client connection interval" - , type => emqttb:duration_us() - , default_ref => [interval] + {[value, cli_param, autorate], + (emqttb_group:conninterval_model('sub/sub', [?SK(sub), metrics, conn_latency, pending])) + #{ default_ref => [interval] , cli_operand => "conninterval" , cli_short => $I }} @@ -104,6 +102,8 @@ model() -> , cli_operand => "clean-start" , cli_short => $c }} + , metrics => + emqttb_behavior_sub:model('sub/sub') }. run() -> @@ -112,15 +112,16 @@ run() -> , expiry => my_conf([expiry]) , parse_metadata => my_conf([parse_metadata]) , clean_start => my_conf([clean_start]) + , metrics => my_conf_key([metrics]) }, emqttb_group:ensure(#{ id => ?GROUP , client_config => my_conf([group]) , behavior => {emqttb_behavior_sub, SubOpts} + , conn_interval => emqttb_autorate:from_model(my_conf_key([conninterval])) }), - Interval = my_conf([conninterval]), set_stage(ramp_up), N = my_conf([n_clients]), - {ok, _} = emqttb_group:set_target(?GROUP, N, Interval), + {ok, _} = emqttb_group:set_target(?GROUP, N), set_stage(run_traffic), loiter(), complete(ok). diff --git a/src/scenarios/emqttb_scenario_sub_flapping.erl b/src/scenarios/emqttb_scenario_sub_flapping.erl index 881231b..7fa7328 100644 --- a/src/scenarios/emqttb_scenario_sub_flapping.erl +++ b/src/scenarios/emqttb_scenario_sub_flapping.erl @@ -30,7 +30,7 @@ -include("emqttb.hrl"). -include_lib("typerefl/include/types.hrl"). --import(emqttb_scenario, [complete/1, loiter/0, my_conf/1, set_stage/2, set_stage/1]). +-import(emqttb_scenario, [complete/1, loiter/0, my_conf/1, my_conf_key/1, set_stage/2, set_stage/1]). %%================================================================================ %% Type declarations @@ -55,10 +55,9 @@ model() -> , cli_short => $t }} , conninterval => - {[value, cli_param], - #{ oneliner => "Client connection interval" - , type => emqttb:duration_us() - , default_ref => [interval] + {[value, cli_param, autorate], + (emqttb_group:conninterval_model('sub_flapping/sub', [?SK(sub_flapping), metrics, conn_latency, pending])) + #{ default_ref => [interval] , cli_operand => "conninterval" , cli_short => $I }} @@ -108,6 +107,8 @@ model() -> , cli_operand => "cycles" , cli_short => $C }} + , metrics => + emqttb_behavior_sub:model('sub_flapping/sub') }. run() -> @@ -115,22 +116,24 @@ run() -> , qos => my_conf([qos]) , expiry => my_conf([expiry]) , clean_start => my_conf([clean_start]) + , metrics => my_conf_key([metrics]) }, emqttb_group:ensure(#{ id => ?GROUP , client_config => my_conf([group]) , behavior => {emqttb_behavior_sub, SubOpts} + , conn_interval => emqttb_autorate:from_model(my_conf_key([conninterval])) }), - cycle(0, my_conf([n_cycles]), my_conf([conninterval])). + cycle(0, my_conf([n_cycles])). -cycle(Cycle, Max, _Interval) when Cycle >= Max -> +cycle(Cycle, Max) when Cycle >= Max -> complete(ok); -cycle(Cycle, Max, Interval) -> +cycle(Cycle, Max) -> set_stage(ramp_up), N = my_conf([n_clients]), - {ok, _} = emqttb_group:set_target(?GROUP, N, Interval), + {ok, _} = emqttb_group:set_target(?GROUP, N), set_stage(ramp_down), {ok, _} = emqttb_group:set_target(?GROUP, 0, undefined), - cycle(Cycle + 1, Max, undefined). + cycle(Cycle + 1, Max). %%================================================================================ %% Internal exports diff --git a/test/escript_SUITE.erl b/test/escript_SUITE.erl index 2b4ab3a..6a0d27b 100644 --- a/test/escript_SUITE.erl +++ b/test/escript_SUITE.erl @@ -27,9 +27,24 @@ suite() -> t_no_args(Config) when is_list(Config) -> ?assertMatch(0, run("")). -t_basic_scenarios(Config) when is_list(Config) -> +t_pub(Config) when is_list(Config) -> ?assertMatch(0, run("--loiter 0 @pub -t foo -I 1000 -N 0")). +t_sub(Config) when is_list(Config) -> + ?assertMatch(0, run("--loiter 0 @sub -t foo -N 0")). + +t_conn(Config) when is_list(Config) -> + ?assertMatch(0, run("--loiter 0 @conn -N 0")). + +t_pubsub_fwd(Config) when is_list(Config) -> + ?assertMatch(0, run("--loiter 0 @pubsub_fwd")). + +t_sub_flapping(Config) when is_list(Config) -> + ?assertMatch(0, run("--loiter 0 @sub_flapping -t foo --cycles 1 -N 0")). + +t_persistent_session(Config) when is_list(Config) -> + ?assertMatch(0, run("@persistent_session --cycles 1 --pubtime 1ms")). + t_set_group_config(Config) when is_list(Config) -> ?assertMatch(0, run("@g -p 9090")), ?assertMatch(0, run("@g -g my_group -p 9090")), From 63b245172d30e63d50ede11f8ff01cbca2fe9d7b Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Wed, 6 Dec 2023 14:34:26 +0100 Subject: [PATCH 03/10] feat: Treat --inflight as receive-maximum --- src/framework/emqttb_worker.erl | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/framework/emqttb_worker.erl b/src/framework/emqttb_worker.erl index a6bb582..0e28103 100644 --- a/src/framework/emqttb_worker.erl +++ b/src/framework/emqttb_worker.erl @@ -143,15 +143,16 @@ connect(ConnOpstat, Properties0, CustomOptions, CustomTcpOptions, CustomSslOptio HostShift = maps:get(host_shift, Properties0, 0), HostSelection = maps:get(host_selection, Properties0, random), Properties = maps:without([host_shift, host_selection], Properties0), - Username = my_cfg([client, username]), - Password = my_cfg([client, password]), - SSL = my_cfg([ssl, enable]), - KeepAlive = my_cfg([connection, keepalive]), + Username = my_cfg([client, username]), + Password = my_cfg([client, password]), + SSL = my_cfg([ssl, enable]), + KeepAlive = my_cfg([connection, keepalive]), + MaxInflight = my_cfg([connection, inflight]), Options = [ {username, Username} || Username =/= undefined] ++ [ {password, Password} || Password =/= undefined] ++ [ {ssl_opts, CustomSslOptions ++ ssl_opts()} || SSL] ++ [ {clientid, my_clientid()} - , {max_inflight, my_cfg([connection, inflight])} + , {max_inflight, MaxInflight} , {hosts, broker_hosts(HostSelection, HostShift)} , {port, get_port()} , {proto_ver, my_cfg([connection, proto_ver])} @@ -159,7 +160,7 @@ connect(ConnOpstat, Properties0, CustomOptions, CustomTcpOptions, CustomSslOptio , {owner, self()} , {ssl, SSL} , {tcp_opts, CustomTcpOptions ++ tcp_opts()} - , {properties, Properties} + , {properties, Properties #{'Receive-Maximum' => MaxInflight}} , {keepalive, KeepAlive} ], {ok, Client} = emqtt:start_link(CustomOptions ++ Options), From 1edf747dc203a0618e9653eef3424f48e27567c7 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Thu, 7 Dec 2023 04:37:13 +0100 Subject: [PATCH 04/10] feat: Verify autorate names --- src/conf/emqttb_conf_model.erl | 20 +++++++++++- src/emqttb.erl | 2 +- src/framework/emqttb_app.erl | 23 ++++++++++++- src/framework/emqttb_autorate.erl | 54 +++++++++++++++++++++++++------ test/emqttb_worker_SUITE.erl | 2 ++ test/escript_SUITE.erl | 4 +-- 6 files changed, 90 insertions(+), 15 deletions(-) diff --git a/src/conf/emqttb_conf_model.erl b/src/conf/emqttb_conf_model.erl index 93a70ac..8000521 100644 --- a/src/conf/emqttb_conf_model.erl +++ b/src/conf/emqttb_conf_model.erl @@ -18,7 +18,7 @@ %% API: -export([model/0]). --export_type([]). +-export_type([object_type/0]). -include("emqttb.hrl"). -include_lib("typerefl/include/types.hrl"). @@ -27,6 +27,9 @@ %% Type declarations %%================================================================================ +-type object_type() :: autorate | metric. +-reflect_type([object_type/0]). + %%================================================================================ %% API funcions %%================================================================================ @@ -239,6 +242,21 @@ model() -> }} } , scenarios => emqttb_scenario:model() + , actions => + #{ ls => + {[map, cli_action], + #{ cli_operand => "ls" + , key_elements => [] + , oneliner => "List objects" + }, + #{ what => + {[value, cli_positional], + #{ oneliner => "Type of objects to list" + , type => object_type() + , cli_arg_position => 1 + }} + }} + } , groups => {[map, cli_action, default_instance], #{ cli_operand => "g" diff --git a/src/emqttb.erl b/src/emqttb.erl index da953cd..eee1e7d 100644 --- a/src/emqttb.erl +++ b/src/emqttb.erl @@ -26,7 +26,7 @@ parse_addresses/1, parse_duration_us/1, parse_duration_ms/1, parse_duration_s/1, parse_byte_size/1, wait_time/0]). --export_type([n_clients/0]). +-export_type([n_clients/0, autorate/0]). -include("emqttb.hrl"). -include_lib("typerefl/include/types.hrl"). diff --git a/src/framework/emqttb_app.erl b/src/framework/emqttb_app.erl index cb10137..added83 100644 --- a/src/framework/emqttb_app.erl +++ b/src/framework/emqttb_app.erl @@ -14,8 +14,9 @@ start(_StartType, _StartArgs) -> emqttb_conf:load_model(), - emqttb_conf:load_conf(), Sup = emqttb_sup:start_link(), + emqttb_conf:load_conf(), + maybe_perform_special_action(), emqttb_autorate:create_autorates(), emqttb_scenario:run_scenarios(), CLIArgs = application:get_env(?APP, cli_args, []), @@ -52,3 +53,23 @@ maybe_start_distr() -> Opts = #{dist_listen => true}, net_kernel:start(Name, Opts) end. + +maybe_perform_special_action() -> + case ?CFG_LIST([actions, ls, {}]) of + [] -> + ok; + [Key] -> + What = ?CFG(Key ++ [what]), + Keys = lee_model:get_metatype_index(What, ?MYMODEL), + MP = case What of + metric -> id; + autorate -> autorate_id + end, + lists:foreach( + fun(K) -> + #mnode{metaparams = #{MP := Id}} = lee_model:get(K, ?MYMODEL), + io:format("~p~n", [Id]) + end, + Keys), + emqttb:terminate() + end. diff --git a/src/framework/emqttb_autorate.erl b/src/framework/emqttb_autorate.erl index 66c2093..334b880 100644 --- a/src/framework/emqttb_autorate.erl +++ b/src/framework/emqttb_autorate.erl @@ -29,7 +29,7 @@ %% gen_server callbacks: -export([init/1, handle_call/3, handle_cast/2, handle_info/2]). %% lee_metatype callbacks: --export([names/1, metaparams/1, meta_validate_node/4, meta_validate/2]). +-export([names/1, metaparams/1, meta_validate_node/4, meta_validate/2, read_patch/2, validate_node/5]). %% internal exports: -export([start_link/1, model/0, from_model/1]). @@ -120,7 +120,7 @@ start_link(Conf = #{id := Id}) -> model() -> #{ id => - {[value, cli_param], + {[value, cli_param, autorate_id], #{ type => atom() , default => default , cli_operand => "autorate" @@ -187,14 +187,16 @@ model() -> %%================================================================================ names(_) -> - [autorate]. + [autorate, autorate_id]. metaparams(autorate) -> %% TBD: make it possible to configure alternative targets. [ {mandatory, autorate_id, atom()} , {mandatory, process_variable, lee:model_key()} , {optional, error_coeff, number()} - ]. + ]; +metaparams(autorate_id) -> + []. meta_validate_node(autorate, Model, _Key, #mnode{metaparams = #{process_variable := ProcVarKey}}) -> @@ -207,7 +209,9 @@ meta_validate_node(autorate, Model, _Key, #mnode{metaparams = #{process_variable end catch _:_ -> Error - end. + end; +meta_validate_node(autorate_id, _, _, _) -> + {[], []}. meta_validate(autorate, Model) -> Ids = [begin @@ -216,23 +220,52 @@ meta_validate(autorate, Model) -> end || Key <- lee_model:get_metatype_index(autorate, Model)], case length(lists:usort(Ids)) =:= length(Ids) of false -> {["Autorate IDs must be unique"], [], []}; - true -> {[], [], []} - end. + true -> {[], [], [{set, autorate_ids, Ids}]} + end; +meta_validate(autorate_id, _) -> + {[], [], []}. + + +-spec validate_node(lee:metatype(), lee:model(), _Staging :: lee:data(), lee:key(), #mnode{}) -> + lee_lib:check_result(). +validate_node(autorate_id, Model, Data, Key, _) -> + Id = lee:get(Model, Data, Key), + {ok, Ids} = lee_model:get_meta(autorate_ids, Model), + case lists:member(Id, Ids) of + true -> {[], []}; + false -> {["Unknown autorate " ++ atom_to_list(Id)], []} + end; +validate_node(autorate, _, _, _, _) -> + {[], []}. + +read_patch(autorate, Model) -> + %% Create default instances: + Prio = -99999, + {ok, Prio, + lists:flatmap( + fun(Key) -> + #mnode{metaparams = MPs} = lee_model:get(Key, ?MYMODEL), + Id = ?m_attr(autorate, autorate_id, MPs), + lee_lib:make_nested_patch(Model, [autorate], + #{ [id] => Id + }) + end, + lee_model:get_metatype_index(autorate, ?MYMODEL))}; +read_patch(autorate_id, _) -> + {ok, 0, []}. create_autorates() -> [begin #mnode{metaparams = MPs} = lee_model:get(Key, ?MYMODEL), Id = ?m_attr(autorate, autorate_id, MPs), - emqttb_conf:patch([{set, [autorate, {Id}], []}]), {ok, _} = emqttb_autorate_sup:ensure(#{ id => Id , error => make_error_fun(Key) , init_val => ?CFG(Key) , conf_root => Id }) - end || Key <- lee_model:get_metatype_index(autorate, ?MYMODEL)], %% TODO: wrong + end || Key <- lee_model:get_metatype_index(autorate, ?MYMODEL)], ok. - -spec from_model(lee:key()) -> atom(). from_model(Key) -> #mnode{metaparams = #{autorate_id := Id}} = lee_model:get(Key, ?MYMODEL), @@ -255,6 +288,7 @@ from_model(Key) -> }). init(Config = #{id := Id, conf_root := ConfRoot, error := ErrF}) -> + logger:info("Starting autorate ~p", [Id]), MRef = case Config of #{parent := Parent} -> monitor(process, Parent); _ -> undefined diff --git a/test/emqttb_worker_SUITE.erl b/test/emqttb_worker_SUITE.erl index 8ea203d..bf1c50f 100644 --- a/test/emqttb_worker_SUITE.erl +++ b/test/emqttb_worker_SUITE.erl @@ -47,6 +47,7 @@ t_group(_Config) -> {ok, Pid} = emqttb_group:start_link(#{ id => Group , client_config => #{} , behavior => {emqttb_dummy_behavior, #{}} + , conn_interval => 'conn/conn' }), {ok, NActual} = emqttb_group:set_target(Group, NClients, 1), ?assert(NActual >= NClients), @@ -91,6 +92,7 @@ error_scenario(Config) -> {ok, Pid} = emqttb_group:start_link(#{ id => Group , client_config => #{} , behavior => {emqttb_dummy_behavior, Config} + , conn_interval => 'conn/conn' }), emqttb_group:set_target_async(Group, NClients, 1), %% Wait until the first worker start: diff --git a/test/escript_SUITE.erl b/test/escript_SUITE.erl index 6a0d27b..a6d75df 100644 --- a/test/escript_SUITE.erl +++ b/test/escript_SUITE.erl @@ -37,13 +37,13 @@ t_conn(Config) when is_list(Config) -> ?assertMatch(0, run("--loiter 0 @conn -N 0")). t_pubsub_fwd(Config) when is_list(Config) -> - ?assertMatch(0, run("--loiter 0 @pubsub_fwd")). + ?assertMatch(0, run("--loiter 0 @pubsub_fwd -n 0")). t_sub_flapping(Config) when is_list(Config) -> ?assertMatch(0, run("--loiter 0 @sub_flapping -t foo --cycles 1 -N 0")). t_persistent_session(Config) when is_list(Config) -> - ?assertMatch(0, run("@persistent_session --cycles 1 --pubtime 1ms")). + ?assertMatch(0, run("@persistent_session --cycles 1 --pubtime 1ms -P 0 -S 0")). t_set_group_config(Config) when is_list(Config) -> ?assertMatch(0, run("@g -p 9090")), From 82202edb7322cb5b79948f157368a125ff7c9be0 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Thu, 7 Dec 2023 06:45:38 +0100 Subject: [PATCH 05/10] feat: Make process variable configurable --- src/framework/emqttb_autorate.erl | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/framework/emqttb_autorate.erl b/src/framework/emqttb_autorate.erl index 334b880..7d9e09b 100644 --- a/src/framework/emqttb_autorate.erl +++ b/src/framework/emqttb_autorate.erl @@ -126,9 +126,22 @@ model() -> , cli_operand => "autorate" , cli_short => $a }} + , process_variable => + {[value, cli_param], + #{ oneliner => "Key of the metric that the autorate uses as a process variable" + , type => lee:model_key() + , cli_operand => "pvar" + }} + , error_coeff => + {[value, cli_param], + #{ oneliner => "Multiply error by this coefficient" + , type => number() + , default => 1 + , cli_operand => "error-coeff" + }} , set_point => {[value, cli_param], - #{ oneliner => "Value that the autorate will try to approach" + #{ oneliner => "Value of the process variable that the autorate will try to keep" , type => number() , default => 0 , cli_operand => "setpoint" @@ -246,8 +259,10 @@ read_patch(autorate, Model) -> fun(Key) -> #mnode{metaparams = MPs} = lee_model:get(Key, ?MYMODEL), Id = ?m_attr(autorate, autorate_id, MPs), + Pvar = ?m_attr(autorate, process_variable, MPs), lee_lib:make_nested_patch(Model, [autorate], #{ [id] => Id + , [process_variable] => Pvar }) end, lee_model:get_metatype_index(autorate, ?MYMODEL))}; From e2a7b730e86d7da90816e40d93ed6b36880b6d42 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Thu, 7 Dec 2023 09:17:20 +0100 Subject: [PATCH 06/10] feat(autorate): Allow to change the process variable in the runtime --- src/conf/emqttb_conf.erl | 11 ++- src/conf/emqttb_mt_scenario.erl | 37 -------- src/framework/emqttb_autorate.erl | 95 +++++++------------ src/framework/emqttb_group.erl | 12 +-- src/framework/emqttb_scenario.erl | 33 ++++++- src/metrics/emqttb_metrics.erl | 24 ++++- src/scenarios/emqttb_scenario_conn.erl | 10 +- .../emqttb_scenario_persistent_session.erl | 12 ++- src/scenarios/emqttb_scenario_pub.erl | 7 +- src/scenarios/emqttb_scenario_pubsub_fwd.erl | 11 ++- src/scenarios/emqttb_scenario_sub.erl | 10 +- .../emqttb_scenario_sub_flapping.erl | 12 ++- test/emqttb_worker_SUITE.erl | 4 +- 13 files changed, 144 insertions(+), 134 deletions(-) delete mode 100644 src/conf/emqttb_mt_scenario.erl diff --git a/src/conf/emqttb_conf.erl b/src/conf/emqttb_conf.erl index a5b0901..807a253 100644 --- a/src/conf/emqttb_conf.erl +++ b/src/conf/emqttb_conf.erl @@ -16,7 +16,7 @@ -module(emqttb_conf). %% API: --export([load_model/0, load_conf/0, get/1, list_keys/1, reload/0, patch/1]). +-export([load_model/0, load_conf/0, get/1, list_keys/1, reload/0, patch/1, string2patch/1]). -export([compile_model/1]). -export_type([]). @@ -136,7 +136,7 @@ metamodel() -> , priority => -110 , file => "/etc/emqttb/emqttb.conf" }) - , lee_metatype:create(emqttb_mt_scenario) + , lee_metatype:create(emqttb_scenario) , lee_metatype:create(emqttb_metrics) , lee_metatype:create(emqttb_autorate) ]. @@ -144,6 +144,13 @@ metamodel() -> cli_args_getter() -> application:get_env(emqttb, cli_args, []). +string2patch(Str) -> + %% Lazy attempt to parse string to a list of command line arguments + {match, L0} = re:run(Str, "'(.+)'|([^ ]+) *", [global, {capture, all_but_first, list}]), + L = lists:map(fun lists:append/1, L0), + {ok, Patch} = lee_cli:read(?MYMODEL, L), + Patch. + %% maybe_show_help_and_exit() -> %% ?CFG([help]) %% andalso open_port({spawn, "man -l docs/EMQTT\\ bench\\ daemon.man"}, [nouse_stdio, out]), diff --git a/src/conf/emqttb_mt_scenario.erl b/src/conf/emqttb_mt_scenario.erl deleted file mode 100644 index ce1e450..0000000 --- a/src/conf/emqttb_mt_scenario.erl +++ /dev/null @@ -1,37 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- --module(emqttb_mt_scenario). -%% With the magic of Lee callbacks, this module starts or stops -%% scenarios when they are added to or removed from the config. - --behavior(lee_metatype). - -%% behavior callbacks: --export([names/1]). - --include("emqttb.hrl"). - -%%================================================================================ -%% behavior callbacks -%%================================================================================ - -names(_) -> - [scenario]. - -%% post_patch(scenario, _, _, _, {set, [?SK(Scenario)], _}) -> -%% emqttb_scenario:run(Scenario); -%% post_patch(scenario, _, _, _, {rm, [?SK(Scenario)]}) -> -%% emqttb_scenario:stop(Scenario). diff --git a/src/framework/emqttb_autorate.erl b/src/framework/emqttb_autorate.erl index 7d9e09b..deac74f 100644 --- a/src/framework/emqttb_autorate.erl +++ b/src/framework/emqttb_autorate.erl @@ -29,7 +29,7 @@ %% gen_server callbacks: -export([init/1, handle_call/3, handle_cast/2, handle_info/2]). %% lee_metatype callbacks: --export([names/1, metaparams/1, meta_validate_node/4, meta_validate/2, read_patch/2, validate_node/5]). +-export([names/1, metaparams/1, meta_validate/2, validate_node/5]). %% internal exports: -export([start_link/1, model/0, from_model/1]). @@ -127,7 +127,7 @@ model() -> , cli_short => $a }} , process_variable => - {[value, cli_param], + {[value, cli_param, metric_id], #{ oneliner => "Key of the metric that the autorate uses as a process variable" , type => lee:model_key() , cli_operand => "pvar" @@ -136,7 +136,7 @@ model() -> {[value, cli_param], #{ oneliner => "Multiply error by this coefficient" , type => number() - , default => 1 + , default => -1 , cli_operand => "error-coeff" }} , set_point => @@ -203,29 +203,10 @@ names(_) -> [autorate, autorate_id]. metaparams(autorate) -> - %% TBD: make it possible to configure alternative targets. - [ {mandatory, autorate_id, atom()} - , {mandatory, process_variable, lee:model_key()} - , {optional, error_coeff, number()} - ]; + [{mandatory, autorate_id, atom()}]; metaparams(autorate_id) -> []. - -meta_validate_node(autorate, Model, _Key, #mnode{metaparams = #{process_variable := ProcVarKey}}) -> - Error = {["process_variable parameter must point at a node of `metric' metatype"], []}, - try lee_model:get(ProcVarKey, Model) of - #mnode{metatypes = MTs} -> - case lists:member(metric, MTs) of - true -> {[], []}; - false -> Error - end - catch - _:_ -> Error - end; -meta_validate_node(autorate_id, _, _, _) -> - {[], []}. - meta_validate(autorate, Model) -> Ids = [begin #mnode{metaparams = #{autorate_id := Id}} = lee_model:get(Key, Model), @@ -242,39 +223,27 @@ meta_validate(autorate_id, _) -> -spec validate_node(lee:metatype(), lee:model(), _Staging :: lee:data(), lee:key(), #mnode{}) -> lee_lib:check_result(). validate_node(autorate_id, Model, Data, Key, _) -> + %% Check that ID matches with some of the autorates in the model: Id = lee:get(Model, Data, Key), {ok, Ids} = lee_model:get_meta(autorate_ids, Model), case lists:member(Id, Ids) of true -> {[], []}; false -> {["Unknown autorate " ++ atom_to_list(Id)], []} end; -validate_node(autorate, _, _, _, _) -> - {[], []}. - -read_patch(autorate, Model) -> - %% Create default instances: - Prio = -99999, - {ok, Prio, - lists:flatmap( - fun(Key) -> - #mnode{metaparams = MPs} = lee_model:get(Key, ?MYMODEL), - Id = ?m_attr(autorate, autorate_id, MPs), - Pvar = ?m_attr(autorate, process_variable, MPs), - lee_lib:make_nested_patch(Model, [autorate], - #{ [id] => Id - , [process_variable] => Pvar - }) - end, - lee_model:get_metatype_index(autorate, ?MYMODEL))}; -read_patch(autorate_id, _) -> - {ok, 0, []}. +validate_node(autorate, Model, Data, _Key, #mnode{metaparams = #{autorate_id := Id}}) -> + %% Check that the configuration is present: + case lee:list(Model, Data, [autorate, {Id}]) of + [_] -> + {[], []}; + [] -> + {[lee_lib:format("Configuration for autorate ~p is missing", [Id])], []} + end. create_autorates() -> [begin #mnode{metaparams = MPs} = lee_model:get(Key, ?MYMODEL), Id = ?m_attr(autorate, autorate_id, MPs), {ok, _} = emqttb_autorate_sup:ensure(#{ id => Id - , error => make_error_fun(Key) , init_val => ?CFG(Key) , conf_root => Id }) @@ -302,8 +271,14 @@ from_model(Key) -> , last_err :: number() }). -init(Config = #{id := Id, conf_root := ConfRoot, error := ErrF}) -> +init(Config = #{id := Id, conf_root := ConfRoot}) -> logger:info("Starting autorate ~p", [Id]), + case Config of + #{error := ErrF} -> + ok; + _ -> + ErrF = make_error_fun(Id, ConfRoot) + end, MRef = case Config of #{parent := Parent} -> monitor(process, Parent); _ -> undefined @@ -415,24 +390,18 @@ clamp(Val, _, _) -> my_cfg(ConfRoot, Key) -> ?CFG([autorate, {ConfRoot} | Key]). --spec make_error_fun(lee:key()) -> fun(() -> number()). -make_error_fun(Key) -> - ModelKey = lee_model:get_model_key(Key), - #mnode{metaparams = MP} = lee_model:get(ModelKey, ?MYMODEL), - ProcessVarKey = ?m_attr(autorate, process_variable, MP), - Id = ?m_attr(autorate, autorate_id, MP), - Coeff = ?m_attr(autorate, error_coeff, MP, 1), - MetricKey = emqttb_metrics:from_model(ProcessVarKey), - %% - #mnode{metaparams = PVarMPs} = lee_model:get(ProcessVarKey, ?MYMODEL), - case ?m_attr(metric, metric_type, PVarMPs) of - counter -> - fun() -> - SetPoint = my_cfg(Id, [set_point]), - Coeff * (SetPoint - emqttb_metrics:get_counter(MetricKey)) - end; - rolling_average -> - fun() -> +-spec make_error_fun(emqttb:autorate(), lee:key()) -> fun(() -> number()). +make_error_fun(Id, ConfRoot) -> + fun() -> + ProcessVarKey = my_cfg(ConfRoot, [process_variable]), + SetPoint = my_cfg(Id, [set_point]), + Coeff = my_cfg(ConfRoot, [error_coeff]), + MetricKey = emqttb_metrics:from_model(ProcessVarKey), + #mnode{metaparams = PVarMPs} = lee_model:get(ProcessVarKey, ?MYMODEL), + case ?m_attr(metric, metric_type, PVarMPs) of + counter -> + Coeff * (SetPoint - emqttb_metrics:get_counter(MetricKey)); + rolling_average -> AvgWindow = 250, SetPoint = my_cfg(Id, [set_point]), Coeff * (SetPoint - emqttb_metrics:get_rolling_average(MetricKey, AvgWindow)) diff --git a/src/framework/emqttb_group.erl b/src/framework/emqttb_group.erl index a46cde0..c9105d0 100644 --- a/src/framework/emqttb_group.erl +++ b/src/framework/emqttb_group.erl @@ -18,8 +18,7 @@ -behavior(gen_server). %% API: --export([ensure/1, stop/1, set_target/2, set_target/3, set_target_async/3, broadcast/2, report_dead_id/2, info/0, - conninterval_model/2]). +-export([ensure/1, stop/1, set_target/2, set_target/3, set_target_async/3, broadcast/2, report_dead_id/2, info/0]). %% gen_server callbacks: -export([init/1, handle_call/3, handle_cast/2, terminate/2, handle_info/2]). @@ -56,15 +55,6 @@ %% API funcions %%================================================================================ --spec conninterval_model(emqttb:group_id(), lee:model_key()) -> map(). -conninterval_model(Group, AutoscaleMetric) -> - #{ oneliner => "Client connection interval" - , autorate_id => Group - , type => emqttb:duration_us() - , error_coeff => -1 - , process_variable => AutoscaleMetric - }. - -spec ensure(group_config()) -> ok. ensure(Conf) -> emqttb_group_sup:ensure(Conf#{parent => self()}). diff --git a/src/framework/emqttb_scenario.erl b/src/framework/emqttb_scenario.erl index cfc5ae0..fb15937 100644 --- a/src/framework/emqttb_scenario.erl +++ b/src/framework/emqttb_scenario.erl @@ -15,12 +15,18 @@ %%-------------------------------------------------------------------- -module(emqttb_scenario). +-behavior(lee_metatype). +-behavior(gen_server). + %% API: -export([set_stage/1, set_stage/2, stage/1, complete/1, loiter/0, model/0, list_enabled_scenarios/0, run/1, stop/1, my_scenario/0, my_conf_key/1, my_conf/1, module/1, name/1, info/0]). +%% lee_metatype callbacks: +-export([names/1, read_patch/2]). + %% gen_server callbacks: -export([init/1, handle_call/3, handle_cast/2, handle_continue/2, terminate/2]). @@ -44,10 +50,15 @@ %% Behavior declaration %%================================================================================ --callback run() -> ok. - +%% Model fragment for the scenario: -callback model() -> lee:lee_module(). +%% Callback that creates the default configuration for the scenario: +-callback initial_config() -> lee:patch(). + +%% This callback executes the scenario: +-callback run() -> ok. + %% -callback name() -> atom(). %%================================================================================ @@ -152,6 +163,24 @@ info() -> end, all_scenario_modules()). +%%================================================================================ +%% Behavior callbacks +%%================================================================================ + +names(_) -> + [scenario]. + +read_patch(scenario, _Model) -> + %% Populate configuration with the default data: + Prio = -99999, + Patch = lists:flatmap( + fun(Module) -> + Module:initial_config() + end, + all_scenario_modules()), + {ok, Prio, Patch}. + + %%================================================================================ %% External exports %%================================================================================ diff --git a/src/metrics/emqttb_metrics.erl b/src/metrics/emqttb_metrics.erl index ab41010..45313b9 100644 --- a/src/metrics/emqttb_metrics.erl +++ b/src/metrics/emqttb_metrics.erl @@ -30,7 +30,7 @@ -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2]). %% lee_metatype callbacks: --export([names/1, metaparams/1, meta_validate/2]). +-export([names/1, metaparams/1, meta_validate/2, validate_node/5]). %% internal exports: -export([start_link/0]). @@ -182,7 +182,7 @@ get_rolling_average(Key, Window) -> %%================================================================================ names(_) -> - [metric]. + [metric, metric_id]. metaparams(metric) -> [ {mandatory, metric_type, typerefl:union([counter, gauge, rolling_average])} @@ -190,7 +190,9 @@ metaparams(metric) -> , {mandatory, oneliner, string()} , {mandatory, labels, [atom()]} , {optional, unit, string()} - ]. + ]; +metaparams(metric_id) -> + []. meta_validate(metric, Model) -> Ids = [begin @@ -200,7 +202,21 @@ meta_validate(metric, Model) -> case length(lists:usort(Ids)) =:= length(Ids) of false -> {["Metric IDs must be unique"], [], []}; true -> {[], [], []} - end. + end; +meta_validate(metric_id, _) -> + {[], [], []}. + +validate_node(metric_id, Model, Data, Key, _) -> + Metric = lee:get(Model, Data, Key), + case lists:member(Metric, lee_model:get_metatype_index(metric, Model)) of + true -> + {[], []}; + false -> + Err = lee_lib:format("Unknown metric key: ~p", [Metric]), + {[Err], []} + end; +validate_node(metric, _, _, _, _) -> + {[], []}. %%================================================================================ %% gen_server callbacks diff --git a/src/scenarios/emqttb_scenario_conn.erl b/src/scenarios/emqttb_scenario_conn.erl index 04bd7d7..29e7f3a 100644 --- a/src/scenarios/emqttb_scenario_conn.erl +++ b/src/scenarios/emqttb_scenario_conn.erl @@ -21,6 +21,7 @@ %% behavior callbacks: -export([ model/0 , run/0 + , initial_config/0 ]). %% internal exports: @@ -46,10 +47,12 @@ model() -> #{ conninterval => {[value, cli_param, autorate], - (emqttb_group:conninterval_model('conn/conn', [?SK(conn), metrics, conn_latency, pending])) - #{ default_ref => [interval] + #{ oneliner => "Client connection interval" + , type => emqttb:duration_us() + , default_ref => [interval] , cli_operand => "conninterval" , cli_short => $I + , autorate_id => 'conn/conninterval' }} , n_clients => {[value, cli_param], @@ -88,6 +91,9 @@ model() -> emqttb_behavior_conn:model('conn/conn') }. +initial_config() -> + emqttb_conf:string2patch("@a -a conn/conninterval --pvar '[scenarios,conn,{},metrics,conn_latency,pending]'"). + run() -> GroupId = ?GROUP, Opts = #{ expiry => my_conf([expiry]) diff --git a/src/scenarios/emqttb_scenario_persistent_session.erl b/src/scenarios/emqttb_scenario_persistent_session.erl index 7970bbc..037c09e 100644 --- a/src/scenarios/emqttb_scenario_persistent_session.erl +++ b/src/scenarios/emqttb_scenario_persistent_session.erl @@ -33,6 +33,7 @@ %% behavior callbacks: -export([ model/0 , run/0 + , initial_config/0 ]). %% internal exports: @@ -152,11 +153,12 @@ model() -> } , conninterval => {[value, cli_param, autorate], - (emqttb_group:conninterval_model('persistent_session/pub', - [?SK(persistent_session), pub, metrics, conn_latency, pending])) - #{ cli_operand => "conninterval" + #{ oneliner => "Client connection interval" + , type => emqttb:duration_us() + , cli_operand => "conninterval" , cli_short => $I , default_str => "10ms" + , autorate_id => 'persistent_session/conninterval' }} , group => {[value, cli_param], @@ -183,6 +185,10 @@ model() -> }} }. +initial_config() -> + emqttb_conf:string2patch("@a -a persistent_session/pubinterval --pvar '[scenarios,persistent_session,{},pub,metrics,pub_latency,pending]'") ++ + emqttb_conf:string2patch("@a -a persistent_session/conninterval --pvar '[scenarios,persistent_session,{},pub,metrics,conn_latency,pending]'"). + run() -> prometheus_summary:declare([ {name, ?PUB_THROUGHPUT} , {help, <<"Write throughput for the persistent session">>} diff --git a/src/scenarios/emqttb_scenario_pub.erl b/src/scenarios/emqttb_scenario_pub.erl index 0a314a9..28c97a2 100644 --- a/src/scenarios/emqttb_scenario_pub.erl +++ b/src/scenarios/emqttb_scenario_pub.erl @@ -21,6 +21,7 @@ %% behavior callbacks: -export([ model/0 , run/0 + , initial_config/0 ]). %% internal exports: @@ -80,7 +81,7 @@ model() -> , default_ref => [interval] , cli_operand => "conninterval" , cli_short => $I - , autorate_id => 'pub/conn' + , autorate_id => 'pub/conninterval' , process_variable => [?SK(pub), metrics, conn_latency, pending] }} , pubinterval => @@ -129,6 +130,10 @@ model() -> emqttb_behavior_pub:model('pub/pub') }. +initial_config() -> + emqttb_conf:string2patch("@a -a pub/pubinterval --pvar '[scenarios,pub,{},metrics,pub_latency,pending]'") ++ + emqttb_conf:string2patch("@a -a pub/conninterval --pvar '[scenarios,pub,{},metrics,conn_latency,pending]'"). + run() -> PubOpts = #{ topic => my_conf([topic]) , pubinterval => my_conf_key([pubinterval]) diff --git a/src/scenarios/emqttb_scenario_pubsub_fwd.erl b/src/scenarios/emqttb_scenario_pubsub_fwd.erl index d2dc2d4..2ef606c 100644 --- a/src/scenarios/emqttb_scenario_pubsub_fwd.erl +++ b/src/scenarios/emqttb_scenario_pubsub_fwd.erl @@ -26,6 +26,7 @@ %% behavior callbacks: -export([ model/0 , run/0 + , initial_config/0 ]). %% internal exports: @@ -102,10 +103,12 @@ model() -> } , conninterval => {[value, cli_param, autorate], - (emqttb_group:conninterval_model('pubsub_fwd/pub', [?SK(pubsub_fwd), pub, metrics, conn_latency, pending])) - #{ cli_operand => "conninterval" + #{ oneliner => "Client connection interval" + , type => emqttb:duration_us() + , cli_operand => "conninterval" , cli_short => $I , default_str => "10ms" + , autorate_id => 'pubsub_fwd/conninterval' }} , group => {[value, cli_param], @@ -146,6 +149,10 @@ model() -> }} }. +initial_config() -> + emqttb_conf:string2patch("@a -a pubsub_fwd/pubinterval --pvar '[scenarios,pubsub_fwd,{},pub,metrics,pub_latency,pending]'") ++ + emqttb_conf:string2patch("@a -a pubsub_fwd/conninterval --pvar '[scenarios,pubsub_fwd,{},pub,metrics,conn_latency,pending]'"). + run() -> set_stage(subscribe), subscribe_stage(), diff --git a/src/scenarios/emqttb_scenario_sub.erl b/src/scenarios/emqttb_scenario_sub.erl index aac62e2..de7a25c 100644 --- a/src/scenarios/emqttb_scenario_sub.erl +++ b/src/scenarios/emqttb_scenario_sub.erl @@ -19,6 +19,7 @@ %% behavior callbacks: -export([ model/0 + , initial_config/0 , run/0 ]). @@ -52,10 +53,12 @@ model() -> }} , conninterval => {[value, cli_param, autorate], - (emqttb_group:conninterval_model('sub/sub', [?SK(sub), metrics, conn_latency, pending])) - #{ default_ref => [interval] + #{ oneliner => "Client connection interval" + , type => emqttb:duration_us() + , default_ref => [interval] , cli_operand => "conninterval" , cli_short => $I + , autorate_id => 'sub/conninterval' }} , n_clients => {[value, cli_param], @@ -106,6 +109,9 @@ model() -> emqttb_behavior_sub:model('sub/sub') }. +initial_config() -> + emqttb_conf:string2patch("@a -a sub/conninterval --pvar '[scenarios,sub,{},metrics,conn_latency,pending]'"). + run() -> SubOpts = #{ topic => my_conf([topic]) , qos => my_conf([qos]) diff --git a/src/scenarios/emqttb_scenario_sub_flapping.erl b/src/scenarios/emqttb_scenario_sub_flapping.erl index 7fa7328..2a397b3 100644 --- a/src/scenarios/emqttb_scenario_sub_flapping.erl +++ b/src/scenarios/emqttb_scenario_sub_flapping.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%%Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ %% behavior callbacks: -export([ model/0 + , initial_config/0 , run/0 ]). @@ -56,10 +57,12 @@ model() -> }} , conninterval => {[value, cli_param, autorate], - (emqttb_group:conninterval_model('sub_flapping/sub', [?SK(sub_flapping), metrics, conn_latency, pending])) - #{ default_ref => [interval] + #{ oneliner => "Client connection interval" + , type => emqttb:duration_us() + , default_ref => [interval] , cli_operand => "conninterval" , cli_short => $I + , autorate_id => 'sub_flapping/conninterval' }} , n_clients => {[value, cli_param], @@ -111,6 +114,9 @@ model() -> emqttb_behavior_sub:model('sub_flapping/sub') }. +initial_config() -> + emqttb_conf:string2patch("@a -a sub_flapping/conninterval --pvar '[scenarios,sub_flapping,{},metrics,conn_latency,pending]'"). + run() -> SubOpts = #{ topic => my_conf([topic]) , qos => my_conf([qos]) diff --git a/test/emqttb_worker_SUITE.erl b/test/emqttb_worker_SUITE.erl index bf1c50f..6b21598 100644 --- a/test/emqttb_worker_SUITE.erl +++ b/test/emqttb_worker_SUITE.erl @@ -47,7 +47,7 @@ t_group(_Config) -> {ok, Pid} = emqttb_group:start_link(#{ id => Group , client_config => #{} , behavior => {emqttb_dummy_behavior, #{}} - , conn_interval => 'conn/conn' + , conn_interval => 'conn/conninterval' }), {ok, NActual} = emqttb_group:set_target(Group, NClients, 1), ?assert(NActual >= NClients), @@ -92,7 +92,7 @@ error_scenario(Config) -> {ok, Pid} = emqttb_group:start_link(#{ id => Group , client_config => #{} , behavior => {emqttb_dummy_behavior, Config} - , conn_interval => 'conn/conn' + , conn_interval => 'conn/conninterval' }), emqttb_group:set_target_async(Group, NClients, 1), %% Wait until the first worker start: From 10d74c8473845ba643a7a9bcecdd6cc46f869c81 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Thu, 7 Dec 2023 09:57:45 +0100 Subject: [PATCH 07/10] refactor: Replace ad-hoc listing of metatypes with API functions --- src/framework/emqttb_app.erl | 19 +++++-------- src/framework/emqttb_autorate.erl | 28 ++++++++++--------- src/metrics/emqttb_metrics.erl | 11 +++++--- .../emqttb_scenario_persistent_session.erl | 4 +-- src/scenarios/emqttb_scenario_pub.erl | 3 -- src/scenarios/emqttb_scenario_pubsub_fwd.erl | 2 -- 6 files changed, 30 insertions(+), 37 deletions(-) diff --git a/src/framework/emqttb_app.erl b/src/framework/emqttb_app.erl index added83..ff5d28b 100644 --- a/src/framework/emqttb_app.erl +++ b/src/framework/emqttb_app.erl @@ -59,17 +59,12 @@ maybe_perform_special_action() -> [] -> ok; [Key] -> - What = ?CFG(Key ++ [what]), - Keys = lee_model:get_metatype_index(What, ?MYMODEL), - MP = case What of - metric -> id; - autorate -> autorate_id - end, - lists:foreach( - fun(K) -> - #mnode{metaparams = #{MP := Id}} = lee_model:get(K, ?MYMODEL), - io:format("~p~n", [Id]) - end, - Keys), + case ?CFG(Key ++ [what]) of + metric -> + Objs = emqttb_metrics:ls(?MYMODEL); + autorate -> + {Objs, _} = lists:unzip(emqttb_autorate:ls(?MYMODEL)) + end, + lists:foreach(fun(K) -> io:format("~p~n", [K]) end, Objs), emqttb:terminate() end. diff --git a/src/framework/emqttb_autorate.erl b/src/framework/emqttb_autorate.erl index deac74f..0f61990 100644 --- a/src/framework/emqttb_autorate.erl +++ b/src/framework/emqttb_autorate.erl @@ -24,7 +24,7 @@ %% https://controlguru.com/integral-reset-windup-jacketing-logic-and-the-velocity-pi-form/ %% API: --export([ensure/1, get_counter/1, reset/2, info/0, create_autorates/0]). +-export([ls/1, ensure/1, get_counter/1, reset/2, info/0, create_autorates/0]). %% gen_server callbacks: -export([init/1, handle_call/3, handle_cast/2, handle_info/2]). @@ -79,6 +79,13 @@ %% API funcions %%================================================================================ +-spec ls(lee:model()) -> [{emqttb:autorate(), lee:model_key()}]. +ls(Model) -> + [begin + #mnode{metaparams = #{autorate_id := Id}} = lee_model:get(Key, Model), + {Id, Key} + end || Key <- lee_model:get_metatype_index(autorate, Model)]. + -spec ensure(config()) -> {auto, counter:counters_ref()}. ensure(Conf) -> {ok, Pid} = emqttb_autorate_sup:ensure(Conf#{parent => self()}), @@ -142,7 +149,7 @@ model() -> , set_point => {[value, cli_param], #{ oneliner => "Value of the process variable that the autorate will try to keep" - , type => number() + , type => emqttb:duration_us() , default => 0 , cli_operand => "setpoint" , cli_short => $s @@ -150,7 +157,7 @@ model() -> , min => {[value, cli_param], #{ oneliner => "Minimum value of the controlled parameter" - , type => integer() + , type => emqttb:duration_us() , default => 0 , cli_operand => "min" , cli_short => $m @@ -158,14 +165,14 @@ model() -> , max => {[value, cli_param], #{ oneliner => "Maximum value of the controlled parameter" - , type => integer() - , default => 100_000_000 % 100s + , type => emqttb:duration_us() + , default_str => "10s" , cli_operand => "max" , cli_short => $M }} , speed => {[value, cli_param], - #{ type => integer() + #{ type => non_neg_integer() , default => 0 , cli_operand => "speed" , cli_short => $V @@ -208,10 +215,7 @@ metaparams(autorate_id) -> []. meta_validate(autorate, Model) -> - Ids = [begin - #mnode{metaparams = #{autorate_id := Id}} = lee_model:get(Key, Model), - Id - end || Key <- lee_model:get_metatype_index(autorate, Model)], + {Ids, _} = lists:unzip(ls(Model)), case length(lists:usort(Ids)) =:= length(Ids) of false -> {["Autorate IDs must be unique"], [], []}; true -> {[], [], [{set, autorate_ids, Ids}]} @@ -241,13 +245,11 @@ validate_node(autorate, Model, Data, _Key, #mnode{metaparams = #{autorate_id := create_autorates() -> [begin - #mnode{metaparams = MPs} = lee_model:get(Key, ?MYMODEL), - Id = ?m_attr(autorate, autorate_id, MPs), {ok, _} = emqttb_autorate_sup:ensure(#{ id => Id , init_val => ?CFG(Key) , conf_root => Id }) - end || Key <- lee_model:get_metatype_index(autorate, ?MYMODEL)], + end || {Id, Key} <- ls(?MYMODEL)], ok. -spec from_model(lee:key()) -> atom(). diff --git a/src/metrics/emqttb_metrics.erl b/src/metrics/emqttb_metrics.erl index 45313b9..94b5bc6 100644 --- a/src/metrics/emqttb_metrics.erl +++ b/src/metrics/emqttb_metrics.erl @@ -19,7 +19,7 @@ -behavior(lee_metatype). %% API: --export([from_model/1, opstat_from_model/1, opstat/2, call_with_counter/4, +-export([ls/1, from_model/1, opstat_from_model/1, opstat/2, call_with_counter/4, new_counter/2, counter_inc/2, counter_dec/2, get_counter/1, new_gauge/2, gauge_set/2, gauge_ref/1, new_rolling_average/2, rolling_average_observe/2, @@ -178,7 +178,7 @@ get_rolling_average(Key, Window) -> end. %%================================================================================ -%% lee_metatype callbacks +%% lee_metatype callbacks and helpers %%================================================================================ names(_) -> @@ -198,7 +198,7 @@ meta_validate(metric, Model) -> Ids = [begin #mnode{metaparams = #{id := Id}} = lee_model:get(Key, Model), Id - end || Key <- lee_model:get_metatype_index(metric, Model)], + end || Key <- ls(Model)], case length(lists:usort(Ids)) =:= length(Ids) of false -> {["Metric IDs must be unique"], [], []}; true -> {[], [], []} @@ -208,7 +208,7 @@ meta_validate(metric_id, _) -> validate_node(metric_id, Model, Data, Key, _) -> Metric = lee:get(Model, Data, Key), - case lists:member(Metric, lee_model:get_metatype_index(metric, Model)) of + case lists:member(Metric, ls(Model)) of true -> {[], []}; false -> @@ -218,6 +218,9 @@ validate_node(metric_id, Model, Data, Key, _) -> validate_node(metric, _, _, _, _) -> {[], []}. +ls(Model) -> + lee_model:get_metatype_index(metric, Model). + %%================================================================================ %% gen_server callbacks %%================================================================================ diff --git a/src/scenarios/emqttb_scenario_persistent_session.erl b/src/scenarios/emqttb_scenario_persistent_session.erl index 037c09e..c347722 100644 --- a/src/scenarios/emqttb_scenario_persistent_session.erl +++ b/src/scenarios/emqttb_scenario_persistent_session.erl @@ -94,9 +94,7 @@ model() -> , default_ref => [interval] , cli_operand => "pubinterval" , cli_short => $i - , autorate_id => 'persistent_session/pubinterval' - , process_variable => [?SK(persistent_session), pub, metrics, pub_latency, pending] - , error_coeff => -1 + , autorate_id => 'persistent_session/pubinterval' }} , n => {[value, cli_param], diff --git a/src/scenarios/emqttb_scenario_pub.erl b/src/scenarios/emqttb_scenario_pub.erl index 28c97a2..7b59a7f 100644 --- a/src/scenarios/emqttb_scenario_pub.erl +++ b/src/scenarios/emqttb_scenario_pub.erl @@ -82,7 +82,6 @@ model() -> , cli_operand => "conninterval" , cli_short => $I , autorate_id => 'pub/conninterval' - , process_variable => [?SK(pub), metrics, conn_latency, pending] }} , pubinterval => {[value, cli_param, autorate], @@ -92,8 +91,6 @@ model() -> , cli_operand => "pubinterval" , cli_short => $i , autorate_id => 'pub/pubinterval' - , process_variable => [?SK(pub), metrics, pub_latency, pending] - , error_coeff => -1 }} , n_clients => {[value, cli_param], diff --git a/src/scenarios/emqttb_scenario_pubsub_fwd.erl b/src/scenarios/emqttb_scenario_pubsub_fwd.erl index 2ef606c..9ccc2d2 100644 --- a/src/scenarios/emqttb_scenario_pubsub_fwd.erl +++ b/src/scenarios/emqttb_scenario_pubsub_fwd.erl @@ -76,8 +76,6 @@ model() -> , cli_operand => "pubinterval" , cli_short => $i , autorate_id => 'pubsub_fwd/pubinterval' - , process_variable => [?SK(pubsub_fwd), pub, metrics, pub_latency, pending] - , error_coeff => -1 }} %% Metrics: , n_published => From 408c1c81446e0118cad133fb5e788d5dc67a58f6 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Thu, 7 Dec 2023 11:26:22 +0100 Subject: [PATCH 08/10] Fix bug with inverted priority of patches --- doc/src/schema.adoc | 18 +++++++----------- src/conf/emqttb_conf.erl | 3 ++- src/conf/emqttb_conf_model.erl | 2 +- src/framework/emqttb_autorate.erl | 31 ++++++++++++++++--------------- src/framework/emqttb_scenario.erl | 5 ++--- 5 files changed, 28 insertions(+), 31 deletions(-) diff --git a/doc/src/schema.adoc b/doc/src/schema.adoc index 6d9999c..76fd97a 100644 --- a/doc/src/schema.adoc +++ b/doc/src/schema.adoc @@ -23,19 +23,20 @@ automatically using https://controlguru.com/integral-reset-windup-jacketing-logi === Autoscale A special autorate controlling the rate of spawning new clients is implicitly created for each client group. -Its name follows the pattern `_autoscale`. +Its name usually follows the pattern `%scenario%/conn_interval`. For example the following command can automatically adjust the rate of connections: [code,bash] ---- -./emqttb @conn -I 10ms -N 10_000 \ - @g --host 172.22.0.13,172.22.0.6 \ - @a -a conn_group_autoscale -V 1000 +./emqttb --pushgw @conn -I 10ms -N 5_000 \ + @a -a conn/conninterval -V 1000 --Ti 0.01 --setpoint 100 ---- -Note: autoscale can be applied to any group used by any scenario. -Additionally, autoscale is used for overload protection, see section about <>. +[id=autorate._.speed] +== Maximum rate of change of the controlled parameter + +Note: by default this parameter is set to 0 for each autorate, effectively locking the control parameter in place. [id=interval] == Default interval between events @@ -310,8 +311,3 @@ It's not desirable to switch between normal and SCRAM connection rate too often. == How often autorate is updated This parameter governs how often error is calculated and control parameter is updated. - -[id=autorate._.speed] -== Maximum rate of change of the controlled parameter - -Note: this parameter can be set to 0 to effectively disable autorate and lock control parameter in place. diff --git a/src/conf/emqttb_conf.erl b/src/conf/emqttb_conf.erl index 807a253..fa90425 100644 --- a/src/conf/emqttb_conf.erl +++ b/src/conf/emqttb_conf.erl @@ -53,8 +53,9 @@ load_conf() -> maybe_load_conf_file(), maybe_dump_conf(), ok; - {error, Errors, _Warnings} -> + {error, Errors, Warnings} -> [logger:critical(E) || E <- Errors], + [logger:warning(E) || E <- Warnings], emqttb:setfail("invalid configuration"), emqttb:terminate() end. diff --git a/src/conf/emqttb_conf_model.erl b/src/conf/emqttb_conf_model.erl index 8000521..e121110 100644 --- a/src/conf/emqttb_conf_model.erl +++ b/src/conf/emqttb_conf_model.erl @@ -247,7 +247,7 @@ model() -> {[map, cli_action], #{ cli_operand => "ls" , key_elements => [] - , oneliner => "List objects" + , oneliner => "List objects and exit" }, #{ what => {[value, cli_positional], diff --git a/src/framework/emqttb_autorate.erl b/src/framework/emqttb_autorate.erl index 0f61990..b55f640 100644 --- a/src/framework/emqttb_autorate.erl +++ b/src/framework/emqttb_autorate.erl @@ -57,7 +57,7 @@ -type config() :: #{ id := emqttb:autorate() , conf_root := atom() - , error := fun(() -> number()) + , error => fun(() -> number()) , scram => scram_fun() , parent => pid() , init_val => integer() @@ -149,7 +149,7 @@ model() -> , set_point => {[value, cli_param], #{ oneliner => "Value of the process variable that the autorate will try to keep" - , type => emqttb:duration_us() + , type => number() , default => 0 , cli_operand => "setpoint" , cli_short => $s @@ -157,7 +157,7 @@ model() -> , min => {[value, cli_param], #{ oneliner => "Minimum value of the controlled parameter" - , type => emqttb:duration_us() + , type => number() , default => 0 , cli_operand => "min" , cli_short => $m @@ -165,8 +165,8 @@ model() -> , max => {[value, cli_param], #{ oneliner => "Maximum value of the controlled parameter" - , type => emqttb:duration_us() - , default_str => "10s" + , type => number() + , default => 100_000_000 % 100s , cli_operand => "max" , cli_short => $M }} @@ -218,7 +218,7 @@ meta_validate(autorate, Model) -> {Ids, _} = lists:unzip(ls(Model)), case length(lists:usort(Ids)) =:= length(Ids) of false -> {["Autorate IDs must be unique"], [], []}; - true -> {[], [], [{set, autorate_ids, Ids}]} + true -> {[], [], []} end; meta_validate(autorate_id, _) -> {[], [], []}. @@ -229,7 +229,7 @@ meta_validate(autorate_id, _) -> validate_node(autorate_id, Model, Data, Key, _) -> %% Check that ID matches with some of the autorates in the model: Id = lee:get(Model, Data, Key), - {ok, Ids} = lee_model:get_meta(autorate_ids, Model), + {Ids, _} = lists:unzip(ls(Model)), case lists:member(Id, Ids) of true -> {[], []}; false -> {["Unknown autorate " ++ atom_to_list(Id)], []} @@ -400,12 +400,13 @@ make_error_fun(Id, ConfRoot) -> Coeff = my_cfg(ConfRoot, [error_coeff]), MetricKey = emqttb_metrics:from_model(ProcessVarKey), #mnode{metaparams = PVarMPs} = lee_model:get(ProcessVarKey, ?MYMODEL), - case ?m_attr(metric, metric_type, PVarMPs) of - counter -> - Coeff * (SetPoint - emqttb_metrics:get_counter(MetricKey)); - rolling_average -> - AvgWindow = 250, - SetPoint = my_cfg(Id, [set_point]), - Coeff * (SetPoint - emqttb_metrics:get_rolling_average(MetricKey, AvgWindow)) - end + ProcessVar = case ?m_attr(metric, metric_type, PVarMPs) of + counter -> + emqttb_metrics:get_counter(MetricKey); + rolling_average -> + AvgWindow = 5000, + emqttb_metrics:get_rolling_average(MetricKey, AvgWindow) + end, + %% logger:error(#{autorate => Id, pvar => ProcessVar, setpoint => SetPoint, pvar_key => ProcessVarKey}), + Coeff * (SetPoint - ProcessVar) end. diff --git a/src/framework/emqttb_scenario.erl b/src/framework/emqttb_scenario.erl index fb15937..ab63abe 100644 --- a/src/framework/emqttb_scenario.erl +++ b/src/framework/emqttb_scenario.erl @@ -172,13 +172,12 @@ names(_) -> read_patch(scenario, _Model) -> %% Populate configuration with the default data: - Prio = -99999, Patch = lists:flatmap( fun(Module) -> Module:initial_config() end, all_scenario_modules()), - {ok, Prio, Patch}. + {ok, 999999, Patch}. %%================================================================================ @@ -224,7 +223,7 @@ handle_call(_, _, S) -> {reply, {error, unknown_call}, S}. handle_cast(_, S) -> - {notrepy, S}. + {noreply, S}. terminate(_, _State) -> persistent_term:erase(?SCENARIO_GROUP_LEADER(group_leader())). From 5ac3cc7888f0cd8d40f2cb9ecf2c1058735883ed Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Thu, 7 Dec 2023 23:35:51 +0100 Subject: [PATCH 09/10] feat: Add generic SCRAM configuration for the autorates --- Makefile | 2 +- doc/src/schema.adoc | 46 +++++++---- src/framework/emqttb_autorate.erl | 77 ++++++++++++++++--- src/framework/emqttb_group.erl | 45 ----------- src/framework/emqttb_worker.erl | 27 ------- src/scenarios/emqttb_scenario_conn.erl | 2 +- .../emqttb_scenario_persistent_session.erl | 2 +- src/scenarios/emqttb_scenario_pub.erl | 2 +- src/scenarios/emqttb_scenario_pubsub_fwd.erl | 2 +- src/scenarios/emqttb_scenario_sub.erl | 2 +- .../emqttb_scenario_sub_flapping.erl | 2 +- 11 files changed, 104 insertions(+), 105 deletions(-) diff --git a/Makefile b/Makefile index 2cc5aa1..46a8feb 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ dialyzer: $(REBAR) .PHONY: test test: $(REBAR) - $(REBAR) do eunit, ct + $(REBAR) do compile, eunit, ct .PHONY: release release: compile docs diff --git a/doc/src/schema.adoc b/doc/src/schema.adoc index 76fd97a..b71f8b9 100644 --- a/doc/src/schema.adoc +++ b/doc/src/schema.adoc @@ -1,4 +1,5 @@ :!sectids: +:stem: = Documentation [id=cluster.node_name] @@ -20,24 +21,54 @@ This can be very expensive in man-hours and computing resources. In order to prevent that, emqttb can tune some parameters (such as message publishing interval) automatically using https://controlguru.com/integral-reset-windup-jacketing-logic-and-the-velocity-pi-form/[PI controller]. +The following formula is used for the error function: + +stem:[e=(a_{SP} - a_{PV}) a_{coeff}] + === Autoscale A special autorate controlling the rate of spawning new clients is implicitly created for each client group. Its name usually follows the pattern `%scenario%/conn_interval`. + +By default, the number of pending (unacked) connections is used as the process variable. +Number of pending connections is a metric that responds very fast to target overload, so it makes a reasonable default. + For example the following command can automatically adjust the rate of connections: [code,bash] ---- ./emqttb --pushgw @conn -I 10ms -N 5_000 \ - @a -a conn/conninterval -V 1000 --Ti 0.01 --setpoint 100 + @a -a conn/conninterval -V 1000 --setpoint 10 ---- + +[id=autorate._.id] +== ID of the autorate configuration + +Autorate configuration can be referred by id. +This value must be equal to one of the elements returned by `emqttb @ls autorate` command. + + [id=autorate._.speed] == Maximum rate of change of the controlled parameter Note: by default this parameter is set to 0 for each autorate, effectively locking the control parameter in place. + +[id=autorate._.process_variable] +== Process variable + +This parameter specifies ID of the metric that senses pressure on the SUT and serves as the process variable (PV). +Its value must be equal to one of the metric IDs returned by `emqttb @ls metric` command. + +[id=autorate._.setpoint] +== Setpoint + +The desired value of the process variable (PV) is called the setpoint. +Autorate adjusts the value of the control variable (CV) to bring the PV close to the setpoint. + + [id=interval] == Default interval between events @@ -51,11 +82,6 @@ Supported units: If unit is not specified then `ms` is assumed. -[id=autorate._.id] -== ID of the autorate configuration - -Autorate configuration can be referred by id. - [id=scenarios.sub] == Run scenario sub @@ -278,14 +304,6 @@ The following substitutions are supported: How often the clients will send `PING` MQTT message to the broker on idle connections. -[id=groups._.target_conn_pending] -== Target number of unacked connections - -In order to optimize the connection rate autoscale relies on the number of unacked (pending) connections. -This parameter configures the value that emqttb autoscale will try to approach. - -Number of pending connections is a metric that responds very fast to target overload, so we use it. - [id=groups._.scram.threshold] == Maximum unacked CONNECT packets diff --git a/src/framework/emqttb_autorate.erl b/src/framework/emqttb_autorate.erl index b55f640..d32108f 100644 --- a/src/framework/emqttb_autorate.erl +++ b/src/framework/emqttb_autorate.erl @@ -200,6 +200,33 @@ model() -> , cli_operand => "update-interval" , cli_short => $u }} + , scram => + #{ enabled => + {[value, cli_param], + #{ type => boolean() + , default => false + , cli_operand => "olp" + }} + , threshold => + {[value, cli_param], + #{ type => non_neg_integer() + , default => 1000 + , cli_operand => "olp-threshold" + }} + , hysteresis => + {[value, cli_param], + #{ oneliner => "Hysteresis (%) of overload detection" + , type => typerefl:range(1, 100) + , default => 50 + , cli_operand => "olp-hysteresis" + }} + , override => + {[value, cli_param], + #{ type => emqttb:duration_us() + , default_str => "10s" + , cli_operand => "olp-override" + }} + } }. %%================================================================================ @@ -298,14 +325,13 @@ init(Config = #{id := Id, conf_root := ConfRoot}) -> Min = my_cfg(ConfRoot, [min]), Current = maps:get(init_val, Config, Min), Err = ErrF(), - ScramFun = maps:get(scram, Config, fun(_) -> false end), set_timer(ConfRoot), {ok, update_rate(#s{ id = Id , parent = MRef , current = Current , conf_root = ConfRoot , error = ErrF - , scram_fun = ScramFun + , scram_fun = make_scram_fun(Id, ConfRoot) , meltdown = false , last_err = Err , last_t = os:system_time(millisecond) @@ -395,18 +421,45 @@ my_cfg(ConfRoot, Key) -> -spec make_error_fun(emqttb:autorate(), lee:key()) -> fun(() -> number()). make_error_fun(Id, ConfRoot) -> fun() -> - ProcessVarKey = my_cfg(ConfRoot, [process_variable]), SetPoint = my_cfg(Id, [set_point]), Coeff = my_cfg(ConfRoot, [error_coeff]), - MetricKey = emqttb_metrics:from_model(ProcessVarKey), - #mnode{metaparams = PVarMPs} = lee_model:get(ProcessVarKey, ?MYMODEL), - ProcessVar = case ?m_attr(metric, metric_type, PVarMPs) of - counter -> - emqttb_metrics:get_counter(MetricKey); - rolling_average -> - AvgWindow = 5000, - emqttb_metrics:get_rolling_average(MetricKey, AvgWindow) - end, + ProcessVar = read_pvar(ConfRoot), %% logger:error(#{autorate => Id, pvar => ProcessVar, setpoint => SetPoint, pvar_key => ProcessVarKey}), Coeff * (SetPoint - ProcessVar) end. + +make_scram_fun(Id, ConfRoot) -> + fun(Meltdown) -> + Enabled = my_cfg(ConfRoot, [scram, enabled]), + Threshold = my_cfg(ConfRoot, [scram, threshold]), + Hysteresis = my_cfg(ConfRoot, [scram, hysteresis]), + Override = my_cfg(ConfRoot, [scram, override]), + PVar = read_pvar(ConfRoot), + if not Enabled -> + false; + Meltdown andalso PVar >= (Threshold * Hysteresis / 100) -> + {true, Override}; + PVar >= Threshold -> + logger:warning("SCRAM is activated for autorate ~p. PV=~p. CV=~p.", + [Id, PVar, Override]), + {true, Override}; + Meltdown -> + logger:warning("SCRAM is deactivated for autorate ~p. PV=~p.", + [Id, PVar]), + false; + true -> + false + end + end. + +read_pvar(ConfRoot) -> + ProcessVarKey = my_cfg(ConfRoot, [process_variable]), + #mnode{metaparams = PVarMPs} = lee_model:get(ProcessVarKey, ?MYMODEL), + MetricKey = emqttb_metrics:from_model(ProcessVarKey), + case ?m_attr(metric, metric_type, PVarMPs) of + counter -> + emqttb_metrics:get_counter(MetricKey); + rolling_average -> + AvgWindow = 5000, + emqttb_metrics:get_rolling_average(MetricKey, AvgWindow) + end. diff --git a/src/framework/emqttb_group.erl b/src/framework/emqttb_group.erl index c9105d0..ef20015 100644 --- a/src/framework/emqttb_group.erl +++ b/src/framework/emqttb_group.erl @@ -380,50 +380,5 @@ maybe_monitor_parent(#{parent := Pid}) -> maybe_monitor_parent(_) -> undefined. -%% create_autorate(GroupID, ConfID) -> -%% ID = my_autorate(GroupID), -%% DefaultConf = #{ [id] => ID -%% }, -%% emqttb_conf:patch(lee_lib:make_nested_patch(?MYMODEL, [autorate], DefaultConf)), -%% AutorateConf = #{ id => ID -%% , conf_root => ID -%% , error => autoscale_error(Metric, GroupID) -%% , scram => fun(Meltdown) -> autoscale_scram(GroupID, ConfID, Meltdown) end -%% }, -%% emqttb_autorate:ensure(AutorateConf). - -%% autoscale_scram(Group, ConfID, Meltdown) -> -%% MaxPending = ?CFG([groups, {ConfID}, scram, threshold]), -%% Hysteresis = ?CFG([groups, {ConfID}, scram, hysteresis]), -%% Override = ?CFG([groups, {ConfID}, scram, override]), -%% Pending = emqttb_metrics:get_counter(?GROUP_N_PENDING(Group, connect)), -%% if Meltdown andalso Pending >= (MaxPending * Hysteresis / 100) -> -%% {true, Override}; -%% Pending >= MaxPending -> -%% logger:warning("SCRAM is activated for group ~p. Unacked connections: ~p. Connection interval is dropped to ~p us.", -%% [Group, Pending, Override]), -%% {true, Override}; -%% Meltdown -> -%% logger:warning("SCRAM is deactivated for group ~p. Unacked connections: ~p. Connection rate is restored to normal value.", -%% [Group, Pending]), -%% false; -%% true -> -%% false -%% end. - -%% autoscale_error(MetricKey, Group) -> -%% %% Note that dependency of number of pending operations on connect -%% %% interval is inverse: lesser interval -> clients connect more -%% %% often -> more load -> more pending operations -%% %% -%% %% So the control must be reversed, and error is the negative of what one usually -%% %% expects: -%% %% Current - Target instead of Target - Current. -%% Metric = emqttb_metrics:from_model(MetricKey), -%% fun() -> -%% Target = ?CFG([groups, {Group}, target_conn_pending]), -%% emqttb_metrics:get_counter(?GROUP_N_PENDING(Group, connect)) - Target -%% end. - dead_id_pool(Group) -> Group. diff --git a/src/framework/emqttb_worker.erl b/src/framework/emqttb_worker.erl index 0e28103..2e7c9dc 100644 --- a/src/framework/emqttb_worker.erl +++ b/src/framework/emqttb_worker.erl @@ -368,33 +368,6 @@ model() -> , default => verify_none }} } - , scram => - #{ threshold => - {[value, cli_param], - #{ type => non_neg_integer() - , default => 100 - , cli_operand => "olp-threshold" - }} - , hysteresis => - {[value, cli_param], - #{ oneliner => "Hysteresis (%) of overload detection" - , type => typerefl:range(1, 100) - , default => 50 - , cli_operand => "olp-hysteresis" - }} - , override => - {[value, cli_param], - #{ type => emqttb:duration_us() - , default_str => "10s" - , cli_operand => "olp-override" - }} - } - , target_conn_pending => - {[value, cli_param], - #{ type => non_neg_integer() - , default => 10 - , cli_operand => "target-conn-pending" - }} }. %%================================================================================ diff --git a/src/scenarios/emqttb_scenario_conn.erl b/src/scenarios/emqttb_scenario_conn.erl index 29e7f3a..1c3fd70 100644 --- a/src/scenarios/emqttb_scenario_conn.erl +++ b/src/scenarios/emqttb_scenario_conn.erl @@ -92,7 +92,7 @@ model() -> }. initial_config() -> - emqttb_conf:string2patch("@a -a conn/conninterval --pvar '[scenarios,conn,{},metrics,conn_latency,pending]'"). + emqttb_conf:string2patch("@a -a conn/conninterval --pvar '[scenarios,conn,{},metrics,conn_latency,pending]' --olp"). run() -> GroupId = ?GROUP, diff --git a/src/scenarios/emqttb_scenario_persistent_session.erl b/src/scenarios/emqttb_scenario_persistent_session.erl index c347722..ebb49ad 100644 --- a/src/scenarios/emqttb_scenario_persistent_session.erl +++ b/src/scenarios/emqttb_scenario_persistent_session.erl @@ -185,7 +185,7 @@ model() -> initial_config() -> emqttb_conf:string2patch("@a -a persistent_session/pubinterval --pvar '[scenarios,persistent_session,{},pub,metrics,pub_latency,pending]'") ++ - emqttb_conf:string2patch("@a -a persistent_session/conninterval --pvar '[scenarios,persistent_session,{},pub,metrics,conn_latency,pending]'"). + emqttb_conf:string2patch("@a -a persistent_session/conninterval --pvar '[scenarios,persistent_session,{},pub,metrics,conn_latency,pending]' --olp"). run() -> prometheus_summary:declare([ {name, ?PUB_THROUGHPUT} diff --git a/src/scenarios/emqttb_scenario_pub.erl b/src/scenarios/emqttb_scenario_pub.erl index 7b59a7f..bae8094 100644 --- a/src/scenarios/emqttb_scenario_pub.erl +++ b/src/scenarios/emqttb_scenario_pub.erl @@ -129,7 +129,7 @@ model() -> initial_config() -> emqttb_conf:string2patch("@a -a pub/pubinterval --pvar '[scenarios,pub,{},metrics,pub_latency,pending]'") ++ - emqttb_conf:string2patch("@a -a pub/conninterval --pvar '[scenarios,pub,{},metrics,conn_latency,pending]'"). + emqttb_conf:string2patch("@a -a pub/conninterval --pvar '[scenarios,pub,{},metrics,conn_latency,pending]' --olp"). run() -> PubOpts = #{ topic => my_conf([topic]) diff --git a/src/scenarios/emqttb_scenario_pubsub_fwd.erl b/src/scenarios/emqttb_scenario_pubsub_fwd.erl index 9ccc2d2..e244f74 100644 --- a/src/scenarios/emqttb_scenario_pubsub_fwd.erl +++ b/src/scenarios/emqttb_scenario_pubsub_fwd.erl @@ -149,7 +149,7 @@ model() -> initial_config() -> emqttb_conf:string2patch("@a -a pubsub_fwd/pubinterval --pvar '[scenarios,pubsub_fwd,{},pub,metrics,pub_latency,pending]'") ++ - emqttb_conf:string2patch("@a -a pubsub_fwd/conninterval --pvar '[scenarios,pubsub_fwd,{},pub,metrics,conn_latency,pending]'"). + emqttb_conf:string2patch("@a -a pubsub_fwd/conninterval --pvar '[scenarios,pubsub_fwd,{},pub,metrics,conn_latency,pending]' --olp"). run() -> set_stage(subscribe), diff --git a/src/scenarios/emqttb_scenario_sub.erl b/src/scenarios/emqttb_scenario_sub.erl index de7a25c..9fa315b 100644 --- a/src/scenarios/emqttb_scenario_sub.erl +++ b/src/scenarios/emqttb_scenario_sub.erl @@ -110,7 +110,7 @@ model() -> }. initial_config() -> - emqttb_conf:string2patch("@a -a sub/conninterval --pvar '[scenarios,sub,{},metrics,conn_latency,pending]'"). + emqttb_conf:string2patch("@a -a sub/conninterval --pvar '[scenarios,sub,{},metrics,conn_latency,pending]' --olp"). run() -> SubOpts = #{ topic => my_conf([topic]) diff --git a/src/scenarios/emqttb_scenario_sub_flapping.erl b/src/scenarios/emqttb_scenario_sub_flapping.erl index 2a397b3..59ab1f3 100644 --- a/src/scenarios/emqttb_scenario_sub_flapping.erl +++ b/src/scenarios/emqttb_scenario_sub_flapping.erl @@ -115,7 +115,7 @@ model() -> }. initial_config() -> - emqttb_conf:string2patch("@a -a sub_flapping/conninterval --pvar '[scenarios,sub_flapping,{},metrics,conn_latency,pending]'"). + emqttb_conf:string2patch("@a -a sub_flapping/conninterval --pvar '[scenarios,sub_flapping,{},metrics,conn_latency,pending]' --olp"). run() -> SubOpts = #{ topic => my_conf([topic]) From 5a2338e9befff73d450286125b5c7c6d505c367c Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Thu, 7 Dec 2023 23:59:35 +0100 Subject: [PATCH 10/10] autorate: Add APIs to freeze and thaw the autorate --- src/framework/emqttb_autorate.erl | 46 +++++++++++++++++++++---------- src/framework/emqttb_group.erl | 1 + 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/src/framework/emqttb_autorate.erl b/src/framework/emqttb_autorate.erl index d32108f..1e90c6e 100644 --- a/src/framework/emqttb_autorate.erl +++ b/src/framework/emqttb_autorate.erl @@ -24,7 +24,7 @@ %% https://controlguru.com/integral-reset-windup-jacketing-logic-and-the-velocity-pi-form/ %% API: --export([ls/1, ensure/1, get_counter/1, reset/2, info/0, create_autorates/0]). +-export([ls/1, ensure/1, get_counter/1, reset/2, info/0, create_autorates/0, activate/1, deactivate/1]). %% gen_server callbacks: -export([init/1, handle_call/3, handle_cast/2, handle_info/2]). @@ -92,21 +92,21 @@ ensure(Conf) -> {auto, get_counter(Pid)}. -spec get_counter(lee:key() | emqttb:autorate() | pid()) -> counters:counters_ref(). -get_counter(Pid) when is_pid(Pid) -> - gen_server:call(Pid, get_counter); -get_counter(Id) when is_atom(Id) -> - gen_server:call(?via(Id), get_counter); -get_counter(Key) -> - #mnode{metaparams = #{autorate_id := Id}} = lee_model:get(Key, ?MYMODEL), - gen_server:call(?via(Id), get_counter). +get_counter(Autorate) -> + gen_server:call(server(Autorate), get_counter). %% Set the current value to the specified value -spec reset(atom() | pid() | lee:ley(), integer()) -> ok. -reset(Key, Val) when is_list(Key) -> - #mnode{metaparams = #{autorate_id := Id}} = lee_model:get(Key, ?MYMODEL), - reset(Id, Val); -reset(Id, Val) -> - gen_server:call(?via(Id), {reset, Val}). +reset(Autorate, Val) -> + gen_server:call(server(Autorate), {reset, Val}). + +%% Freeze the value of autorate +deactivate(Autorate) -> + gen_server:call(server(Autorate), deactivate). + +%% Thaw the value of autorate +activate(Autorate) -> + gen_server:call(server(Autorate), activate). -spec info() -> [info()]. info() -> @@ -291,6 +291,7 @@ from_model(Key) -> -record(s, { id :: atom() , parent :: reference() | undefined + , active :: boolean() , current :: float() , conf_root :: atom() , error :: fun(() -> number()) @@ -327,6 +328,7 @@ init(Config = #{id := Id, conf_root := ConfRoot}) -> Err = ErrF(), set_timer(ConfRoot), {ok, update_rate(#s{ id = Id + , active = true , parent = MRef , current = Current , conf_root = ConfRoot @@ -337,6 +339,10 @@ init(Config = #{id := Id, conf_root := ConfRoot}) -> , last_t = os:system_time(millisecond) })}. +handle_call(activate, _From, S) -> + {reply, ok, S#s{active = true}}; +handle_call(deactivate, _From, S) -> + {reply, ok, S#s{active = false}}; handle_call(get_counter, _From, S) -> {reply, emqttb_metrics:gauge_ref(?AUTORATE_RATE(S#s.id)), S}; handle_call({reset, Val}, _From, S) -> @@ -347,9 +353,11 @@ handle_call(_, _From, S) -> handle_cast(_, S) -> {noreply, S}. -handle_info(tick, S) -> +handle_info(tick, S = #s{active = Active}) -> set_timer(S#s.conf_root), - {noreply, update_rate(S)}; + if Active -> {noreply, update_rate(S)}; + true -> {noreply, S} + end; handle_info({'DOWN', MRef, _, _, _}, S = #s{parent = MRef}) -> {stop, normal, S}; handle_info(_, S) -> @@ -463,3 +471,11 @@ read_pvar(ConfRoot) -> AvgWindow = 5000, emqttb_metrics:get_rolling_average(MetricKey, AvgWindow) end. + +server(Pid) when is_pid(Pid) -> + Pid; +server(Id) when is_atom(Id) -> + ?via(Id); +server(Key) when is_list(Key) -> + #mnode{metaparams = #{autorate_id := Id}} = lee_model:get(Key, ?MYMODEL), + ?via(Id). diff --git a/src/framework/emqttb_group.erl b/src/framework/emqttb_group.erl index ef20015..fe3470d 100644 --- a/src/framework/emqttb_group.erl +++ b/src/framework/emqttb_group.erl @@ -254,6 +254,7 @@ do_set_target(Target, InitInterval, OnComplete, S = #s{ scaling = Scaling true -> down end, maybe_cancel_previous(Scaling), + emqttb_autorate:activate(Autorate), case Direction of stay -> OnComplete({ok, N}),