diff --git a/doc/operation-and-maintenance/MongooseIM-metrics.md b/doc/operation-and-maintenance/MongooseIM-metrics.md index ea17a54d558..4887225c599 100644 --- a/doc/operation-and-maintenance/MongooseIM-metrics.md +++ b/doc/operation-and-maintenance/MongooseIM-metrics.md @@ -50,16 +50,17 @@ All metrics are divided into the following groups:

`histogram`

- A histogram collects values and groups them in buckets. + A histogram collects values over a 60-second sliding window and exposes the number of observations and their sum (as counters), along with the 50th, 75th, 90th, 95th, 99th, and 99.9th percentiles with 1% accuracy. **Example:** ``` - # TYPE xmpp_element_in_byte_size histogram + # TYPE xmpp_element_in_byte_size summary # HELP xmpp_element_in_byte_size Event: xmpp_element_in, Metric: byte_size - xmpp_element_in_byte_size_bucket{connection_type="c2s",host_type="localhost",le="1"} 0 + xmpp_element_in_byte_size_count{connection_type="c2s",host_type="localhost"} 0 + xmpp_element_in_byte_size_sum{connection_type="c2s",host_type="localhost"} 0 + xmpp_element_in_byte_size{connection_type="c2s",host_type="localhost",quantile="0.5"} 0 ... - xmpp_element_in_byte_size_bucket{connection_type="c2s",host_type="localhost",le="1073741824"} 0 - xmpp_element_in_byte_size_bucket{connection_type="c2s",host_type="localhost",le="+Inf"} 0 + xmpp_element_in_byte_size{connection_type="c2s",host_type="localhost",quantile="0.999"} 0 ``` === "Exometer" diff --git a/rebar.config b/rebar.config index 65a959a38a2..c597110540e 100644 --- a/rebar.config +++ b/rebar.config @@ -56,8 +56,9 @@ {exometer_report_statsd, {git, "https://github.com/esl/exometer_report_statsd.git", {branch, "master"}}}, {syslogger, "0.3.0"}, {flatlog, "0.1.2"}, - {prometheus, "5.0.0"}, - {prometheus_cowboy, "0.1.9"}, + {ddskerl, "0.4.3"}, + {prometheus, "6.1.2"}, + {prometheus_cowboy, "0.2.0"}, %%% Stateless libraries {opuntia, "1.1.2"}, @@ -218,7 +219,7 @@ {plt_extra_apps, [jid, cowboy, cowlib, lasse, p1_utils, ranch, gen_fsm_compat, epgsql, cqerl, common_test, tools, amqp_client, jiffy, erl_csv, inets, compiler, jsx, rabbit_common, mysql, eredis, erlcloud, telemetry, - nksip, nklib, nkservice, nkpacket, prometheus, exometer_core, + nksip, nklib, nkservice, nkpacket, ddskerl, prometheus, exometer_core, mnesia, cets, cpool, tirerl, erlang_doctor]}]}. {cover_print_enabled, true}. diff --git a/rebar.lock b/rebar.lock index 12d8886864b..68bac0aae92 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,5 +1,5 @@ {"1.2.0", -[{<<"accept">>,{pkg,<<"accept">>,<<"0.3.6">>},2}, +[{<<"accept">>,{pkg,<<"accept">>,<<"0.3.7">>},2}, {<<"amqp_client">>,{pkg,<<"amqp_client">>,<<"4.2.1">>},0}, {<<"backoff">>,{pkg,<<"backoff">>,<<"1.1.6">>},1}, {<<"base16">>,{pkg,<<"base16">>,<<"2.0.1">>},1}, @@ -19,6 +19,7 @@ {<<"credentials_obfuscation">>, {pkg,<<"credentials_obfuscation">>,<<"3.5.0">>}, 1}, + {<<"ddskerl">>,{pkg,<<"ddskerl">>,<<"0.4.3">>},0}, {<<"eini">>,{pkg,<<"eini">>,<<"1.2.9">>},1}, {<<"epgsql">>,{pkg,<<"epgsql">>,<<"4.7.1">>},0}, {<<"eredis">>,{pkg,<<"eredis">>,<<"1.7.1">>},0}, @@ -86,10 +87,9 @@ 0}, {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.4.0">>},1}, {<<"pooler">>,{pkg,<<"pooler">>,<<"1.5.3">>},1}, - {<<"prometheus">>,{pkg,<<"prometheus">>,<<"5.0.0">>},0}, - {<<"prometheus_cowboy">>,{pkg,<<"prometheus_cowboy">>,<<"0.1.9">>},0}, - {<<"prometheus_httpd">>,{pkg,<<"prometheus_httpd">>,<<"2.1.13">>},1}, - {<<"quantile_estimator">>,{pkg,<<"quantile_estimator">>,<<"1.0.2">>},1}, + {<<"prometheus">>,{pkg,<<"prometheus">>,<<"6.1.2">>},0}, + {<<"prometheus_cowboy">>,{pkg,<<"prometheus_cowboy">>,<<"0.2.0">>},0}, + {<<"prometheus_httpd">>,{pkg,<<"prometheus_httpd">>,<<"2.1.15">>},1}, {<<"quickrand">>,{pkg,<<"quickrand">>,<<"2.0.7">>},1}, {<<"rabbit_common">>,{pkg,<<"rabbit_common">>,<<"4.2.1">>},1}, {<<"ranch">>,{pkg,<<"ranch">>,<<"2.2.0">>},0}, @@ -117,7 +117,7 @@ {<<"worker_pool">>,{pkg,<<"worker_pool">>,<<"6.4.0">>},0}]}. [ {pkg_hash,[ - {<<"accept">>, <<"AD44AC7D704BF70EF8FB2E313EF5B978F9D1330BDDAC64509E93AFDA13281215">>}, + {<<"accept">>, <<"CD6E34A2D7E28CA38B2D3CB233734CA0C221EFBC1F171F91FEC5F162CC2D18DA">>}, {<<"amqp_client">>, <<"CFF0CC13186E57457DC5745F1B3A4127C6857717CB8F5920DC457C84D0AD00A2">>}, {<<"backoff">>, <<"83B72ED2108BA1EE8F7D1C22E0B4A00CFE3593A67DBC792799E8CCE9F42F796B">>}, {<<"base16">>, <<"F0549F732E03BE8124ED0D19FD5EE52146CC8BE24C48CBC3F23AB44B157F11A2">>}, @@ -131,6 +131,7 @@ {<<"cowlib">>, <<"623791C56C1CC9DF54A71A9C55147A401549917F00A2E48A6AE12B812C586CED">>}, {<<"cpool">>, <<"76222AA1DAC0F8089707167BD69D221EDA63DB65B8BD67DBF6E133075392EEDC">>}, {<<"credentials_obfuscation">>, <<"61E282ADFB4439486B3994FAAEC69543C7EE6CC7E70C6340E8853FD9DEAF8219">>}, + {<<"ddskerl">>, <<"BB97B90EEF1B5906520CEBDD38DA29B18A770B5567F6297927F53566EAE9BEBB">>}, {<<"eini">>, <<"FCC3CBD49BBDD9A1D9735C7365DAFFCD84481CCE81E6CB80537883AA44AC4895">>}, {<<"epgsql">>, <<"D4E47CAE46C18C8AFA88E34D59A9B4BAE16368D7CE1EB3DA24FA755EB28393EB">>}, {<<"eredis">>, <<"39E31AA02ADCD651C657F39AAFD4D31A9B2F63C6C700DC9CECE98D4BC3C897AB">>}, @@ -162,10 +163,9 @@ {<<"p1_utils">>, <<"67B0C4AC9FA3BA3EF563B31AA111B0A004439A37FAC85E027F1C3617E1C7EC6C">>}, {<<"parse_trans">>, <<"BB87AC362A03CA674EBB7D9D498F45C03256ADED7214C9101F7035EF44B798C7">>}, {<<"pooler">>, <<"898CD1FA301FC42D4A8ED598CE139B71CA85B54C16AB161152B5CC5FBDCFA1A8">>}, - {<<"prometheus">>, <<"8A37A3216D8DB019D19068602669C9819C099120F8E39994DD1BD3A3F5553376">>}, - {<<"prometheus_cowboy">>, <<"D9D5B300516A61ED5AE31391F8EEEEB202230081D32A1813F2D78772B6F274E1">>}, - {<<"prometheus_httpd">>, <<"F086390B4E4E3F41112889B745BAC53D26437B6139496E6700C2508858F5985B">>}, - {<<"quantile_estimator">>, <<"ECD281D40110FDD9BA62685531E4435E0839A52FD1058DA5564F1763E4642EF7">>}, + {<<"prometheus">>, <<"E5C3C567CD8B0994425920763405635211183C15BDC47D0CD524807DDE20CA1D">>}, + {<<"prometheus_cowboy">>, <<"526F75D9850A9125496F78BCEECCA0F237BC7B403C976D44508543AE5967DAD9">>}, + {<<"prometheus_httpd">>, <<"8F767D819A5D36275EAB9264AFF40D87279151646776069BF69FBDBBD562BD75">>}, {<<"quickrand">>, <<"D2BD76676A446E6A058D678444B7FDA1387B813710D1AF6D6E29BB92186C8820">>}, {<<"rabbit_common">>, <<"1D64E391E12116B76B1425EB96B7552DE51F0301093EBA669B5334F4759CC1E8">>}, {<<"ranch">>, <<"25528F82BC8D7C6152C57666CA99EC716510FE0925CB188172F41CE93117B1B0">>}, @@ -184,7 +184,7 @@ {<<"uuid">>, <<"B2078D2CC814F53AFA52D36C91E08962C7E7373585C623F4C0EA6DFB04B2AF94">>}, {<<"worker_pool">>, <<"0347B805A8E5804B5676A9885FB3B9B6C1627099C449C3C67C0E8E6AF79E9AA6">>}]}, {pkg_hash_ext,[ - {<<"accept">>, <<"A5167FA1AE90315C3F1DD189446312F8A55D00EFA357E9C569BDA47736B874C3">>}, + {<<"accept">>, <<"CA69388943F5DAD2E7232A5478F16086E3C872F48E32B88B378E1885A59F5649">>}, {<<"amqp_client">>, <<"8AE00B055A58500E0557F73D9C0FFE257487131E603F7F84FE72CBFAAF03838A">>}, {<<"backoff">>, <<"CF0CFFF8995FB20562F822E5CC47D8CCF664C5ECDC26A684CBE85C225F9D7C39">>}, {<<"base16">>, <<"06EA2D48343282E712160BA89F692B471DB8B36ABE8394F3445FF9032251D772">>}, @@ -198,6 +198,7 @@ {<<"cowlib">>, <<"0AF652D1550C8411C3B58EED7A035A7FB088C0B86AFF6BC504B0BC3B7F791AA2">>}, {<<"cpool">>, <<"430E18DF4A9D584EB1ED0D196A87CC02E878AF5B4888BFDC9B65F86A96480E30">>}, {<<"credentials_obfuscation">>, <<"843ADBE3246861CE0F1A0FA3222F384834EB31DEFD8D6B9CBA7AFD2977C957BC">>}, + {<<"ddskerl">>, <<"4E0F6047C6A002CE38F6EA155276DD918CD635FD0A5EDB9E0B46AEB6F7AFF2C2">>}, {<<"eini">>, <<"DA64AE8DB7C2F502E6F20CDF44CD3D9BE364412B87FF49FEBF282540F673DFCB">>}, {<<"epgsql">>, <<"B6D86B7DC42C8555B1D4E20880E5099D6D6D053148000E188E548F98E4E01836">>}, {<<"eredis">>, <<"7C2B54C566FED55FEEF3341CA79B0100A6348FD3F162184B7ED5118D258C3CC1">>}, @@ -229,10 +230,9 @@ {<<"p1_utils">>, <<"D0379E8C1156B98BD64F8129C1DE022FCCA4F2FDB7486CE73BF0ED2C3376B04C">>}, {<<"parse_trans">>, <<"F99E368830BEA44552224E37E04943A54874F08B8590485DE8D13832B63A2DC3">>}, {<<"pooler">>, <<"058D85C5081289B90E97E4DDDBC3BB5A3B4A19A728AB3BC88C689EFCC36A07C7">>}, - {<<"prometheus">>, <<"80D29564A5DC4490B53FD225D752B65FB0DBEBA41497F96D62223338127C5659">>}, - {<<"prometheus_cowboy">>, <<"5F71C039DEB9E9FF9DD6366BC74C907A463872B85286E619EFF0BDA15111695A">>}, - {<<"prometheus_httpd">>, <<"9B5A44D1F6FBB3C3FE6F85F06DAFE680AD9FFD591EC65A10BB51DFF0FBBE45D2">>}, - {<<"quantile_estimator">>, <<"DB404793D6384995A1AC6DD973E2CEE5BE9FCC128765BDBA53D87C564E296B64">>}, + {<<"prometheus">>, <<"109662001328112B860DE112D49B575310BE8FF33FE04BF0482EEC4B1E5A1278">>}, + {<<"prometheus_cowboy">>, <<"2C7EB12F4B970D91E3B47BAAD0F138F6ADC34E53EEB0AE18068FF0AFAB441B24">>}, + {<<"prometheus_httpd">>, <<"67736D000745184D5013C58A63E947821AB90CB9320BC2E6AE5D3061C6FFE039">>}, {<<"quickrand">>, <<"B8ACBF89A224BC217C3070CA8BEBC6EB236DBE7F9767993B274084EA044D35F0">>}, {<<"rabbit_common">>, <<"FF509B07E639B1784898C28031E5204FEA14260172E4FC339F94405586037E40">>}, {<<"ranch">>, <<"FA0B99A1780C80218A4197A59EA8D3BDAE32FBFF7E88527D7D8A4787EFF4F8E7">>}, diff --git a/src/ejabberd_sup.erl b/src/ejabberd_sup.erl index 7750ed1103f..a3d00db9924 100644 --- a/src/ejabberd_sup.erl +++ b/src/ejabberd_sup.erl @@ -69,7 +69,8 @@ init(noargs) -> [StartIdServer, PG, Hooks, - Instrument, + Instrument + ] ++ prometheus_sliding_window_spec() ++ [ Cleaner, SMBackendSupervisor, OutgoingPoolsSupervisor @@ -122,6 +123,12 @@ template_supervisor_spec(Name, Module) -> supervisor_spec(Mod) -> {Mod, {Mod, start_link, []}, permanent, infinity, supervisor, [Mod]}. +prometheus_sliding_window_spec() -> + case mongoose_config:get_opt([instrumentation, prometheus], undefined) of + undefined -> []; + #{} -> [mongoose_prometheus_sliding_window:child_spec()] + end. + worker_spec(Mod) -> worker_spec(Mod, []). diff --git a/src/instrument/mongoose_instrument_exometer.erl b/src/instrument/mongoose_instrument_exometer.erl index eb2109fa151..567a74e59e1 100644 --- a/src/instrument/mongoose_instrument_exometer.erl +++ b/src/instrument/mongoose_instrument_exometer.erl @@ -153,7 +153,7 @@ handle_metric_event(EventName, Labels, MetricName, MetricType, Measurements) -> ok end. --spec update_metric(exometer:name(), spiral | histogram, integer()) -> ok. +-spec update_metric(exometer:name(), mongoose_instrument:metric_type(), integer()) -> ok. update_metric(Name, gauge, Value) when is_integer(Value) -> ok = exometer:update(Name, Value); update_metric(Name, counter, Value) when is_integer(Value) -> diff --git a/src/instrument/mongoose_instrument_prometheus.erl b/src/instrument/mongoose_instrument_prometheus.erl index f0843d37625..6e74991cebf 100644 --- a/src/instrument/mongoose_instrument_prometheus.erl +++ b/src/instrument/mongoose_instrument_prometheus.erl @@ -13,6 +13,7 @@ start(#{}) -> Apps = [prometheus, prometheus_httpd, prometheus_cowboy], {ok, _} = application:ensure_all_started(Apps, permanent), + prometheus_registry:register_collector(mongoose_prometheus_sliding_window_collector), ok. -spec set_up(mongoose_instrument:event_name(), mongoose_instrument:labels(), @@ -57,7 +58,7 @@ declare_metric(MetricSpec, counter) -> declare_metric(MetricSpec, spiral) -> prometheus_counter:declare(MetricSpec); declare_metric(MetricSpec, histogram) -> - prometheus_histogram:declare([{buckets, histogram_buckets()} | MetricSpec]). + mongoose_prometheus_sliding_window:declare(MetricSpec). -spec reset_metric(name(), [mongoose_instrument:label_value()], mongoose_instrument:metric_type()) -> boolean(). @@ -68,7 +69,7 @@ reset_metric(Name, LabelValues, counter) -> reset_metric(Name, LabelValues, spiral) -> prometheus_counter:remove(Name, LabelValues); reset_metric(Name, LabelValues, histogram) -> - prometheus_histogram:remove(Name, LabelValues). + mongoose_prometheus_sliding_window:remove(Name, LabelValues). -spec initialize_metric(name(), [mongoose_instrument:label_value()], mongoose_instrument:metric_type()) -> ok. @@ -92,15 +93,6 @@ metric_spec(EventName, LabelKeys, MetricName) -> {duration_unit, false} % prevent unwanted implicit conversions, e.g. seconds -> microseconds ]. --spec histogram_buckets() -> [integer()]. -histogram_buckets() -> - histogram_buckets([], 1 bsl 30). % ~1.07 * 10^9 - -histogram_buckets(AccBuckets, Val) when Val > 0 -> - histogram_buckets([Val | AccBuckets], Val bsr 1); -histogram_buckets(AccBuckets, _Val) -> - AccBuckets. - -spec handle_metric_event(mongoose_instrument:event_name(), [mongoose_instrument:label_value()], mongoose_instrument:metric_name(), mongoose_instrument:metric_type(), mongoose_instrument:measurements()) -> ok. @@ -139,4 +131,4 @@ update_metric(Name, Labels, counter, Value) when is_integer(Value) -> update_metric(Name, Labels, spiral, Value) when is_integer(Value), Value >= 0 -> ok = prometheus_counter:inc(Name, Labels, Value); update_metric(Name, Labels, histogram, Value) when is_integer(Value) -> - ok = prometheus_histogram:observe(Name, Labels, Value). + ok = mongoose_prometheus_sliding_window:observe(Name, Labels, Value). diff --git a/src/instrument/mongoose_prometheus_sliding_window.erl b/src/instrument/mongoose_prometheus_sliding_window.erl new file mode 100644 index 00000000000..16505d90759 --- /dev/null +++ b/src/instrument/mongoose_prometheus_sliding_window.erl @@ -0,0 +1,351 @@ +-module(mongoose_prometheus_sliding_window). + +%% Manages a sliding window using DDSketch instances for quantile summaries. +%% Window parameters are configurable via window_size_ms and window_step_ms constants. + +-export([child_spec/0, + declare/1, + observe/3, + value/2, + values/1, + remove/2, + get_all_metric_names/0, + get_metric_spec/1]). + +-export([start_link/0, + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3]). + +-export([window_size_ms/0, window_step_ms/0, window_count/0]). % Exported for mocking in tests + +-ignore_xref([start_link/0, value/2, window_size_ms/0, window_step_ms/0, window_count/0]). + +-behaviour(gen_server). + +%% Window parameters + +-spec window_size_ms() -> pos_integer(). +window_size_ms() -> 60000. % Total window size: 60 seconds + +-spec window_step_ms() -> pos_integer(). +window_step_ms() -> 3000. % Step size: 3 seconds per sub-window + +-spec window_count() -> pos_integer(). +window_count() -> ?MODULE:window_size_ms() div ?MODULE:window_step_ms(). + +%% DDSketch parameters + +-spec default_quantiles() -> [number()]. +default_quantiles() -> [0.5, 0.75, 0.90, 0.95, 0.99, 0.999]. + +-spec default_error() -> number(). +default_error() -> 0.01. + +%% Setting the default bound to 1260 means that: +%% * measuring in µs suffices for actions lasting up to a day (with 1% accuracy), +%% * measuring in bytes suffices for sizes up to 81 GB (with 1% accuracy). +%% See: https://hexdocs.pm/ddskerl/ddskerl_ets.html +-spec default_bound() -> non_neg_integer(). +default_bound() -> 1260. + +-record(state, { + timer_ref :: {ok, timer:tref()} | {error, term()} | undefined, + ets_table :: ets:tab() | undefined, + metrics = #{} :: #{name() => #{{name(), label_values()} => metric_state()}}, + metric_specs = #{} :: #{name() => proplists:proplist()} +}). + +-type name() :: string(). +-type label_values() :: [mongoose_instrument:label_value()]. +-type window_data() :: #{sketch := ddskerl_ets:ddsketch(), + ref := ets:tab(), + name := term()}. +-type metric_state() :: #{windows => [window_data()], + current_index => non_neg_integer()}. + +%% Public API + +-spec declare(proplists:proplist()) -> boolean(). +declare(MetricSpec) -> + Name = proplists:get_value(name, MetricSpec), + gen_server:call(?MODULE, {declare, Name, MetricSpec}). + +-spec observe(name(), label_values(), number()) -> ok. +observe(Name, LabelValues, Value) -> + gen_server:cast(?MODULE, {observe, Name, LabelValues, Value}). + +-spec value(name(), label_values()) -> undefined | {non_neg_integer(), number(), [{number(), number()}]}. +value(Name, LabelValues) -> + gen_server:call(?MODULE, {value, Name, LabelValues}). + +-spec values(name()) -> [{label_values(), {non_neg_integer(), number(), [{number(), number()}]}}]. +values(Name) -> + [{LV, V} || LV <- get_label_values(Name), V <- [value(Name, LV)], V =/= undefined]. + +-spec get_label_values(name()) -> [label_values()]. +get_label_values(Name) -> + gen_server:call(?MODULE, {get_label_values, Name}). + +-spec remove(name(), label_values()) -> boolean(). +remove(Name, LabelValues) -> + gen_server:call(?MODULE, {remove, Name, LabelValues}). + +-spec get_all_metric_names() -> [name()]. +get_all_metric_names() -> + gen_server:call(?MODULE, get_all_metric_names). + +-spec get_metric_spec(name()) -> proplists:proplist() | undefined. +get_metric_spec(Name) -> + gen_server:call(?MODULE, {get_metric_spec, Name}). + +%% gen_server callbacks + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +init([]) -> + EtsTable = ets:new(?MODULE, [set, private, {read_concurrency, true}, {write_concurrency, true}]), + TimerRef = timer:send_interval(?MODULE:window_step_ms(), rotate), + {ok, #state{timer_ref = TimerRef, ets_table = EtsTable}}. + +handle_call({declare, Name, MetricSpec}, _From, State) -> + case maps:is_key(Name, State#state.metrics) of + true -> + {reply, false, State}; + false -> + NewMetrics = maps:put(Name, #{}, State#state.metrics), + NewSpecs = maps:put(Name, MetricSpec, State#state.metric_specs), + {reply, true, State#state{metrics = NewMetrics, metric_specs = NewSpecs}} + end; +handle_call({value, Name, LabelValues}, _From, State) -> + Result = get_value(Name, LabelValues, State), + {reply, Result, State}; +handle_call({get_label_values, Name}, _From, State) -> + Result = do_get_label_values(Name, State), + {reply, Result, State}; +handle_call({remove, Name, LabelValues}, _From, State) -> + NewState = do_remove(Name, LabelValues, State), + {reply, true, NewState}; +handle_call(get_all_metric_names, _From, State) -> + Names = maps:keys(State#state.metrics), + {reply, Names, State}; +handle_call({get_metric_spec, Name}, _From, State) -> + Spec = maps:get(Name, State#state.metric_specs, undefined), + {reply, Spec, State}; +handle_call(_Request, _From, State) -> + {reply, ok, State}. + +handle_cast({observe, Name, LabelValues, Value}, State) -> + NewState = do_observe(Name, LabelValues, Value, State), + {noreply, NewState}; +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(rotate, State) -> + NewState = rotate_all_windows(State), + {noreply, NewState}; +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, #state{timer_ref = {ok, TRef}}) -> + timer:cancel(TRef), + ok; +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%% Child spec for supervision tree + +-spec child_spec() -> supervisor:child_spec(). +child_spec() -> + #{id => ?MODULE, + start => {?MODULE, start_link, []}, + restart => permanent, + shutdown => timer:seconds(5), + type => worker, + modules => [?MODULE]}. + +%% Internal functions + +-spec do_observe(name(), label_values(), number(), #state{}) -> #state{}. +do_observe(Name, LabelValues, Value, State) -> + case maps:find(Name, State#state.metrics) of + error -> + erlang:error({unknown_metric, Name}); + {ok, MetricState} -> + Key = {Name, LabelValues}, + {Windows, CurrentIndex} = + ensure_windows_initialized(Key, State), + + %% Get current window and add observation + CurrentWindow = lists:nth(CurrentIndex + 1, Windows), + Sketch = maps:get(sketch, CurrentWindow), + UpdatedSketch = ddskerl_ets:insert(Sketch, Value), + UpdatedWindow = CurrentWindow#{sketch := UpdatedSketch}, + + %% Update windows list and metric state + UpdatedWindows = set_nth(CurrentIndex + 1, UpdatedWindow, Windows), + UpdatedMetricState = maps:put(Key, + #{windows => UpdatedWindows, + current_index => CurrentIndex}, + MetricState), + + UpdatedMetrics = maps:put(Name, UpdatedMetricState, State#state.metrics), + State#state{metrics = UpdatedMetrics} + end. + +-spec ensure_windows_initialized({name(), label_values()}, #state{}) -> {[window_data()], non_neg_integer()}. +ensure_windows_initialized(Key, State) -> + WindowCount = ?MODULE:window_count(), + case get_metric_state(Key, State) of + undefined -> + Windows = [new_window(Key, Index, State) || Index <- lists:seq(0, WindowCount - 1)], + {Windows, 0}; + #{windows := Windows, current_index := CurrentIndex} -> + {Windows, CurrentIndex} + end. + +-spec get_value(name(), label_values(), #state{}) -> undefined | {non_neg_integer(), number(), [{number(), number()}]}. +get_value(Name, LabelValues, State) -> + Key = {Name, LabelValues}, + case get_metric_state(Key, State) of + undefined -> + undefined; + #{windows := Windows} -> + WindowTuples = [window_tuple(Window) || Window <- Windows], + Merged = lists:foldl(fun + (undefined, Acc) -> Acc; + (Val, undefined) -> Val; + (Val, Acc) -> ddskerl_ets:merge_tuples(Acc, Val) + end, undefined, WindowTuples), + case Merged of + undefined -> + undefined; + _ -> + Count = ddskerl_ets:total_tuple(Merged), + case Count of + 0 -> + undefined; + _ -> + Sum = ddskerl_ets:sum_tuple(Merged), + Quantiles = [{Q, ddskerl_ets:quantile_tuple(Merged, Q)} || Q <- default_quantiles()], + {Count, Sum, Quantiles} + end + end + end. + +-spec do_get_label_values(name(), #state{}) -> [label_values()]. +do_get_label_values(Name, State) -> + case State#state.metrics of + #{Name := MetricState} -> + [LabelValues || {_, LabelValues} <- maps:keys(MetricState)]; + #{} -> + [] + end. + +-spec do_remove(name(), label_values(), #state{}) -> #state{}. +do_remove(Name, LabelValues, State) -> + Key = {Name, LabelValues}, + case maps:find(Name, State#state.metrics) of + error -> + State; + {ok, MetricState} -> + UpdatedMetricState = remove_metric_state(Key, MetricState), + UpdatedMetrics = maps:put(Name, UpdatedMetricState, State#state.metrics), + State#state{metrics = UpdatedMetrics} + end. + +-spec get_metric_state({name(), label_values()}, #state{}) -> undefined | metric_state(). +get_metric_state({Name, LabelValues}, State) -> + case State#state.metrics of + #{Name := MetricState} -> + maps:get({Name, LabelValues}, MetricState, undefined); + #{} -> + undefined + end. + +-spec set_nth(pos_integer(), T, [T]) -> [T]. +set_nth(1, Value, [_ | Rest]) -> + [Value | Rest]; +set_nth(N, Value, [H | Rest]) when N > 1 -> + [H | set_nth(N - 1, Value, Rest)]. + +%% Window helpers + +-spec metric_options(name(), #state{}) -> {number(), non_neg_integer()}. +metric_options(Name, State) -> + MetricSpec = maps:get(Name, State#state.metric_specs, []), + Error = proplists:get_value(error, MetricSpec, default_error()), + Bound = proplists:get_value(bound, MetricSpec, default_bound()), + {Error, Bound}. + +-spec window_name({name(), label_values()}, non_neg_integer()) -> term(). +window_name({Name, LabelValues}, Index) -> + {Name, LabelValues, Index}. + +-spec new_window({name(), label_values()}, non_neg_integer(), #state{}) -> window_data(). +new_window(Key = {Name, _LabelValues}, Index, State) -> + {Error, Bound} = metric_options(Name, State), + WindowName = window_name(Key, Index), + Sketch = ddskerl_ets:new(#{ets_table => State#state.ets_table, + name => WindowName, + error => Error, + bound => Bound}), + #{sketch => Sketch, + ref => State#state.ets_table, + name => WindowName}. + +-spec rotate_all_windows(#state{}) -> #state{}. +rotate_all_windows(State) -> + WindowCount = ?MODULE:window_count(), + UpdatedMetrics = maps:map( + fun(_Name, MetricState) -> + maps:map( + fun(_Key, #{windows := Windows, current_index := CurrentIndex}) -> + %% Move to next window and reset it + NewIndex = (CurrentIndex + 1) rem WindowCount, + WindowToReset = lists:nth(NewIndex + 1, Windows), + ResetWindow = reset_window(WindowToReset), + UpdatedWindows = set_nth(NewIndex + 1, ResetWindow, Windows), + #{windows => UpdatedWindows, current_index => NewIndex} + end, MetricState) + end, State#state.metrics), + State#state{metrics = UpdatedMetrics}. + +-spec reset_window(window_data()) -> window_data(). +reset_window(WindowData) -> + Sketch = maps:get(sketch, WindowData), + ResetSketch = ddskerl_ets:reset(Sketch), + WindowData#{sketch := ResetSketch}. + +-spec window_tuple(window_data()) -> ddskerl_ets:object() | undefined. +window_tuple(WindowData) -> + Ref = maps:get(ref, WindowData), + Name = maps:get(name, WindowData), + case ets:lookup(Ref, Name) of + [Val] -> Val; + [] -> undefined + end. + +-spec remove_metric_state({name(), label_values()}, #{}) -> #{}. +remove_metric_state(Key, MetricState) -> + case maps:find(Key, MetricState) of + error -> + MetricState; + {ok, #{windows := Windows}} -> + lists:foreach(fun(Window) -> delete_window(Window) end, Windows), + maps:remove(Key, MetricState) + end. + +-spec delete_window(window_data()) -> ok. +delete_window(WindowData) -> + Ref = maps:get(ref, WindowData), + Name = maps:get(name, WindowData), + ets:delete(Ref, Name), + ok. diff --git a/src/instrument/mongoose_prometheus_sliding_window_collector.erl b/src/instrument/mongoose_prometheus_sliding_window_collector.erl new file mode 100644 index 00000000000..7dd46613338 --- /dev/null +++ b/src/instrument/mongoose_prometheus_sliding_window_collector.erl @@ -0,0 +1,53 @@ +-module(mongoose_prometheus_sliding_window_collector). + +-behaviour(prometheus_collector). + +-export([deregister_cleanup/1, + collect_mf/2]). + +%% Collector behavior callbacks + +deregister_cleanup(_) -> + ok. + +collect_mf(_Registry, Callback) -> + %% Get all metric names from the sliding window manager + MetricNames = mongoose_prometheus_sliding_window:get_all_metric_names(), + lists:foreach(fun(Name) -> + collect_metric_family(Name, Callback) + end, MetricNames), + ok. + +%% Internal functions + +collect_metric_family(Name, Callback) -> + Values = mongoose_prometheus_sliding_window:values(Name), + case Values of + [] -> + ok; + _ -> + SWName = sliding_window_metric_name(Name), + MetricSpec = mongoose_prometheus_sliding_window:get_metric_spec(Name), + Help = case MetricSpec of + undefined -> ""; + Spec -> proplists:get_value(help, Spec, "") + end, + LabelKeys = case MetricSpec of + undefined -> []; + Spec2 -> proplists:get_value(labels, Spec2, []) + end, + %% Convert to Prometheus format + Metrics = [create_summary_metric(LabelKeys, LabelValues, Count, Sum, Quantiles) + || {LabelValues, {Count, Sum, Quantiles}} <- Values], + %% Create and send the metric family + MF = prometheus_model_helpers:create_mf(SWName, Help, summary, Metrics), + Callback(MF) + end. + +sliding_window_metric_name(Name) -> Name. + +create_summary_metric(LabelKeys, LabelValues, Count, Sum, Quantiles) -> + %% Convert label keys and values to label pairs format + LabelPairs = lists:zip(LabelKeys, LabelValues), + QuantilePairs = [{Q, V} || {Q, V} <- Quantiles], + prometheus_model_helpers:summary_metric(LabelPairs, Count, Sum, QuantilePairs). diff --git a/test/mongoose_instrument_metrics_SUITE.erl b/test/mongoose_instrument_metrics_SUITE.erl index 69b7a7d39a1..b3293226cd8 100644 --- a/test/mongoose_instrument_metrics_SUITE.erl +++ b/test/mongoose_instrument_metrics_SUITE.erl @@ -10,11 +10,30 @@ -define(HOST_TYPE, <<"localhost">>). -define(HOST_TYPE2, <<"test type">>). +-define(assertRelEqual(Expect, Actual, Tolerance), + (fun() -> + __Exp = (Expect), + __Act = (Actual), + __Tol = (Tolerance), + case __Exp == 0 of + true -> ?assertEqual(__Exp, __Act); + false -> + __RelErr = abs(__Exp - __Act) / __Exp, + __Check = case __RelErr < __Tol of + true -> ok; + false -> {fail, [{expected, __Exp}, {actual, __Act}, + {rel_error, __RelErr}, {tolerance, __Tol}]} + end, + ?assertEqual(ok, __Check) + end + end)()). + -import(mongoose_instrument_exometer, [exometer_metric_name/3]). %% Setup and teardown all() -> [{group, prometheus}, + {group, prometheus_sliding_window}, % separate, because it can't be run in parallel {group, exometer}, {group, exometer_global}, {group, prometheus_and_exometer} @@ -30,8 +49,10 @@ groups() -> prometheus_counter_cannot_be_decreased, prometheus_counter_is_updated_separately_for_different_labels, prometheus_histogram_is_created_and_updated, + prometheus_histogram_is_calculated_correctly, prometheus_histogram_is_updated_separately_for_different_labels, multiple_prometheus_metrics_are_updated]}, + {prometheus_sliding_window, [], [prometheus_histogram_sliding_window_expires_data]}, {exometer, [parallel], [exometer_skips_non_metric_event, exometer_gauge_is_created_and_updated, exometer_gauge_is_updated_separately_for_different_labels, @@ -61,10 +82,16 @@ init_per_group(Group, Config) -> mongoose_config:set_opts(#{hosts => [?HOST_TYPE], host_types => [?HOST_TYPE2], instrumentation => opts(Group)}), - Config1 = async_helper:start(Config, mongoose_instrument, start_link, []), + Config1 = async_helper:start(Config, extra_processes(Group) ++ + [{mongoose_instrument, start_link, []}]), mongoose_instrument:persist(), Config1 ++ extra_config(Group). +extra_processes(prometheus) -> [{mongoose_prometheus_sliding_window, start_link, []}]; +extra_processes(prometheus_sliding_window) -> [{mongoose_prometheus_sliding_window, start_link, []}]; +extra_processes(prometheus_and_exometer) -> [{mongoose_prometheus_sliding_window, start_link, []}]; +extra_processes(_) -> []. + end_per_group(Group, Config) -> async_helper:stop_all(Config), mongoose_config:erase_opts(), @@ -78,12 +105,15 @@ end_per_testcase(_Case, _Config) -> log_helper:unsubscribe(). apps(prometheus) -> [prometheus, prometheus_httpd, prometheus_cowboy]; +apps(prometheus_sliding_window) -> apps(prometheus); apps(exometer) -> [exometer_core]; apps(exometer_global) -> [exometer_core]; apps(prometheus_and_exometer) -> apps(prometheus) ++ apps(exometer). opts(prometheus) -> #{prometheus => #{}}; +opts(prometheus_sliding_window) -> + opts(prometheus); opts(exometer) -> #{exometer => #{all_metrics_are_global => false, report => #{}}}; opts(exometer_global) -> @@ -184,13 +214,33 @@ prometheus_histogram_is_created_and_updated(Config) -> ok = mongoose_instrument:set_up(Event, ?LABELS, #{metrics => #{time => histogram}}), %% Prometheus histogram shows no value if there is no data - ?assertEqual(undefined, prometheus_histogram:value(Metric, [?HOST_TYPE])), + ?assertEqual(undefined, mongoose_prometheus_sliding_window:value(Metric, [?HOST_TYPE])), ok = mongoose_instrument:execute(Event, ?LABELS, #{time => 1}), - ?assertMatch({[1, 0|_], 1}, prometheus_histogram:value(Metric, [?HOST_TYPE])), + ?assertMatch({1, 1, _}, mongoose_prometheus_sliding_window:value(Metric, [?HOST_TYPE])), ok = mongoose_instrument:execute(Event, ?LABELS, #{time => 1}), - ?assertMatch({[2, 0|_], 2}, prometheus_histogram:value(Metric, [?HOST_TYPE])), + ?assertMatch({2, 2, _}, mongoose_prometheus_sliding_window:value(Metric, [?HOST_TYPE])), ok = mongoose_instrument:execute(Event, ?LABELS, #{time => 2}), - ?assertMatch({[2, 1|_], 4}, prometheus_histogram:value(Metric, [?HOST_TYPE])). + ?assertMatch({3, 4, _}, mongoose_prometheus_sliding_window:value(Metric, [?HOST_TYPE])). + +prometheus_histogram_is_calculated_correctly(Config) -> + Event = ?config(event, Config), + Metric = prom_name(Event, time), + ok = mongoose_instrument:set_up(Event, ?LABELS, #{metrics => #{time => histogram}}), + + %% Prometheus histogram shows no value if there is no data + ?assertEqual(undefined, mongoose_prometheus_sliding_window:value(Metric, [?HOST_TYPE])), + Values = [1, 2, 2, 2, 2, 2, 2, 9, 9, 10], + lists:foreach(fun(V) -> ok = mongoose_instrument:execute(Event, ?LABELS, #{time => V}) end, Values), + Sum = lists:sum(Values), + Length = length(Values), + ?assertMatch({Length, Sum, _}, mongoose_prometheus_sliding_window:value(Metric, [?HOST_TYPE])), + %% Check some quantiles + {_, _, Quantiles} = mongoose_prometheus_sliding_window:value(Metric, [?HOST_TYPE]), + Tolerance = 0.01, + ?assertRelEqual(2, proplists:get_value(0.5, Quantiles), Tolerance), + ?assertRelEqual(9, proplists:get_value(0.9, Quantiles), Tolerance), + ?assertRelEqual(10, proplists:get_value(0.95, Quantiles), Tolerance), + ?assertRelEqual(10, proplists:get_value(0.99, Quantiles), Tolerance). prometheus_histogram_is_updated_separately_for_different_labels(Config) -> Event = ?config(event, Config), @@ -199,8 +249,36 @@ prometheus_histogram_is_updated_separately_for_different_labels(Config) -> ok = mongoose_instrument:set_up(Event, ?LABELS2, #{metrics => #{time => histogram}}), ok = mongoose_instrument:execute(Event, ?LABELS, #{time => 1}), ok = mongoose_instrument:execute(Event, ?LABELS2, #{time => 2}), - ?assertMatch({[1, 0|_], 1}, prometheus_histogram:value(Metric, [?HOST_TYPE])), - ?assertMatch({[0, 1|_], 2}, prometheus_histogram:value(Metric, [?HOST_TYPE2])). + ?assertMatch({1, 1, _}, mongoose_prometheus_sliding_window:value(Metric, [?HOST_TYPE])), + ?assertMatch({1, 2, _}, mongoose_prometheus_sliding_window:value(Metric, [?HOST_TYPE2])). + +prometheus_histogram_sliding_window_expires_data(Config) -> + Event = ?config(event, Config), + Metric = prom_name(Event, time), + + %% We mock the window parameters to use small values and wait a short time. + meck:new(mongoose_prometheus_sliding_window, [passthrough]), + meck:expect(mongoose_prometheus_sliding_window, window_size_ms, fun() -> 1000 end), + meck:expect(mongoose_prometheus_sliding_window, window_step_ms, fun() -> 100 end), + + %% Restart the gen_server to make sure that the new window parameters are applied + ok = gen_server:stop(mongoose_prometheus_sliding_window), + {ok, _} = mongoose_prometheus_sliding_window:start_link(), + + try + ok = mongoose_instrument:set_up(Event, ?LABELS, #{metrics => #{time => histogram}}), + ok = mongoose_instrument:execute(Event, ?LABELS, #{time => 10}), + ?assertMatch({1, 10, _}, mongoose_prometheus_sliding_window:value(Metric, [?HOST_TYPE])), + timer:sleep(500), + ok = mongoose_instrument:execute(Event, ?LABELS, #{time => 20}), + ?assertMatch({2, 30, _}, mongoose_prometheus_sliding_window:value(Metric, [?HOST_TYPE])), + timer:sleep(700), % Wait enough for the first value to expire + ?assertMatch({1, 20, _}, mongoose_prometheus_sliding_window:value(Metric, [?HOST_TYPE])), + timer:sleep(500), % Wait enough for the second value to expire + ?assertEqual(undefined, mongoose_prometheus_sliding_window:value(Metric, [?HOST_TYPE])) + after + meck:unload(mongoose_prometheus_sliding_window) + end. multiple_prometheus_metrics_are_updated(Config) -> Event = ?config(event, Config), @@ -211,18 +289,18 @@ multiple_prometheus_metrics_are_updated(Config) -> %% Update both metrics ok = mongoose_instrument:execute(Event, ?LABELS, #{count => 1, time => 2}), ?assertEqual(1, prometheus_counter:value(Counter, [?HOST_TYPE])), - HistogramValue = prometheus_histogram:value(Histogram, [?HOST_TYPE]), - ?assertMatch({[0, 1|_], 2}, HistogramValue), + HistogramValue = mongoose_prometheus_sliding_window:value(Histogram, [?HOST_TYPE]), + ?assertMatch({1, 2, _}, HistogramValue), %% Update only one metric ok = mongoose_instrument:execute(Event, ?LABELS, #{count => 2}), ?assertEqual(3, prometheus_counter:value(Counter, [?HOST_TYPE])), - ?assertEqual(HistogramValue, prometheus_histogram:value(Histogram, [?HOST_TYPE])), + ?assertEqual(HistogramValue, mongoose_prometheus_sliding_window:value(Histogram, [?HOST_TYPE])), %% No update ok = mongoose_instrument:execute(Event, ?LABELS, #{something => irrelevant}), ?assertEqual(3, prometheus_counter:value(Counter, [?HOST_TYPE])), - ?assertEqual(HistogramValue, prometheus_histogram:value(Histogram, [?HOST_TYPE])). + ?assertEqual(HistogramValue, mongoose_prometheus_sliding_window:value(Histogram, [?HOST_TYPE])). exometer_skips_non_metric_event(Config) -> Event = ?config(event, Config), @@ -386,7 +464,7 @@ prometheus_and_exometer_metrics_are_updated(Config) -> ?assertEqual({ok, [{count, 1}]}, exometer:get_value([?HOST_TYPE, Event, count], count)), ?assertEqual({ok, [{mean, 2}]}, exometer:get_value([?HOST_TYPE, Event, time], mean)), ?assertEqual(1, prometheus_counter:value(prom_name(Event, count), [?HOST_TYPE])), - ?assertMatch({[0, 1|_], 2}, prometheus_histogram:value(prom_name(Event, time), [?HOST_TYPE])). + ?assertMatch({1, 2, _}, mongoose_prometheus_sliding_window:value(prom_name(Event, time), [?HOST_TYPE])). %% Helpers diff --git a/test/prometheus_endpoint_SUITE.erl b/test/prometheus_endpoint_SUITE.erl index 20bbebcf996..ac601d78f07 100644 --- a/test/prometheus_endpoint_SUITE.erl +++ b/test/prometheus_endpoint_SUITE.erl @@ -23,7 +23,8 @@ init_per_suite(Config) -> mongoose_config:set_opts(opts()), Config1 = async_helper:start(Config, [{mongoose_instrument, start_link, []}, {mim_ct_sup, start_link, [ejabberd_sup]}, - {mongoose_listener_sup, start_link, []}]), + {mongoose_listener_sup, start_link, []}, + {mongoose_prometheus_sliding_window, start_link, []}]), mongoose_listener:start(), Config1. @@ -54,7 +55,7 @@ test_metrics(Event, Labels) -> check_counter(Event, requests, Labels, 1, Scraped), % 'spiral' is a Prometheus counter check_gauge(Event, sessions, Labels, 5, Scraped), % 'counter' is a Prometheus gauge check_gauge(Event, seconds, Labels, 10, Scraped), - check_histogram(Event, time, Labels, #{count => 1, sum => 2, bucket_num => 32}, Scraped). + check_histogram(Event, time, Labels, #{count => 1, sum => 2, median => 2}, Scraped). %% Checks for the parsed metrics @@ -71,38 +72,17 @@ check_gauge(Event, Metric, Labels, ExpValue, Scraped) -> ?assertEqual({Labels, ExpValue}, Value). check_histogram(Event, Metric, Labels, ExpValues, Scraped) -> - #{count := ExpCount, sum := ExpSum, bucket_num := ExpBucketNum} = ExpValues, - [Type, Help] = get_metric([Event, Metric], Scraped), - ?assertEqual({<<"TYPE">>, <<"histogram">>}, Type), + #{count := ExpCount, sum := ExpSum, median := ExpMedian} = ExpValues, + [Type, Help | _] = get_metric([Event, Metric], Scraped), + ?assertEqual({<<"TYPE">>, <<"summary">>}, Type), ?assertEqual({<<"HELP">>, help(Event, Metric)}, Help), [Count] = get_metric([Event, Metric, count], Scraped), ?assertEqual({Labels, ExpCount}, Count), [Sum] = get_metric([Event, Metric, sum], Scraped), ?assertEqual({Labels, ExpSum}, Sum), - Buckets = get_metric([Event, Metric, bucket], Scraped), - ?assertEqual(ExpBucketNum, length(Buckets)), - check_buckets(ExpCount, Labels, Buckets). - -%% Check that the histogram buckets have growing thresholds and counts, -%% and that the last bucket has the expected total count (because they are cumulative). -check_buckets(ExpCount, Labels, Buckets) -> - InitState = #{labels => Labels, last_count => 0, last_threshold => 0}, - #{final_count := LastCount} = lists:foldl(fun check_bucket/2, InitState, Buckets), - ?assertEqual(ExpCount, LastCount). - -check_bucket({Labels, Count}, State) -> - #{labels := BaseLabels, last_count := LastCount, last_threshold := LastThreshold} = State, - {ThresholdBin, Labels1} = maps:take(le, Labels), - ?assertEqual(Labels1, BaseLabels), - ?assert(Count >= LastCount), - case ThresholdBin of - <<"+Inf">> -> - #{final_count => Count}; - _ -> - Threshold = binary_to_integer(ThresholdBin), - ?assert(Threshold > LastThreshold), - State#{last_count => Count, last_threshold => Threshold} - end. + MedianLabels = Labels#{quantile => <<"0.5">>}, + [{MedianLabels, Median}] = get_metric([Event, Metric], MedianLabels, Scraped), + ?assert(abs(Median - ExpMedian) < 0.01). help(Event, Metric) -> <<"Event: ", (atom_to_binary(Event))/binary, ", Metric: ", (atom_to_binary(Metric))/binary>>. @@ -112,6 +92,11 @@ get_metric(Parts, Scraped) when is_list(Parts) -> get_metric(Name, Scraped) when is_binary(Name) -> [{LabelsOrKey, Value} || {N, LabelsOrKey, Value} <- Scraped, N =:= Name]. +get_metric(Parts, Labels, Scraped) when is_list(Parts) -> + get_metric(binary_name(Parts), Labels, Scraped); +get_metric(Name, Labels, Scraped) when is_binary(Name) -> + [{L, Value} || {N, L, Value} <- Scraped, N =:= Name, is_map(L), maps:merge(L, Labels) =:= L]. + binary_name(Atoms) -> list_to_binary(string:join([atom_to_list(Atom) || Atom <- Atoms], "_")).