possible traffic shaping or throttling issues with custom module

[olcrazypete]

MongooseIM version: 3.7.1
Installed from: src
Erlang/OTP version: EPEL - 03.18.el7

Finding issues with a mongooseIM install that we are looking to put into production. We currently have a jabberd2 XMPP server as a backend to a custom alert client for only ~2000 users.
In testing the mongooseIM replacement, all functions of our alert systems seem to work fine for a low number of users, around ~20, but we quickly run into issues with users above 30 or so logged in.
The system seems to become unresponsive around the API endpoint responses, running on port 8088. It really seems like a throttled response but I can't find where to open that up and have removed or raised every throttle with no response.
Config is attached.

mongooseim.cfg.zip

[arcusfelis]

Hi!

When debugging performance issues, open mongooseim shell (i.e. mongooseim debug command).
And insert these commands:

Processes with the longest queue:

    rp([{Len, case erlang:process_info(TopPid, registered_name) of {_,X} -> X; _ -> TopPid end, erlang:process_info(TopPid, current_stacktrace)} || {Len, TopPid} <- lists:sublist(lists:reverse(lists:keysort(1, [{try element(2,erlang:process_info(Pid, message_queue_len)) + proplists:get_value('$internal_queue_len', element(2, erlang:process_info(Pid, dictionary)), 0) catch _:_ -> -1 end, Pid} || Pid <- erlang:processes()])), 20)]).

Processes, consuming the most memory:

    rp(lists:reverse([{Len, case erlang:process_info(TopPid, registered_name) of {_,X} -> X; _ -> TopPid end, erlang:process_info(TopPid, current_stacktrace)} || {Len, TopPid} <- lists:sublist(lists:reverse(lists:keysort(1, [{try element(2,erlang:process_info(Pid, memory)) catch _:_ -> -1 end, Pid} || Pid <- erlang:processes()])), 20)])).

And post the results of the both commands here.
(You have to do that when the system is under the load).

[olcrazypete]

I'm not having a lot of luck in deciphering the output but here is what I have. This was 90 bot clients and three of our regular client apps logged in to confirm messages weren't getting thru.

([email protected])14> rp([{Len, case erlang:process_info(TopPid, registered_name) of {_,X} -> X; _ -> TopPid end, erlang:process_info(TopPid, current_stacktrace)} || {Len, TopPid} <- lists:sublist(lists:reverse(lists:keysort(1, [{try element(2,erlang:process_info(Pid, message_queue_len)) + proplists:get_value('$internal_queue_len', element(2, erlang:process_info(Pid, dictionary)), 0) catch : -> -1 end, Pid} || Pid <- erlang:processes()])), 20)]).
[{0,<0.25291.4>,
{current_stacktrace,
[{gen,do_call,4,[{file,"gen.erl"},{line,167}]},
{gen_server,call,3,[{file,"gen_server.erl"},{line,219}]},
{mongoose_http_client,make_request,8,
[{file,
"/home/esolutions/MongooseIM/src/mongoose_http_client.erl"},
{line,64}]},
{mod_event_pusher_http,make_req,7,
[{file,
"/home/esolutions/MongooseIM/src/event_pusher/mod_event_pusher_http.erl"},
{line,97}]},
{mod_event_pusher_http,push_event,6,
[{file,
"/home/esolutions/MongooseIM/src/event_pusher/mod_event_pusher_http.erl"},
{line,71}]},
{lists,map,2,[{file,"lists.erl"},{line,1239}]},
{mod_event_pusher_http,push_event,3,
[{file,
"/home/esolutions/MongooseIM/src/event_pusher/mod_event_pusher_http.erl"},
{line,60}]},
{lists,foldl,3,[{file,"lists.erl"},{line,1263}]}]}},
{0,<0.25290.4>,
{current_stacktrace,
[{cowboy_http,loop,1,
[{file,
"/home/esolutions/MongooseIM/build/default/lib/cowboy/src/cowboy_http.erl"},
{line,203}]},
{proc_lib,init_p_do_apply,3,
[{file,"proc_lib.erl"},{line,249}]}]}},
{0,<0.24912.4>,{current_stacktrace,[]}},
{0,<0.24911.4>,{current_stacktrace,[]}},
{0,<0.24896.4>,{current_stacktrace,[]}},
{0,<0.24895.4>,{current_stacktrace,[]}},
{0,<0.24874.4>,
{current_stacktrace,
[{gen,do_call,4,[{file,"gen.erl"},{line,167}]},
{gen_server,call,3,[{file,"gen_server.erl"},{line,219}]},
{mongoose_http_client,make_request,8,
[{file,
"/home/esolutions/MongooseIM/src/mongoose_http_client.erl"},
{line,64}]},
{mod_event_pusher_http,make_req,7,
[{file,
"/home/esolutions/MongooseIM/src/event_pusher/mod_event_pusher_http.erl"},
{line,97}]},
{mod_event_pusher_http,push_event,6,
[{file,
"/home/esolutions/MongooseIM/src/event_pusher/mod_event_pusher_http.erl"},
{line,71}]},
{lists,map,2,[{file,"lists.erl"},{line,1239}]},
{mod_event_pusher_http,push_event,3,
[{file,
"/home/esolutions/MongooseIM/src/event_pusher/mod_event_pusher_http.erl"},
{line,60}]},
{lists,foldl,3,[{file,"lists.erl"},{line,1263}]}]}},
{0,<0.24873.4>,{current_stacktrace,[]}},
{0,<0.24784.4>,{current_stacktrace,[]}},
{0,<0.24783.4>,{current_stacktrace,[]}},
{0,<0.24782.4>,{current_stacktrace,[]}},
{0,<0.24781.4>,{current_stacktrace,[]}},
{0,<0.24780.4>,{current_stacktrace,[]}},
{0,<0.24779.4>,{current_stacktrace,[]}},
{0,<0.24778.4>,{current_stacktrace,[]}},
{0,<0.24777.4>,{current_stacktrace,[]}},
{0,<0.24776.4>,{current_stacktrace,[]}},
{0,<0.24775.4>,{current_stacktrace,[]}},
{0,<0.24774.4>,{current_stacktrace,[]}},
{0,<0.24773.4>,{current_stacktrace,[]}}]
ok
([email protected])15> rp(lists:reverse([{Len, case erlang:process_info(TopPid, registered_name) of {,X} -> X; _ -> TopPid end, erlang:process_info(TopPid, current_stacktrace)} || {Len, TopPid} <- lists:sublist(lists:reverse(lists:keysort(1, [{try element(2,erlang:process_info(Pid, memory)) catch : -> -1 end, Pid} || Pid <- erlang:processes()])), 20)])). [{109332,<0.794.0>,
{current_stacktrace,[{exometer_probe,loop,1,
[{file,"/home/esolutions/MongooseIM/_build/default/lib/exometer_core/src/exometer_probe.erl"},
{line,673}]},
{proc_lib,init_p,3,[{file,"proc_lib.erl"},{line,234}]}]}},
{109448,<0.388.0>,
{current_stacktrace,[{application_master,loop_it,4,
[{file,"application_master.erl"},{line,351}]}]}},
{111516,
'wpool_pool-mongoose_wpool$http$global$auth-process-sup',
{current_stacktrace,[{gen_server,loop,7,
[{file,"gen_server.erl"},{line,394}]},
{proc_lib,init_p_do_apply,3,
[{file,"proc_lib.erl"},{line,249}]}]}},
{111516,
'wpool_pool-mongoose_wpool$http$global$push_messages-process-sup',
{current_stacktrace,[{gen_server,loop,7,
[{file,"gen_server.erl"},{line,394}]},
{proc_lib,init_p_do_apply,3,
[{file,"proc_lib.erl"},{line,249}]}]}},
{142768,<0.247.0>,
{current_stacktrace,[{application_master,loop_it,4,
[{file,"application_master.erl"},{line,351}]}]}},
{142812,code_server,
{current_stacktrace,[{code_server,loop,1,
[{file,"code_server.erl"},{line,151}]}]}},
{142812,release_handler,
{current_stacktrace,[{gen_server,loop,7,
[{file,"gen_server.erl"},{line,394}]},
{proc_lib,init_p_do_apply,3,
[{file,"proc_lib.erl"},{line,249}]}]}},
{143000,<0.127.0>,
{current_stacktrace,[{application_master,loop_it,4,
[{file,"application_master.erl"},{line,351}]}]}},
{147248,<0.901.0>,
{current_stacktrace,[{gen_server,loop,7,
[{file,"gen_server.erl"},{line,394}]},
{proc_lib,init_p_do_apply,3,
[{file,"proc_lib.erl"},{line,249}]}]}},
{176380,httpc_manager,
{current_stacktrace,[{gen_server,loop,7,
[{file,"gen_server.erl"},{line,394}]},
{proc_lib,init_p_do_apply,3,
[{file,"proc_lib.erl"},{line,249}]}]}},
{176456,<0.753.0>,
{current_stacktrace,[{gun,loop,1,
[{file,"/home/esolutions/MongooseIM/_build/default/lib/gun/src/gun.erl"},
{line,764}]},
{gun,proc_lib_hack,5,
[{file,"/home/esolutions/MongooseIM/_build/default/lib/gun/src/gun.erl"},
{line,649}]},
{proc_lib,init_p_do_apply,3,
[{file,"proc_lib.erl"},{line,249}]}]}},
{176552,mnesia_monitor,
{current_stacktrace,[{gen_server,loop,7,
[{file,"gen_server.erl"},{line,394}]},
{proc_lib,init_p_do_apply,3,
[{file,"proc_lib.erl"},{line,249}]}]}},
{197004,<0.820.0>,
{current_stacktrace,[{exometer_probe,loop,1,
[{file,"/home/esolutions/MongooseIM/_build/default/lib/exometer_core/src/exometer_probe.erl"},
{line,673}]},
{proc_lib,init_p,3,[{file,"proc_lib.erl"},{line,234}]}]}},
{197052,<0.818.0>,
{current_stacktrace,[{exometer_probe,loop,1,
[{file,"/home/esolutions/MongooseIM/_build/default/lib/exometer_core/src/exometer_probe.erl"},
{line,673}]},
{proc_lib,init_p,3,[{file,"proc_lib.erl"},{line,234}]}]}},
{230564,exometer_report,
{current_stacktrace,[{gen_server,loop,7,
[{file,"gen_server.erl"},{line,394}]},
{proc_lib,init_p_do_apply,3,
[{file,"proc_lib.erl"},{line,249}]}]}},
{230660,<0.71.0>,
{current_stacktrace,[{shell,get_command1,5,
[{file,"shell.erl"},{line,303}]},
{shell,server_loop,7,[{file,"shell.erl"},{line,230}]}]}},
{263964,erl_prim_loader,
{current_stacktrace,[{erl_prim_loader,loop,3,[]}]}},
{274364,exometer_admin,
{current_stacktrace,[{gen_server,loop,7,
[{file,"gen_server.erl"},{line,394}]},
{proc_lib,init_p_do_apply,3,
[{file,"proc_lib.erl"},{line,249}]}]}},
{284676,<0.823.0>,
{current_stacktrace,[{exometer_probe,loop,1,
[{file,"/home/esolutions/MongooseIM/_build/default/lib/exometer_core/src/exometer_probe.erl"},
{line,673}]},
{proc_lib,init_p,3,[{file,"proc_lib.erl"},{line,234}]}]}},
{285980,application_controller,
{current_stacktrace,[{gen_server,loop,7,
[{file,"gen_server.erl"},{line,394}]}]}}]
ok