Skip to content

Commit 08be41c

Browse files
leondavileondavi
authored
Merge pull request #420 from leondavi/perf_stats
memory perf works
2 parents 82316bf + 123c477 commit 08be41c

File tree

3 files changed

+57
-49
lines changed

3 files changed

+57
-49
lines changed

‎src_erl/NerlnetApp/src/Client/clientStatem.erl

Lines changed: 33 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -267,14 +267,17 @@ training(cast, In = {sample,Body}, State = #client_statem_state{etsRef = EtsRef}
267267
gen_statem:cast(WorkerPid, {sample, SourceName ,BatchID ,BatchOfSamples}),
268268
stats:increment_messages_sent(ClientStatsEts),
269269
BatchSize = nerl_tools:calculate_size(BatchOfSamples),
270-
stats:increment_bytes_sent(ClientStatsEts , BatchSize);
270+
stats:increment_bytes_sent(ClientStatsEts , BatchSize),
271+
perf_stats_memory_usage_update_train();
271272
true -> ?LOG_ERROR("Given worker ~p isn't found in client ~p",[WorkerName, ClientName]) end,
272273
{next_state, training, State#client_statem_state{etsRef = EtsRef}};
273274

274275
% This action is used for start_stream triggered from a clients' worker and not source
275276
training(cast, {start_stream , {worker, WorkerName, TargetPair}}, State = #client_statem_state{etsRef = EtsRef}) ->
276277
ListOfActiveWorkersSources = ets:lookup_element(EtsRef, active_workers_streams, ?DATA_IDX),
277278
ets:update_element(EtsRef, active_workers_streams, {?DATA_IDX, ListOfActiveWorkersSources ++ [{WorkerName, TargetPair}]}),
279+
280+
perf_stats_memory_usage_update_train(),
278281
{keep_state, State};
279282

280283
% This action is used for start_stream triggered from a source per worker
@@ -287,6 +290,8 @@ training(cast, In = {start_stream , Data}, State = #client_statem_state{etsRef =
287290
stats:increment_bytes_received(ClientStatsEts , nerl_tools:calculate_size(In)),
288291
WorkerPid = clientWorkersFunctions:get_worker_pid(EtsRef , WorkerName),
289292
gen_statem:cast(WorkerPid, {start_stream, SourceName}),
293+
294+
perf_stats_memory_usage_update_train(),
290295
{keep_state, State};
291296

292297

@@ -296,7 +301,9 @@ training(cast, In = {end_stream , Data}, State = #client_statem_state{etsRef = E
296301
stats:increment_messages_received(ClientStatsEts),
297302
stats:increment_bytes_received(ClientStatsEts , nerl_tools:calculate_size(In)),
298303
WorkerPid = clientWorkersFunctions:get_worker_pid(EtsRef , WorkerName),
299-
gen_statem:cast(WorkerPid, {end_stream, SourceName}),
304+
gen_statem:cast(WorkerPid, {end_stream, SourceName}),
305+
306+
perf_stats_memory_usage_update_train(),
300307
{keep_state, State};
301308

302309
training(cast, In = {stream_ended , Pair}, State = #client_statem_state{etsRef = EtsRef}) ->
@@ -371,7 +378,8 @@ predict(cast, In = {sample,Body}, State = #client_statem_state{etsRef = EtsRef})
371378
WorkerPid = clientWorkersFunctions:get_worker_pid(EtsRef , WorkerName),
372379
gen_statem:cast(WorkerPid, {sample, SourceName ,BatchID ,BatchOfSamples}),
373380
stats:increment_messages_sent(ClientStatsEts),
374-
stats:increment_bytes_sent(ClientStatsEts , nerl_tools:calculate_size(BatchOfSamples));
381+
stats:increment_bytes_sent(ClientStatsEts , nerl_tools:calculate_size(BatchOfSamples)),
382+
perf_stats_memory_usage_update_predict();
375383
true -> ?LOG_ERROR("Given worker ~p isn't found in client ~p",[WorkerName, ClientName])
376384
end,
377385
{next_state, predict, State#client_statem_state{etsRef = EtsRef}};
@@ -380,6 +388,8 @@ predict(cast, In = {sample,Body}, State = #client_statem_state{etsRef = EtsRef})
380388
predict(cast, {start_stream , {worker, WorkerName, TargetName}}, State = #client_statem_state{etsRef = EtsRef}) ->
381389
ListOfActiveWorkersSources = ets:lookup_element(EtsRef, active_workers_streams, ?DATA_IDX),
382390
ets:update_element(EtsRef, active_workers_streams, {?DATA_IDX, ListOfActiveWorkersSources ++ [{WorkerName, TargetName}]}),
391+
392+
perf_stats_memory_usage_update_predict(),
383393
{keep_state, State};
384394

385395
% This action is used for start_stream triggered from a source per worker
@@ -392,6 +402,8 @@ predict(cast, In = {start_stream , Data}, State = #client_statem_state{etsRef =
392402
stats:increment_bytes_received(ClientStatsEts , nerl_tools:calculate_size(In)),
393403
WorkerPid = clientWorkersFunctions:get_worker_pid(EtsRef , WorkerName),
394404
gen_statem:cast(WorkerPid, {start_stream, SourceName}),
405+
406+
perf_stats_memory_usage_update_predict(),
395407
{keep_state, State};
396408

397409
predict(cast, In = {end_stream , Data}, State = #client_statem_state{etsRef = EtsRef}) ->
@@ -400,7 +412,9 @@ predict(cast, In = {end_stream , Data}, State = #client_statem_state{etsRef = Et
400412
stats:increment_messages_received(ClientStatsEts),
401413
stats:increment_bytes_received(ClientStatsEts , nerl_tools:calculate_size(In)),
402414
WorkerPid = clientWorkersFunctions:get_worker_pid(EtsRef , WorkerName),
403-
gen_statem:cast(WorkerPid, {end_stream, SourceName}),
415+
gen_statem:cast(WorkerPid, {end_stream, SourceName}),
416+
417+
perf_stats_memory_usage_update_predict(),
404418
{keep_state, State};
405419

406420
predict(cast, In = {stream_ended , Pair}, State = #client_statem_state{etsRef = EtsRef}) ->
@@ -547,4 +561,18 @@ handle_w2w_msg(EtsRef, FromWorker, ToWorker, Data) ->
547561
nerl_tools:http_router_request(RouterHost, RouterPort, [DestClient], atom_to_list(worker_to_worker_msg), MessageBody),
548562
stats:increment_messages_sent(ClientStatsEts),
549563
stats:increment_bytes_sent(ClientStatsEts , nerl_tools:calculate_size(MessageBody))
550-
end.
564+
end.
565+
566+
perf_stats_memory_usage_update_train() ->
567+
% memory usage update
568+
PerformanceStatsEts = get(performance_stats_ets),
569+
MemoryUsageValue = stats:query_memory_usage(),
570+
stats:update_memory_peak_usage_train(PerformanceStatsEts, MemoryUsageValue),
571+
stats:update_memory_train_ema_usage(PerformanceStatsEts, MemoryUsageValue).
572+
573+
perf_stats_memory_usage_update_predict() ->
574+
% memory usage update
575+
PerformanceStatsEts = get(performance_stats_ets),
576+
MemoryUsageValue = stats:query_memory_usage(),
577+
stats:update_memory_peak_usage_predict(PerformanceStatsEts, MemoryUsageValue),
578+
stats:update_memory_predict_ema_usage(PerformanceStatsEts, MemoryUsageValue).

‎src_erl/NerlnetApp/src/Stats/stats.erl

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -293,14 +293,14 @@ query_memory_usage() ->
293293
%% Get the memory usage of the Erlang VM
294294
[{system_total_memory,SystemTotalMemory},
295295
{free_memory, FreeMemory},
296-
{total_memory, TotalMemory},
297-
{buffered_memory, BufferedMemory},
298-
{cached_memory, CachedMemory},
299-
{total_swap, TotalSwap},
300-
{free_swap, FreeSwap},
301-
{available_memory, AvailableMemory}] = memsup:get_system_memory_data(),
296+
{total_memory, _TotalMemory},
297+
{buffered_memory, _BufferedMemory},
298+
{cached_memory, _CachedMemory},
299+
{total_swap, _TotalSwap},
300+
{free_swap, _FreeSwap},
301+
{available_memory, _AvailableMemory}] = memsup:get_system_memory_data(),
302302
%% Calculate the used memory
303-
UsedMemory = SystemTotalMemory - FreeMemory.
303+
SystemTotalMemory - FreeMemory.
304304

305305
reset_query_cpu_util_cores() ->
306306
%% Reset the CPU utilization data

‎src_erl/NerlnetApp/src/nerlnetApp_app.erl

Lines changed: 17 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
-behaviour(application).
2121
-include("nerl_tools.hrl").
2222

23-
-define(NERLNET_APP_VERSION, "1.5.3").
23+
-define(NERLNET_APP_VERSION, "1.5.4").
2424
-define(NERLPLANNER_TESTED_VERSION,"1.0.4").
2525

2626
-export([start/2, stop/1]).
@@ -49,42 +49,22 @@
4949
welcome_print() ->
5050
io:format(
5151
"
52-
53-
__ __ __ __
54-
/ | _ / | / |/ |
55-
$$ | / \\ $$ | ______ $$ |$$ | _______ ______ _____ ____ ______
56-
$$ |/$ \\$$ | / \\ $$ |$$ | / | / \\ / \\/ \\ / \\
57-
$$ /$$$ $$ |/$$$$$$ |$$ |$$ |/$$$$$$$/ /$$$$$$ |$$$$$$ $$$$ |/$$$$$$ |
58-
$$ $$/$$ $$ |$$ $$ |$$ |$$ |$$ | $$ | $$ |$$ | $$ | $$ |$$ $$ |
59-
$$$$/ $$$$ |$$$$$$$$/ $$ |$$ |$$ \\_____ $$ \\__$$ |$$ | $$ | $$ |$$$$$$$$/
60-
$$$/ $$$ |$$ |$$ |$$ |$$ |$$ $$/ $$ | $$ | $$ |$$ |
61-
$$/ $$/ $$$$$$$/ $$/ $$/ $$$$$$$/ $$$$$$/ $$/ $$/ $$/ $$$$$$$/
62-
63-
64-
65-
__
66-
/ |
67-
_$$ |_ ______
68-
/ $$ | / \\
69-
$$$$$$/ /$$$$$$ |
70-
$$ | __ $$ | $$ |
71-
$$ |/ |$$ \\__$$ |
72-
$$ $$/ $$ $$/
73-
$$$$/ $$$$$$/
74-
75-
76-
77-
__ __ ________ __ __ __ __
78-
/ \\ / |/ | / |/ \\ / | / |
79-
$$ \\ $$ |$$$$$$$$/ ______ $$ |$$ \\ $$ | ______ _$$ |_
80-
$$$ \\$$ |$$ |__ / \\ $$ |$$$ \\$$ | / \\ / $$ |
81-
$$$$ $$ |$$ | /$$$$$$ |$$ |$$$$ $$ |/$$$$$$ |$$$$$$/
82-
$$ $$ $$ |$$$$$/ $$ | $$/ $$ |$$ $$ $$ |$$ $$ | $$ | __
83-
$$ |$$$$ |$$ |_____ $$ | $$ |$$ |$$$$ |$$$$$$$$/ $$ |/ |
84-
$$ | $$$ |$$ |$$ | $$ |$$ | $$$ |$$ | $$ $$/
85-
$$/ $$/ $$$$$$$$/ $$/ $$/ $$/ $$/ $$$$$$$/ $$$$/
86-
87-
~n~n").
52+
╔═══════════════════════════════════════════════════════════════════════════════╗
53+
║ ║
54+
║ \033[38;5;88m███╗ ██╗███████╗██████╗ ██╗ ███╗ ██╗███████╗████████╗\033[0m ║
55+
║ \033[38;5;88m████╗ ██║██╔════╝██╔══██╗██║ ████╗ ██║██╔════╝╚══██╔══╝\033[0m ║
56+
║ \033[38;5;88m██╔██╗ ██║█████╗ ██████╔╝██║ ██╔██╗ ██║█████╗ ██║\033[0m ║
57+
║ \033[38;5;88m██║╚██╗██║██╔══╝ ██╔══██╗██║ ██║╚██╗██║██╔══╝ ██║\033[0m ║
58+
║ \033[38;5;88m██║ ╚████║███████╗██║ ██║███████╗██║ ╚████║███████╗ ██║\033[0m ║
59+
║ \033[38;5;88m╚═╝ ╚═══╝╚══════╝╚═╝ ╚═╝╚══════╝╚═╝ ╚═══╝╚══════╝ ╚═╝\033[0m ║
60+
║ ║
61+
║ \033[1mDistributed Neural Network Framework\033[0m ║
62+
║ \033[38;5;88mVersion ~s\033[0m ║
63+
║ \033[32mDevice is Ready\033[0m ║
64+
║ ║
65+
╚═══════════════════════════════════════════════════════════════════════════════╝
66+
67+
~n", [?NERLNET_APP_VERSION]).
8868

8969
legal_print() ->
9070
io:format("Nerlnet OTP Application is given without any warranty.~n"),

0 commit comments

Comments
 (0)