Skip to content

Commit 60303fb

Browse files
leondavileondavi
authored
Merge pull request #278 from leondavi/integrationNerlplanner
Integration nerlplanner
2 parents 140f14b + 83faddc commit 60303fb

File tree

123 files changed

+4646
-4030
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

123 files changed

+4646
-4030
lines changed

‎CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ set(PROJECT_BINARY_DIR build)
1212
set(CMAKE_CXX_STANDARD 14)
1313
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -latomic")
1414
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -latomic")
15-
set(CMAKE_CXX_FLAGS_RELEASE "-O3")
15+
set(CMAKE_CXX_FLAGS_RELEASE "-O2")
1616

1717
# Options
1818
option(NERLWOLF "Use Wolfram Engine workers extension" OFF)

‎NerlnetBuild.sh

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,14 +153,18 @@ if command -v python3 >/dev/null 2>&1; then
153153
echo "$NERLNET_BUILD_PREFIX Python 3 is installed"
154154
# Generate auto-generated files
155155
set -e
156-
AUTOGENERATED_WORKER_DEFINITIONS_PATH="`pwd`/src_cpp/opennnBridge/worker_definitions_ag.h"
156+
AUTOGENERATED_WORKER_DEFINITIONS_PATH="`pwd`/src_cpp/common/worker_definitions_ag.h"
157157
AUTOGENERATED_WORKER_DEFINITIONS_PATH_HRL="`pwd`/src_erl/NerlnetApp/src/worker_definitions_ag.hrl"
158158
AUTOGENERATED_DC_DEFINITIONS_PATH_HRL="`pwd`/src_erl/NerlnetApp/src/dc_definitions_ag.hrl"
159+
AUTOGENERATED_SOURCE_DEFINITIONS_PATH_HRL="`pwd`/src_erl/NerlnetApp/src/source_definitions_ag.hrl"
160+
AUTOGENERATED_ROUTER_DEFINITIONS_PATH_HRL="`pwd`/src_erl/NerlnetApp/src/router_definitions_ag.hrl"
159161

160162
echo "$NERLNET_BUILD_PREFIX Generate auto-generated files"
161163
python3 src_py/nerlPlanner/CppHeadersExporter.py --output $AUTOGENERATED_WORKER_DEFINITIONS_PATH #--debug
162164
python3 src_py/nerlPlanner/ErlHeadersExporter.py --gen_worker_fields_hrl --output $AUTOGENERATED_WORKER_DEFINITIONS_PATH_HRL #--debug
163165
python3 src_py/nerlPlanner/ErlHeadersExporter.py --gen_dc_fields_hrl --output $AUTOGENERATED_DC_DEFINITIONS_PATH_HRL #--debug
166+
python3 src_py/nerlPlanner/ErlHeadersExporter.py --gen_source_fields_hrl --output $AUTOGENERATED_SOURCE_DEFINITIONS_PATH_HRL #--debug
167+
python3 src_py/nerlPlanner/ErlHeadersExporter.py --gen_router_fields_hrl --output $AUTOGENERATED_ROUTER_DEFINITIONS_PATH_HRL #--debug
164168
set +e
165169
else
166170
echo "$NERLNET_BUILD_PREFIX Python 3 is not installed"

‎NerlnetMonitor.sh

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/bin/bash
2+
3+
MONITOR_PATH="src_erl/NerlMonitor"
4+
GUI_PATH="src_erl/NerlMonitor/src"
5+
6+
echo "NerlnetMonitor Activated"
7+
8+
9+
cd $MONITOR_PATH
10+
rebar3 shell --name [email protected] --setcookie COOKIE
11+
12+
cd ../../

‎NerlnetPlanner.sh

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,9 @@ DOT_EXIST=$(dpkg-query -W --showformat='${Status}\n' $DOT_PACKAGE | grep "instal
1818
print "Checking for $DOT_PACKAGE"
1919
if [ "" = "$DOT_EXIST" ]; then
2020
print "Please run: sudo apt-get install $DOT_PACKAGE"
21+
exit 1
2122
else
22-
print "installed"
23+
print "$DOT_PACKAGE installed"
2324
fi
2425

2526

@@ -28,8 +29,19 @@ GRAPHVIZ_EXIST=$(dpkg-query -W --showformat='${Status}\n' $GRAPHVIZ_PACKAGE|grep
2829
print "Checking for $GRAPHVIZ_PACKAGE"
2930
if [ "" = "$GRAPHVIZ_EXIST" ]; then
3031
print "Please run: sudo apt-get install $GRAPHVIZ_PACKAGE"
32+
exit 1
3133
else
32-
print "installed"
34+
print "$GRAPHVIZ_PACKAGE installed"
35+
fi
36+
37+
TKINTER_PACKAGE="python3-tk"
38+
TKINTER_EXIST=$(dpkg-query -W --showformat='${Status}\n' $TKINTER_PACKAGE|grep "install ok installed")
39+
print "Checking for $TKINTER_PACKAGE"
40+
if [ "" = "$TKINTER_EXIST" ]; then
41+
print "Please run: sudo apt-get install $TKINTER_PACKAGE"
42+
exit 1
43+
else
44+
print "$TKINTER_PACKAGE installed"
3345
fi
3446

3547
# set python environment to run Nerlnet Flow

‎inputJsonsFiles/ConnectionMap/conn_test_synt_1d_2c_1s_4r_4w.json

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"connectionsMap":
3+
{
4+
"r1":["mainServer", "r2"],
5+
"r2":["r3", "s1"],
6+
"r3":["r4", "c1"],
7+
"r4":["r1", "c2"]
8+
}
9+
}

‎inputJsonsFiles/DistributedConfig/dc_test_synt_1d_2c_1s_4r_4w.json

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
{
2+
"nerlnetSettings": {
3+
"frequency": "60",
4+
"batchSize": "50"
5+
},
6+
"mainServer": {
7+
"port": "8081",
8+
"args": ""
9+
},
10+
"apiServer": {
11+
"port": "8082",
12+
"args": ""
13+
},
14+
"devices": [
15+
{
16+
"name": "pc1",
17+
"ipv4": "10.211.55.3",
18+
"entities": "c1,c2,r2,r1,r3,r4,s1,apiServer,mainServer"
19+
}
20+
],
21+
"routers": [
22+
{
23+
"name": "r1",
24+
"port": "8086",
25+
"policy": "0"
26+
},
27+
{
28+
"name": "r2",
29+
"port": "8087",
30+
"policy": "0"
31+
},
32+
{
33+
"name": "r3",
34+
"port": "8088",
35+
"policy": "0"
36+
},
37+
{
38+
"name": "r4",
39+
"port": "8089",
40+
"policy": "0"
41+
}
42+
],
43+
"sources": [
44+
{
45+
"name": "s1",
46+
"port": "8085",
47+
"frequency": "60",
48+
"policy": "0",
49+
"epochs": "1",
50+
"type": "0"
51+
}
52+
],
53+
"clients": [
54+
{
55+
"name": "c1",
56+
"port": "8083",
57+
"workers": "w1,w2"
58+
},
59+
{
60+
"name": "c2",
61+
"port": "8084",
62+
"workers": "w3,w4"
63+
}
64+
],
65+
"workers": [
66+
{
67+
"name": "w1",
68+
"model_sha": "d8df752e0a2e8f01de8f66e9cec941cdbc65d144ecf90ab7713e69d65e7e82aa"
69+
},
70+
{
71+
"name": "w2",
72+
"model_sha": "d8df752e0a2e8f01de8f66e9cec941cdbc65d144ecf90ab7713e69d65e7e82aa"
73+
},
74+
{
75+
"name": "w3",
76+
"model_sha": "d8df752e0a2e8f01de8f66e9cec941cdbc65d144ecf90ab7713e69d65e7e82aa"
77+
},
78+
{
79+
"name": "w4",
80+
"model_sha": "d8df752e0a2e8f01de8f66e9cec941cdbc65d144ecf90ab7713e69d65e7e82aa"
81+
}
82+
],
83+
"model_sha": {
84+
"d8df752e0a2e8f01de8f66e9cec941cdbc65d144ecf90ab7713e69d65e7e82aa": {
85+
"modelType": "0",
86+
"_doc_modelType": " nn:0 | approximation:1 | classification:2 | forecasting:3 | image-classification:4 | text-classification:5 | text-generation:6 | auto-association:7 | autoencoder:8 | ae-classifier:9 |",
87+
"layersSizes": "5,10,5,3",
88+
"_doc_layersSizes": "List of postive integers [L0, L1, ..., LN]",
89+
"layerTypesList": "1,3,3,3",
90+
"_doc_LayerTypes": " Default:0 | Scaling:1 | CNN:2 | Perceptron:3 | Pooling:4 | Probabilistic:5 | LSTM:6 | Reccurrent:7 | Unscaling:8 |",
91+
"layers_functions": "1,6,6,11",
92+
"_doc_layers_functions_activation": " Threshold:1 | Sign:2 | Logistic:3 | Tanh:4 | Linear:5 | ReLU:6 | eLU:7 | SeLU:8 | Soft-plus:9 | Soft-sign:10 | Hard-sigmoid:11 |",
93+
"_doc_layer_functions_pooling": " none:1 | Max:2 | Avg:3 |",
94+
"_doc_layer_functions_probabilistic": " Binary:1 | Logistic:2 | Competitive:3 | Softmax:4 |",
95+
"_doc_layer_functions_scaler": " none:1 | MinMax:2 | MeanStd:3 | STD:4 | Log:5 |",
96+
"lossMethod": "2",
97+
"_doc_lossMethod": " SSE:1 | MSE:2 | NSE:3 | MinkowskiE:4 | WSE:5 | CEE:6 |",
98+
"lr": "0.01",
99+
"_doc_lr": "Positve float",
100+
"epochs": "1",
101+
"_doc_epochs": "Positve Integer",
102+
"optimizer": "5",
103+
"_doc_optimizer": " GD:0 | CGD:1 | SGD:2 | QuasiNeuton:3 | LVM:4 | ADAM:5 |",
104+
"optimizerArgs": "",
105+
"_doc_optimizerArgs": "String",
106+
"infraType": "0",
107+
"_doc_infraType": " opennn:0 | wolfengine:1 |",
108+
"distributedSystemType": "0",
109+
"_doc_distributedSystemType": " none:0 | fedClientAvg:1 | fedServerAvg:2 |",
110+
"distributedSystemArgs": "",
111+
"_doc_distributedSystemArgs": "String",
112+
"distributedSystemToken": "none",
113+
"_doc_distributedSystemToken": "Token that associates distributed group of workers and parameter-server"
114+
}
115+
}
116+
}

‎inputJsonsFiles/Workers/distributed_worker.json

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
{
2+
"modelType": "5",
3+
"_doc_modelType": " approximation:1 | classification:2 | forecasting:3 | encoder_decoder:4 | nn:5 | autoencoder:6 | ae-classifier:7 |",
4+
"layersSizes": "512,254,64,32,2,1",
5+
"_doc_layersSizes": "List of postive integers [L0, L1, ..., LN]",
6+
"layerTypesList": "1,3,3,3,3,3",
7+
"_doc_LayerTypes": " Default:0 | Scaling:1 | CNN:2 | Perceptron:3 | Pooling:4 | Probabilistic:5 | LSTM:6 | Reccurrent:7 | Unscaling:8 | Bounding:9 |",
8+
"layers_functions": "1,6,6,6,6,6",
9+
"_doc_layers_functions_activation": " Threshold:1 | Sign:2 | Logistic:3 | Tanh:4 | Linear:5 | ReLU:6 | eLU:7 | SeLU:8 | Soft-plus:9 | Soft-sign:10 | Hard-sigmoid:11 |",
10+
"_doc_layer_functions_pooling": " none:1 | Max:2 | Avg:3 |",
11+
"_doc_layer_functions_probabilistic": " Binary:1 | Logistic:2 | Competitive:3 | Softmax:4 |",
12+
"_doc_layer_functions_scaler": " none:1 | MinMax:2 | MeanStd:3 | STD:4 | Log:5 |",
13+
"lossMethod": "2",
14+
"_doc_lossMethod": " SSE:1 | MSE:2 | NSE:3 | MinkowskiE:4 | WSE:5 | CEE:6 |",
15+
"lr": "0.001",
16+
"_doc_lr": "Positve float",
17+
"epochs": "1",
18+
"_doc_epochs": "Positve Integer",
19+
"optimizer": "6",
20+
"_doc_optimizer": " none:0 | SGD:1 | Mini-Batch:2 | Momentum:3 | NAG:4 | Adagrad:5 | ADAM:6 |",
21+
"infraType": "0",
22+
"_doc_infraType": " opennn:0 | wolfengine:1 |",
23+
"distributedSystemType": "0",
24+
"_doc_distributedSystemType": " none:0 | fedClientAvg:1 | fedServerAvg:2 |",
25+
"distributedSystemToken": "0",
26+
"_doc_distributedSystemToken": "Token that associates distributed group of workers and parameter-server"
27+
}

‎inputJsonsFiles/Workers/testworker_dc_test_synt_1d_2c_1s_4r_4w.json

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
{
2+
"modelType": "5",
3+
"_doc_modelType": " approximation:1 | classification:2 | forecasting:3 | encoder_decoder:4 | nn:5 |autoencoder:6 | ae-classifier:7 |",
4+
"layersSizes": "5,10,5,3",
5+
"_doc_layersSizes": "List of postive integers [L0, L1, ..., LN]",
6+
"layerTypesList": "1,3,3,3",
7+
"_doc_LayerTypes": " Default:0 | Scaling:1 | CNN:2 | Perceptron:3 | Pooling:4 | Probabilistic:5 | LSTM:6 | Reccurrent:7 | Unscaling:8 | Bounding:9 |",
8+
"layers_functions": "1,6,11,11",
9+
"_doc_layers_functions_activation": " Threshold:1 | Sign:2 | Logistic:3 | Tanh:4 | Linear:5 | ReLU:6 | eLU:7 | SeLU:8 | Soft-plus:9 | Soft-sign:10 | Hard-sigmoid:11 |",
10+
"_doc_layer_functions_pooling": " none:1 | Max:2 | Avg:3 |",
11+
"_doc_layer_functions_probabilistic": " Binary:1 | Logistic:2 | Competitive:3 | Softmax:4 |",
12+
"_doc_layer_functions_scaler": " none:1 | MinMax:2 | MeanStd:3 | STD:4 | Log:5 |",
13+
"lossMethod": "2",
14+
"_doc_lossMethod": " SSE:1 | MSE:2 | NSE:3 | MinkowskiE:4 | WSE:5 | CEE:6 |",
15+
"lr": "0.01",
16+
"_doc_lr": "Positve float",
17+
"epochs": "1",
18+
"_doc_epochs": "Positve Integer",
19+
"optimizer": "6",
20+
"_doc_optimizer": " none:0 | SGD:1 | Mini-Batch:2 | Momentum:3 | NAG:4 | Adagrad:5 | ADAM:6 |",
21+
"optimizerArgs": "",
22+
"_doc_optimizerArgs": "String",
23+
"infraType": "0",
24+
"_doc_infraType": " opennn:0 | wolfengine:1 |",
25+
"distributedSystemType": "0",
26+
"_doc_distributedSystemType": " none:0 | fedClientAvg:1 | fedServerAvg:2 |",
27+
"distributedSystemArgs" : "",
28+
"_doc_distributedSystemArgs": "String",
29+
"distributedSystemToken": "none",
30+
"_doc_distributedSystemToken": "Token that associates distributed group of workers and parameter-server"
31+
}

‎inputJsonsFiles/Workers/worker_latest.json

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
{
2+
"modelType": "0",
3+
"_doc_modelType": " nn:0 | approximation:1 | classification:2 | forecasting:3 | image-classification:4 | text-classification:5 | text-generation:6 | auto-association:7 | autoencoder:8 | ae-classifier:9 |",
4+
"layersSizes": "100,80,60,40,20,1",
5+
"_doc_layersSizes": "List of postive integers [L0, L1, ..., LN]",
6+
"layerTypesList": "1,5,5,5,5,1",
7+
"_doc_LayerTypes": " Default:0 | Scaling:1 | CNN:2 | Perceptron:3 | Pooling:4 | Probabilistic:5 | LSTM:6 | Reccurrent:7 | Unscaling:8 | Bounding:9 |",
8+
"layers_functions": "1,6,6,6,6,1",
9+
"_doc_layers_functions_activation": " Threshold:1 | Sign:2 | Logistic:3 | Tanh:4 | Linear:5 | ReLU:6 | eLU:7 | SeLU:8 | Soft-plus:9 | Soft-sign:10 | Hard-sigmoid:11 |",
10+
"_doc_layer_functions_pooling": " none:1 | Max:2 | Avg:3 |",
11+
"_doc_layer_functions_probabilistic": " Binary:1 | Logistic:2 | Competitive:3 | Softmax:4 |",
12+
"_doc_layer_functions_scaler": " none:1 | MinMax:2 | MeanStd:3 | STD:4 | Log:5 |",
13+
"lossMethod": "2",
14+
"_doc_lossMethod": " SSE:1 | MSE:2 | NSE:3 | MinkowskiE:4 | WSE:5 | CEE:6 |",
15+
"lr": "0.01",
16+
"_doc_lr": "Positve float",
17+
"epochs": "1",
18+
"_doc_epochs": "Positve Integer",
19+
"optimizer": "2",
20+
"_doc_optimizer": " GD:0 | CGD:1 | SGD:2 | QuasiNeuton:3 | LVM:4 | ADAM:5 |",
21+
"optimizerArgs": "",
22+
"_doc_optimizerArgs": "String",
23+
"infraType": "0",
24+
"_doc_infraType": " opennn:0 | wolfengine:1 |",
25+
"distributedSystemType": "0",
26+
"_doc_distributedSystemType": " none:0 | fedClientAvg:1 | fedServerAvg:2 |",
27+
"distributedSystemArgs": "none",
28+
"_doc_distributedSystemArgs": "String",
29+
"distributedSystemToken": "none",
30+
"_doc_distributedSystemToken": "Token that associates distributed group of workers and parameter-server"
31+
}

‎inputJsonsFiles/experimentsFlow/exp_test_synt_1d_2c_1s_4r_4w.json

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
{
2+
"Features": 5,
3+
"Labels": ["Norm(0,1)", "Norm(4,1)", "Norm(10,3)"],
4+
"CSV path": "synthetic",
5+
"Batches per source":
6+
{
7+
"Training": 2000,
8+
"Prediction": 2000
9+
},
10+
"Training": [
11+
{
12+
"source name": "s1",
13+
"workers": "w1,w2,w3,w4",
14+
"CSV path": "synthetic_training"
15+
}
16+
],
17+
"Prediction": [
18+
{
19+
"source name": "s1",
20+
"workers": "w1,w2,w3,w4",
21+
"CSV path": "synthetic_prediction"
22+
}
23+
]
24+
}
25+

0 commit comments

Comments
 (0)