Merge pull request #278 from leondavi/integrationNerlplanner

Integration nerlplanner

CMakeLists.txt

@@ -12,7 +12,7 @@ set(PROJECT_BINARY_DIR build)
 set(CMAKE_CXX_STANDARD 14)
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -latomic")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -latomic")
-set(CMAKE_CXX_FLAGS_RELEASE "-O3")
+set(CMAKE_CXX_FLAGS_RELEASE "-O2")
 
 # Options
 option(NERLWOLF "Use Wolfram Engine workers extension" OFF)

NerlnetBuild.sh

@@ -153,14 +153,18 @@ if command -v python3 >/dev/null 2>&1; then
     echo "$NERLNET_BUILD_PREFIX Python 3 is installed"
 	# Generate auto-generated files
 	set -e
-	AUTOGENERATED_WORKER_DEFINITIONS_PATH="`pwd`/src_cpp/opennnBridge/worker_definitions_ag.h"
+	AUTOGENERATED_WORKER_DEFINITIONS_PATH="`pwd`/src_cpp/common/worker_definitions_ag.h"
 	AUTOGENERATED_WORKER_DEFINITIONS_PATH_HRL="`pwd`/src_erl/NerlnetApp/src/worker_definitions_ag.hrl"
 	AUTOGENERATED_DC_DEFINITIONS_PATH_HRL="`pwd`/src_erl/NerlnetApp/src/dc_definitions_ag.hrl"
+	AUTOGENERATED_SOURCE_DEFINITIONS_PATH_HRL="`pwd`/src_erl/NerlnetApp/src/source_definitions_ag.hrl"
+	AUTOGENERATED_ROUTER_DEFINITIONS_PATH_HRL="`pwd`/src_erl/NerlnetApp/src/router_definitions_ag.hrl"
 
 	echo "$NERLNET_BUILD_PREFIX Generate auto-generated files"
 	python3 src_py/nerlPlanner/CppHeadersExporter.py --output $AUTOGENERATED_WORKER_DEFINITIONS_PATH #--debug
 	python3 src_py/nerlPlanner/ErlHeadersExporter.py --gen_worker_fields_hrl --output $AUTOGENERATED_WORKER_DEFINITIONS_PATH_HRL #--debug
 	python3 src_py/nerlPlanner/ErlHeadersExporter.py --gen_dc_fields_hrl --output $AUTOGENERATED_DC_DEFINITIONS_PATH_HRL #--debug
+	python3 src_py/nerlPlanner/ErlHeadersExporter.py --gen_source_fields_hrl --output $AUTOGENERATED_SOURCE_DEFINITIONS_PATH_HRL #--debug
+	python3 src_py/nerlPlanner/ErlHeadersExporter.py --gen_router_fields_hrl --output $AUTOGENERATED_ROUTER_DEFINITIONS_PATH_HRL #--debug
 	set +e
 else
     echo "$NERLNET_BUILD_PREFIX Python 3 is not installed"

NerlnetMonitor.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+MONITOR_PATH="src_erl/NerlMonitor"
+GUI_PATH="src_erl/NerlMonitor/src"
+
+echo "NerlnetMonitor Activated"
+
+
+cd $MONITOR_PATH
+rebar3 shell --name [email protected] --setcookie COOKIE 
+
+cd ../../

NerlnetPlanner.sh

@@ -18,8 +18,9 @@ DOT_EXIST=$(dpkg-query -W --showformat='${Status}\n' $DOT_PACKAGE | grep "instal
 print "Checking for $DOT_PACKAGE"
 if [ "" = "$DOT_EXIST" ]; then
   print "Please run: sudo apt-get install $DOT_PACKAGE"
+  exit 1
 else
-  print "installed"
+  print "$DOT_PACKAGE installed"
 fi
 
 
@@ -28,8 +29,19 @@ GRAPHVIZ_EXIST=$(dpkg-query -W --showformat='${Status}\n' $GRAPHVIZ_PACKAGE|grep
 print "Checking for $GRAPHVIZ_PACKAGE"
 if [ "" = "$GRAPHVIZ_EXIST" ]; then
   print "Please run: sudo apt-get install $GRAPHVIZ_PACKAGE"
+  exit 1
 else
-  print "installed"
+  print "$GRAPHVIZ_PACKAGE installed"
+fi
+
+TKINTER_PACKAGE="python3-tk"
+TKINTER_EXIST=$(dpkg-query -W --showformat='${Status}\n' $TKINTER_PACKAGE|grep "install ok installed")
+print "Checking for $TKINTER_PACKAGE"
+if [ "" = "$TKINTER_EXIST" ]; then
+  print "Please run: sudo apt-get install $TKINTER_PACKAGE"
+  exit 1
+else
+  print "$TKINTER_PACKAGE installed"
 fi
 
 # set python environment to run Nerlnet Flow

inputJsonsFiles/ConnectionMap/conn_test_synt_1d_2c_1s_4r_4w.json

@@ -0,0 +1,9 @@
+{
+    "connectionsMap":
+    {
+        "r1":["mainServer", "r2"],
+        "r2":["r3", "s1"],
+        "r3":["r4", "c1"],
+        "r4":["r1", "c2"]
+    }
+}

inputJsonsFiles/DistributedConfig/dc_test_synt_1d_2c_1s_4r_4w.json

@@ -0,0 +1,116 @@
+{
+    "nerlnetSettings": {
+        "frequency": "60",
+        "batchSize": "50"
+    },
+    "mainServer": {
+        "port": "8081",
+        "args": ""
+    },
+    "apiServer": {
+        "port": "8082",
+        "args": ""
+    },
+    "devices": [
+        {
+            "name": "pc1",
+            "ipv4": "10.211.55.3",
+            "entities": "c1,c2,r2,r1,r3,r4,s1,apiServer,mainServer"
+        }
+    ],
+    "routers": [
+        {
+            "name": "r1",
+            "port": "8086",
+            "policy": "0"
+        },
+        {
+            "name": "r2",
+            "port": "8087",
+            "policy": "0"
+        },
+        {
+            "name": "r3",
+            "port": "8088",
+            "policy": "0"
+        },
+        {
+            "name": "r4",
+            "port": "8089",
+            "policy": "0"
+        }
+    ],
+    "sources": [
+        {
+            "name": "s1",
+            "port": "8085",
+            "frequency": "60",
+            "policy": "0",
+            "epochs": "1",
+            "type": "0"
+        }
+    ],
+    "clients": [
+        {
+            "name": "c1",
+            "port": "8083",
+            "workers": "w1,w2"
+        },
+        {
+            "name": "c2",
+            "port": "8084",
+            "workers": "w3,w4"
+        }
+    ],
+    "workers": [
+        {
+            "name": "w1",
+            "model_sha": "d8df752e0a2e8f01de8f66e9cec941cdbc65d144ecf90ab7713e69d65e7e82aa"
+        },
+        {
+            "name": "w2",
+            "model_sha": "d8df752e0a2e8f01de8f66e9cec941cdbc65d144ecf90ab7713e69d65e7e82aa"
+        },
+        {
+            "name": "w3",
+            "model_sha": "d8df752e0a2e8f01de8f66e9cec941cdbc65d144ecf90ab7713e69d65e7e82aa"
+        },
+        {
+            "name": "w4",
+            "model_sha": "d8df752e0a2e8f01de8f66e9cec941cdbc65d144ecf90ab7713e69d65e7e82aa"
+        }
+    ],
+    "model_sha": {
+        "d8df752e0a2e8f01de8f66e9cec941cdbc65d144ecf90ab7713e69d65e7e82aa": {
+            "modelType": "0",
+            "_doc_modelType": " nn:0 | approximation:1 | classification:2 | forecasting:3 | image-classification:4 | text-classification:5 | text-generation:6 | auto-association:7 | autoencoder:8 | ae-classifier:9 |",
+            "layersSizes": "5,10,5,3",
+            "_doc_layersSizes": "List of postive integers [L0, L1, ..., LN]",
+            "layerTypesList": "1,3,3,3",
+            "_doc_LayerTypes": " Default:0 | Scaling:1 | CNN:2 | Perceptron:3 | Pooling:4 | Probabilistic:5 | LSTM:6 | Reccurrent:7 | Unscaling:8 |",
+            "layers_functions": "1,6,6,11",
+            "_doc_layers_functions_activation": " Threshold:1 | Sign:2 | Logistic:3 | Tanh:4 | Linear:5 | ReLU:6 | eLU:7 | SeLU:8 | Soft-plus:9 | Soft-sign:10 | Hard-sigmoid:11 |",
+            "_doc_layer_functions_pooling": " none:1 | Max:2 | Avg:3 |",
+            "_doc_layer_functions_probabilistic": " Binary:1 | Logistic:2 | Competitive:3 | Softmax:4 |",
+            "_doc_layer_functions_scaler": " none:1 | MinMax:2 | MeanStd:3 | STD:4 | Log:5 |",
+            "lossMethod": "2",
+            "_doc_lossMethod": " SSE:1 | MSE:2 | NSE:3 | MinkowskiE:4 | WSE:5 | CEE:6 |",
+            "lr": "0.01",
+            "_doc_lr": "Positve float",
+            "epochs": "1",
+            "_doc_epochs": "Positve Integer",
+            "optimizer": "5",
+            "_doc_optimizer": " GD:0 | CGD:1 | SGD:2 | QuasiNeuton:3 | LVM:4 | ADAM:5 |",
+            "optimizerArgs": "",
+            "_doc_optimizerArgs": "String",
+            "infraType": "0",
+            "_doc_infraType": " opennn:0 | wolfengine:1 |",
+            "distributedSystemType": "0",
+            "_doc_distributedSystemType": " none:0 | fedClientAvg:1 | fedServerAvg:2 |",
+            "distributedSystemArgs": "",
+            "_doc_distributedSystemArgs": "String",
+            "distributedSystemToken": "none",
+            "_doc_distributedSystemToken": "Token that associates distributed group of workers and parameter-server"
+        }
+    }
+}

inputJsonsFiles/Workers/distributed_worker.json

@@ -0,0 +1,27 @@
+{
+    "modelType": "5",
+    "_doc_modelType": " approximation:1 | classification:2 | forecasting:3 | encoder_decoder:4 | nn:5 | autoencoder:6 | ae-classifier:7 |",
+    "layersSizes": "512,254,64,32,2,1",
+    "_doc_layersSizes": "List of postive integers [L0, L1, ..., LN]",
+    "layerTypesList": "1,3,3,3,3,3",
+    "_doc_LayerTypes": " Default:0 | Scaling:1 | CNN:2 | Perceptron:3 | Pooling:4 | Probabilistic:5 | LSTM:6 | Reccurrent:7 | Unscaling:8 | Bounding:9 |",
+    "layers_functions": "1,6,6,6,6,6",
+    "_doc_layers_functions_activation": " Threshold:1 | Sign:2 | Logistic:3 | Tanh:4 | Linear:5 | ReLU:6 | eLU:7 | SeLU:8 | Soft-plus:9 | Soft-sign:10 | Hard-sigmoid:11 |",
+    "_doc_layer_functions_pooling": " none:1 | Max:2 | Avg:3 |",
+    "_doc_layer_functions_probabilistic": " Binary:1 | Logistic:2 | Competitive:3 | Softmax:4 |",
+    "_doc_layer_functions_scaler": " none:1 | MinMax:2 | MeanStd:3 | STD:4 | Log:5 |",
+    "lossMethod": "2",
+    "_doc_lossMethod": " SSE:1 | MSE:2 | NSE:3 | MinkowskiE:4 | WSE:5 | CEE:6 |",
+    "lr": "0.001",
+    "_doc_lr": "Positve float",
+    "epochs": "1",
+    "_doc_epochs": "Positve Integer",
+    "optimizer": "6",
+    "_doc_optimizer": " none:0 | SGD:1 | Mini-Batch:2 | Momentum:3 | NAG:4 | Adagrad:5 | ADAM:6 |",
+    "infraType": "0",
+    "_doc_infraType": " opennn:0 | wolfengine:1 |",
+    "distributedSystemType": "0",
+    "_doc_distributedSystemType": " none:0 | fedClientAvg:1 | fedServerAvg:2 |",
+    "distributedSystemToken": "0",
+    "_doc_distributedSystemToken": "Token that associates distributed group of workers and parameter-server"
+}

inputJsonsFiles/Workers/testworker_dc_test_synt_1d_2c_1s_4r_4w.json

@@ -0,0 +1,31 @@
+{
+   "modelType": "5",
+            "_doc_modelType": " approximation:1 | classification:2 | forecasting:3 | encoder_decoder:4 | nn:5 |autoencoder:6 | ae-classifier:7 |",
+            "layersSizes": "5,10,5,3",
+            "_doc_layersSizes": "List of postive integers [L0, L1, ..., LN]",
+            "layerTypesList": "1,3,3,3",
+            "_doc_LayerTypes": " Default:0 | Scaling:1 | CNN:2 | Perceptron:3 | Pooling:4 | Probabilistic:5 | LSTM:6 | Reccurrent:7 | Unscaling:8 | Bounding:9 |",
+            "layers_functions": "1,6,11,11",
+            "_doc_layers_functions_activation": " Threshold:1 | Sign:2 | Logistic:3 | Tanh:4 | Linear:5 | ReLU:6 | eLU:7 | SeLU:8 | Soft-plus:9 | Soft-sign:10 | Hard-sigmoid:11 |",
+            "_doc_layer_functions_pooling": " none:1 | Max:2 | Avg:3 |",
+            "_doc_layer_functions_probabilistic": " Binary:1 | Logistic:2 | Competitive:3 | Softmax:4 |",
+            "_doc_layer_functions_scaler": " none:1 | MinMax:2 | MeanStd:3 | STD:4 | Log:5 |",
+            "lossMethod": "2",
+            "_doc_lossMethod": " SSE:1 | MSE:2 | NSE:3 | MinkowskiE:4 | WSE:5 | CEE:6 |",
+            "lr": "0.01",
+            "_doc_lr": "Positve float",
+            "epochs": "1",
+            "_doc_epochs": "Positve Integer",
+            "optimizer": "6",
+    	    "_doc_optimizer": " none:0 | SGD:1 | Mini-Batch:2 | Momentum:3 | NAG:4 | Adagrad:5 | ADAM:6 |",
+            "optimizerArgs": "",
+            "_doc_optimizerArgs": "String",
+            "infraType": "0",
+            "_doc_infraType": " opennn:0 | wolfengine:1 |",
+            "distributedSystemType": "0",
+            "_doc_distributedSystemType": " none:0 | fedClientAvg:1 | fedServerAvg:2 |",
+            "distributedSystemArgs" : "",
+            "_doc_distributedSystemArgs": "String",
+            "distributedSystemToken": "none",
+            "_doc_distributedSystemToken": "Token that associates distributed group of workers and parameter-server"
+}

inputJsonsFiles/Workers/worker_latest.json

@@ -0,0 +1,31 @@
+{
+    "modelType": "0",
+    "_doc_modelType": " nn:0 | approximation:1 | classification:2 | forecasting:3 | image-classification:4 | text-classification:5 | text-generation:6 | auto-association:7 | autoencoder:8 | ae-classifier:9 |",
+    "layersSizes": "100,80,60,40,20,1",
+    "_doc_layersSizes": "List of postive integers [L0, L1, ..., LN]",
+    "layerTypesList": "1,5,5,5,5,1",
+    "_doc_LayerTypes": " Default:0 | Scaling:1 | CNN:2 | Perceptron:3 | Pooling:4 | Probabilistic:5 | LSTM:6 | Reccurrent:7 | Unscaling:8 | Bounding:9 |",
+    "layers_functions": "1,6,6,6,6,1",
+    "_doc_layers_functions_activation": " Threshold:1 | Sign:2 | Logistic:3 | Tanh:4 | Linear:5 | ReLU:6 | eLU:7 | SeLU:8 | Soft-plus:9 | Soft-sign:10 | Hard-sigmoid:11 |",
+    "_doc_layer_functions_pooling": " none:1 | Max:2 | Avg:3 |",
+    "_doc_layer_functions_probabilistic": " Binary:1 | Logistic:2 | Competitive:3 | Softmax:4 |",
+    "_doc_layer_functions_scaler": " none:1 | MinMax:2 | MeanStd:3 | STD:4 | Log:5 |",
+    "lossMethod": "2",
+    "_doc_lossMethod": " SSE:1 | MSE:2 | NSE:3 | MinkowskiE:4 | WSE:5 | CEE:6 |",
+    "lr": "0.01",
+    "_doc_lr": "Positve float",
+    "epochs": "1",
+    "_doc_epochs": "Positve Integer",
+    "optimizer": "2",
+    "_doc_optimizer": " GD:0 | CGD:1 | SGD:2 | QuasiNeuton:3 | LVM:4 | ADAM:5 |",
+    "optimizerArgs": "",
+    "_doc_optimizerArgs": "String",
+    "infraType": "0",
+    "_doc_infraType": " opennn:0 | wolfengine:1 |",
+    "distributedSystemType": "0",
+    "_doc_distributedSystemType": " none:0 | fedClientAvg:1 | fedServerAvg:2 |",
+    "distributedSystemArgs": "none",
+    "_doc_distributedSystemArgs": "String",
+    "distributedSystemToken": "none",
+    "_doc_distributedSystemToken": "Token that associates distributed group of workers and parameter-server"
+}

inputJsonsFiles/experimentsFlow/exp_test_synt_1d_2c_1s_4r_4w.json

@@ -0,0 +1,25 @@
+{
+   "Features": 5,
+   "Labels": ["Norm(0,1)", "Norm(4,1)", "Norm(10,3)"],
+   "CSV path": "synthetic",
+   "Batches per source": 
+   {
+      "Training": 2000,
+      "Prediction": 2000
+   },
+   "Training":  [
+      {
+         "source name": "s1",
+         "workers": "w1,w2,w3,w4",
+         "CSV path": "synthetic_training"
+      }
+   ],
+   "Prediction": [
+      {
+         "source name": "s1",
+         "workers": "w1,w2,w3,w4",
+         "CSV path": "synthetic_prediction"
+      }
+   ]
+}
+

src_cpp/CMakeLists.txt

@@ -7,6 +7,7 @@
 
 project(src_cpp)
 
+add_subdirectory(common)
 add_subdirectory(opennnBridge)
 
 if(NERLWOLF)

src_cpp/common/CMakeLists.txt

@@ -0,0 +1,22 @@
+project(common)
+
+set(NIFPP_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../nifpp/")
+
+set(SRC_CODE
+    "common_definitions.h"
+    "nerlLayer.cpp"
+    "nerlLayer.h"
+    "nerlWorker.cpp"
+    "nerlWorker.h"
+    "utilities.cpp"
+    "utilities.h"
+    "nerlWorkerFunc.h"
+    "worker_definitions_ag.h"
+    "bridgeController.h"
+    )
+
+add_library(common SHARED ${SRC_CODE})
+
+target_include_directories(common PUBLIC .
+    ${NIFPP_PATH}
+)