Crash Experiment

ansible/experiment.yaml

@@ -6,7 +6,7 @@
   tasks:
   - name: Run the Client
     ansible.builtin.shell:
-      cmd: "/root/oblishard/client -duration 10 -conf /root/oblishard -output /root/oblishard/output.txt"
+      cmd: "/root/oblishard/client -duration 30 -conf /root/oblishard -output /root/oblishard/output.txt -logpath /root/oblishard/client.txt"
   - name: Get the output
     ansible.builtin.fetch:
       src: "/root/oblishard/output.txt"

ansible/templates/oblishard-oramnode.service.j2

@@ -7,7 +7,8 @@ After=syslog.target network.target
 Type=simple
 ExecStart=/root/oblishard/oramnode/oramnode \
     -oramnodeid {{ item.id }} \
-    -ip {{ item.local_bind_ip }} \
+    -bindip {{ item.local_bind_ip }} \
+    -advip {{ item.exposed_ip }} \
     -rpcport {{ item.port }} \
     -replicaid {{ item.replicaid }} \
     -raftport {{ item.raftport }} \

ansible/templates/oblishard-shardnode.service.j2

@@ -7,7 +7,8 @@ After=syslog.target network.target
 Type=simple
 ExecStart=/root/oblishard/shardnode/shardnode \
     -shardnodeid {{ item.id }} \
-    -ip {{ item.local_bind_ip }} \
+    -bindip {{ item.local_bind_ip }} \
+    -advip {{ item.exposed_ip }} \
     -rpcport {{ item.port }} \
     -replicaid {{ item.replicaid }} \
     -raftport {{ item.raftport }} \

cmd/client/main.go

@@ -15,7 +15,7 @@ import (
 	"go.opentelemetry.io/otel"
 )
 
-// Usage: go run . -duration=<duration in seconds>  -logpath=<log path> -conf=<configs path>
+// Usage: ./client -h
 func main() {
 	logPath := flag.String("logpath", "", "path to write logs")
 	configsPath := flag.String("conf", "../../configs/default", "configs directory path")
@@ -73,12 +73,8 @@ func main() {
 	defer cancel()
 
 	go c.SendRequestsForever(ctx, readResponseChannel, writeResponseChannel)
-	startTime := time.Now()
-	readOperations, writeOperations := c.GetResponsesForever(ctx, readResponseChannel, writeResponseChannel)
-	elapsed := time.Since(startTime)
-	throughput := float64(readOperations+writeOperations) / elapsed.Seconds()
-	averageLatency := float64(elapsed.Milliseconds()) / float64((readOperations + writeOperations))
-	err = client.WriteOutputToFile(*outputFilePath, throughput, averageLatency)
+	responseCounts := c.GetResponsesForever(ctx, readResponseChannel, writeResponseChannel)
+	err = client.WriteOutputToFile(*outputFilePath, responseCounts)
 	if err != nil {
 		log.Fatal().Msgf("Failed to write output to file; %v", err)
 	}

cmd/oramnode/main.go

@@ -15,10 +15,11 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
-// Usage: ./oramnode -oramnodeid=<oramnodeid> -ip=<ip> -rpcport=<rpcport> -replicaid=<replicaid> -raftport=<raftport> -joinaddr=<ip:port> -conf=<configs path> -logpath=<log path>
+// Usage: ./oramnode -h
 func main() {
 	oramNodeID := flag.Int("oramnodeid", 0, "oramnode id, starting consecutively from zero")
-	ip := flag.String("ip", "127.0.0.1", "ip of this replica")
+	bindIP := flag.String("bindip", "127.0.0.1", "ip of this replica to bind to")
+	advIP := flag.String("advip", "127.0.0.1", "ip of this replica to advertise")
 	replicaID := flag.Int("replicaid", 0, "replica id, starting consecutively from zero")
 	rpcPort := flag.Int("rpcport", 0, "node rpc port")
 	raftPort := flag.Int("raftport", 0, "node raft port")
@@ -67,5 +68,5 @@ func main() {
 		defer cpuProfile.Stop()
 	}
 
-	oramnode.StartServer(*oramNodeID, *ip, *rpcPort, *replicaID, *raftPort, *joinAddr, rpcClients, redisEndpoints, parameters)
+	oramnode.StartServer(*oramNodeID, *bindIP, *advIP, *rpcPort, *replicaID, *raftPort, *joinAddr, rpcClients, redisEndpoints, parameters)
 }

cmd/router/main.go

@@ -15,7 +15,7 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
-// Usage: ./router -routerid=<routerid> -ip=<ip> -port=<port> -conf=<configs path> -logpath=<log path>
+// Usage: ./router -h
 func main() {
 
 	routerID := flag.Int("routerid", 0, "router id, starting consecutively from zero")

cmd/shardnode/main.go

@@ -15,10 +15,11 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
-// Usage: ./shardnode -shardnodeid=<shardnodeid> -ip=<ip> -rpcport=<rpcport> -replicaid=<replicaid> -raftport=<raftport> -joinaddr=<ip:port> -conf=<configs path> -logpath=<log path>
+// Usage: ./shardnode -h
 func main() {
 	shardNodeID := flag.Int("shardnodeid", 0, "shardnode id, starting consecutively from zero")
-	ip := flag.String("ip", "127.0.0.1", "ip of this replica")
+	bindIP := flag.String("bindip", "127.0.0.1", "bind ip of this replica")
+	advIP := flag.String("advip", "127.0.0.1", "advertise ip of this replica")
 	replicaID := flag.Int("replicaid", 0, "replica id, starting consecutively from zero")
 	rpcPort := flag.Int("rpcport", 0, "node rpc port")
 	raftPort := flag.Int("raftport", 0, "node raft port")
@@ -69,5 +70,5 @@ func main() {
 		defer cpuProfile.Stop()
 	}
 
-	shardnode.StartServer(*shardNodeID, *ip, *rpcPort, *replicaID, *raftPort, *joinAddr, rpcClients, parameters, redisEndpoints, *configsPath)
+	shardnode.StartServer(*shardNodeID, *bindIP, *advIP, *rpcPort, *replicaID, *raftPort, *joinAddr, rpcClients, parameters, redisEndpoints, *configsPath)
 }

experiments/crash_experiments/shardnode_crash/hosts

@@ -0,0 +1,3 @@
+host1 ansible_host=tem15 ansible_user=asetayes
+host2 ansible_host=tem17 ansible_user=asetayes
+host3 ansible_host=tem19 ansible_user=asetayes

experiments/crash_experiments/shardnode_crash/jaeger_endpoint.yaml

@@ -0,0 +1,5 @@
+exposed_ip: 192.168.252.115
+local_bind_ip: 192.168.252.115
+deploy_host: host1
+api_port: 16686
+grpc_port: 4317

experiments/crash_experiments/shardnode_crash/oramnode_endpoints.yaml

@@ -0,0 +1,43 @@
+endpoints:
+  - exposed_ip: 192.168.252.115
+    local_bind_ip: 192.168.252.115
+    deploy_host: host1
+    port: 1751
+    id: 0
+    replicaid: 0
+    raftport: 1127
+  - exposed_ip: 192.168.252.117
+    local_bind_ip: 192.168.252.117
+    deploy_host: host2
+    port: 1752
+    id: 0
+    replicaid: 1
+    raftport: 1128
+  - exposed_ip: 192.168.252.119
+    local_bind_ip: 192.168.252.119
+    deploy_host: host3
+    port: 1753
+    id: 0
+    replicaid: 2
+    raftport: 1129
+  # - exposed_ip: 192.168.252.117
+  #   local_bind_ip: 192.168.252.117
+  #   deploy_host: host2
+  #   port: 1754
+  #   id: 1
+  #   replicaid: 0
+  #   raftport: 1130
+  # - exposed_ip: 192.168.252.115
+  #   local_bind_ip: 192.168.252.115
+  #   deploy_host: host1
+  #   port: 1755
+  #   id: 1
+  #   replicaid: 1
+  #   raftport: 1131
+  # - exposed_ip: 192.168.252.119
+  #   local_bind_ip: 192.168.252.119
+  #   deploy_host: host3
+  #   port: 1756
+  #   id: 1
+  #   replicaid: 2
+  #   raftport: 1132

experiments/crash_experiments/shardnode_crash/parameters.yaml

@@ -0,0 +1,15 @@
+max-blocks-to-send: 400 # The maximum number of blocks to send from each shard node to the oram node during evictions
+eviction-rate: 100 # How many ReadPath operations before eviction
+evict-path-count: 200 # How many paths to evict at a time
+batch-timeout: 5 # How many milliseconds to wait before sending a batch of blocks to the oram node 
+epoch-time: 5 # How many milliseconds between each epoch
+trace: false # Whether to use opentelemetry and jaeger
+Z: 1 # number of real blocks per bucket
+S: 6 # number of dummy blocks per bucket
+shift: 1 # 2^shift is the tree branching factor
+tree-height: 20 # height of the tree
+redis-pipeline-size: 5000000 # number of requests to pipeline to redis
+max-requests: 10000 # maximum number of requests in flight at the client
+block-size: 150 # size of each block in bytes
+log: false # whether to log
+profile: false # Whether to profile