From 6111a8b3754e5260285bbd606bfb3ff518e73021 Mon Sep 17 00:00:00 2001 From: Lloyd Date: Mon, 2 Mar 2020 22:05:32 +0900 Subject: [PATCH] [Example] Improve Cipher demo (#16) * Add `MakeFile`s that make it easy to spin up the needed infra, startup the Cipher server, and Go workers * Update README.md for the Cipher demo Signed-off-by: lloydmeta --- example/ciphers/Makefile | 19 ++++++++++ example/ciphers/README.md | 31 +++++++++------- example/ciphers/server/start-server | 15 ++++++++ example/ciphers/worker-go/start-workers | 47 +++++++++++++++++++++++++ 4 files changed, 100 insertions(+), 12 deletions(-) create mode 100644 example/ciphers/Makefile create mode 100755 example/ciphers/server/start-server create mode 100755 example/ciphers/worker-go/start-workers diff --git a/example/ciphers/Makefile b/example/ciphers/Makefile new file mode 100644 index 0000000..463791c --- /dev/null +++ b/example/ciphers/Makefile @@ -0,0 +1,19 @@ +GO_WORKER_COUNT ?= 5 + +install-eck: + @$(MAKE) install-eck -C k8s + +k8s-deploy: + @$(MAKE) deploy -C k8s + +k8s-teardown: + @$(MAKE) teardown -C k8s + +k8s-show-credentials: + @$(MAKE) show-credentials -C k8s + +start-go-workers: + @worker-go/start-workers ${GO_WORKER_COUNT} + +start-cipher-server: + @server/start-server \ No newline at end of file diff --git a/example/ciphers/README.md b/example/ciphers/README.md index ea53dc3..827eb0d 100644 --- a/example/ciphers/README.md +++ b/example/ciphers/README.md @@ -4,28 +4,35 @@ This is an example server-worker app. ### Setup -If you have k8s installed, you can use the Makefile in the `k8s` dir to install ECK, spin up the needed infra, -and get credentials that you can put into the relevant config files. +If you don't have [Kubernetes](https://kubernetes.io) installed, install it. -If you don't you'll need to spin these up separately. +If you don't have EC installed, run `make install-eck` to install the [ECK](https://www.elastic.co/guide/en/cloud-on-k8s/current/index.html) operator. -### Run the Tasques server - -In the project root dir, start a Tasques server (`go run ./app`) +Run `make k8s-deploy` to deploy all the needed infra, such as Tasques, including ES clusters (for tasques and our demo app), +Kibana, and APM. ### Run the Ciphers server -On `example/ciphers/server`, run `go run main.go` +Run `make start-cipher-server` to start the web server for Ciphers. It will wait for the previous services to be ready before +starting. #### Add some Messages to cipher -Go to the server at [localhost:9000](http://localhost:9000) (default) and create messages, which will create jobs. +Once the Ciphers server is up, go to the server at [localhost:9000](http://localhost:9000) (default) and use the form to +create plain text messages, which will create jobs to have them encoded by workers. -### Start the Go worker +### Run the Ciphers Go workers -In `example/ciphers/worker-go`, run `go run main.go --worker-id worker1` replacing `worker1` with a unique worker id -per worker process. +Run `make start-go-workers` to start a number of Go Cipher workers. By default, we start 5, but you can customise this using +`GO_WORKER_COUNT` (e.g. `GO_WORKER_COUNT=10 make start-go-workers`). Also note that half of the workers are configured +to have a chance of failure, and all have an artificial pause to simulate load. #### Workers in other languages -There are example workers in Java and Rust as well (`worker-$lang`) \ No newline at end of file +There are example workers in Java and Rust as well (`worker-$lang`) + +### Kibana Dashboard + +1. To log into the Tasques server's Kibana, run `make k8s-show-credentials` and get the password of the **Tasques** cluster. +2. Then go to [localhost:5601](http://localhost:5601) and use `elastic` and the password retrieved in the previous step. +3. Use the Kibana navigation bar to go to Dashboards, then select the `tasques` dashboard from the list. \ No newline at end of file diff --git a/example/ciphers/server/start-server b/example/ciphers/server/start-server new file mode 100755 index 0000000..d0b7fb8 --- /dev/null +++ b/example/ciphers/server/start-server @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +cd "$(dirname "$0")" + +while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8080/recurring_tasques)" != "200" ]]; do + echo "Waiting for Tasques server to come online ..." + sleep 5; +done + +while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:9201)" != "200" ]]; do + echo "Waiting for Cipher ES server to come online ..." + sleep 5; +done + +go run main.go \ No newline at end of file diff --git a/example/ciphers/worker-go/start-workers b/example/ciphers/worker-go/start-workers new file mode 100755 index 0000000..fc6b161 --- /dev/null +++ b/example/ciphers/worker-go/start-workers @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +cd "$(dirname "$0")" +worker_process_ids=() + +function kill_workers() { + echo "Sending SIGINT to workers workers" + for pid in "${worker_process_ids[@]}"; do + kill -2 $pid + done + exit 0 +} + +number_of_workers=$1 + +number_re='^[0-9]+$' +if ! [[ $number_of_workers =~ $number_re ]]; then + echo "Error: Worker number argument (first and only) not a number" >&2; exit 1 +fi + +trap kill_workers SIGINT SIGTERM SIGHUP + +while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8080/recurring_tasques)" != "200" ]]; do + echo "Waiting for Tasques server to come online ..." + sleep 5; +done + +while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:9201)" != "200" ]]; do + echo "Waiting for Cipher ES server to come online ..." + sleep 5; +done + +for ((i=0; i