#704 new runit config files, need some fixes for integration test

scripts/deploy-conflux.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -x
+set -e
+export CELOS_USER=obaskakov
+
+#scripts/build.sh
+#ansible-playbook scripts/playbooks/kinit.yaml -c ssh -u ${CELOS_USER} -i ${INVENTORY_SERVER}
+
+export INVENTORY_SERVER=scripts/inventory/conflux-server
+export INVENTORY_UI=scripts/inventory/conflux-ui
+./scripts/server-and-ui-action.sh stop
+
+export INVENTORY_SERVER=scripts/inventory/testing-server
+export INVENTORY_UI=scripts/inventory/testing-ui
+./scripts/server-and-ui-action.sh stop

scripts/deploy.sh

@@ -1,12 +1,12 @@
 #!/usr/bin/env bash
 set -x
 set -e
-[[ -z ${GIT_COMMIT} ]] && echo pls specify GIT_COMMIT && exit 1
-export ANSIBLE_SSH_ARGS=""
+export CELOS_USER=celos
+export INVENTORY_SERVER=scripts/inventory/production-server
+export INVENTORY_UI=scripts/inventory/production-ui
 scripts/build.sh
-ansible-playbook scripts/prod/kinit.yaml -c ssh -u celos -i scripts/prod/conf/inventory-server -e "@scripts/prod/conf/celos-testing.json" -e "@scripts/conf/common-params-server.json"
-ansible-playbook scripts/celos-deploy.yaml -c ssh -u celos -i scripts/prod/conf/inventory-server -e "@scripts/prod/conf/celos-testing.json" -e "@scripts/conf/common-params-server.json" -e service_version=${GIT_COMMIT}
-ansible-playbook scripts/celos-deploy.yaml -c ssh -u celos -i scripts/prod/conf/inventory-ui     -e "@scripts/prod/conf/celos-testing.json" -e "@scripts/conf/common-params-ui.json" -e service_version=${GIT_COMMIT}
+ansible-playbook scripts/playbooks/kinit.yaml -c ssh -u ${CELOS_USER} -i ${INVENTORY_SERVER}
+./scripts/server-and-ui-action.sh deploy
 
 MINUTES_JAN01_2015=23667720
 MINUTES_SINCE_JAN01_2015=$(($(date +%s) / 60 - MINUTES_JAN01_2015))

scripts/inventory/conflux-server

@@ -0,0 +1,7 @@
+[host]
+oj002.ewr004.collective-media.net
+
+[host:vars]
+celos_wrk_dir="/home/{{ ansible_ssh_user }}/conflux"
+service_name=conflux-server
+service_port=11091

scripts/env/inventory-conflux-ui → scripts/inventory/conflux-ui

@@ -6,4 +6,3 @@ service_name="conflux-ui"
 service_port=11092
 ui_hue_url="http://cldmgr001.ewr004.collective-media.net:8888/oozie"
 ui_config_file="/home/celos/prod/ui/config.json"
-celos_server_url="oj002.ewr004.collective-media.net"

scripts/inventory/integration-server

@@ -0,0 +1,7 @@
+[host]
+oj002.ewr004.collective-media.net
+
+[host:vars]
+service_name=celos-integration-server
+service_port=19091
+celos_wrk_dir="/home/{{ ansible_ssh_user }}/integration"

scripts/env/testing/inventory-ui → scripts/inventory/integration-ui

@@ -2,8 +2,7 @@
 oj002.ewr004.collective-media.net
 
 [host:vars]
-celos_server_host="oj002.ewr004.collective-media.net"
-service_name={{ui_service_name}}
-service_port={{ui_service_port}}
+service_name=celos-integration-ui
+service_port=19092
 ui_hue_url="http://cldmgr001.ewr004.collective-media.net:8888/oozie"
 ui_config_file="/home/celos/prod/ui/config.json"

scripts/env/prod/inventory-server → scripts/inventory/production-server

@@ -2,9 +2,10 @@
 celos002.ewr004.collective-media.net
 
 [host:vars]
+service_name=celos-server
+service_port=9091
+celos_wrk_dir=/home/celos/prod
 celos_workflows_dir=/etc/celos/workflows
 celos_defaults_dir=/etc/celos/defaults
 celos_db_dir=/var/lib/celos/db
 celos_log_dir=/var/log/celos
-service_name={{server_service_name}}
-service_port={{server_service_port}}

scripts/env/conflux/inventory-ui → scripts/inventory/production-ui

@@ -2,8 +2,7 @@
 oj002.ewr004.collective-media.net
 
 [host:vars]
-celos_server_host="oj002.ewr004.collective-media.net"
-service_name={{ui_service_name}}
-service_port={{ui_service_port}}
+service_name="celos-ui"
+service_port=9092
 ui_hue_url="http://cldmgr001.ewr004.collective-media.net:8888/oozie"
 ui_config_file="/home/celos/prod/ui/config.json"

scripts/inventory/testing-server

@@ -0,0 +1,7 @@
+[host]
+oj002.ewr004.collective-media.net
+
+[host:vars]
+service_name=testing-server
+service_port=11093
+celos_wrk_dir="/home/{{ ansible_ssh_user }}/testing"

scripts/env/prod/inventory-ui → scripts/inventory/testing-ui

@@ -2,8 +2,7 @@
 oj002.ewr004.collective-media.net
 
 [host:vars]
-celos_server_host="celos002.ewr004.collective-media.net"
-service_name={{ui_service_name}}
-service_port={{ui_service_port}}
+service_name=testing-ui
+service_port=11094
 ui_hue_url="http://cldmgr001.ewr004.collective-media.net:8888/oozie"
 ui_config_file="/home/celos/prod/ui/config.json"

scripts/params/common-server.json

@@ -0,0 +1,9 @@
+{
+  "service_args": "--workflows {{ celos_workflows_dir }} --defaults {{ celos_defaults_dir }} --db {{ celos_db_dir }} --logs {{ celos_log_dir }}",
+  "jar_path": "../../celos-server/build/libs/celos-server.jar",
+  "main_class": "com.collective.celos.server.Main",
+  "celos_workflows_dir": "{{ celos_wrk_dir }}/workflows",
+  "celos_defaults_dir": "{{ celos_wrk_dir }}/defaults",
+  "celos_db_dir": "{{ celos_wrk_dir }}/db",
+  "celos_log_dir": "{{ celos_wrk_dir }}/logs"
+}

scripts/params/common-ui.json

@@ -0,0 +1,5 @@
+{
+  "jar_path": "{{ root }}/celos-ui/build/libs/celos-ui.jar",
+  "main_class": "com.collective.celos.ui.Main",
+  "service_args": "--celos {{ service_discovery }} --hue {{ ui_hue_url }}  --config {{ ui_config_file }}"
+}

scripts/conf/common-params-server.json → scripts/params/custom-server.json

@@ -1,5 +1,5 @@
 {
   "service_args": "--workflows {{ celos_workflows_dir }} --defaults {{ celos_defaults_dir }} --db {{ celos_db_dir }} --logs {{ celos_log_dir }}",
-  "jar_path": "../celos-server/build/libs/celos-server.jar",
+  "jar_path": "../../celos-server/build/libs/celos-server.jar",
   "main_class": "com.collective.celos.server.Main"
 }

scripts/playbooks/celos-deploy.yaml

@@ -6,7 +6,7 @@
 - hosts: host
   gather_facts: no
   vars:
-    root: ".."
+    root: "../.."
     service_user: "{{ ansible_ssh_user }}"
     prefix: "/home/{{ service_user }}/local"
     service_dir: "{{ prefix }}/service/{{ service_name }}"
@@ -42,11 +42,11 @@
     ignore_errors: yes
 
   - file: state=directory dest={{ prefix }}/bin
-  - template: src=templates/service-start.tpl dest={{ start_script_path }} owner={{ service_user }} mode=0755
+  - template: src={{root}}/scripts/templates/service-start.tpl dest={{ start_script_path }} owner={{ service_user }} mode=0755
 
-  - template: src=templates/run.tpl dest="{{ service_dir }}/run" owner={{ service_user }} mode=0755
-  - template: src=templates/check.tpl dest="{{ service_dir }}/check" owner={{ service_user }} mode=0755
-  - template: src=templates/log-run.tpl dest="{{ service_dir }}/log/run" owner={{ service_user }} mode=0755
+  - template: src={{root}}/scripts/templates/run.tpl dest="{{ service_dir }}/run" owner={{ service_user }} mode=0755
+  - template: src={{root}}/scripts/templates/check.tpl dest="{{ service_dir }}/check" owner={{ service_user }} mode=0755
+  - template: src={{root}}/scripts/templates/log-run.tpl dest="{{ service_dir }}/log/run" owner={{ service_user }} mode=0755
 
   - file: state=directory dest={{ service_dir }}/supervise mode=0755
   - shell: "[ -p {{ service_dir }}/supervise/ok ]      || mkfifo {{ service_dir }}/supervise/ok"

scripts/playbooks/celos-purge.yaml

@@ -10,7 +10,6 @@
     service_user: "{{ ansible_ssh_user }}"
     prefix: "/home/{{ service_user }}/local"
     service_dir: "{{ prefix }}/service/{{ service_name }}"
-    runit_restart: "{{ root }}/scripts/runit-restart-service.sh"
     start_script_path: "{{ prefix }}/bin/{{ service_name}}"
     dest_jar_path: "{{ prefix }}/lib/{{ service_name}}.jar"
     stdout_log_path: "{{ prefix }}/log/{{ service_name }}"

scripts/server-and-ui-action.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+set -x
+set -e
+
+[[ -z ${CELOS_USER} ]] && echo pls specify CELOS_USER && exit 1
+[[ -z ${GIT_COMMIT} ]] && echo pls specify GIT_COMMIT && exit 1
+if [[ -f ${INVENTORY_SERVER} ]]
+then true
+else echo "pls specify INVENTORY_SERVER and make sure INVENTORY_SERVER exists" && exit 1
+fi
+if [[ -f ${INVENTORY_UI} ]]
+then true
+else echo "pls specify INVENTORY_UI and make sure INVENTORY_UI exists" && exit 1
+fi
+
+CELOS_ACTION=$1
+
+[[ -z ${CELOS_ACTION} ]] && echo 'usage ./scripts/server-and-ui-action.sh CELOS_ACTION (deploy, check, purge, start, stop)' && exit 1
+if [ $CELOS_ACTION = "deploy" ] ||
+   [ $CELOS_ACTION = "check"  ] ||
+   [ $CELOS_ACTION = "purge"  ] ||
+   [ $CELOS_ACTION = "start"  ] ||
+   [ $CELOS_ACTION = "stop"   ]
+then true
+else echo 'usage ./scripts/server-and-ui-action.sh CELOS_ACTION (deploy, check, purge, start, stop)' && exit 1
+fi
+
+export ANSIBLE_SSH_ARGS=""
+# use custom params from inventory file it they are not specified
+if [ -z "$(grep celos_workflows_dir ${INVENTORY_SERVER})" ]
+then ansible-playbook "scripts/playbooks/celos-${CELOS_ACTION}.yaml" -c ssh -u ${CELOS_USER} -i ${INVENTORY_SERVER} -e "@scripts/params/custom-server.json" -e service_version=${GIT_COMMIT}
+else ansible-playbook "scripts/playbooks/celos-${CELOS_ACTION}.yaml" -c ssh -u ${CELOS_USER} -i ${INVENTORY_SERVER} -e "@scripts/params/common-server.json" -e service_version=${GIT_COMMIT}
+fi
+# parse server host and port from inventory file
+TMP1="$(grep service_name ${INVENTORY_SERVER} | cut -d '=' -f 2)"
+TMP2="$(grep service_port ${INVENTORY_SERVER} | cut -d '=' -f 2)"
+SERVER_URL="http://${TMP1}:${TMP2}"
+
+ansible-playbook "scripts/playbooks/celos-${CELOS_ACTION}.yaml" -c ssh -u ${CELOS_USER} -i ${INVENTORY_UI} -e "@scripts/params/common-ui.json" -e service_discovery=${SERVER_URL} -e service_version=${GIT_COMMIT}

scripts/start.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 set -x
 set -e
-ansible-playbook ./scripts/prod/celos-start.yaml -u celos -i scripts/prod/conf/inventory-server -e "@scripts/prod/conf/celos-testing.json"
-ansible-playbook ./scripts/prod/celos-start.yaml -u celos -i scripts/prod/conf/inventory-ui -e "@scripts/prod/conf/celos-testing.json"
+export CELOS_USER=celos
+export INVENTORY_SERVER=scripts/inventory/production-server
+export INVENTORY_UI=scripts/inventory/production-ui
+./scripts/server-and-ui-action.sh start

scripts/stop.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 set -x
 set -e
-ansible-playbook ./scripts/prod/celos-stop.yaml -u celos -i scripts/prod/conf/inventory-server -e "@scripts/prod/conf/celos-testing.json"
-ansible-playbook ./scripts/prod/celos-stop.yaml -u celos -i scripts/prod/conf/inventory-ui -e "@scripts/prod/conf/celos-testing.json"
+export CELOS_USER=celos
+export INVENTORY_SERVER=scripts/inventory/production-server
+export INVENTORY_UI=scripts/inventory/production-ui
+./scripts/server-and-ui-action.sh stop

scripts/test/integration-test.sh

@@ -1,10 +1,10 @@
 #!/usr/bin/env bash
-export ANSIBLE_SSH_ARGS=""
-SERVER_PARAMS="-c ssh -u celos-ci -i scripts/test/conf/inventory-server -e @scripts/test/conf/integration-params.json -e @scripts/conf/common-params-server.json -e service_version=${GIT_COMMIT}"
-UI_PARAMS="    -c ssh -u celos-ci -i scripts/test/conf/inventory-ui     -e @scripts/test/conf/integration-params.json -e @scripts/conf/common-params-ui.json     -e service_version=${GIT_COMMIT}"
+[[ -z ${GIT_COMMIT} ]] && echo pls specify GIT_COMMIT && exit 1
+SERVER_PARAMS="-c ssh -u celos-ci -i scripts/test/conf/integration-server -e @scripts/test/conf/integration-params.json -e @scripts/conf/common-server.json -e service_version=${GIT_COMMIT}"
+UI_PARAMS="    -c ssh -u celos-ci -i scripts/test/conf/testing-ui     -e @scripts/test/conf/integration-params.json -e @scripts/conf/common-ui.json     -e service_version=${GIT_COMMIT}"
 set -x
 set -e
-[[ -z ${GIT_COMMIT} ]] && echo pls specify GIT_COMMIT && exit 1
+export ANSIBLE_SSH_ARGS=""
 scripts/build.sh
 ansible-playbook scripts/prod/kinit.yaml ${SERVER_PARAMS}
 ansible-playbook scripts/celos-deploy.yaml ${SERVER_PARAMS}

scripts/test/stop-start-test.sh

@@ -1,15 +1,16 @@
 #!/usr/bin/env bash
-SERVER_PARAMS="-c ssh -u celos-ci -i scripts/test/conf/inventory-server -e @scripts/test/conf/integration-params.json -e @scripts/conf/common-params-server.json"
-UI_PARAMS="    -c ssh -u celos-ci -i scripts/test/conf/inventory-ui     -e @scripts/test/conf/integration-params.json -e @scripts/conf/common-params-ui.json"
+SERVER_PARAMS="-c ssh -u celos-ci -i scripts/inventory/integration-server -e @scripts/params/common-server.json"
+UI_PARAMS="    -c ssh -u celos-ci -i scripts/inventory/integration-ui     -e @scripts/params/common-ui.json"
 set -e
 set -x
-ansible-playbook scripts/prod/celos-check.yaml ${SERVER_PARAMS}
-ansible-playbook scripts/prod/celos-check.yaml ${UI_PARAMS}
-ansible-playbook scripts/prod/celos-stop.yaml ${SERVER_PARAMS}
-ansible-playbook scripts/prod/celos-stop.yaml ${UI_PARAMS}
-! ansible-playbook scripts/prod/celos-check.yaml ${SERVER_PARAMS}
-! ansible-playbook scripts/prod/celos-check.yaml ${UI_PARAMS}
-ansible-playbook scripts/prod/celos-start.yaml ${SERVER_PARAMS}
-ansible-playbook scripts/prod/celos-start.yaml ${UI_PARAMS}
-ansible-playbook scripts/prod/celos-check.yaml ${SERVER_PARAMS}
-ansible-playbook scripts/prod/celos-check.yaml ${UI_PARAMS}
+export ANSIBLE_SSH_ARGS=""
+ansible-playbook scripts/playbooks/celos-check.yaml ${SERVER_PARAMS}
+ansible-playbook scripts/playbooks/celos-check.yaml ${UI_PARAMS}
+ansible-playbook scripts/playbooks/celos-stop.yaml ${SERVER_PARAMS}
+ansible-playbook scripts/playbooks/celos-stop.yaml ${UI_PARAMS}
+! ansible-playbook scripts/playbooks/celos-check.yaml ${SERVER_PARAMS}
+! ansible-playbook scripts/playbooks/celos-check.yaml ${UI_PARAMS}
+ansible-playbook scripts/playbooks/celos-start.yaml ${SERVER_PARAMS}
+ansible-playbook scripts/playbooks/celos-start.yaml ${UI_PARAMS}
+ansible-playbook scripts/playbooks/celos-check.yaml ${SERVER_PARAMS}
+ansible-playbook scripts/playbooks/celos-check.yaml ${UI_PARAMS}