Issue 2329: Update travis.yml to enable system test execution on AWS (p…

…ravega#2351)

* Modifies travis.yml to include commands for executing system tests on AWS.

* Collects the test logs and store them in Amazon S3 bucket.

* Destroys EC2 cluster after the run.

Signed-off-by: PrabhaVeerubhotla <[email protected]>

.travis.yml

@@ -14,6 +14,43 @@ env:
   global:
     - GRADLE_OPTS="-Xms128m"
 
+jobs:
+  include:
+    - stage: build
+      if: type IN (push, pull_request)
+      install: ./gradlew assemble
+      script: ./gradlew check
+      after_success:
+           - bash <(curl -s https://codecov.io/bash) -t ccceafaf-7c60-4a02-9165-480174b535a2
+
+    - stage: system tests
+      if: repo != pravega/pravega
+      env:
+         - secure: xMd2acI6P4cIrVH1ha+CkwkHkomjjanKOWth8rqrLvLjSCA+K/zOqMp0VtDMyDFWSPLUnpa/D2oViGZD/q1rtLTs8O3aVQcQn9MiFL3VpwZKZ3PNEI9MhSgCMGh4oem4jIOu/A7xfTPFTOJsFyOFWi627MHW7EZmUsZOkzmMU04ms5i2Q63pboTd7W2ym9rzTWpGe3eb6IuqEbSyElWNDwKcVRT55YYvyahRL+lcOkObPXkiS2EcS098jfmofKDhkNFZ58m51ukOg7117pnQw0J8hXCtlcfJHfJC4+Mjao8IKeAzeoXkJL/lA1qSCAdkcg85+OeUjJ2HOTWsV+y+aS8VPe0bth2YMKn3/EfWVcrusbgVsbV1bUbje9C5OUOiBZ+0mgcBI2FMP9fk1lHOPJ/XXPfup6g1N67mB3kCK+YI3eTyhFaGnTaN+jz3qn86on8sXw9ionC3KJrUQP0C0JriPVMIAqRQoTBGP8BqtNvCHnEyhriJHuEeRB2XAOAjrge2qJ9MP7heaFmSlG/zqTUZDpHP4CVWX8OTc5jTVWKZ5mfB8XNXVLfVZjaVOXva4eQWJ1KGvoEMwBu+pCThyA6AKHikFSjPxUymdTscyIt7Ma+0YzTy0ejT1OHjdo2eLN2W5swSCA+SWM4oKkR5MAle6NHdHwnw6lPNfLyHzug=
+         - secure: hMyMR2tf9kNsWJ1+6aQWiW2X+vc6dF0mdB4sCXUk//wCqwZ81ri8AOndXdDWv+PZAE0ZCbNynRX5BtXyY1St0cY49+FFFy0ExvrD0Ydro/zoEw2vaAX6f5PSNISRwFXPZv13He516Ssj0733QHs+d0cSywd+6+KXJtaIbgxtzRcqI38pD6BXoT67x1LthkGDOdIt/+N6JQbuQVHsPchABRRr0eZcwU/SjdW0vXXoGZ5b0odHVtRnioFq2Q87zmcCr5rfvFmnerGrnhXywO4MzCA83K3j9vlkAoapV8WWMb9uTKwnxCj4XIKsH6nXDzWAsO2sNlwBu4QtkNXGfoS0H7vUSBZcjDaWSV2YZNl9YUVcjzgzHApoE63NAAm7XwESqAl7AE8GkZX8ZZIC7mH0nbQ3I312d6AzV69o6Npxr8q8sZjK94Qp9jUkgM7/GkMN0R6x8QbByTH5BY8KQaXO4XEF7/GaMkkklJARy66FjO8mOWk3z0cE1QG49zY6l+fatMJzKjDbPXDAoJNqiDqhtfXpHSq2CM+BHMRwp+SHMihl3kus0LzahFSPAqpdT6sJo5XYQi4Ygtkrxya/e15TZQeP4/uvEBwPcCf25LA8GhJ60L1poA2I9ptU1V79w7b72/rJeOWrVvqV+M9tviL1+j9VzSAeMXkBufa7J5W5uak=
+         - secure: hgu6ik1qGduzifEKVdzYyKNawyBmGClyDDw2SaNJqh/FVipT7nFPhi+O5v49OcyUuwXwW/BSYUNqlxjfSUWrZ7dN/kSnTUGFYYqs6RAjhY15NldoXYnmFZHqT3kS38ZQLyTMIeWd0OllR/nx0cQylkK1DNu42MmImH7MXHxjKw/FTN02tSba674ZaHVABxLGuqFtVVx2sORXkU64RCcwg4KSbSQlwUzUIInpkOyKkoI+RugcEHd8YItWiLrJUgvyBwU4IcBKvCgXDOWXlfii+/6eAZdw+2QiANtsWAkz0mxXejyyOAl4iUTMIuhuxjZ5eYcwyLoPXt14WXwuBh2wCZyzuU3h/sa6G+YR1gti4fMNjg2u1X+5uBD8TdHfCQ67D+1Pkx/7AzHYinrxEmx2o4XUs+6wkGcHGbzMUu/njeHSiVpedKAlIv+FuVeNJ73/CF1F2QCuEusUYnj7AXJFikYEWOK1FTI2ip3nQ5fAh/qFoHJNdWNHNhvF8XnRyCTdMK0GIUSXiHP7ciQtl8Sgl2N6Y0nvK9Hwk6T1nJBZr3rxdzPfwW0QBSFN4yyZkR6J4rq4mPi56xl72KV+zrcNsYWehcz8GWVspESn7EAXbwymHGdeJCel3JQ7Abxdf6mrDmTLP6POUtI3RjaIoQAulWBArNxe7e/wiVwWSPGRedE=
+      before_install:
+         - openssl aes-256-cbc -K $encrypted_7840d5b8aa7c_key -iv $encrypted_7840d5b8aa7c_iv -in aws-key-pair.pem.enc -out aws-key-pair.pem -d
+         - sudo wget https://releases.hashicorp.com/terraform/0.11.3/terraform_0.11.3_linux_amd64.zip
+         - unzip terraform_0.11.3_linux_amd64.zip
+         - sudo mv terraform /usr/local/bin/
+         - chmod +x /usr/local/bin/terraform
+         - sudo terraform --version
+      install:
+         - echo "Executing system tests on AWS"
+      script:
+         - ./gradlew startSystemTestsWithDocker -DskipServiceInstallation=false
+               -Dlog.level=DEBUG -DexecType=DOCKER -PawsExecution=true -Paws_access_key="$AWS_ACCESS_KEY"
+               -Paws_secret_key="$AWS_SECRET_KEY" -Paws_region=us-east-2  -Paws_key_name=aws-key-pair
+               -Pconfig_path=/home/travis/build/pravega/pravega/test/system/aws -Pcred_path=/home/travis/build/pravega/pravega/aws-key-pair.pem
+               -Ppravega_org="$TRAVIS_REPO_SLUG" -Ppravega_branch="$TRAVIS_BRANCH"  -DimageVersion="$TRAVIS_BRANCH"
+               -DawsExec=true
+         - ./gradlew --info collectSystemTestLogsFromAws -Paws_access_key="$AWS_ACCESS_KEY" -Paws_secret_key="$AWS_SECRET_KEY"
+               -Paws_region=us-east-2 -Paws_key_name=aws-key-pair
+               -Pcred_path=/home/travis/build/pravega/pravega/aws-key-pair.pem -Pconfig_path=/home/travis/build/pravega/pravega/test/system/aws
+               -Ppravega_org="$TRAVIS_REPO_SLUG" -Ppravega_branch="$TRAVIS_BRANCH" -Ptravis_commit="$TRAVIS_COMMIT"
+
+
 sudo: required
 
 services:
@@ -36,10 +73,6 @@ notifications:
     - [email protected]
     - [email protected]
     - [email protected]
-    - [email protected]
     - [email protected]
     - [email protected]
-    - [email protected]
     - [email protected]
-after_success:
-  - bash <(curl -s https://codecov.io/bash) -t ccceafaf-7c60-4a02-9165-480174b535a2

build.gradle

@@ -741,6 +741,11 @@ project('test:system') {
         systemProperty "dockerImageRegistry", "${dockerRegistryUrl}"
 
     }
+
+    task collectSystemTestLogsFromAws(type: Exec) {
+        commandLine './aws/postTestScript.sh'
+        args "$aws_access_key", "$aws_secret_key", "$aws_region", "$aws_key_name", "$cred_path", "$config_path", "$pravega_org", "$pravega_branch", "$travis_commit"
+    }
 }
 
 subprojects {

gradle.properties

@@ -78,3 +78,4 @@ cred_path=/home/ubuntu
 config_path=/home/ubuntu
 pravega_org=pravega
 pravega_branch=master
+travis_commit=0

test/system/aws/logCollectionScript.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+# Copyright (c) 2017 Dell Inc., or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+ACCESSKEY=${1:-null}
+SECRETKEY=${2:-null}
+RUNNAME=${3:-null}
+COMMIT=${4:-null}
+host="$(hostname)"
+
+####copy all container logs in a given host.
+containersResult="$(docker ps -a -q|xargs)"
+echo "Containers in $host are ${containersResult}"
+containers=(${containersResult/%$'\r'/})
+for container in "${containers[@]}"
+do
+   echo "Executing commands for $container"
+   mkdir -p logs/$container
+   docker cp $container:/opt/pravega/logs/. logs/$container/
+   if [ $? -ne 0 ]; then
+     echo "$container is not a controller/pravega node"
+     continue
+   fi
+   tar -zcvf logs_$container.tar.gz ./logs/$container/
+
+done
+index=1
+find / -name logs |grep "/merged/opt/pravega/logs"| while read -r path ; do
+    echo "Copying logs from $path $index"
+    mkdir -p logs/$host-container-$index
+    cp -r $path/. logs/$host-container-$index/
+    index=$((index+1))
+done
+if [ -d logs ]; then
+  tar -zcvf logs_$host.tar.gz logs/
+else
+  echo "No pravega containers running on $host"
+fi
+#### copy all test logs on the given mesos host.
+find / -name 'server.log' -print0 | tar -czvf testLogsIn-$host.tar.gz --null -T -
+
+echo "Uploading logs to s3 bucket:pravega-systemtests-logs"
+for file in ./*.tar.gz; do  sudo AWS_ACCESS_KEY_ID=$ACCESSKEY AWS_SECRET_ACCESS_KEY=$SECRETKEY AWS_DEFAULT_REGION=us-east-2  aws s3 cp $file  s3://pravega-systemtests-logs/$RUNNAME/$COMMIT/; done
+echo "Logs upload completed"

test/system/aws/master_script.sh

@@ -27,8 +27,9 @@ ipAddress=$(ifconfig eth0 | grep "inet addr" | cut -d ':' -f 2 | cut -d ' ' -f 1
 sudo docker swarm init --advertise-addr $ipAddress
 sudo docker network create -d overlay --attachable docker-network
 sudo docker swarm join-token worker > token.sh
-cd /tmp && git clone https://github.com/$pravegaOrg/pravega.git && cd pravega/
+cd /tmp && git clone https://github.com/$pravegaOrg.git && cd pravega/
 git checkout $branch
 sudo add-apt-repository ppa:openjdk-r/ppa -y && sudo apt-get -y update && sudo apt-get install -y openjdk-8-jdk
 sudo chmod 777 /var/run/docker.sock
-export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 && sudo ./gradlew clean buildPravegaImage -PpravegaBaseTag=nautilus/pravega -PpravegaVersion=$branch  && sudo ./gradlew clean buildBookkeeperImage -PbookkeeperBaseTag=nautilus/bookkeeper -PpravegaVersion=$branch
+export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 && sudo ./gradlew --warn buildPravegaImage -PpravegaBaseTag=nautilus/pravega -PpravegaVersion=$branch  && sudo ./gradlew --warn buildBookkeeperImage -PbookkeeperBaseTag=nautilus/bookkeeper -PpravegaVersion=$branch
+sudo apt -y install awscli

test/system/aws/output.tf

@@ -11,3 +11,11 @@
 output "ip" {
   value = "${aws_instance.swarm_master.*.public_ip}"
 }
+
+output "master_public_dns" {
+  value = "${aws_instance.swarm_master.*.public_dns}"
+}
+
+output "slave_public_dns" {
+  value = "${aws_instance.swarm_slaves.*.public_dns}"
+}

test/system/aws/postTestScript.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+# Copyright (c) 2017 Dell Inc., or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+aws_access_key=${1:-null}
+aws_secret_key=${2:-null}
+aws_region=${3:-null}
+aws_key_name=${4:-aws-key-pair}
+cred_path=${5:-null}
+config_path=${6:-null}
+pravega_org=${7:-pravega/pravega}
+pravega_branch=${8:-master}
+travis_commit=${9:-null}
+cd aws/
+sed -i 's/,/ /g' public_dns.txt
+var=`cat public_dns.txt`
+sudo chmod 400 $cred_path
+for i in $var; do
+        ssh -o StrictHostKeyChecking=no -i $cred_path root@$i "bash -s" -- < ./logCollectionScript.sh $aws_access_key $aws_secret_key $pravega_branch $travis_commit
+done
+
+TF_LOG=INFO terraform destroy -force -var aws_access_key=$aws_access_key \
+ -var aws_secret_key=$aws_secret_key \
+  -var aws_region=$aws_region  \
+  -var aws_key_name=$aws_key_name \
+-var cred_path=$cred_path \
+ -var config_path=$config_path \
+  -var pravega_org=$pravega_org  \
+  -var pravega_branch=$pravega_branch

test/system/aws/pravega.tf

@@ -17,7 +17,7 @@ provider "aws" {
 
 #security group, allow all incoming and outgoing traffic
 resource "aws_security_group" "pravega_default" {
- name = "pravega_default"
+ name_prefix = "pravega_default"
  ingress {
   from_port = 0
   to_port = 0
@@ -40,7 +40,7 @@ resource "aws_instance" "swarm_master" {
  ami = "${lookup(var.pravega_aws_amis, var.aws_region)}"
  instance_type = "${lookup(var.pravega_instance_type, var.aws_region)}"
  key_name = "${var.aws_key_name}"
- security_groups = ["pravega_default"]
+ vpc_security_group_ids = ["${aws_security_group.pravega_default.id}"]
  provisioner "local-exec" {
    command = "chmod 400 ${var.cred_path}"
  }
@@ -78,7 +78,7 @@ resource "aws_instance" "swarm_slaves" {
  ami = "${lookup(var.pravega_aws_amis, var.aws_region)}"
  instance_type = "${lookup(var.pravega_instance_type, var.aws_region)}"
  key_name = "${var.aws_key_name}"
- security_groups = ["pravega_default"]
+ vpc_security_group_ids = ["${aws_security_group.pravega_default.id}"]
  provisioner "local-exec" {
    command = "chmod 400 ${var.cred_path}"
  }

test/system/aws/preTestScript.sh

@@ -14,15 +14,20 @@ aws_region=${3:-null}
 aws_key_name=${4:-aws-key-pair}
 cred_path=${5:-null}
 config_path=${6:-null}
-pravega_org=${7:-pravega}
+pravega_org=${7:-pravega/pravega}
 pravega_branch=${8:-master}
 cd aws/
-terraform init
-terraform apply -auto-approve -var aws_access_key=$aws_access_key \
+TF_LOG=INFO terraform init
+TF_LOG=INFO terraform apply -auto-approve -var aws_access_key=$aws_access_key \
  -var aws_secret_key=$aws_secret_key \
   -var aws_region=$aws_region  \
   -var aws_key_name=$aws_key_name \
 -var cred_path=$cred_path \
  -var config_path=$config_path \
   -var pravega_org=$pravega_org  \
   -var pravega_branch=$pravega_branch
+touch public_dns.txt
+master_public_dns=`terraform output master_public_dns`
+echo $master_public_dns >> $config_path/public_dns.txt
+slave_public_dns=`terraform output slave_public_dns`
+echo $slave_public_dns >> $config_path/public_dns.txt

test/system/aws/slave_script.sh

@@ -19,8 +19,9 @@ sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubun
 sudo apt-get update
 apt-cache madison docker-ce
 sudo apt-get -y install docker-ce=17.09.0~ce-0~ubuntu 
-cd /tmp && git clone https://github.com/$pravegaOrg/pravega.git && cd pravega/
+cd /tmp && git clone https://github.com/$pravegaOrg.git && cd pravega/
 git checkout $branch
 sudo add-apt-repository ppa:openjdk-r/ppa -y && sudo apt-get -y update && sudo apt-get install -y openjdk-8-jdk
 sudo chmod 777 /var/run/docker.sock
-export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 && sudo ./gradlew clean buildPravegaImage -PpravegaBaseTag=nautilus/pravega -PpravegaVersion=$branch  && sudo ./gradlew clean buildBookkeeperImage -PbookkeeperBaseTag=nautilus/bookkeeper -PpravegaVersion=$branch
+export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 && sudo ./gradlew --warn buildPravegaImage -PpravegaBaseTag=nautilus/pravega -PpravegaVersion=$branch  && sudo ./gradlew --warn buildBookkeeperImage -PbookkeeperBaseTag=nautilus/bookkeeper -PpravegaVersion=$branch
+sudo apt -y install awscli