Merge pull request #20 from docksal/develop

Release 2.0.0
docksal · Dec 31, 2019 · 0b27183 · 0b27183
2 parents aedf7ba + ea65553
commit 0b27183
Show file tree

Hide file tree

Showing 8 changed files with 231 additions and 107 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -9,41 +9,19 @@ env:
     - AWS_ACCESS_KEY_ID=AKIAIPJXMNOEV65OMURA
     # Encrypted AWS_SECRET_ACCESS_KEY
     - secure: "KGqbZdK65Z/fwuM8XkIO7awap+hastKvpAXJPe1FvwwZ9QGK+t9ZDljyIN8oSZdEPH4te7opo5D5LxNVVbFuTfnQ4P3Mikl7aOoGQi39wx5XmBueeQB+r/fl1GjkOoRcHyLmRRbeP4Ux/VDG2cCNUsS9q/s6kqsf8k0RmcpDMn0tBef/CixdnTfoQxO8QOgDCoBt+ErBamgt/HFWM89bw4uaehXwoevgArVkaiFyyJv3xBpKRAWKYPuwxCc9iadbb13SiPbs5qx0lQulrYLzf3qkQKBBJRGAPysC7uITW10TY/KjxmHA4S6+Z67P5GK5khZ55xsC86gz0fgWHFCdflbu46kS2bkAozEf9HnU3cDV3sHMgPyIOaAgGwmYjcDjwE+oxa54sTfzEwZ1yAKY3twwDiKue/4hdaHVknoWUJdjL7s5fOS3XXl3g8pHsRvzAx9XO8ZAOv7n2Pf0UvGbgklbTgDQO+Vvi7jDJb1yXGT6o9YFTpazpj5R3kU1Nf3V28zNQ8/OGqKssKghgT/9Cm1DqZIWWiPHmbkF6Q7e/HBSgTLaRd/PI//C/pSlc25TO/vQ+hBoqdxZ/1FCDDiCpLCXxd1b33ea3HUKBtmKKtSWDYGo6VVI8X+PCx7DvzpWIX85xgLhvloTUeNUL4ly/dcVhuEDiq5eWGHdcw20NmY="
+    - S3_BUCKET="docksal-aws-templates"
+    - UPLOAD_DIR="sandbox-server"
+    - LOCAL_DIR="build"
 
   matrix:
-    - VERSION=basic
-    - VERSION=advanced
+    - TEMPLATE_TYPE=basic
+    - TEMPLATE_TYPE=advanced
 
 install:
   - pip install awscli --upgrade
 
 script:
-  - cd aws-cloudformation && ./scripts/bash2yaml startup-${VERSION}.sh tpl-${VERSION}.yaml
-  - mkdir -p build && mv -f template.yaml build/${VERSION}.yaml
-  - aws cloudformation validate-template --region ${AWS_REGION} --template-body file://build/${VERSION}.yaml
-
-deploy:
-  - provider: s3
-    region: "${AWS_REGION}"
-    access_key_id: "${AWS_ACCESS_KEY_ID}"
-    secret_access_key: "${AWS_SECRET_ACCESS_KEY}"
-    bucket: docksal-aws-templates
-    skip_cleanup: true
-    acl: public_read
-    local_dir: build
-    upload-dir: sandbox-server/edge
-    on:
-      repo: docksal/sandbox-server
-      branch: develop
-  - provider: s3
-    region: "${AWS_REGION}"
-    access_key_id: "${AWS_ACCESS_KEY_ID}"
-    secret_access_key: "${AWS_SECRET_ACCESS_KEY}"
-    bucket: docksal-aws-templates
-    skip_cleanup: true
-    acl: public_read
-    local_dir: build
-    upload-dir: sandbox-server/stable
-    on:
-      repo: docksal/sandbox-server
-      branch: master
+  - cd aws-cloudformation && ./scripts/bash2yaml startup-${TEMPLATE_TYPE}.sh tpl-${TEMPLATE_TYPE}.yaml
+  - mkdir -p ${LOCAL_DIR} && mv -f template.yaml ${LOCAL_DIR}/${TEMPLATE_TYPE}.yaml
+  - aws cloudformation validate-template --region ${AWS_REGION} --template-body file://${LOCAL_DIR}/${TEMPLATE_TYPE}.yaml
+  - ${TRAVIS_BUILD_DIR}/scripts/upload-to-s3.sh
diff --git a/aws-cloudformation/scripts/find-amis b/aws-cloudformation/scripts/find-amis
@@ -1,11 +1,15 @@
 #!/bin/bash
 
+# ubuntu official owner 099720109477
+latest_ami="ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-????????"
+# latest_ami="ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20191002"
+
 AWS_REGION=$(aws ec2 describe-regions --output text | awk '{print $3}' | xargs)
 
-echo '    "Region2AMI" : {'
+echo "  Region2AMI:"
 for region in ${AWS_REGION}
 do
-    ami=`aws ec2 describe-images --region ${region} --owners 099720109477 --filters --output json 'Name=name,Values=ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-????????' 'Name=state,Values=available' | jq -r '.Images | sort_by(.CreationDate) | last(.[]).ImageId'`
-    echo '      "'${region}'" : { "AMI" : "'${ami}'" },'
+    ami=`aws ec2 describe-images --region ${region} --owners 099720109477 --filters --output json 'Name=name,Values='${latest_ami}'' 'Name=state,Values=available' | jq -r '.Images | sort_by(.CreationDate) | last(.[]).ImageId'`
+    echo "    ${region}:"
+    echo "      AMI: ${ami}"
 done
-echo '    }'
diff --git a/aws-cloudformation/startup-advanced.sh b/aws-cloudformation/startup-advanced.sh
@@ -22,6 +22,15 @@ PROJECTS_ROOT="${BUILD_USER_HOME}/builds"
 
 ##########
 # Functions begin
+reread_device()
+{
+    DEVICE=$1
+    hdparm -z ${DEVICE} >>/var/log/device.log 2>&1 || true
+    file -s ${DEVICE} >>/var/log/device.log 2>&1 || true
+    partprobe ${DEVICE} >>/var/log/device.log 2>&1 || true
+    blockdev --rereadpt -v ${DEVICE} >>/var/log/device.log 2>&1 || true
+    fdisk -l ${DEVICE} >>/var/log/device.log 2>&1 || true
+}
 
 mount_part()
 {
@@ -44,22 +53,30 @@ create_fs()
     mkfs.ext4 -m 0 -F -E lazy_itable_init=0,lazy_journal_init=0,discard ${DATA_DISK} -L ${DISK_LABEL} >/dev/null 2>&1
 }
 
-get_part_list()
+get_disk_info()
 {
-    # get disk partitions info.
+    # get disk/partitions info.
     # the result will contain strings: NAME="/dev/nvme1n1";TYPE="disk";FSTYPE="";LABEL="";MOUNTPOINT="" NAME="/dev/nvme0n1";TYPE="disk";FSTYPE="";LABEL="";MOUNTPOINT=""
-    # in our case will be only one string
     DATA_DISK=$1
-    partprobe
-    lsblk -p -n -P -o NAME,TYPE,FSTYPE,LABEL,MOUNTPOINT ${DATA_DISK} | grep part | sed 's/ /;/g'
+    reread_device "$DATA_DISK"
+    device_list="$(lsblk -p -n -P -o NAME,TYPE,FSTYPE,LABEL,MOUNTPOINT ${DATA_DISK} 2>&1 | sed 's/ /;/g')"
+    while read device_info
+    do
+      eval ${device_info}
+      # exit if device already mounted
+      [[ ${MOUNTPOINT} != "" ]] && return
+      # exit if device has ext4 fs
+      [[ ${FSTYPE} == "ext4" ]] && return
+    done <<< ${device_list}
+    # if not found ext4 fs return empty data for creating new fs
+    NAME=""; TYPE=""; FSTYPE=""; LABEL=""; MOUNTPOINT=""
 }
 
 create_part()
 {
     # create msdos partition table and create primary partition used 100% disk size
     DATA_DISK=$1
     /sbin/parted ${DATA_DISK} -s mklabel msdos
-    /sbin/parted ${DATA_DISK} -s -a optimal mkpart primary 0% 100%
 }
 
 ##########
@@ -131,17 +148,15 @@ then
     # find additional data disk, format it and mount
     for disk in $(lsblk -d -p -n -o NAME,TYPE | grep disk | cut -d' ' -f1)
     do
-        # partitioning disk if disk is clean
-        [[ $(get_part_list "${disk}") == "" ]] && { echo "Disk ${disk} is clean! Creating partition..."; create_part "${disk}"; }
-        eval $(echo $(get_part_list "${disk}"))
-        # skip disk if his partition is mounted
-        [[ "$MOUNTPOINT" != "" ]] && { echo "Disk $disk have partition $NAME, and it already mounted! Skipping..."; continue; }
-        # mount disk partition if ext4 fs found, but not mounted (volume was added from another instance)
-        [[ "$FSTYPE" == "ext4" ]] && { echo "Disk $disk have partition $NAME with FS, but not mounted! Mounting..."; mount_part "$NAME"; continue; }
-        # create fs and mount when we already have partition, but fs not created yet
-        echo "Disk $disk have partition $NAME, but does not have FS! Creating FS and mounting..."
-        create_fs "${NAME}"
-        mount_part "${NAME}"
+        # get partition info
+        get_disk_info "${disk}"
+        [[ "$MOUNTPOINT" != "" ]] && { echo "Disk $NAME already mounted! Skipping..."; continue; }
+        [[ "$FSTYPE" == "ext4" ]] && { echo "Disk $NAME have ext4 filesystem but not mounted! Mounting..."; mount_part "$NAME"; continue; }
+        echo "Disk ${disk} is clean! Creating partition..."
+        wipefs -fa ${disk} >/dev/null
+        create_part "${disk}"
+        create_fs "${disk}"
+        mount_part "${disk}"
     done
 fi
 

diff --git a/aws-cloudformation/startup-basic.sh b/aws-cloudformation/startup-basic.sh
@@ -20,6 +20,18 @@ PROJECT_INACTIVITY_TIMEOUT="0.5h"
 PROJECT_DANGLING_TIMEOUT="168h"
 PROJECTS_ROOT="${BUILD_USER_HOME}/builds"
 
+##########
+# Functions begin
+reread_device()
+{
+    DEVICE=$1
+    hdparm -z ${DEVICE} >>/var/log/device.log 2>&1 || true
+    file -s ${DEVICE} >>/var/log/device.log 2>&1 || true
+    partprobe ${DEVICE} >>/var/log/device.log 2>&1 || true
+    blockdev --rereadpt -v ${DEVICE} >>/var/log/device.log 2>&1 || true
+    fdisk -l ${DEVICE} >>/var/log/device.log 2>&1 || true
+}
+
 mount_part()
 {
     DATA_DISK=$1
@@ -41,24 +53,35 @@ create_fs()
     mkfs.ext4 -m 0 -F -E lazy_itable_init=0,lazy_journal_init=0,discard ${DATA_DISK} -L ${DISK_LABEL} >/dev/null 2>&1
 }
 
-get_part_list()
+get_disk_info()
 {
-    # get disk partitions info.
+    # get disk/partitions info.
     # the result will contain strings: NAME="/dev/nvme1n1";TYPE="disk";FSTYPE="";LABEL="";MOUNTPOINT="" NAME="/dev/nvme0n1";TYPE="disk";FSTYPE="";LABEL="";MOUNTPOINT=""
-    # in our case will be only one string
     DATA_DISK=$1
-    partprobe
-    lsblk -p -n -P -o NAME,TYPE,FSTYPE,LABEL,MOUNTPOINT ${DATA_DISK} | grep part | sed 's/ /;/g'
+    reread_device "$DATA_DISK"
+    device_list="$(lsblk -p -n -P -o NAME,TYPE,FSTYPE,LABEL,MOUNTPOINT ${DATA_DISK} 2>&1 | sed 's/ /;/g')"
+    while read device_info
+    do
+      eval ${device_info}
+      # exit if device already mounted
+      [[ ${MOUNTPOINT} != "" ]] && return
+      # exit if device has ext4 fs
+      [[ ${FSTYPE} == "ext4" ]] && return
+    done <<< ${device_list}
+    # if not found ext4 fs return empty data for creating new fs
+    NAME=""; TYPE=""; FSTYPE=""; LABEL=""; MOUNTPOINT=""
 }
 
 create_part()
 {
     # create msdos partition table and create primary partition used 100% disk size
     DATA_DISK=$1
     /sbin/parted ${DATA_DISK} -s mklabel msdos
-    /sbin/parted ${DATA_DISK} -s -a optimal mkpart primary 0% 100%
 }
 
+##########
+# Functions end
+
 # Create build-agent user with no-password sudo access
 # Forcing the uid to avoid race conditions with GCP creating project level users at the same time.
 # (Otherwise, we may run into something like "useradd: UID 1001 is not unique")
@@ -90,17 +113,15 @@ done
 # find additional data disk, format it and mount
 for disk in $(lsblk -d -p -n -o NAME,TYPE | grep disk | cut -d' ' -f1)
 do
-    # partitioning disk if disk is clean
-    [[ $(get_part_list "${disk}") == "" ]] && { echo "Disk ${disk} is clean! Creating partition..."; create_part "${disk}"; }
-    eval $(echo $(get_part_list "${disk}"))
-    # skip disk if his partition is mounted
-    [[ "$MOUNTPOINT" != "" ]] && { echo "Disk $disk have partition $NAME, and it already mounted! Skipping..."; continue; }
-    # mount disk partition if ext4 fs found, but not mounted (volume was added from another instance)
-    [[ "$FSTYPE" == "ext4" ]] && { echo "Disk $disk have partition $NAME with FS, but not mounted! Mounting..."; mount_part "$NAME"; continue; }
-    # create fs and mount when we already have partition, but fs not created yet
-    echo "Disk $disk have partition $NAME, but does not have FS! Creating FS and mounting..."
-    create_fs "${NAME}"
-    mount_part "${NAME}"
+    # get partition info
+    get_disk_info "${disk}"
+    [[ "$MOUNTPOINT" != "" ]] && { echo "Disk $NAME already mounted! Skipping..."; continue; }
+    [[ "$FSTYPE" == "ext4" ]] && { echo "Disk $NAME have ext4 filesystem but not mounted! Mounting..."; mount_part "$NAME"; continue; }
+    echo "Disk ${disk} is clean! Creating partition..."
+    wipefs -fa ${disk} >/dev/null
+    create_part "${disk}"
+    create_fs "${disk}"
+    mount_part "${disk}"
 done
 
 if [ -d $MOUNT_POINT ]