Fix build (#36)

* Regression test for sudo functionality

* Add Ubuntu 20.04 and propagate exit code

* Fix for sudo bug

* Fix build failures

* Make sure shellcheck container errors are bubbled up

* Remove debug statement

* Fix docker-compose call for shellcheck

Author: yorinasub17

.circleci/config.yml

@@ -4,7 +4,7 @@ jobs:
     machine: true
     steps:
       - checkout
-      - run: docker-compose up shellcheck
+      - run: docker-compose up --exit-code-from shellcheck shellcheck
   integration_test:
     docker:
       - image: 087285199408.dkr.ecr.us-east-1.amazonaws.com/circle-ci-test-image-base:go1.14
@@ -29,14 +29,22 @@ jobs:
     machine: true
     steps:
       - checkout
-      - run: docker-compose up bats_ubuntu1604
+      - run: docker-compose up --exit-code-from bats_ubuntu1604 bats_ubuntu1604
   bats_ubuntu1804:
     # We need to run Docker Compose with privileged settings, which isn't supported by CircleCI's Docker executor, so
     # we have to use the machine executor instead.
     machine: true
     steps:
       - checkout
-      - run: docker-compose up bats_ubuntu1804
+      - run: docker-compose up --exit-code-from bats_ubuntu1804 bats_ubuntu1804
+  bats_ubuntu2004:
+    # We need to run Docker Compose with privileged settings, which isn't supported by CircleCI's Docker executor, so
+    # we have to use the machine executor instead.
+    machine: true
+    steps:
+      - checkout
+      - run: docker-compose up --exit-code-from bats_ubuntu2004 bats_ubuntu2004
+
 workflows:
   version: 2
   checks:
@@ -47,3 +55,4 @@ workflows:
             - Gruntwork Admin
       - bats_ubuntu1604
       - bats_ubuntu1804
+      - bats_ubuntu2004

Dockerfile.ubuntu16.04.bats

@@ -3,17 +3,16 @@ MAINTAINER Gruntwork <[email protected]>
 
 # Install basic dependencies
 RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
-    apt-get install -y vim python-pip jq sudo curl libffi-dev python3-dev && \
+    apt-get install -y vim git python-pip jq sudo curl libffi-dev python3-dev && \
     update-alternatives --install /usr/bin/python python /usr/bin/python3 1 && \
     update-alternatives --config python && \
     curl https://bootstrap.pypa.io/pip/3.5/get-pip.py -o /tmp/get-pip.py && \
     python /tmp/get-pip.py
 
 # Install Bats
-RUN apt-get install -y software-properties-common && \
-    add-apt-repository ppa:duggan/bats && \
-    DEBIAN_FRONTEND=noninteractive apt-get update && \
-    apt-get install -y bats
+RUN git clone https://github.com/bats-core/bats-core.git /tmp/bats-core && \
+    /tmp/bats-core/install.sh /usr/local && \
+    rm -r /tmp/bats-core
 
 # Upgrade pip
 RUN pip install -U pip
@@ -25,13 +24,13 @@ RUN pip install awscli --upgrade --user
 RUN pip install flask moto moto[server] networkx==2.2
 
 # Install tools we'll need to create a mock EC2 metadata server
-RUN apt-get install -y net-tools iptables
+RUN DEBIAN_FRONTEND=noninteractive apt-get install -y net-tools iptables
 
 # Copy mock AWS CLI into the PATH
 COPY ./.circleci/aws-local.sh /usr/local/bin/aws
 
-# These have been added to resolve some encoding error issues with the tests. These were introduced during the upgrade to Python 3.6, 
-# which is known to have some sensitivity around locale issues. These variables should correct that, per examples like this SO thread: 
+# These have been added to resolve some encoding error issues with the tests. These were introduced during the upgrade to Python 3.6,
+# which is known to have some sensitivity around locale issues. These variables should correct that, per examples like this SO thread:
 # https://stackoverflow.com/questions/51026315/how-to-solve-unicodedecodeerror-in-python-3-6/51027262#51027262.
 ENV LC_ALL=C.UTF-8
-ENV LANG=C.UTF-8
+ENV LANG=C.UTF-8

Dockerfile.ubuntu18.04.bats

@@ -2,35 +2,32 @@ FROM ubuntu:18.04
 MAINTAINER Gruntwork <[email protected]>
 
 # Install basic dependencies
-RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
-    apt-get install -y vim python-pip jq sudo curl libffi-dev python3-dev
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y git vim python3-pip jq sudo curl libffi-dev python3-dev
 
 # Install Bats
-RUN apt-get install -y software-properties-common && \
-    add-apt-repository ppa:duggan/bats && \
-    DEBIAN_FRONTEND=noninteractive apt-get update && \
-    apt-get install -y bats
+RUN git clone https://github.com/bats-core/bats-core.git /tmp/bats-core && \
+    /tmp/bats-core/install.sh /usr/local && \
+    rm -r /tmp/bats-core
+
+# Upgrade pip
+RUN pip3 install -U pip
 
 # Install AWS CLI
-RUN apt install python3-distutils python3-apt -y && \
-    update-alternatives --install /usr/bin/python python /usr/bin/python3 1 && \
-    update-alternatives --config python && \
-    curl https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py && \
-    python /tmp/get-pip.py && \
-    pip install awscli --upgrade --user
+RUN pip3 install awscli --upgrade --user
 
 # Install moto: https://github.com/spulec/moto
 # Lock cfn-lint and pysistent to last known working versions
-RUN pip install flask moto moto[server] cfn-lint==0.35.1 pyrsistent==0.16.0
+RUN pip3 install flask moto moto[server] cfn-lint==0.35.1 pyrsistent==0.16.0
 
 # Install tools we'll need to create a mock EC2 metadata server
-RUN apt-get install -y net-tools iptables
+RUN DEBIAN_FRONTEND=noninteractive apt-get install -y net-tools iptables
 
 # Copy mock AWS CLI into the PATH
 COPY ./.circleci/aws-local.sh /usr/local/bin/aws
 
-# These have been added to resolve some encoding error issues with the tests. These were introduced during the upgrade to Python 3.6, 
-# which is known to have some sensitivity around locale issues. These variables should correct that, per examples like this SO thread: 
+# These have been added to resolve some encoding error issues with the tests. These were introduced during the upgrade to Python 3.6,
+# which is known to have some sensitivity around locale issues. These variables should correct that, per examples like this SO thread:
 # https://stackoverflow.com/questions/51026315/how-to-solve-unicodedecodeerror-in-python-3-6/51027262#51027262.
 ENV LC_ALL=C.UTF-8
-ENV LANG=C.UTF-8
+ENV LANG=C.UTF-8

Dockerfile.ubuntu20.04.bats

@@ -0,0 +1,33 @@
+FROM ubuntu:20.04
+MAINTAINER Gruntwork <[email protected]>
+
+# Install basic dependencies
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y git vim python3-pip jq sudo curl libffi-dev python3-dev
+
+# Install Bats
+RUN git clone https://github.com/bats-core/bats-core.git /tmp/bats-core && \
+    /tmp/bats-core/install.sh /usr/local && \
+    rm -r /tmp/bats-core
+
+# Upgrade pip
+RUN pip3 install -U pip
+
+# Install AWS CLI
+RUN pip3 install awscli --upgrade --user
+
+# Install moto: https://github.com/spulec/moto
+# Lock cfn-lint and pysistent to last known working versions
+RUN pip3 install flask moto moto[server] cfn-lint==0.35.1 pyrsistent==0.16.0
+
+# Install tools we'll need to create a mock EC2 metadata server
+RUN DEBIAN_FRONTEND=noninteractive apt-get install -y net-tools iptables
+
+# Copy mock AWS CLI into the PATH
+COPY ./.circleci/aws-local.sh /usr/local/bin/aws
+
+# These have been added to resolve some encoding error issues with the tests. These were introduced during the upgrade to Python 3.6,
+# which is known to have some sensitivity around locale issues. These variables should correct that, per examples like this SO thread:
+# https://stackoverflow.com/questions/51026315/how-to-solve-unicodedecodeerror-in-python-3-6/51027262#51027262.
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8

docker-compose.yml

@@ -30,3 +30,14 @@ services:
     command: bats test
     # Necessary so we can run a mock EC2 metadata service on port 80 on a special IP
     privileged: true
+  bats_ubuntu2004:
+    build:
+      context: ./
+      dockerfile: Dockerfile.ubuntu20.04.bats
+    volumes:
+      # Mount all the files so you have "hot reload" of all changes from the host
+      - ./:/usr/local/src/bash-commons
+    working_dir: /usr/local/src/bash-commons
+    command: bats test
+    # Necessary so we can run a mock EC2 metadata service on port 80 on a special IP
+    privileged: true

modules/bash-commons/src/os.sh

@@ -105,7 +105,7 @@ function os_create_user {
     log_info "User $username already exists. Will not create again."
   else
     log_info "Creating user named $username"
-    "$exuseradd" "$username"
+    $exuseradd "$username"
   fi
 }
 
@@ -121,5 +121,5 @@ function os_change_dir_owner {
   fi
 
   log_info "Changing ownership of $dir to $username"
-  "$exchown" -R "$username:$username" "$dir"
+  $exchown -R "$username:$username" "$dir"
 }

test/aws-cli.bats

@@ -16,7 +16,7 @@ function teardown {
   run aws_get_instance_tags "fake-id" "us-east-1"
   assert_success
 
-  local readonly expected=$(cat <<END_HEREDOC
+  local -r expected=$(cat <<END_HEREDOC
 {
   "Tags": []
 }
@@ -27,16 +27,16 @@ END_HEREDOC
 }
 
 @test "aws_get_instance_tags non-empty" {
-  local readonly tag_key="foo"
-  local readonly tag_value="bar"
+  local -r tag_key="foo"
+  local -r tag_value="bar"
 
   local instance_id
   instance_id=$(create_mock_instance_with_tags "$tag_key" "$tag_value")
 
   run aws_get_instance_tags "$instance_id" "us-east-1"
   assert_success
 
-  local readonly expected=$(cat <<END_HEREDOC
+  local -r expected=$(cat <<END_HEREDOC
 {
    "Tags": [
      {
@@ -72,7 +72,7 @@ END_HEREDOC
   run aws_describe_asg "fake-asg-name" "us-east-1"
   assert_success
 
-  local readonly expected=$(cat <<END_HEREDOC
+  local -r expected=$(cat <<END_HEREDOC
 {
   "AutoScalingGroups": []
 }
@@ -83,11 +83,11 @@ END_HEREDOC
 }
 
 @test "aws_describe_asg non-empty" {
-  local readonly asg_name="foo"
-  local readonly min_size=1
-  local readonly max_size=3
-  local readonly region="us-east-1"
-  local readonly azs="${region}a"
+  local -r asg_name="foo"
+  local -r min_size=1
+  local -r max_size=3
+  local -r region="us-east-1"
+  local -r azs="${region}a"
 
   create_mock_asg "$asg_name" "$min_size" "$max_size" "$azs"
 
@@ -111,7 +111,7 @@ END_HEREDOC
   run aws_describe_instances_in_asg "fake-asg-name" "us-east-1"
   assert_success
 
-  local readonly expected=$(cat <<END_HEREDOC
+  local -r expected=$(cat <<END_HEREDOC
 {
   "Reservations": []
 }
@@ -122,11 +122,11 @@ END_HEREDOC
 }
 
 @test "aws_describe_instances_in_asg non-empty" {
-  local readonly asg_name="foo"
-  local readonly min_size=1
-  local readonly max_size=3
-  local readonly region="us-east-1"
-  local readonly azs="${region}a"
+  local -r asg_name="foo"
+  local -r min_size=1
+  local -r max_size=3
+  local -r region="us-east-1"
+  local -r azs="${region}a"
 
   create_mock_asg "$asg_name" "$min_size" "$max_size" "$azs"
 
@@ -142,7 +142,7 @@ END_HEREDOC
   run aws_get_instances_with_tag "Name" "Value" "us-east-1"
   assert_success
 
-  local readonly expected=$(cat <<END_HEREDOC
+  local -r expected=$(cat <<END_HEREDOC
 {
   "Reservations": []
 }

test/aws-helper.bash

@@ -19,7 +19,7 @@ function start_ec2_metadata_mock {
   export meta_data_local_hostname="$3"
   export meta_data_public_hostname="$4"
   export meta_data_instance_id="$5"
-  local readonly region="$6"
+  local -r region="$6"
   export meta_data_placement__availability_zone="$7"
   export dynamic_data_instance_identity__document=$(cat <<END_HEREDOC
 {
@@ -71,7 +71,7 @@ END_HEREDOC
 function stop_ec2_metadata_mock {
   # Stop ec2-metadata-mock if it's running
   if [[ -f "$EC2_METADATA_MOCK_PID_PATH" ]]; then
-    local readonly pid=$(cat "$EC2_METADATA_MOCK_PID_PATH")
+    local -r pid=$(cat "$EC2_METADATA_MOCK_PID_PATH")
     kill "$pid" 2>&1 > "$EC2_METADATA_MOCK_LOG_FILE_PATH"
     rm -f "$EC2_METADATA_MOCK_PID_PATH"
 
@@ -108,7 +108,7 @@ function start_moto {
 function stop_moto {
   # Stop moto if it's running
   if [[ -f "$MOTO_PID_FILE_PATH" ]]; then
-    local readonly pid=$(cat "$MOTO_PID_FILE_PATH")
+    local -r pid=$(cat "$MOTO_PID_FILE_PATH")
     kill "$pid" 2>&1 > /dev/null
     rm -f "$MOTO_PID_FILE_PATH"
 
@@ -118,8 +118,8 @@ function stop_moto {
 }
 
 function create_mock_instance_with_tags {
-  local readonly tag_key="$1"
-  local readonly tag_value="$2"
+  local -r tag_key="$1"
+  local -r tag_value="$2"
 
   local mock_instance
   mock_instance=$(aws ec2 run-instances)
@@ -133,11 +133,11 @@ function create_mock_instance_with_tags {
 }
 
 function create_mock_asg {
-  local readonly asg_name="$1"
-  local readonly min_size="$2"
-  local readonly max_size="$3"
-  local readonly azs="$4"
+  local -r asg_name="$1"
+  local -r min_size="$2"
+  local -r max_size="$3"
+  local -r azs="$4"
 
-  aws autoscaling create-launch-configuration --launch-configuration-name "$asg_name"
+  aws autoscaling create-launch-configuration --launch-configuration-name "$asg_name" --image-id ami-fake --instance-type t3.micro
   aws autoscaling create-auto-scaling-group --auto-scaling-group-name "$asg_name" --min-size "$min_size" --max-size "$max_size" --availability-zones "$azs" --launch-configuration-name "$asg_name"
-}
+}

test/aws-wrapper.bats

@@ -255,25 +255,7 @@ END_HEREDOC
 
   load_aws_mock "$tags" "" ""
 
-  run naws_wrapper_wait_for_instance_tags "i-1234567890abcdef0" "us-east-1" "$max_retries" "$sleep_between_retries"
-
-  assert_failure
-}
-
-@test "aws_wrapper_wait_for_instance_tags no tags" {
-  local reaodnly max_retries=1
-  local readonly sleep_between_retries=0
-  local readonly tags=$(cat <<END_HEREDOC
-{
-   "Tags": []
- }
-END_HEREDOC
-)
-
-  load_aws_mock "$tags" "" ""
-
-  run naws_wrapper_wait_for_instance_tags "i-1234567890abcdef0" "us-east-1" "$max_retries" "$sleep_between_retries"
-
+  run aws_wrapper_wait_for_instance_tags "i-1234567890abcdef0" "us-east-1" "$max_retries" "$sleep_between_retries"
   assert_failure
 }
 
@@ -1596,4 +1578,4 @@ END_HEREDOC
 
   run aws_wrapper_get_asg_rally_point $asg_name $aws_region $use_public_hostname $retries $sleep_between_retries
   assert_failure
-}
+}