run-tests

#!/usr/bin/env bash
#
# Test Framework for update-systemd-resolved.
# Copyright (C) 2016, Jonathan Wright <jon@than.io>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# Set colour escape sequences

PS4='+ ${BASH_SOURCE:-$0}@${LINENO:-0}${FUNCNAME:+#${FUNCNAME}()}: '

if [[ -n ${NO_COLOR-} ]] || ! colors="$(tput colors 2> /dev/null)" || (("${colors:-0}" < 16)); then
  RED=''
  YELLOW=''
  ORANGE=''
  GREEN=''
  DARK=''
  RESET=''
else
  RED='\033[0;31m'
  YELLOW='\033[1;33m'
  ORANGE='\033[0;33m'
  GREEN='\033[0;32m'
  DARK='\033[1;30m'
  RESET='\033[0m'
fi

# Set Pass/Fail signatures
PASS="✓"
FAIL="✗"

# Counters
COUNT_PASS=0
COUNT_FAIL=0

# Names of files in which test cases failed
declare -A FAILED

# Flag for determining whether a test script called the `runtest' function
RUNTEST_CALLED=0

AUTOMATED_TESTING=1

if ! toplevel="$(git rev-parse --show-toplevel 2> /dev/null)"; then
  if ! toplevel="$(readlink -f "${BASH_SOURCE[0]%/*}/.." 2> /dev/null)"; then
    toplevel="${PWD:-$(pwd)}"
  fi
fi

# Ensure that things like "source update-systemd-resolved" pick up the script
# in this repository's toplevel rather than, say,
# /usr/bin/update-systemd-resolved.
export PATH="${toplevel}${PATH:+:${PATH}}"
unset toplevel

busctl2var() {
  if (("$#" != 2)); then
    printf 1>&2 -- 'usage: %s VARIABLE_NAME BUSCTL_CALL\n' "${FUNCNAME[0]}"
    return 1
  fi

  local -n var_ref="$1"
  shift

  # Compatible with busybox; with GNU sed's `\U` extension we could all the
  # string-munging work with sed alone and not require `tr` in the pipeline.
  local varname_suffix
  varname_suffix="$(
    printf -- '%s' "$1" |
      sed -e '
      s/[^[:alnum:]]//g
      s/\([a-z]\)\([A-Z]\)/\1_\2/g
      s/\([A-Z]\)\([A-Z][a-z]\)/\1_\2/g
    ' |
      tr '[:lower:]' '[:upper:]' |
      sed 's/^SET_LINK_//'
  )"

  # `var_ref` is a reference variable; we use it by assigning to it.
  # shellcheck disable=SC2034
  var_ref="TEST_BUSCTL_${varname_suffix}"
}

busctl() {
  # Short-circuit if we got "busctl status <...>".  If TEST_BUSCTL_STATUS_RC is
  # unset or empty, assume this is the call from the "main" function in
  # update-systemd-resolved, rather than an automated test checking the
  # behaviour of "busctl status <...>".
  case "${1-}" in
    status)
      if [[ -n ${TEST_BUSCTL_STATUS_RC-} ]]; then
        busctl_called=1
        _log "busctl status: returning ${TEST_BUSCTL_STATUS_RC}"
        return "$TEST_BUSCTL_STATUS_RC"
      else
        return 0
      fi
      ;;
  esac

  shift 4
  _log "busctl called with: ${*@Q}"
  # Set that busctl has been called
  busctl_called=1

  if (("$#" < 1)); then
    _fail "busctl called without arguments"
    return
  fi

  local call="$1"
  shift

  local indir
  busctl2var indir "$call" || return

  if [[ -v $indir ]]; then
    local -a expected=()
    local -a expanded
    read -r -a expanded <<< "${!indir}"

    if (("$#" > 0)); then
      local signature="$1"

      local elem etype
      local -i i
      for ((i = 0; i < "${#expanded[@]}"; i++)); do
        elem="${expanded[${i}]}"
        etype="${signature:i:1}"

        # Support "true" and "false" for boolstrings; we expect these to be
        # coerced to "yes" and "no", respectively.  Similarly, support "yes"
        # and "no" for booleans; we expect these to be coerced to "true" and
        # "false", respectively.
        case "${elem,,}" in
          true)
            if [[ $etype == s ]]; then
              elem=yes
            fi
            ;;
          false)
            if [[ $etype == s ]]; then
              elem=no
            fi
            ;;
          yes)
            if [[ $etype == b ]]; then
              elem=true
            fi
            ;;
          no)
            if [[ $etype == b ]]; then
              elem=false
            fi
            ;;
          default)
            elem=""
            ;;
        esac

        expected+=("$elem")
      done
    else
      expected=("${expanded[@]}")
    fi
  fi

  case "$call" in
    SetLinkDNS) ;;
    FlushCaches | ResetServerFeatures | ResetStatistics)
      argdesc=zero
      argcount=0
      ;;
    SetLinkDomains | SetLinkDNSSEC | SetLinkDNSOverTLS | SetLinkDefaultRoute | SetLinkLLMNR | SetLinkMulticastDNS)
      argdesc=three
      argcount=3
      ;;
    RevertLink)
      # Called upon `down` action
      return
      ;;
    *)
      _fail "Unknown command called on busctl: ${call}"
      ;;
  esac

  # XXX *NOT* `[[ -v expected ]]`, which returns a nonzero status when the
  # `expected` is declared but empty...
  if ! declare -p expected &> /dev/null; then
    _fail "${call} was called unexpectedly"
    return
  elif [[ ${expected[0]:-} == SKIP ]]; then
    return
  elif (("$#" != "${argcount:-${#}}")); then
    _fail "${call} must be called with exactly ${argdesc?} (${argcount?}) arguments"
  elif (("${argcount:-0}" > 0)) && [[ ${expected[0]} != "${if_index?}" ]]; then
    _fail "${call} not called with the expected interface index as first argument: expected ${if_index}, got ${expected[0]}"
  else
    local -a actual=("${@:2:$(("$#" - 1))}")

    local actual_size="${#expected[@]}"
    local expected_size="${#actual[@]}"
    local max

    if ((actual_size > expected_size)); then
      max="$actual_size"
    else
      max="$expected_size"
    fi

    local i actual_arg expected_arg
    for ((i = 0; i < max; i++)); do
      actual_arg="${actual[i]:-'<undef>'}"
      expected_arg="${expected[i]:-'<undef>'}"
      if [[ $actual_arg != "$expected_arg" ]]; then
        _fail "${call} not called with the expected setting value in position ${i}: expected ${expected_arg@Q}, got ${actual_arg@Q}"
      fi
    done
  fi
}

ip() {
  _log "ip called with: ${*@Q}"

  if [[ "${1} ${2} ${3} ${4}" == "link show dev ${dev}" ]]; then
    _pass "ip was called correctly"
  else
    _fail "ip was called with incorrect or unknown arguments"
  fi

  # Return fake ip statement
  echo -e "${ip_ifindex}: ${dev}: <POINTOPOINT,MULTICAST,NOARP,UP,LOWER_UP>" \
    " mtu 1500 qdisc fq_codel state UNKNOWN mode DEFAULT group default qlen" \
    " 100\n    link/none"
}

resolvectl() {
  _log "resolvectl called with: ${*@Q}"
}

logger() {
  # Remove standard options
  if [[ $* == *' --' ]]; then
    set --
  else
    while (("$#" > 0)); do
      case "$1" in
        --)
          shift
          break
          ;;
        *)
          shift
          ;;
      esac
    done
  fi

  if (("$#" == 0)) && ! [[ -t 0 ]]; then
    local message
    while read -r message; do
      _log "-- ${message}"
    done
  else
    _log "-- $*"
  fi
}

exit() {
  # Override "exit" builtin.  Note that "exit" is equivalent to "exit $?", so
  # handle that case.
  _log "exit called with status ${1:-$?}"
}

_log() {
  (echo >&2 -e "    ${DARK}${*}${RESET}")
}

_pass() {
  COUNT_PASS=$((COUNT_PASS + 1))
  (echo >&2 -e "  ${GREEN}${PASS}${RESET} ${*}")
}

_fail() {
  COUNT_FAIL=$((COUNT_FAIL + 1))
  if [[ -v TEST ]]; then
    FAILED["$TEST"]=1
  fi
  (echo >&2 -e "  ${RED}${FAIL} ${*}${RESET}")
}

checktest() {
  # Increment counter so that we don't double-execute if a test script calls
  # this function.
  : ${RUNTEST_CALLED:=0}
  ((RUNTEST_CALLED += 1))

  echo -e "${GREEN}- Testing ${TEST_TITLE:-a nameless test}${RESET}"

  if (($# < 1)); then
    set -- source update-systemd-resolved
  fi

  # Source, don't run, so we don't need to export and internal functions override
  # external calls out to system commands
  exit_status=0
  "$@" || exit_status="$?"
  exit_message="script exited with a ${exit_status} exit status"

  if [[ "$((exit_status > 0))" == "${EXPECT_FAILURE:-0}" ]]; then
    _pass "$exit_message"
  else
    _fail "$exit_message"
  fi
}

runtest() {
  checktest

  if [[ ${TEST_BUSCTL_CALLED-} == 0 ]]; then
    if ((busctl_called == 0)); then
      _pass "busctl was not called, as expected"
    else
      _fail "busctl was called, not expected"
    fi
  elif ((busctl_called == 0)); then
    _fail "busctl was not called, not expected"
  fi

  echo
}

evaltest() {
  TEST="${1?}"

  # Set/Reset loop variables
  RUNTEST_CALLED=0
  EXPECT_FAILURE=0
  busctl_called=0
  # Set/Reset expected results
  # We don't expect any `busctl` calls by default...
  unset "${!TEST_BUSCTL_@}"
  # Except for `FlushCaches`, which should be called with no arguments.
  TEST_BUSCTL_FLUSH_CACHES=""
  # Set/Reset expected `busctl` exit status
  unset TEST_BUSCTL_STATUS_RC

  # Keep this random, as we will never know the ifindex up-front
  ip_ifindex=$((RANDOM %= 64))

  # Same for the device
  dev="tun${RANDOM}"

  # Clear foreign_option_*
  unset "${!foreign_option_@}"

  # Import the test configuration
  # shellcheck source-path=SCRIPTDIR source=tests/01_no_updates.sh
  source "$TEST" || return

  if ((RUNTEST_CALLED > 0)); then
    return
  fi

  declare -a foreign_options || return

  local i=0
  local opt
  for opt in "${foreign_options[@]}" in; do
    declare "foreign_option_$((i += 1))=${opt}"
  done

  runtest
}

echo "update-systemd-resolved Test Suite"
echo

if (("$#" < 1)); then
  set -- ./tests
fi

for path in "$@"; do
  if [[ -d $path ]]; then
    for test in "$path"/*.sh; do
      if [[ -f $test ]]; then
        evaltest "$test"
      fi
    done
  else
    evaltest "$path"
  fi
done

echo -e "  ${GREEN}${PASS} ${COUNT_PASS} Passed${RESET}"
echo -e "  ${RED}${FAIL} ${COUNT_FAIL} Failed${RESET}"

if [[ -v FAILED ]] && (("${#FAILED[@]}" > 0)); then
  echo -e "\n  ${YELLOW}The following files have failing test cases:${RESET}"

  for failed in "${!FAILED[@]}"; do
    echo -e "    ${ORANGE}${failed}${RESET}"
  done
fi

# Make sure we fail if there are failed tests
((COUNT_FAIL == 0))