Merge pull request #1738 from alphagov/sengi/faster-tests

Speed up the pre-merge checks.

.github/workflows/lint.yaml

@@ -0,0 +1,115 @@
+name: lint
+
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: The branch, tag or SHA to checkout
+        default: main
+        type: string
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - ".git**"
+  pull_request:
+
+jobs:
+  check-docs:
+    name: Docs up to date
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - name: terraform-docs
+        env:
+          TERRAFORM_DOCS_VERSION: '0.16.0'
+        run: |
+          echo "Downloading terraform-docs-v${TERRAFORM_DOCS_VERSION} binary."
+          curl -Lsf \
+            "https://github.com/terraform-docs/terraform-docs/releases/download/v${TERRAFORM_DOCS_VERSION}/terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-amd64.tar.gz" | tar -zxf - terraform-docs
+
+          echo "Checking whether auto-generated README files are up to date."
+          PATH="$PATH:$PWD" tools/update-docs.sh
+
+          if ! git diff --exit-code; then
+            echo "The documentation isn't up to date. Run tools/update-docs.sh and commit the results."
+            exit 1
+          fi
+
+  shellcheck:
+    name: Shellcheck
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - name: Run ShellCheck
+        uses: ludeeus/action-shellcheck@00cae500b08a931fb5698e11e79bfbd38e612a38  # 2.0.0
+        env:
+          SHELLCHECK_OPTS: -e SC2086 -e SC1117
+
+  terraform-fmt:
+    name: terraform fmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - uses: actions/cache@v3
+        with:
+          path: ~/tfenv
+          key: tfenv-${{ runner.os }}
+      - name: terraform install
+        env:
+          TFENV_VERSION: '3.0.0'
+        run: |
+          echo "Installing tfenv."
+          mkdir -p tfenv
+          curl -L --silent --fail \
+            "https://github.com/tfutils/tfenv/archive/refs/tags/v${TFENV_VERSION}.tar.gz" \
+            | tar -zxf - --strip-components=1 -C tfenv
+          export PATH="$PATH:$PWD/tfenv/bin"
+
+          echo "Installing default Terraform version."
+          tfenv install
+          tfenv use
+
+          echo "Installing other necessary Terraform versions."
+          for version in $(find terraform/ -type f -name '.terraform-version' | xargs sort -u); do
+            tfenv install "$version"
+          done
+      - name: terraform fmt
+        run: |
+          find terraform/ \
+            -type f -name main.tf -not -path "*.terraform*" \
+            -execdir "$PWD/tfenv/bin/terraform" fmt -recursive . \;
+          if ! git diff --exit-code; then
+            echo "Run terraform fmt -recursive . to fix formatting."
+            exit 1
+          fi
+
+  other-linters:
+    name: Other linters
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - uses: ruby/setup-ruby@v1
+      - uses: actions/cache@v3
+        with:
+          path: vendor/bundle
+          key: bundle-${{ hashFiles('**/Gemfile.lock') }}
+          restore-keys: bundle
+      - name: bundle install
+        run: bundle install -j8 --deployment --without development
+      - name: RSpec
+        run: bundle exec rspec spec/validate_resources_spec.rb
+      - name: Lint resource names
+        run: bundle exec lib/resource_name_lint.rb
+      - name: Validate JSON
+        run: find . -type -f -name '*.json' | xargs tools/json-check.sh
+      - name: Check ADRs
+        run: tools/adr-check.sh

Gemfile

@@ -3,4 +3,6 @@ source 'https://rubygems.org'
 gem 'rspec'
 gem 'colorize'
 
-gem 'pry'
+group :development do
+  gem 'pry'
+end

terraform/lambda/DownloadLogsAnalytics/build_deployment_package.sh

@@ -1,6 +1,8 @@
 #! /bin/bash
+set -eu
 
 virtualenv -p python3.7 .venv
+# shellcheck disable=SC1091
 . .venv/bin/activate
 
 mkdir -p wheelhouse

terraform/lambda/SendPublicAPIEventsToGA/build_deployment_package.sh

@@ -1,6 +1,8 @@
 #! /bin/bash
+set -eu
 
 virtualenv -p python3.7 .venv
+# shellcheck disable=SC1091
 . .venv/bin/activate
 
 mkdir -p wheelhouse

terraform/projects/fastly-datagovuk/fastly.sh

@@ -4,6 +4,8 @@ set -e
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
+# TODO: FIXME
+# shellcheck disable=SC2207
 fastly_raw_ips=( $(curl https://api.fastly.com/public-ip-list 2>/dev/null | jq -r ".addresses[]") )
 
 fastly_ips_snippet=""

tools/json-check.sh

@@ -1,11 +1,9 @@
-#!/usr/bin/env bash
+#!/bin/dash
 set -eu
 
 for file in "$@"; do
-  lint=$(jq -r '.' $file >/dev/null 2>/dev/null || echo "failed")
-
-  if [[ $lint == "failed" ]]; then
-    echo "${file} has invalid JSON"
+  if jq -r . "$file" >/dev/null 2>&1; then
+    echo "$file has invalid JSON"
     exit 1
   fi
 done

tools/update-docs.sh

@@ -2,11 +2,21 @@
 #
 # Update docs across all projects and modules
 #
-if [[ ! $(command -v terraform-docs) ]]; then
-  echo "Must install terraform-docs: https://github.com/segmentio/terraform-docs"
+set -euo pipefail
+
+if ! terraform-docs -v; then
+  echo >&2 "Must install terraform-docs: https://github.com/terraform-docs/terraform-docs"
   exit 1
 fi
 
-for i in $(find terraform/ -name main.tf |sed 's/\/main.tf//g'); do
-  terraform-docs --lockfile=false md $i > $i/README.md
+this_script_dir=$(dirname -- "${BASH_SOURCE[0]}")
+terraform_dir="${this_script_dir}/../terraform"
+
+find "$terraform_dir" -type f -name main.tf | while read -r main_tf; do
+  module=$(dirname "$main_tf")
+  terraform-docs --lockfile=false md "$module" > "${module}/README.md" &
 done
+wait
+
+# Ugly workaround for lack of shell job control when running on GitHub Actions.
+while pgrep terraform-docs >/dev/null; do sleep 1; done