diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 2268f78..68f3e18 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -37,3 +37,24 @@ jobs: go-version: ${{ matrix.go }} - name: Test run: go test ./... + e2e: + name: End-to-End test (UBI 10) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + go: + - 'stable' + - 'oldstable' + container: registry.access.redhat.com/ubi10:latest + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Setup Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + go-version: ${{ matrix.go }} + - name: Install Dependencies + run: dnf install -y make jq util-linux + - name: E2E Tests + run: make e2e diff --git a/.gitignore b/.gitignore index 97b89a5..46c3d7c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ /feeds /reports /vendor -/cvetool \ No newline at end of file +/cvetool +/*.db* diff --git a/Makefile b/Makefile index 7cbc234..0340532 100644 --- a/Makefile +++ b/Makefile @@ -15,3 +15,7 @@ cvetool: vendor clean: rm -f cvetool rm -rf vendor + +.PHONY: e2e +e2e: cvetool + bash tests/e2e.sh diff --git a/tests/e2e.sh b/tests/e2e.sh new file mode 100644 index 0000000..2aaf60e --- /dev/null +++ b/tests/e2e.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +set -u -o pipefail + +cvetool="./cvetool" +failures=0 + +fail() { + echo "" + echo "==========================================" + echo "FAIL: $1" + echo "==========================================" + if [ -n "${2:-}" ]; then + echo "--- output ---" + echo "$2" + echo "--- end output ---" + fi + echo "" + failures=$((failures + 1)) +} + +# Test: `cvetool --version` exits successfully +echo "Test: cvetool --version exits successfully..." +version_output=$("$cvetool" --version 2>&1) || { + fail "cvetool --version exited non-zero" "${version_output:-}" +} +if [ -z "${version_output:-}" ]; then + fail "cvetool --version produced no output" +else + echo "PASS: cvetool --version output: $version_output" +fi + +# Test: `cvetool update` should not produce any warnings +echo "Test: cvetool update produces no warnings..." +tmpdb=$(mktemp) +unwritable_dir="" +cleanup() { + [ -n "$unwritable_dir" ] && chmod 0755 "$unwritable_dir" 2>/dev/null && rm -rf "$unwritable_dir" + rm -f "$tmpdb" +} +trap cleanup EXIT +update_ok=true +update_output=$("$cvetool" -l debug update --db-path "$tmpdb" 2>&1) || { + fail "cvetool update exited non-zero" "${update_output:-}" + update_ok=false +} +if echo "${update_output:-}" | grep -qi "WARN"; then + fail "cvetool update produced warnings" "$update_output" +else + echo "PASS: no warnings in cvetool update output" +fi + +if [ "$update_ok" = true ]; then + # Test: `cvetool scan` exits successfully (warnings are acceptable) + echo "Test: cvetool scan exits successfully..." + scan_output=$("$cvetool" -l debug scan --db-path "$tmpdb" 2>&1) || { + fail "cvetool scan exited non-zero" "${scan_output:-}" + } + if echo "${scan_output:-}" | grep -qi "ERR"; then + fail "cvetool scan produced errors" "$scan_output" + else + echo "PASS: cvetool scan exited successfully" + fi + + # Test: `cvetool scan --format sarif` produces valid JSON + echo "Test: cvetool scan --format sarif produces valid JSON..." + json_output=$("$cvetool" -l debug scan --db-path "$tmpdb" --format sarif 2>/dev/null) || { + fail "cvetool scan --format sarif exited non-zero" "${json_output:-}" + } + if ! echo "${json_output:-}" | jq . >/dev/null 2>&1; then + fail "cvetool scan --format sarif produced invalid JSON" "$json_output" + else + echo "PASS: cvetool scan --format sarif output is valid JSON" + fi + + # Test: `cvetool scan --format quay` produces valid JSON + echo "Test: cvetool scan --format quay produces valid JSON..." + json_output=$("$cvetool" -l debug scan --db-path "$tmpdb" --format quay 2>/dev/null) || { + fail "cvetool scan --format quay exited non-zero" "${json_output:-}" + } + if ! echo "${json_output:-}" | jq . >/dev/null 2>&1; then + fail "cvetool scan --format quay produced invalid JSON" "$json_output" + else + echo "PASS: cvetool scan --format quay output is valid JSON" + fi +else + echo "SKIP: scan tests skipped because cvetool update failed" +fi + +# Test: `cvetool scan` with a bad db path should fail +echo "Test: cvetool scan with bad db path fails..." +if bad_db_output=$("$cvetool" scan --db-path /nonexistent/bad.db 2>&1); then + fail "cvetool scan with bad db path exited 0 (expected failure)" "$bad_db_output" +else + echo "PASS: cvetool scan with bad db path exits non-zero" +fi + +# Test: `cvetool update` with an unwritable db path should fail +echo "Test: cvetool update with unwritable db path fails..." +unwritable_dir=$(mktemp -d) +chmod 0000 "$unwritable_dir" +if [ "$(id -u)" -eq 0 ]; then + if unwritable_output=$(runuser -u nobody -- "$cvetool" update --db-path "$unwritable_dir/db" 2>&1); then + fail "cvetool update with unwritable db path exited 0 (expected failure)" "${unwritable_output:-}" + else + echo "PASS: cvetool update with unwritable db path exits non-zero" + fi +else + if unwritable_output=$("$cvetool" update --db-path "$unwritable_dir/db" 2>&1); then + fail "cvetool update with unwritable db path exited 0 (expected failure)" "${unwritable_output:-}" + else + echo "PASS: cvetool update with unwritable db path exits non-zero" + fi +fi + +# Summary +echo "" +echo "==========================================" +if [ $failures -gt 0 ]; then + echo "DONE: $failures test(s) FAILED" + echo "==========================================" + exit 1 +else + echo "DONE: all tests passed" + echo "==========================================" +fi