Add -script, -skip-must-fail

For the benefit running in CI via https://github.com/toml-lang/setup-toml-test

Author: arp242

README.md

@@ -38,41 +38,15 @@ that directory by setting `GOBIN`; for example to use the current directory:
 
 [release page]: https://github.com/toml-lang/toml-test/releases
 
-### Installation in CI
-There are two options:
-
-1. Build from source as per the above; this is typically more than fast enough,
-   but requires Go to be installed. For example in GitHub actions:
-
-       steps:
-         - {uses: actions/setup-go@v4, with: {go-version: '1.24'}}
-         - run: 'go install github.com/toml-lang/toml-test/cmd/toml-test@latest'
-         # ~/go/bin should already be in $PATH on GitHub actions.
-
-2. Alternatively, you can download a release binary. These are statically linked
-   and should run anywhere. For example in GitHub actions:
-
-       steps:
-         - name: 'install toml-test'
-           run: |
-             version=v1.6.0
-             platform=linux-amd64
-             curl -sL https://github.com/toml-lang/toml-test/releases/download/$version/toml-test-$version-$platform.gz |
-                 gzip -d >toml-test
-             chmod a+x toml-test
-
-Many implementations don't pass all the tests, because they are "known bugs" or
-by choice (especially for "invalid" tests as some are a tad pedantic). You can
-use `-skip` to skip some tests; for example with a small shell script:
-
-    #!/usr/bin/env bash
-    skip=(
-        -skip 'invalid/foo'
-        -skip 'invalid/bar'
-    )
-    toml-test -toml=1.0 ${skip[@]} my-parser-cmd
-
-Use `toml-test my-parser-cmd -print-skip` to generate a script for all failures.
+Running in CI
+-------------
+The [setup-toml-test] action can be used in GitHub. See the README for more
+details.
+
+For other CI systems: the action essentially just downloads a release binary.
+See index.js. Should be easy enough to reproduce elsewhere.
+
+[setup-toml-test]: https://github.com/toml-lang/setup-toml-test
 
 Usage
 -----

cmd/toml-test/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	_ "embed"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -12,6 +13,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"text/template"
 	"time"
 
 	"github.com/BurntSushi/toml"
@@ -22,33 +24,42 @@ import (
 
 var hlErr = zli.Color256(224).Bg() | zli.Color256(0) | zli.Bold
 
+//go:embed script.gotxt
+var script []byte
+
+var scriptTemplate = template.Must(template.New("").Option("missingkey=error").Parse(string(script)))
+
 func parseFlags() (tomltest.Runner, []string, int, string, bool, bool, bool) {
 	f := zli.NewFlags(os.Args)
 	var (
-		help        = f.Bool(false, "help", "h")
-		versionFlag = f.IntCounter(0, "version", "V")
-		tomlVersion = f.String(tomltest.DefaultVersion, "toml")
-		encoder     = f.Bool(false, "encoder")
-		testDir     = f.String("", "testdir")
-		showAll     = f.IntCounter(0, "v")
-		color       = f.String("always", "color")
-		skip        = f.StringList(nil, "skip")
-		run         = f.StringList(nil, "run")
-		listFiles   = f.Bool(false, "list-files")
-		cat         = f.Int(0, "cat")
-		copyFiles   = f.Bool(false, "copy")
-		parallel    = f.Int(runtime.NumCPU(), "parallel")
-		printSkip   = f.Bool(false, "print-skip")
-		intAsFloat  = f.Bool(false, "int-as-float")
-		errors      = f.String("", "errors")
-		timeout     = f.String("1s", "timeout")
-		noNumber    = f.Bool(false, "no-number", "no_number")
+		help          = f.Bool(false, "help", "h")
+		versionFlag   = f.IntCounter(0, "version", "V")
+		tomlVersion   = f.String(tomltest.DefaultVersion, "toml")
+		encoder       = f.Bool(false, "encoder")
+		testDir       = f.String("", "testdir")
+		showAll       = f.IntCounter(0, "v")
+		color         = f.String("always", "color")
+		skip          = f.StringList(nil, "skip")
+		run           = f.StringList(nil, "run")
+		listFiles     = f.Bool(false, "list-files")
+		cat           = f.Int(0, "cat")
+		copyFiles     = f.Bool(false, "copy")
+		parallel      = f.Int(runtime.NumCPU(), "parallel")
+		script        = f.Bool(false, "script", "print-skip") // -print-skip is the old name
+		intAsFloat    = f.Bool(false, "int-as-float")
+		errors        = f.String("", "errors")
+		timeout       = f.String("1s", "timeout")
+		noNumber      = f.Bool(false, "no-number", "no_number")
+		skipMustError = f.Bool(false, "skip-error")
 	)
 	zli.F(f.Parse())
 	if help.Bool() {
 		fmt.Printf(usage, filepath.Base(os.Args[0]))
 		zli.Exit(0)
 	}
+	if script.Bool() && encoder.Bool() {
+		zli.Fatalf("cannot use -script and -encoder; generate a script without -encoder and fill in the encoder binary")
+	}
 	if tomlVersion.String() == "latest" {
 		*tomlVersion.Pointer() = tomltest.DefaultVersion
 	}
@@ -85,16 +96,17 @@ func parseFlags() (tomltest.Runner, []string, int, string, bool, bool, bool) {
 	}
 
 	r := tomltest.Runner{
-		Encoder:    encoder.Bool(),
-		RunTests:   run.StringsSplit(","),
-		SkipTests:  skip.StringsSplit(","),
-		Version:    tomlVersion.String(),
-		Parallel:   parallel.Int(),
-		Files:      fsys,
-		Parser:     tomltest.NewCommandParser(fsys, f.Args),
-		Timeout:    dur,
-		IntAsFloat: intAsFloat.Bool(),
-		Errors:     errs,
+		Encoder:       encoder.Bool(),
+		RunTests:      run.StringsSplit(","),
+		SkipTests:     skip.StringsSplit(","),
+		Version:       tomlVersion.String(),
+		Parallel:      parallel.Int(),
+		Files:         fsys,
+		Parser:        tomltest.NewCommandParser(fsys, f.Args),
+		Timeout:       dur,
+		IntAsFloat:    intAsFloat.Bool(),
+		SkipMustError: skipMustError.Bool(),
+		Errors:        errs,
 	}
 	if intAsFloat.Bool() {
 		r.SkipTests = append(r.SkipTests, "valid/integer/long")
@@ -132,7 +144,7 @@ func parseFlags() (tomltest.Runner, []string, int, string, bool, bool, bool) {
 		}
 	}
 
-	return r, f.Args, showAll.Int(), testDir.String(), listFiles.Bool(), printSkip.Bool(), noNumber.Bool()
+	return r, f.Args, showAll.Int(), testDir.String(), listFiles.Bool(), script.Bool(), noNumber.Bool()
 }
 
 func getFS(testDir string, set bool) fs.FS {
@@ -181,7 +193,7 @@ func getList(r tomltest.Runner) []string {
 }
 
 func main() {
-	runner, cmd, showAll, testDir, listFiles, printSkip, noNumber := parseFlags()
+	runner, cmd, showAll, testDir, listFiles, script, noNumber := parseFlags()
 
 	if listFiles {
 		l := getList(runner)
@@ -194,6 +206,28 @@ func main() {
 	tests, err := runner.Run()
 	zli.F(err)
 
+	//if script && (tests.FailedValid > 0 || tests.FailedInvalid > 0) {
+	if script {
+		var failedValid, failedInvalid []string
+		for _, f := range tests.Tests {
+			if f.Failed() {
+				if f.Type() == tomltest.TypeValid {
+					failedValid = append(failedValid, f.Path)
+				} else {
+					failedInvalid = append(failedInvalid, f.Path)
+				}
+			}
+		}
+		err := scriptTemplate.Execute(os.Stdout, struct {
+			Decoder       string
+			TOML          string
+			FailedValid   []string
+			FailedInvalid []string
+		}{strings.Join(cmd, " "), runner.Version, failedValid, failedInvalid})
+		zli.F(err)
+		return
+	}
+
 	for _, t := range tests.Tests {
 		if t.Failed() || showAll > 1 {
 			fmt.Print(detailed(runner, t, noNumber))
@@ -213,21 +247,6 @@ func main() {
 		fmt.Printf(", %2d skipped", tests.Skipped)
 	}
 
-	if printSkip && (tests.FailedValid > 0 || tests.FailedInvalid > 0) {
-		fmt.Print("\n\n    #!/usr/bin/env bash\n    # Also compatible with zsh.\n    skip=(\n")
-		for _, f := range tests.Tests {
-			if f.Failed() {
-				fmt.Printf("        -skip '%s'\n", f.Path)
-			}
-		}
-		fmt.Println("    )")
-		fmt.Printf("    toml-test -toml=%s ${skip[@]} %s", runner.Version, strings.Join(cmd, " "))
-		if runner.Encoder {
-			fmt.Print(" -encoder")
-		}
-		fmt.Println()
-	}
-
 	fmt.Println()
 	if runner.Encoder {
 		fmt.Printf("encoder tests: %3d passed, %2d failed\n", tests.PassedValid, tests.FailedValid)

cmd/toml-test/script.gotxt

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# Also compatible with zsh, but not POSIX sh.
+#
+# Run the toml-test compliance tests: https://github.com/toml-lang/toml-test
+
+decoder="{{.Decoder}}"
+
+# Leave empty to not test encoder tests.
+encoder=
+
+# Version of the TOML specification to test.
+toml={{.TOML}}
+
+# Skip known failures.
+skip=(
+{{- if .FailedValid}}
+	# Failing "valid" tests
+{{range $t := .FailedValid}}{{"\t"}}-skip '{{$t}}'
+{{$t}}{{end}}{{end}}
+{{- if .FailedInvalid}}
+	# Failing "invalid" tests
+{{range $t := .FailedInvalid}}{{"\t"}}-skip '{{$t}}'
+{{end}}{{end -}}
+)
+
+# Find toml-test
+tt=
+if [[ -x "./toml-test" ]]; then
+	tt="./toml-test"
+elif command -v "toml-test" >/dev/null; then
+	tt="toml-test"
+elif [[ -n "$(go env GOBIN)" ]] && [[ -x "$(go env GOBIN)/toml-test" ]]; then
+	tt="$(go env GOPATH)/toml-test"
+elif [[ -n "$(go env GOPATH)" ]] && [[ -x "$(go env GOPATH)/bin/toml-test" ]]; then
+	tt="$(go env GOPATH)/bin/toml-test"
+elif [[ -x "$HOME/go/bin/toml-test" ]]; then
+	tt="$HOME/go/bin/toml-test"
+fi
+if ! command -v "$tt" >/dev/null; then
+	echo >&2 'toml-test not in current dir, $PATH, $GOBIN, $GOPATH/bin, or $HOME/go/bin; install with:'
+	echo >&2 '    % go install github.com/toml-lang/toml-test/cmd/toml-test@latest'
+	echo >&2
+	echo >&2 'Or download a binary from:'
+	echo >&2 '    https://github.com/toml-lang/toml-test/releases'
+	exit 1
+fi
+
+fail=0
+"$tt" -toml="$toml" -skip-must-error ${skip[@]} -- "$decoder" "$@" || fail=1
+if [[ -n "$encoder" ]]; then
+	"$tt" -toml="$toml" -skip-must-error ${skip[@]} -encoder -- "$encoder" "$@" || fail=1
+fi
+exit $fail