From 523908407578971e5960514ac41d519a566e7d7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacob=20L=C3=A4rfors?= <1135394+jlarfors@users.noreply.github.com> Date: Tue, 24 Sep 2024 10:15:48 +0300 Subject: [PATCH] add compile checks to terragen tests Use `gotype` to check that generated Go code compiles. --- pkg/terragen/gowrapper_test.go | 291 ++++++++++++++++++++++----------- pkg/testutil/timeout.go | 69 ++++++++ 2 files changed, 267 insertions(+), 93 deletions(-) create mode 100644 pkg/testutil/timeout.go diff --git a/pkg/terragen/gowrapper_test.go b/pkg/terragen/gowrapper_test.go index b4bbbe1..bb3cfaf 100644 --- a/pkg/terragen/gowrapper_test.go +++ b/pkg/terragen/gowrapper_test.go @@ -7,10 +7,14 @@ import ( "context" "encoding/json" "flag" + "fmt" + "log/slog" "os" + "os/exec" "path/filepath" "slices" "testing" + "time" "github.com/golingon/lingon/pkg/internal/terrajen" tu "github.com/golingon/lingon/pkg/testutil" @@ -18,7 +22,7 @@ import ( "golang.org/x/tools/txtar" ) -var update = flag.Bool("update", false, "update golden files") +var goldenTestDir = filepath.Join("testdata", "golden") type ProviderTestCase struct { Name string @@ -31,116 +35,139 @@ type ProviderTestCase struct { FilterDataSources []string } -// TestGenerateProvider tests the generation of Terraform provider schemas into -// Go code. -// -// When using the -update flag, it updates the golden files which are used as a -// baseline for detecting drift in the generated code. -// It is quite challenging to verify the correctness of the generated code, -// and it is out of scope of this pkg and test. -func TestGenerateProvider(t *testing.T) { - goldenTestDir := filepath.Join("testdata", "golden") +var providerTests = []ProviderTestCase{ + { + Name: "aws_emr_cluster", + ProviderName: "aws", + ProviderSource: "hashicorp/aws", + ProviderVersion: "5.44.0", - tests := []ProviderTestCase{ - { - Name: "aws_emr_cluster", - ProviderName: "aws", - ProviderSource: "hashicorp/aws", - ProviderVersion: "5.44.0", + FilterResources: []string{"aws_emr_cluster"}, + FilterDataSources: []string{"aws_emr_cluster"}, + }, + { + Name: "aws_iam_role", + ProviderName: "aws", + ProviderSource: "hashicorp/aws", + ProviderVersion: "5.44.0", - FilterResources: []string{"aws_emr_cluster"}, - FilterDataSources: []string{"aws_emr_cluster"}, - }, - { - Name: "aws_iam_role", - ProviderName: "aws", - ProviderSource: "hashicorp/aws", - ProviderVersion: "5.44.0", + FilterResources: []string{"aws_iam_role"}, + FilterDataSources: []string{"aws_iam_role"}, + }, + { + Name: "aws_securitylake_subscriber", + ProviderName: "aws", + ProviderSource: "hashicorp/aws", + ProviderVersion: "5.44.0", - FilterResources: []string{"aws_iam_role"}, - FilterDataSources: []string{"aws_iam_role"}, - }, - { - Name: "aws_securitylake_subscriber", - ProviderName: "aws", - ProviderSource: "hashicorp/aws", - ProviderVersion: "5.44.0", + FilterResources: []string{"aws_securitylake_subscriber"}, + FilterDataSources: []string{"aws_securitylake_subscriber"}, + }, + { + Name: "aws_globalaccelerator_cross_account_attachment", + ProviderName: "aws", + ProviderSource: "hashicorp/aws", + ProviderVersion: "5.47.0", - FilterResources: []string{"aws_securitylake_subscriber"}, - FilterDataSources: []string{"aws_securitylake_subscriber"}, + FilterResources: []string{ + "aws_globalaccelerator_cross_account_attachment", }, - { - Name: "aws_globalaccelerator_cross_account_attachment", - ProviderName: "aws", - ProviderSource: "hashicorp/aws", - ProviderVersion: "5.47.0", - - FilterResources: []string{ - "aws_globalaccelerator_cross_account_attachment", - }, - FilterDataSources: []string{ - "aws_globalaccelerator_cross_account_attachment", - }, + FilterDataSources: []string{ + "aws_globalaccelerator_cross_account_attachment", }, - } + }, +} + +func TestMain(m *testing.M) { + update := flag.Bool("update", false, "update golden files") + flag.Parse() if *update { - t.Log("running update") - // Generate "golden" files. Start be deleting the directory. - err := os.RemoveAll(goldenTestDir) - tu.AssertNoError(t, err, "removing golden test dir") - - for _, test := range tests { - ctx := context.Background() - ps, err := GenerateProviderSchema( - ctx, Provider{ - Name: test.ProviderName, - Source: test.ProviderSource, - Version: test.ProviderVersion, - }, - ) - tu.AssertNoError(t, err, "generating provider schema:", test.Name) + slog.Info("updating golden files and skipping tests") + if err := generatedGoldenFiles(); err != nil { + slog.Error("generating golden files", "error", err) + os.Exit(1) + } + // Skip running tests if updating golden files. + return + } + os.Exit(m.Run()) +} - // Filter resources and data sources. - for rName := range ps.ResourceSchemas { - if !slices.Contains(test.FilterResources, rName) { - delete(ps.ResourceSchemas, rName) - } - } - for dName := range ps.DataSourceSchemas { - if !slices.Contains(test.FilterDataSources, dName) { - delete(ps.DataSourceSchemas, dName) - } - } +func generatedGoldenFiles() error { + if err := os.RemoveAll(goldenTestDir); err != nil { + return fmt.Errorf("removing golden test dir: %w", err) + } - providerGenerator := terrajen.ProviderGenerator{ - GeneratedPackageLocation: "out", - ProviderName: test.ProviderName, - ProviderSource: test.ProviderSource, - ProviderVersion: test.ProviderVersion, + for _, test := range providerTests { + ctx := context.Background() + ps, err := GenerateProviderSchema( + ctx, Provider{ + Name: test.ProviderName, + Source: test.ProviderSource, + Version: test.ProviderVersion, + }, + ) + if err != nil { + return fmt.Errorf("generating provider schema: %w", err) + } + + // Filter resources and data sources. + for rName := range ps.ResourceSchemas { + if !slices.Contains(test.FilterResources, rName) { + delete(ps.ResourceSchemas, rName) + } + } + for dName := range ps.DataSourceSchemas { + if !slices.Contains(test.FilterDataSources, dName) { + delete(ps.DataSourceSchemas, dName) } + } - ar, err := generateProviderTxtar(providerGenerator, ps) - tu.AssertNoError(t, err, "generating provider txtar") + providerGenerator := terrajen.ProviderGenerator{ + GeneratedPackageLocation: "out", + ProviderName: test.ProviderName, + ProviderSource: test.ProviderSource, + ProviderVersion: test.ProviderVersion, + } - testDir := filepath.Join(goldenTestDir, test.Name) - err = os.MkdirAll(testDir, os.ModePerm) - tu.AssertNoError(t, err, "creating golden test dir: ", testDir) + ar, err := generateProviderTxtar(providerGenerator, ps) + if err != nil { + return fmt.Errorf("generating provider txtar: %w", err) + } - schemaPath := filepath.Join(testDir, "schema.json") - schemaFile, err := os.Create(schemaPath) - tu.AssertNoError(t, err, "creating schema file") - err = json.NewEncoder(schemaFile).Encode(ps) - tu.AssertNoError(t, err, "writing schema file") + testDir := filepath.Join(goldenTestDir, test.Name) + if err := os.MkdirAll(testDir, os.ModePerm); err != nil { + return fmt.Errorf("creating golden test dir: %w", err) + } - txtarPath := filepath.Join(testDir, "provider.txtar") - err = os.WriteFile(txtarPath, txtar.Format(ar), 0o644) - tu.AssertNoError(t, err, "writing txtar file") + schemaPath := filepath.Join(testDir, "schema.json") + schemaFile, err := os.Create(schemaPath) + if err != nil { + return fmt.Errorf("creating schema file: %w", err) + } + if err := json.NewEncoder(schemaFile).Encode(ps); err != nil { + return fmt.Errorf("writing schema file: %w", err) } - t.SkipNow() + txtarPath := filepath.Join(testDir, "provider.txtar") + if err := os.WriteFile(txtarPath, txtar.Format(ar), 0o644); err != nil { + return fmt.Errorf("writing txtar file: %w", err) + } } + return nil +} - for _, test := range tests { +// TestGenerateProvider tests that the generated Go code for a Terraform +// provider matches the golden tests. +// This is to ensure that the generated code is consistent and does not change, +// unless we want it to. +// +// It is quite challenging to verify the correctness of the generated code, +// and it is out of scope of this pkg and test. +func TestGenerateProvider(t *testing.T) { + goldenTestDir := filepath.Join("testdata", "golden") + + for _, test := range providerTests { t.Run(test.Name, func(t *testing.T) { schemaPath := filepath.Join(goldenTestDir, test.Name, "schema.json") schemaFile, err := os.Open(schemaPath) @@ -224,3 +251,81 @@ func TestParseProvider(t *testing.T) { ) } } + +var ( + // gotype is for linting code + gotypeCmd = "golang.org/x/tools/cmd/gotype" + gotypeVersion = "@v0.25.0" + gotype = gotypeCmd + gotypeVersion +) + +// TestCompileGenGoCode tests that the generated Go code compiles. +// This test is slow because compiles the generated Go code using `gotype`. +// https://pkg.go.dev/golang.org/x/tools/cmd/gotype +// +// This test has a dependency on `gotype`. It uses `go run ...` to execute +// `gotype`. +func TestCompileGenGoCode(t *testing.T) { + ctx := tu.WithTimeout(t, context.Background(), time.Minute*10) + for _, test := range providerTests { + t.Run(test.Name, func(t *testing.T) { + outDir := filepath.Join("out", test.Name) + schemaPath := filepath.Join(goldenTestDir, test.Name, "schema.json") + schemaFile, err := os.Open(schemaPath) + tu.AssertNoError(t, err, "opening schema file") + var providerSchema tfjson.ProviderSchema + err = json.NewDecoder(schemaFile).Decode(&providerSchema) + tu.AssertNoError(t, err, "decoding schema file") + + if err := GenerateGoCode( + GenerateGoArgs{ + ProviderName: test.ProviderName, + ProviderSource: test.ProviderSource, + ProviderVersion: test.ProviderVersion, + OutDir: outDir, + Force: false, + Clean: true, + }, + &providerSchema, + ); err != nil { + tu.AssertNoError(t, err) + } + dirs := dirsForProvider(outDir, &providerSchema) + for _, dir := range dirs { + t.Logf("running gotype on %s", dir) + goTypeExec(t, ctx, dir) + } + }) + } +} + +// goTypeExec executes `gotype` in the given directory. +func goTypeExec(t *testing.T, ctx context.Context, dir string) { + cmd := exec.CommandContext(ctx, "go", "run", gotype, "-v", ".") + cmd.Dir = dir + out, err := cmd.CombinedOutput() + if err != nil { + t.Logf("gotype output:\n%s", string(out)) + tu.AssertNoError(t, err) + } +} + +// dirsForProvider returns a list of directories that contain generated Go code +// based on a terraform provider. +func dirsForProvider(root string, schema *tfjson.ProviderSchema) []string { + dirsMap := map[string]struct{}{ + // Include root directory because it contains the provider.go file. + root: {}, + } + for key := range schema.ResourceSchemas { + dirsMap[filepath.Join(root, key)] = struct{}{} + } + for key := range schema.DataSourceSchemas { + dirsMap[filepath.Join(root, key)] = struct{}{} + } + dirs := make([]string, 0, len(dirsMap)) + for dir := range dirsMap { + dirs = append(dirs, dir) + } + return dirs +} diff --git a/pkg/testutil/timeout.go b/pkg/testutil/timeout.go new file mode 100644 index 0000000..a697578 --- /dev/null +++ b/pkg/testutil/timeout.go @@ -0,0 +1,69 @@ +package testutil + +import ( + "context" + "fmt" + "testing" + "time" +) + +// WithTimeout returns a context that will be canceled after the test timeout. +// If the test timeout given via `go test -timeout ` is less than the +// required timeout, the test will fail immediately. +// This is to prevent tests that you know require a long time to run from +// starting and then timing out. Might as well fail early. +func WithTimeout( + t *testing.T, + ctx context.Context, + timeout time.Duration, +) context.Context { + t.Helper() + deadline, hasDeadline := t.Deadline() + testTimeout := roundDurationUpToSecond(time.Until(deadline)) + // Is test timeout duration longer than the required timeout. + // Minus 1 second to avoid rounding errors. + isTestTimeoutOK := testTimeout >= timeout + + if hasDeadline && !isTestTimeoutOK { + t.Fatal( + Callers(), + fmt.Sprintf( + "test timeout (%s) less than required timeout (%s)", + testTimeout, + timeout, + ), + ) + } + + ctx, cancel := context.WithTimeout(ctx, time.Hour) + t.Cleanup(cancel) + + go func() { + tickAt := time.Minute * 5 + runningFor := time.Duration(0) + + ticker := time.NewTicker(tickAt) + defer ticker.Stop() + for { + select { + case <-ticker.C: + runningFor += tickAt + t.Logf( + "%s: test in progress: running for (%s), timeout in (%s)", + Callers(), + runningFor, + testTimeout-runningFor, + ) + case <-ctx.Done(): + return + } + } + }() + + return ctx +} + +// roundDurationUpToSecond rounds the duration up to the nearest second. +func roundDurationUpToSecond(d time.Duration) time.Duration { + return time.Duration(d.Seconds())*time.Second + time.Second +}