From 71a19f3eef8820c0d6cb2bbb41d5b1da68f7de16 Mon Sep 17 00:00:00 2001 From: Jieyu Tian Date: Fri, 13 Sep 2024 17:44:18 +0000 Subject: [PATCH] Automatically generate type info for output in metadata file --- cli/bpmetadata/cmd.go | 77 ++++++++- cli/bpmetadata/parser/state_parser.go | 90 +++++++++++ cli/bpmetadata/parser/state_parser_test.go | 180 +++++++++++++++++++++ 3 files changed, 339 insertions(+), 8 deletions(-) create mode 100644 cli/bpmetadata/parser/state_parser.go create mode 100644 cli/bpmetadata/parser/state_parser_test.go diff --git a/cli/bpmetadata/cmd.go b/cli/bpmetadata/cmd.go index 5430a2bf3a3..c2e3eafdf45 100644 --- a/cli/bpmetadata/cmd.go +++ b/cli/bpmetadata/cmd.go @@ -2,27 +2,35 @@ package bpmetadata import ( "errors" + "flag" "fmt" "os" "path" + "path/filepath" "strings" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/cli/bpmetadata/parser" "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/cli/util" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" "github.com/itchyny/json2yaml" + testingiface "github.com/mitchellh/go-testing-interface" "github.com/spf13/cobra" "github.com/spf13/viper" + "github.com/stretchr/testify/assert" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" + structpb "google.golang.org/protobuf/types/known/structpb" "sigs.k8s.io/yaml" ) var mdFlags struct { - path string - nested bool - force bool - display bool - validate bool - quiet bool + path string + nested bool + force bool + display bool + validate bool + quiet bool + genOutputType bool } const ( @@ -49,6 +57,7 @@ func init() { Cmd.Flags().BoolVar(&mdFlags.nested, "nested", true, "Flag for generating metadata for nested blueprint, if any.") Cmd.Flags().BoolVarP(&mdFlags.validate, "validate", "v", false, "Validate metadata against the schema definition.") Cmd.Flags().BoolVarP(&mdFlags.quiet, "quiet", "q", false, "Run in quiet mode suppressing all prompts.") + Cmd.Flags().BoolVarP(&mdFlags.genOutputType, "generate-output-type", "g", false, "Automatically generate type field for outputs.") } var Cmd = &cobra.Command{ @@ -148,6 +157,21 @@ func generateMetadataForBpPath(bpPath string) error { return fmt.Errorf("error creating metadata for blueprint at path: %s. Details: %w", bpPath, err) } + // If the flag is set, extract output types + if mdFlags.genOutputType { + outputTypes, err := extractOutputTypesFromState(bpPath) + if err != nil { + return fmt.Errorf("error extracting output types for blueprint at path: %s. Details: %w", bpPath, err) + } + + // Update the bpMetadata with the extracted output types + for i, output := range bpMetaObj.Spec.Interfaces.Outputs { + if outputType, ok := outputTypes[output.Name]; ok { + bpMetaObj.Spec.Interfaces.Outputs[i].Type = outputType + } + } + } + // write core metadata to disk err = WriteMetadata(bpMetaObj, bpPath, metadataFileName) if err != nil { @@ -180,6 +204,43 @@ func generateMetadataForBpPath(bpPath string) error { return nil } +// Extract output types from terraform.tfstate +func extractOutputTypesFromState(bpPath string) (map[string]*structpb.Value, error) { + outputTypes := make(map[string]*structpb.Value) + + // Construct the path to the test/setup directory + tfDir := filepath.Join(bpPath) + + // testing.T chekcs verbose flag to determine it's mode. Add this line as a flags initializer + // so the program doesn't panic. + flag.Parse() + runtimeT := testingiface.RuntimeT{} + + root := tft.NewTFBlueprintTest( + &runtimeT, + tft.WithTFDir(tfDir), // Setup test at the blueprint path + ) + + root.DefineVerify(func(assert *assert.Assertions) { + stateFilePath := path.Join(bpPath, "terraform.tfstate") + stateData, err := os.ReadFile(stateFilePath) + if err != nil { + assert.FailNowf("Failed to read terraform.tfstate", "Error reading state file: %v", err) + } + + // Parse the state file and extract output types + parsedOutputTypes, err := parser.ParseOutputTypesFromState(stateData) + if err != nil { + assert.FailNowf("Failed to parse output types", "Error parsing output types: %v", err) + } + outputTypes = parsedOutputTypes + }) + + root.Test() // This will run terraform init and apply, then the DefineVerify function, and finally destroy + + return outputTypes, nil +} + func CreateBlueprintMetadata(bpPath string, bpMetadataObj *BlueprintMetadata) (*BlueprintMetadata, error) { // Verify that readme is present. readmeContent, err := os.ReadFile(path.Join(bpPath, readmeFileName)) @@ -243,8 +304,8 @@ func CreateBlueprintMetadata(bpPath string, bpMetadataObj *BlueprintMetadata) (* return nil, fmt.Errorf("error creating blueprint interfaces: %w", err) } - // Merge existing connections (if any) into the newly generated interfaces - mergeExistingConnections(bpMetadataObj.Spec.Interfaces, existingInterfaces) + // Merge existing connections (if any) into the newly generated interfaces + mergeExistingConnections(bpMetadataObj.Spec.Interfaces, existingInterfaces) // get blueprint requirements rolesCfgPath := path.Join(repoDetails.Source.BlueprintRootPath, tfRolesFileName) diff --git a/cli/bpmetadata/parser/state_parser.go b/cli/bpmetadata/parser/state_parser.go new file mode 100644 index 00000000000..33314042e98 --- /dev/null +++ b/cli/bpmetadata/parser/state_parser.go @@ -0,0 +1,90 @@ +package parser + +import ( + "bytes" + "encoding/json" + "fmt" + + "google.golang.org/protobuf/types/known/structpb" + + tfjson "github.com/hashicorp/terraform-json" + "github.com/zclconf/go-cty/cty" +) + +func ParseOutputTypesFromState(stateData []byte) (map[string]*structpb.Value, error) { + // Unmarshal the state data into a map[string]interface{} first + var rawState map[string]interface{} + err := json.Unmarshal(stateData, &rawState) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal state data: %w", err) + } + + // Check if "format_version" key exists + if _, ok := rawState["format_version"]; !ok { + // If not present, add it with a default value + rawState["format_version"] = "1.0" + } + + // Create a "values" field with "outputs" inside + rawState["values"] = map[string]interface{}{ + "outputs": rawState["outputs"], + } + + // Remove the top-level "outputs" field + delete(rawState, "outputs") + + // Now marshal the updated map back to JSON + updatedStateData, err := json.MarshalIndent(rawState, "", " ") + if err != nil { + return nil, fmt.Errorf("failed to marshal updated state data: %w", err) + } + + var state tfjson.State + + // Unmarshal the updated JSON into tfjson.State + err = json.Unmarshal(updatedStateData, &state) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal state data: %w", err) + } + + outputTypeMap := make(map[string]*structpb.Value) + for name, output := range state.Values.Outputs { + pbValue, err := convertOutputTypeToStructpb(output) + if err != nil { + return nil, fmt.Errorf("failed to convert output %q to structpb.Value: %w", name, err) + } + outputTypeMap[name] = pbValue + } + + return outputTypeMap, nil +} + +func convertOutputTypeToStructpb(output *tfjson.StateOutput) (*structpb.Value, error) { + // Handle nil values explicitly + if output.Value == nil { + return structpb.NewNullValue(), nil + } + + // Handle cases where output.Type is NilType + if output.Type == cty.NilType { + return structpb.NewNullValue(), nil + } + + // Marshal the output value to JSON + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + err := enc.Encode(output.Type) + if err != nil { + return nil, fmt.Errorf("failed to marshal output type to JSON: %w", err) + } + + // Unmarshal the JSON into a structpb.Value + pbValue := &structpb.Value{} + err = pbValue.UnmarshalJSON(buf.Bytes()) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal JSON into structpb.Value: %w", err) + } + + return pbValue, nil +} diff --git a/cli/bpmetadata/parser/state_parser_test.go b/cli/bpmetadata/parser/state_parser_test.go new file mode 100644 index 00000000000..e8027d56780 --- /dev/null +++ b/cli/bpmetadata/parser/state_parser_test.go @@ -0,0 +1,180 @@ +package parser + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/protobuf/types/known/structpb" +) + +func TestParseOutputTypesFromState_WithSimpleTypes(t *testing.T) { + t.Parallel() + stateData := []byte(` +{ + "version": 4, + "terraform_version": "0.12.0", + "serial": 8, + "lineage": "491fd8f4-81b5-9890-520c-8a173c36e483", + "outputs": { + "boolean_output": { + "value": true, + "type": "bool" + }, + "number_output": { + "value": 42, + "type": "number" + }, + "string_output": { + "value": "foo", + "type": "string" + } + }, + "resources": [] +} +`) + want := map[string]*structpb.Value{ + "boolean_output": structpb.NewStringValue("bool"), + "number_output": structpb.NewStringValue("number"), + "string_output": structpb.NewStringValue("string"), + } + got, err := ParseOutputTypesFromState(stateData) + if err != nil { + t.Errorf("ParseOutputTypesFromState() error = %v", err) + return + } + if diff := cmp.Diff(got, want, cmp.Comparer(compareStructpbValues)); diff != "" { + t.Errorf("ParseOutputTypesFromState() mismatch (-got +want):\n%s", diff) + } +} + +func TestParseOutputTypesFromState_WithComplexTypes(t *testing.T) { + t.Parallel() + stateData := []byte(` +{ + "version": 4, + "terraform_version": "0.12.0", + "serial": 8, + "lineage": "491fd8f4-81b5-9890-520c-8a173c36e483", + "outputs": { + "interpolated_deep": { + "value": { + "foo": "bar", + "map": { + "bar": "baz", + "id": "424881806176056736" + }, + "number": 42 + }, + "type": [ + "object", + { + "foo": "string", + "map": [ + "object", + { + "bar": "string", + "id": "string" + } + ], + "number": "number" + } + ] + }, + "list": { + "value": [ + "foo", + "bar" + ], + "type": [ + "tuple", + [ + "string", + "string" + ] + ] + }, + "map": { + "value": { + "foo": "bar", + "number": 42 + }, + "type": [ + "object", + { + "foo": "string", + "number": "number" + } + ] + } + }, + "resources": [] +} +`) + want := map[string]*structpb.Value{ + "interpolated_deep": structpb.NewListValue(&structpb.ListValue{Values: []*structpb.Value{ + structpb.NewStringValue("object"), + structpb.NewStructValue(&structpb.Struct{Fields: map[string]*structpb.Value{ + "foo": structpb.NewStringValue("string"), + "map": structpb.NewListValue(&structpb.ListValue{Values: []*structpb.Value{structpb.NewStringValue("object"), structpb.NewStructValue(&structpb.Struct{Fields: map[string]*structpb.Value{"bar": structpb.NewStringValue("string"), "id": structpb.NewStringValue("string")}})}}), + "number": structpb.NewStringValue("number"), + }}), + }}), + "list": structpb.NewListValue(&structpb.ListValue{Values: []*structpb.Value{ + structpb.NewStringValue("tuple"), + structpb.NewListValue(&structpb.ListValue{Values: []*structpb.Value{structpb.NewStringValue("string"), structpb.NewStringValue("string")}}), + }}), + "map": structpb.NewListValue(&structpb.ListValue{Values: []*structpb.Value{ + structpb.NewStringValue("object"), + structpb.NewStructValue(&structpb.Struct{Fields: map[string]*structpb.Value{ + "foo": structpb.NewStringValue("string"), + "number": structpb.NewStringValue("number"), + }}), + }}), + } + got, err := ParseOutputTypesFromState(stateData) + if err != nil { + t.Errorf("ParseOutputTypesFromState() error = %v", err) + return + } + if diff := cmp.Diff(got, want, cmp.Comparer(compareStructpbValues)); diff != "" { + t.Errorf("ParseOutputTypesFromState() mismatch (-got +want):\n%s", diff) + } +} + +func TestParseOutputTypesFromState_WithoutTypes(t *testing.T) { + t.Parallel() + stateData := []byte(` +{ + "version": 4, + "terraform_version": "0.12.0", + "serial": 8, + "lineage": "491fd8f4-81b5-9890-520c-8a173c36e483", + "outputs": { + "no_type_output": { + "value": "some_value" + } + }, + "resources": [] +} +`) + want := map[string]*structpb.Value{ + "no_type_output": structpb.NewNullValue(), // Expecting null value when type is missing + } + + got, err := ParseOutputTypesFromState(stateData) + if err != nil { + t.Errorf("ParseOutputTypesFromState() error = %v", err) + return + } + if diff := cmp.Diff(got, want, cmp.Comparer(compareStructpbValues)); diff != "" { + t.Errorf("ParseOutputTypesFromState() mismatch (-got +want):\n%s", diff) + } +} + +// compareStructpbValues is a custom comparer for structpb.Value +func compareStructpbValues(x, y *structpb.Value) bool { + // Marshal to JSON and compare the JSON strings + xJSON, _ := x.MarshalJSON() + yJSON, _ := y.MarshalJSON() + return string(xJSON) == string(yJSON) +}