From 4211be24afc73079dbe98426f59ba20ddb5b011b Mon Sep 17 00:00:00 2001 From: Jieyu Tian Date: Wed, 4 Sep 2024 13:56:41 +0000 Subject: [PATCH] Add new command to generate output types automatically --- cli/bpmetadata/cmd.go | 75 ++++++++++++++++++++--- cli/bpmetadata/int-test/workflow.sh | 19 +++--- cli/bpmetadata/parser/state_parser.go | 49 +++++++++++++++ infra/blueprint-test/pkg/tft/terraform.go | 60 +++++++++++++++++- 4 files changed, 184 insertions(+), 19 deletions(-) create mode 100644 cli/bpmetadata/parser/state_parser.go diff --git a/cli/bpmetadata/cmd.go b/cli/bpmetadata/cmd.go index 4aff3651bf7d..92ad2e139cc8 100644 --- a/cli/bpmetadata/cmd.go +++ b/cli/bpmetadata/cmd.go @@ -5,24 +5,30 @@ import ( "fmt" "os" "path" + "path/filepath" "strings" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/cli/bpmetadata/parser" "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/cli/util" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" "github.com/itchyny/json2yaml" "github.com/spf13/cobra" "github.com/spf13/viper" + "github.com/stretchr/testify/assert" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" + structpb "google.golang.org/protobuf/types/known/structpb" "sigs.k8s.io/yaml" ) var mdFlags struct { - path string - nested bool - force bool - display bool - validate bool - quiet bool + path string + nested bool + force bool + display bool + validate bool + quiet bool + genOutputType bool } const ( @@ -49,6 +55,7 @@ func init() { Cmd.Flags().BoolVar(&mdFlags.nested, "nested", true, "Flag for generating metadata for nested blueprint, if any.") Cmd.Flags().BoolVarP(&mdFlags.validate, "validate", "v", false, "Validate metadata against the schema definition.") Cmd.Flags().BoolVarP(&mdFlags.quiet, "quiet", "q", false, "Run in quiet mode suppressing all prompts.") + Cmd.Flags().BoolVarP(&mdFlags.genOutputType, "generate-output-type", "g", false, "Automatically generate type field for outputs.") } var Cmd = &cobra.Command{ @@ -148,6 +155,24 @@ func generateMetadataForBpPath(bpPath string) error { return fmt.Errorf("error creating metadata for blueprint at path: %s. Details: %w", bpPath, err) } + // If the flag is set, extract output types + if mdFlags.genOutputType { + Log.Info("jieyutian - genOutputType flag activated.") + outputTypes, err := extractOutputTypesFromState(bpPath) + if err != nil { + return fmt.Errorf("error extracting output types for blueprint at path: %s. Details: %w", bpPath, err) + } + + Log.Info("jieyutian - outputTypes:", outputTypes) + + // Update the bpMetadata with the extracted output types + for i, output := range bpMetaObj.Spec.Interfaces.Outputs { + if outputType, ok := outputTypes[output.Name]; ok { + bpMetaObj.Spec.Interfaces.Outputs[i].Type = outputType + } + } + } + // write core metadata to disk err = WriteMetadata(bpMetaObj, bpPath, metadataFileName) if err != nil { @@ -180,6 +205,40 @@ func generateMetadataForBpPath(bpPath string) error { return nil } +// Extract output types from terraform.tfstate +func extractOutputTypesFromState(bpPath string) (map[string]*structpb.Value, error) { + outputTypes := make(map[string]*structpb.Value) + + // Construct the path to the test/setup directory + tfDir := filepath.Join(bpPath, "test", "setup") + + // Create a fake testing.T for tft.NewTFBlueprintTest + // fakeT := &testing.T{} + + root, _ := tft.NewTFBlueprintTestForNonTest( + tft.WithTFDir(tfDir), // Setup test at the blueprint path + ) + + root.DefineVerify(func(assert *assert.Assertions) { + stateFilePath := path.Join(bpPath, "terraform.tfstate") + stateData, err := os.ReadFile(stateFilePath) + if err != nil { + assert.FailNowf("Failed to read terraform.tfstate", "Error reading state file: %v", err) + } + + // Parse the state file and extract output types + parsedOutputTypes, err := parser.ParseOutputTypesFromState(stateData) + if err != nil { + assert.FailNowf("Failed to parse output types", "Error parsing output types: %v", err) + } + outputTypes = parsedOutputTypes + }) + + root.Test() // This will run terraform init and apply, then the DefineVerify function, and finally destroy + + return outputTypes, nil +} + func CreateBlueprintMetadata(bpPath string, bpMetadataObj *BlueprintMetadata) (*BlueprintMetadata, error) { // Verify that readme is present. readmeContent, err := os.ReadFile(path.Join(bpPath, readmeFileName)) @@ -243,8 +302,8 @@ func CreateBlueprintMetadata(bpPath string, bpMetadataObj *BlueprintMetadata) (* return nil, fmt.Errorf("error creating blueprint interfaces: %w", err) } - // Merge existing connections (if any) into the newly generated interfaces - mergeExistingConnections(bpMetadataObj.Spec.Interfaces, existingInterfaces) + // Merge existing connections (if any) into the newly generated interfaces + mergeExistingConnections(bpMetadataObj.Spec.Interfaces, existingInterfaces) // get blueprint requirements rolesCfgPath := path.Join(repoDetails.Source.BlueprintRootPath, tfRolesFileName) diff --git a/cli/bpmetadata/int-test/workflow.sh b/cli/bpmetadata/int-test/workflow.sh index 8365732901e9..c2c3686e1c04 100755 --- a/cli/bpmetadata/int-test/workflow.sh +++ b/cli/bpmetadata/int-test/workflow.sh @@ -38,20 +38,21 @@ fi # Create a temporary working folder to create assets for # the integration test. If the temp folder already exists, remove it. -if [ -d $WORKING_FOLDER ] -then - rm -r -f $WORKING_FOLDER -fi +# if [ -d $WORKING_FOLDER ] +# then +# rm -r -f $WORKING_FOLDER +# fi -mkdir $WORKING_FOLDER && cd $WORKING_FOLDER +# mkdir $WORKING_FOLDER && cd $WORKING_FOLDER +cd $WORKING_FOLDER # Get the blueprint package for v4.0.0 specifically because the golden metadata # to be validated is for that version. -git config --global advice.detachedHead false -git clone -b v4.0.0 --single-branch https://github.com/terraform-google-modules/terraform-google-cloud-storage.git "./$BLUPRINT_FOLDER/" -../../../bin/cft blueprint metadata -p $BLUPRINT_FOLDER -d -q -f +# git config --global advice.detachedHead false +# git clone -b v4.0.0 --single-branch https://github.com/terraform-google-modules/terraform-google-cloud-storage.git "./$BLUPRINT_FOLDER/" +../../../bin/cft blueprint metadata -p $BLUPRINT_FOLDER -d -q -f -g -mkdir $GIT_FOLDER +# mkdir $GIT_FOLDER cp "../$GOLDENS_FOLDER/$GOLDEN_METADATA" "$GIT_FOLDER/$WORKING_METADATA" cp "../$GOLDENS_FOLDER/$GOLDEN_DISPLAY_METADATA" "$GIT_FOLDER/$WORKING_DISPLAY_METADATA" diff --git a/cli/bpmetadata/parser/state_parser.go b/cli/bpmetadata/parser/state_parser.go new file mode 100644 index 000000000000..e3294238a426 --- /dev/null +++ b/cli/bpmetadata/parser/state_parser.go @@ -0,0 +1,49 @@ +package parser + +import ( + "encoding/json" + + "github.com/golang/protobuf/jsonpb" + "google.golang.org/protobuf/types/known/structpb" +) + +// ParseOutputTypesFromState parses the terraform.tfstate file and extracts the output types +func ParseOutputTypesFromState(stateData []byte) (map[string]*structpb.Value, error) { // Change return type + var state struct { + Outputs map[string]struct { + Type interface{} `json:"type"` + } `json:"outputs"` + } + + if err := json.Unmarshal(stateData, &state); err != nil { + return nil, err + } + + outputTypes := make(map[string]*structpb.Value) // Change map type + for outputName, outputData := range state.Outputs { + outputType, err := convertInterfaceToValue(outputData.Type) // Use convertInterfaceToValue + if err != nil { + return nil, err + } + outputTypes[outputName] = outputType + } + + return outputTypes, nil +} + +// convertInterfaceToValue converts an interface{} to a structpb.Value +func convertInterfaceToValue(v interface{}) (*structpb.Value, error) { + // Marshal the interface{} to JSON + jsonData, err := json.Marshal(v) + if err != nil { + return nil, err + } + + // Unmarshal the JSON into a structpb.Value + var val structpb.Value + if err := jsonpb.UnmarshalString(string(jsonData), &val); err != nil { + return nil, err + } + + return &val, nil +} diff --git a/infra/blueprint-test/pkg/tft/terraform.go b/infra/blueprint-test/pkg/tft/terraform.go index 89c20e44bbaa..e2f360be0dbd 100644 --- a/infra/blueprint-test/pkg/tft/terraform.go +++ b/infra/blueprint-test/pkg/tft/terraform.go @@ -267,11 +267,67 @@ func NewTFBlueprintTest(t testing.TB, opts ...tftOption) *TFBlueprintTest { tft.setupOutputOverrides[k] = v } - tftVersion := gjson.Get(terraform.RunTerraformCommand(tft.t, tft.GetTFOptions(), "version", "-json"), "terraform_version") - tft.logger.Logf(tft.t, "Running tests TF configs in %s with version %s", tft.tfDir, tftVersion) + // tftVersion := gjson.Get(terraform.RunTerraformCommand(tft.t, tft.GetTFOptions(), "version", "-json"), "terraform_version") + // tft.logger.Logf(tft.t, "Running tests TF configs in %s with version %s", tft.tfDir, tftVersion) return tft } +// mockT implements the testing.TB interface with minimal functionality +type mockT struct { + *gotest.T +} + +// Logf is required by testing.TB but does nothing in this mock +func (m *mockT) Logf(format string, args ...any) { + fmt.Printf("mockT.Logf called with format: %s, args: %v\n", format, args) +} + +// Errorf is required by testing.TB but does nothing in this mock +func (m *mockT) Errorf(format string, args ...any) { + fmt.Printf("mockT.Errorf called with format: %s, args: %v\n", format, args) +} + +// Fatal is required by testing.TB but does nothing in this mock +func (m *mockT) Fatal(args ...any) { + fmt.Printf("mockT.Fatal called with args: %v\n", args) +} + +// Fatalf is required by testing.TB but does nothing in this mock +func (m *mockT) Fatalf(format string, args ...any) { + fmt.Printf("mockT.Fatalf called with format: %s, args: %v\n", format, args) +} + +// TFBlueprintTestForNonTest encapsulates the necessary parts of TFBlueprintTest +// for use outside of a testing context. +type TFBlueprintTestForNonTest struct { + *TFBlueprintTest + Assertions *assert.Assertions // Add an Assertions field +} + +// NewTFBlueprintTestForNonTest creates a TFBlueprintTest instance suitable for +// non-testing scenarios. +func NewTFBlueprintTestForNonTest(opts ...tftOption) (*TFBlueprintTestForNonTest, error) { + // Create a mock testing.T implementation + mockT := &mockT{T: &gotest.T{}} + + // Create a custom logger with minimal configuration + customLogger := logger.Discard + + // Append the custom logger option + opts = append(opts, WithLogger(customLogger)) + + // Create the underlying TFBlueprintTest + tft := NewTFBlueprintTest(mockT, opts...) + + // If you need assertions in your non-test context, initialize them here: + assertions := assert.New(mockT) + + return &TFBlueprintTestForNonTest{ + TFBlueprintTest: tft, + Assertions: assertions, + }, nil +} + // sensitiveOutputs returns a map of sensitive output keys for module in dir. func (b *TFBlueprintTest) sensitiveOutputs(dir string) map[string]bool { mod, err := tfconfig.LoadModule(dir)