From c0c3149588e829eb2e4c4831e9a34fb32b53121c Mon Sep 17 00:00:00 2001 From: Jieyu Tian Date: Fri, 13 Sep 2024 17:44:18 +0000 Subject: [PATCH] Automatically generate type info for output in metadata file --- cli/bpmetadata/cmd.go | 70 ++++++++++++++++--- cli/bpmetadata/int-test/workflow.sh | 19 ++--- cli/bpmetadata/parser/state_parser.go | 49 +++++++++++++ infra/blueprint-test/pkg/tft/terraform.go | 36 +++++++++- infra/blueprint-test/pkg/utils/mock_t.go | 49 +++++++++++++ infra/blueprint-test/pkg/utils/mock_t_test.go | 56 +++++++++++++++ 6 files changed, 260 insertions(+), 19 deletions(-) create mode 100644 cli/bpmetadata/parser/state_parser.go create mode 100644 infra/blueprint-test/pkg/utils/mock_t.go create mode 100644 infra/blueprint-test/pkg/utils/mock_t_test.go diff --git a/cli/bpmetadata/cmd.go b/cli/bpmetadata/cmd.go index 5430a2bf3a3f..bf3496b58dc4 100644 --- a/cli/bpmetadata/cmd.go +++ b/cli/bpmetadata/cmd.go @@ -5,24 +5,30 @@ import ( "fmt" "os" "path" + "path/filepath" "strings" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/cli/bpmetadata/parser" "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/cli/util" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" "github.com/itchyny/json2yaml" "github.com/spf13/cobra" "github.com/spf13/viper" + "github.com/stretchr/testify/assert" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" + structpb "google.golang.org/protobuf/types/known/structpb" "sigs.k8s.io/yaml" ) var mdFlags struct { - path string - nested bool - force bool - display bool - validate bool - quiet bool + path string + nested bool + force bool + display bool + validate bool + quiet bool + genOutputType bool } const ( @@ -49,6 +55,7 @@ func init() { Cmd.Flags().BoolVar(&mdFlags.nested, "nested", true, "Flag for generating metadata for nested blueprint, if any.") Cmd.Flags().BoolVarP(&mdFlags.validate, "validate", "v", false, "Validate metadata against the schema definition.") Cmd.Flags().BoolVarP(&mdFlags.quiet, "quiet", "q", false, "Run in quiet mode suppressing all prompts.") + Cmd.Flags().BoolVarP(&mdFlags.genOutputType, "generate-output-type", "g", false, "Automatically generate type field for outputs.") } var Cmd = &cobra.Command{ @@ -148,6 +155,22 @@ func generateMetadataForBpPath(bpPath string) error { return fmt.Errorf("error creating metadata for blueprint at path: %s. Details: %w", bpPath, err) } + // If the flag is set, extract output types + if mdFlags.genOutputType { + Log.Info("jieyutian - genOutputType flag activated.") + outputTypes, err := extractOutputTypesFromState(bpPath) + if err != nil { + return fmt.Errorf("error extracting output types for blueprint at path: %s. Details: %w", bpPath, err) + } + + // Update the bpMetadata with the extracted output types + for i, output := range bpMetaObj.Spec.Interfaces.Outputs { + if outputType, ok := outputTypes[output.Name]; ok { + bpMetaObj.Spec.Interfaces.Outputs[i].Type = outputType + } + } + } + // write core metadata to disk err = WriteMetadata(bpMetaObj, bpPath, metadataFileName) if err != nil { @@ -180,6 +203,37 @@ func generateMetadataForBpPath(bpPath string) error { return nil } +// Extract output types from terraform.tfstate +func extractOutputTypesFromState(bpPath string) (map[string]*structpb.Value, error) { + outputTypes := make(map[string]*structpb.Value) + + // Construct the path to the test/setup directory + tfDir := filepath.Join(bpPath) + + root, _ := tft.NewTFBlueprintTestForNonTest( + tft.WithTFDir(tfDir), // Setup test at the blueprint path + ) + + root.DefineVerify(func(assert *assert.Assertions) { + stateFilePath := path.Join(bpPath, "terraform.tfstate") + stateData, err := os.ReadFile(stateFilePath) + if err != nil { + assert.FailNowf("Failed to read terraform.tfstate", "Error reading state file: %v", err) + } + + // Parse the state file and extract output types + parsedOutputTypes, err := parser.ParseOutputTypesFromState(stateData) + if err != nil { + assert.FailNowf("Failed to parse output types", "Error parsing output types: %v", err) + } + outputTypes = parsedOutputTypes + }) + + root.Test() // This will run terraform init and apply, then the DefineVerify function, and finally destroy + + return outputTypes, nil +} + func CreateBlueprintMetadata(bpPath string, bpMetadataObj *BlueprintMetadata) (*BlueprintMetadata, error) { // Verify that readme is present. readmeContent, err := os.ReadFile(path.Join(bpPath, readmeFileName)) @@ -243,8 +297,8 @@ func CreateBlueprintMetadata(bpPath string, bpMetadataObj *BlueprintMetadata) (* return nil, fmt.Errorf("error creating blueprint interfaces: %w", err) } - // Merge existing connections (if any) into the newly generated interfaces - mergeExistingConnections(bpMetadataObj.Spec.Interfaces, existingInterfaces) + // Merge existing connections (if any) into the newly generated interfaces + mergeExistingConnections(bpMetadataObj.Spec.Interfaces, existingInterfaces) // get blueprint requirements rolesCfgPath := path.Join(repoDetails.Source.BlueprintRootPath, tfRolesFileName) diff --git a/cli/bpmetadata/int-test/workflow.sh b/cli/bpmetadata/int-test/workflow.sh index 8365732901e9..c2c3686e1c04 100755 --- a/cli/bpmetadata/int-test/workflow.sh +++ b/cli/bpmetadata/int-test/workflow.sh @@ -38,20 +38,21 @@ fi # Create a temporary working folder to create assets for # the integration test. If the temp folder already exists, remove it. -if [ -d $WORKING_FOLDER ] -then - rm -r -f $WORKING_FOLDER -fi +# if [ -d $WORKING_FOLDER ] +# then +# rm -r -f $WORKING_FOLDER +# fi -mkdir $WORKING_FOLDER && cd $WORKING_FOLDER +# mkdir $WORKING_FOLDER && cd $WORKING_FOLDER +cd $WORKING_FOLDER # Get the blueprint package for v4.0.0 specifically because the golden metadata # to be validated is for that version. -git config --global advice.detachedHead false -git clone -b v4.0.0 --single-branch https://github.com/terraform-google-modules/terraform-google-cloud-storage.git "./$BLUPRINT_FOLDER/" -../../../bin/cft blueprint metadata -p $BLUPRINT_FOLDER -d -q -f +# git config --global advice.detachedHead false +# git clone -b v4.0.0 --single-branch https://github.com/terraform-google-modules/terraform-google-cloud-storage.git "./$BLUPRINT_FOLDER/" +../../../bin/cft blueprint metadata -p $BLUPRINT_FOLDER -d -q -f -g -mkdir $GIT_FOLDER +# mkdir $GIT_FOLDER cp "../$GOLDENS_FOLDER/$GOLDEN_METADATA" "$GIT_FOLDER/$WORKING_METADATA" cp "../$GOLDENS_FOLDER/$GOLDEN_DISPLAY_METADATA" "$GIT_FOLDER/$WORKING_DISPLAY_METADATA" diff --git a/cli/bpmetadata/parser/state_parser.go b/cli/bpmetadata/parser/state_parser.go new file mode 100644 index 000000000000..e3294238a426 --- /dev/null +++ b/cli/bpmetadata/parser/state_parser.go @@ -0,0 +1,49 @@ +package parser + +import ( + "encoding/json" + + "github.com/golang/protobuf/jsonpb" + "google.golang.org/protobuf/types/known/structpb" +) + +// ParseOutputTypesFromState parses the terraform.tfstate file and extracts the output types +func ParseOutputTypesFromState(stateData []byte) (map[string]*structpb.Value, error) { // Change return type + var state struct { + Outputs map[string]struct { + Type interface{} `json:"type"` + } `json:"outputs"` + } + + if err := json.Unmarshal(stateData, &state); err != nil { + return nil, err + } + + outputTypes := make(map[string]*structpb.Value) // Change map type + for outputName, outputData := range state.Outputs { + outputType, err := convertInterfaceToValue(outputData.Type) // Use convertInterfaceToValue + if err != nil { + return nil, err + } + outputTypes[outputName] = outputType + } + + return outputTypes, nil +} + +// convertInterfaceToValue converts an interface{} to a structpb.Value +func convertInterfaceToValue(v interface{}) (*structpb.Value, error) { + // Marshal the interface{} to JSON + jsonData, err := json.Marshal(v) + if err != nil { + return nil, err + } + + // Unmarshal the JSON into a structpb.Value + var val structpb.Value + if err := jsonpb.UnmarshalString(string(jsonData), &val); err != nil { + return nil, err + } + + return &val, nil +} diff --git a/infra/blueprint-test/pkg/tft/terraform.go b/infra/blueprint-test/pkg/tft/terraform.go index 89c20e44bbaa..7d90b2a56f97 100644 --- a/infra/blueprint-test/pkg/tft/terraform.go +++ b/infra/blueprint-test/pkg/tft/terraform.go @@ -250,6 +250,7 @@ func NewTFBlueprintTest(t testing.TB, opts ...tftOption) *TFBlueprintTest { loadTFEnvVar(tft.tfEnvVars, tft.getTFOutputsAsInputs(outputs)) if credsEnc, exists := tft.tfEnvVars[fmt.Sprintf("TF_VAR_%s", setupKeyOutputName)]; tft.saKey == "" && exists { if credDec, err := b64.StdEncoding.DecodeString(credsEnc); err == nil { + t.Logf("jieyutian - cred: %s", credDec) gcloud.ActivateCredsAndEnvVars(tft.t, string(credDec)) } else { tft.t.Fatalf("Unable to decode setup sa key: %v", err) @@ -267,11 +268,42 @@ func NewTFBlueprintTest(t testing.TB, opts ...tftOption) *TFBlueprintTest { tft.setupOutputOverrides[k] = v } - tftVersion := gjson.Get(terraform.RunTerraformCommand(tft.t, tft.GetTFOptions(), "version", "-json"), "terraform_version") - tft.logger.Logf(tft.t, "Running tests TF configs in %s with version %s", tft.tfDir, tftVersion) + // tftVersion := gjson.Get(terraform.RunTerraformCommand(tft.t, tft.GetTFOptions(), "version", "-json"), "terraform_version") + // tft.logger.Logf(tft.t, "Running tests TF configs in %s with version %s", tft.tfDir, tftVersion) return tft } +// TFBlueprintTestForNonTest encapsulates the necessary parts of TFBlueprintTest +// for use outside of a testing context. +type TFBlueprintTestForNonTest struct { + *TFBlueprintTest + Assertions *assert.Assertions // Add an Assertions field +} + +// NewTFBlueprintTestForNonTest creates a TFBlueprintTest instance suitable for +// non-testing scenarios. +func NewTFBlueprintTestForNonTest(opts ...tftOption) (*TFBlueprintTestForNonTest, error) { + // Create a mock testing.T implementation + mockT := &utils.MockT{T: &gotest.T{}} + + // Create a custom logger with minimal configuration + customLogger := logger.Discard + + // Append the custom logger option + opts = append(opts, WithLogger(customLogger)) + + // Create the underlying TFBlueprintTest + tft := NewTFBlueprintTest(mockT, opts...) + + // If you need assertions in your non-test context, initialize them here: + assertions := assert.New(mockT) + + return &TFBlueprintTestForNonTest{ + TFBlueprintTest: tft, + Assertions: assertions, + }, nil +} + // sensitiveOutputs returns a map of sensitive output keys for module in dir. func (b *TFBlueprintTest) sensitiveOutputs(dir string) map[string]bool { mod, err := tfconfig.LoadModule(dir) diff --git a/infra/blueprint-test/pkg/utils/mock_t.go b/infra/blueprint-test/pkg/utils/mock_t.go new file mode 100644 index 000000000000..4f4f01533a63 --- /dev/null +++ b/infra/blueprint-test/pkg/utils/mock_t.go @@ -0,0 +1,49 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "fmt" + "testing" +) + +// MockT implements the testing.TB interface with minimal functionality +type MockT struct { + *testing.T +} + +// Logf is required by testing.TB but does nothing in this mock +func (m *MockT) Logf(format string, args ...any) { + fmt.Printf("MockT.Logf called with format: %s, args: %v\n", format, args) +} + +// Errorf is required by testing.TB but does nothing in this mock +func (m *MockT) Errorf(format string, args ...any) { + fmt.Printf("MockT.Errorf called with format: %s, args: %v\n", format, args) +} + +// Fatal is required by testing.TB but does nothing in this mock +func (m *MockT) Fatal(args ...any) { + fmt.Printf("MockT.Fatal called with args: %v\n", args) + panic(fmt.Sprint(args...)) // Call panic to stop execution +} + +// Fatalf is required by testing.TB but does nothing in this mock +func (m *MockT) Fatalf(format string, args ...any) { + fmt.Printf("MockT.Fatalf called with format: %s, args: %v\n", format, args) + panic(fmt.Sprintf(format, args...)) // Call panic to stop execution +} diff --git a/infra/blueprint-test/pkg/utils/mock_t_test.go b/infra/blueprint-test/pkg/utils/mock_t_test.go new file mode 100644 index 000000000000..dfeeea28289c --- /dev/null +++ b/infra/blueprint-test/pkg/utils/mock_t_test.go @@ -0,0 +1,56 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils_test + +import ( + "testing" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils" +) + +func TestMockT(t *testing.T) { + mockT := &utils.MockT{T: t} + + // Test Logf + mockT.Logf("test logf with %s", "argument") + // Check the output (you might need to capture it for a more robust test) + + // Test Errorf + mockT.Errorf("test errorf with %d", 123) + // Check the output + + // Test Fatal + // Note: Fatal will stop execution, so use a subtest or defer to recover + t.Run("Test Fatal", func(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Errorf("mockT.Fatal did not panic") + } + }() + mockT.Fatal("test fatal") + }) + + // Test Fatalf + t.Run("Test Fatalf", func(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Errorf("mockT.Fatalf did not panic") + } + }() + mockT.Fatalf("test fatalf with %s", "argument") + }) +}