feat: read plugin parse options from the config plugins blob

go.mod

@@ -15,9 +15,9 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/hashicorp/go-hclog v1.6.3
 	github.com/hashicorp/go-plugin v1.8.0
-	github.com/infracost/config v0.11.0
-	github.com/infracost/go-proto v1.25.1
-	github.com/infracost/proto v1.156.0
+	github.com/infracost/config v0.12.0
+	github.com/infracost/go-proto v1.26.0
+	github.com/infracost/proto v1.159.0
 	github.com/liamg/tml v0.7.1
 	github.com/maruel/panicparse/v2 v2.5.0
 	github.com/mattn/go-runewidth v0.0.16

go.sum

@@ -107,12 +107,12 @@ github.com/hashicorp/yamux v0.1.2 h1:XtB8kyFOyHXYVFnwT5C3+Bdo8gArse7j2AQ0DA0Uey8
 github.com/hashicorp/yamux v0.1.2/go.mod h1:C+zze2n6e/7wshOZep2A70/aQU6QBRWJO/G6FT1wIns=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
-github.com/infracost/config v0.11.0 h1:CYzIOu9bmbZVeMO6jqKn18VG55uTB/2nlgLFSsZDEX8=
-github.com/infracost/config v0.11.0/go.mod h1:IfWRgSkjAuSRowYwzGEw/fYTCDEGc/XU10U3sIj/YWs=
-github.com/infracost/go-proto v1.25.1 h1:34bfSUVRFsbTBASShBSQ+9IGvUmEg8C89X73Hem/Asw=
-github.com/infracost/go-proto v1.25.1/go.mod h1:9m9zdHPl6mME24y3J0CdTBWXr+d03mfgXuy+9JzA/wU=
-github.com/infracost/proto v1.156.0 h1:Kmlapuqqn0POosT0L2VgoW5GSUDU4ll9EfeQeKfkuCM=
-github.com/infracost/proto v1.156.0/go.mod h1:Z8vPWBWblwJlw+/ksO+BtsXwf9NiOcSTWx0WRWNbfUA=
+github.com/infracost/config v0.12.0 h1:woUV/aN10Q8605Y7GFcv2dHRG2ux8jZs1diY7Iv4sBo=
+github.com/infracost/config v0.12.0/go.mod h1:SBUzeK5+gj0mAukviMRxeqDIlmAbjukfdMDjTT3GjYY=
+github.com/infracost/go-proto v1.26.0 h1:DkvnNA3jbYp/i6PALl59JHO552W5Hw+bEhfhg5JH7e0=
+github.com/infracost/go-proto v1.26.0/go.mod h1:9m9zdHPl6mME24y3J0CdTBWXr+d03mfgXuy+9JzA/wU=
+github.com/infracost/proto v1.159.0 h1:fcgE4AApNvK8M4cbumOpRpAq+HlbVXzGaCdqCPkl6kA=
+github.com/infracost/proto v1.159.0/go.mod h1:Z8vPWBWblwJlw+/ksO+BtsXwf9NiOcSTWx0WRWNbfUA=
 github.com/jhump/protoreflect v1.17.0 h1:qOEr613fac2lOuTgWN4tPAtLL7fUSbuJL5X5XumQh94=
 github.com/jhump/protoreflect v1.17.0/go.mod h1:h9+vUUL38jiBzck8ck+6G/aeMX8Z4QUY/NiJPwPNi+8=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=

pkg/scanner/scan.go

@@ -115,22 +115,19 @@ func ScanProject(ctx context.Context, opts *ScanProjectOptions) (*ProjectResult,
 	}
 
 	genericOptions := buildGenericOptions(opts)
-	rawOptions, rawOptionsFormat, err := buildIaCOptions(opts, projectType)
+	rawOptions, err := buildIaCOptions(opts, projectType, genericOptions)
 	if err != nil {
 		return nil, fmt.Errorf("failed to build parser plugin options: %w", err)
 	}
 
-	// Try the parser-results cache first. The fingerprint folds in
-	// RawOptions + format so e.g. a different workspace or tfvars file
-	// invalidates correctly. Skipping the gRPC + HCL parse + module load
-	// is the entire point of this cache; the typical hot path on a 10k-
-	// project repo where one project changed is N-1 hits + 1 miss.
+	// Try the parser-results cache first. The fingerprint folds in RawOptions so e.g. a different
+	// workspace or tfvars file invalidates correctly. Skipping the gRPC + HCL parse + module load
+	// is the entire point of this cache; the typical hot path on a 10k-project repo where one
+	// project changed is N-1 hits + 1 miss.
 	pluginName := parserPlugin.Info.GetName()
 	pluginVersion := parserPlugin.Info.GetVersion()
 
-	fingerprintExtra := append([]byte(rawOptionsFormat), 0)
-	fingerprintExtra = append(fingerprintExtra, rawOptions...)
-	fingerprint, fpErr := fingerprintProject(absoluteProjectPath, fingerprintExtra)
+	fingerprint, fpErr := fingerprintProject(absoluteProjectPath, rawOptions)
 	if fpErr != nil {
 		logging.Debugf("parser fingerprint failed for %q: %s", absoluteProjectPath, fpErr)
 	}
@@ -142,10 +139,10 @@ func ScanProject(ctx context.Context, opts *ScanProjectOptions) (*ProjectResult,
 
 	if response == nil {
 		response, err = parserPlugin.Parse(ctx, &pluginpb.ParseRequest{
-			Path:             absoluteProjectPath,
-			GenericOptions:   genericOptions,
-			RawOptions:       rawOptions,
-			RawOptionsFormat: rawOptionsFormat,
+			Path:           absoluteProjectPath,
+			GenericOptions: genericOptions,
+			RawOptions:     rawOptions,
+			// raw_options_format was removed from the proto - raw_options is always JSON now.
 		})
 		if err != nil {
 			return nil, fmt.Errorf("parser plugin error: %w (run with --debug or set INFRACOST_CLI_LOG_LEVEL=debug for more details)", err)
@@ -292,103 +289,75 @@ func finalProjectType(projectType repoconfig.ProjectType, absoluteProjectPath st
 }
 
 func buildGenericOptions(opts *ScanProjectOptions) *options.GenericOptions {
-	return &options.GenericOptions{
+	genericOptions := &options.GenericOptions{
 		ProjectName:        opts.Project.Name,
 		EnvironmentName:    opts.Project.EnvName,
 		RepoDirectory:      opts.RootDir,
 		TemporaryDirectory: os.TempDir(),
 		CacheDirectory:     opts.CacheDir,
 		WorkingDirectory:   opts.RootDir,
+		// Workspace and the terraform cloud config are caller-sourced runtime options passed via
+		// GenericOptions (not the blob), because they are also read outside the plugin.
+		Workspace: opts.Project.Terraform.Workspace,
 	}
+
+	if c := opts.Project.Terraform.Cloud; c.Org != "" || c.Workspace != "" || c.Host != "" {
+		genericOptions.TerraformCloudConfiguration = &options.TerraformCloudConfiguration{
+			Organization: c.Org,
+			Workspace:    c.Workspace,
+			Hostname:     c.Host,
+		}
+	}
+
+	return genericOptions
 }
 
-func buildIaCOptions(opts *ScanProjectOptions, projectType repoconfig.ProjectType) ([]byte, string, error) {
+// buildIaCOptions returns the raw_options blob to send to the parser plugin for a project. The
+// persistable parse options live in the config plugins.<name> blob (config generates them, and
+// folds hand-written / older configs into them on load), so they are forwarded verbatim. The
+// caller-sourced runtime options are written into the typed GenericOptions fields, which the latest
+// plugins read in preference to the blob - so they are never duplicated into the blob.
+func buildIaCOptions(opts *ScanProjectOptions, projectType repoconfig.ProjectType, genericOptions *options.GenericOptions) ([]byte, error) {
 	switch projectType {
 	case repoconfig.ProjectTypeTerraform, repoconfig.ProjectTypeTerragrunt, repoconfig.ProjectTypeCiscoStacks:
-		data, err := json.Marshal(buildTerraformPluginOptions(opts, namingPolicyAttributeRequirements(opts.FinopsPolicies)))
-		return data, "application/json", err
+		if genericOptions != nil {
+			genericOptions.Env = opts.Project.Env
+			genericOptions.RequiredAttributes = protoAttributeRequirements(namingPolicyAttributeRequirements(opts.FinopsPolicies))
+		}
+		return pluginBlobJSON(opts.Project, string(projectType))
 	case repoconfig.ProjectTypeCloudFormation:
-		data, err := json.Marshal(buildCloudFormationPluginOptions(opts.Project))
-		return data, "application/json", err
+		// cdk_* projects resolve to cloudformation (see finalProjectType) and the config keys their
+		// blob under cloudformation too.
+		return pluginBlobJSON(opts.Project, string(repoconfig.ProjectTypeCloudFormation))
 	default:
-		return nil, "", nil
+		return nil, nil
 	}
 }
 
-type terraformPluginOptions struct {
-	RegexSourceMap              map[string]string            `json:"regexSourceMap,omitempty"`
-	Env                         map[string]string            `json:"env,omitempty"`
-	Vars                        map[string]any               `json:"vars,omitempty"`
-	Workspace                   string                       `json:"workspace,omitempty"`
-	TfVarsFiles                 []string                     `json:"tfVarsFiles,omitempty"`
-	TerraformCloudConfiguration *terraformCloudConfiguration `json:"terraformCloudConfiguration,omitempty"`
-	RequiredAttributes          map[string][]string          `json:"requiredAttributes,omitempty"`
-}
-
-type terraformCloudConfiguration struct {
-	Organization string `json:"organization"`
-	Workspace    string `json:"workspace"`
-	Hostname     string `json:"hostname"`
-}
-
-type cloudFormationPluginOptions struct {
-	InputParameters map[string]any `json:"inputParameters"`
-	AWSContext      *awsContext    `json:"awsContext"`
-}
-
-type awsContext struct {
-	Region    string `json:"region"`
-	AccountID string `json:"accountId"`
-	StackID   string `json:"stackId"`
-	StackName string `json:"stackName"`
-}
-
-func buildTerraformPluginOptions(opts *ScanProjectOptions, requiredAttributes []*parserapi.AttributeRequirement) *terraformPluginOptions {
-	regexSourceMap := make(map[string]string, len(opts.RepoConfig.Terraform.SourceMap))
-	for _, source := range opts.RepoConfig.Terraform.SourceMap {
-		regexSourceMap[source.Match] = source.Replace
-	}
-
-	var cloudConfig *terraformCloudConfiguration
-	if opts.Project.Terraform.Cloud.Org != "" || opts.Project.Terraform.Cloud.Workspace != "" || opts.Project.Terraform.Cloud.Host != "" {
-		cloudConfig = &terraformCloudConfiguration{
-			Organization: opts.Project.Terraform.Cloud.Org,
-			Workspace:    opts.Project.Terraform.Cloud.Workspace,
-			Hostname:     opts.Project.Terraform.Cloud.Host,
-		}
-	}
-
-	tfRequiredAttributes := make(map[string][]string, len(requiredAttributes))
-	for _, attr := range requiredAttributes {
-		tfRequiredAttributes[attr.ResourceType] = attr.Attributes
-	}
-
-	return &terraformPluginOptions{
-		RegexSourceMap:              regexSourceMap,
-		Env:                         opts.Project.Env,
-		Vars:                        opts.Project.Terraform.Vars,
-		Workspace:                   opts.Project.Terraform.Workspace,
-		TfVarsFiles:                 opts.Project.Terraform.VarFiles,
-		TerraformCloudConfiguration: cloudConfig,
-		RequiredAttributes:          tfRequiredAttributes,
+// pluginBlobJSON marshals the project's persisted plugins.<key> blob (keyed by the consuming plugin)
+// as JSON, returning nil when there is no blob for that key.
+func pluginBlobJSON(project *repoconfig.Project, key string) ([]byte, error) {
+	blob := project.Plugins[key]
+	if len(blob) == 0 {
+		return nil, nil
 	}
+	return json.Marshal(blob)
 }
 
-func buildCloudFormationPluginOptions(project *repoconfig.Project) *cloudFormationPluginOptions {
-	var awsCtx *awsContext
-	if project.AWS.Region != "" || project.AWS.AccountID != "" || project.AWS.StackID != "" || project.AWS.StackName != "" {
-		awsCtx = &awsContext{
-			Region:    project.AWS.Region,
-			AccountID: project.AWS.AccountID,
-			StackID:   project.AWS.StackID,
-			StackName: project.AWS.StackName,
-		}
+// protoAttributeRequirements converts the naming-policy attribute requirements into the typed
+// GenericOptions form.
+func protoAttributeRequirements(reqs []*parserapi.AttributeRequirement) []*options.AttributeRequirement {
+	if len(reqs) == 0 {
+		return nil
 	}
-
-	return &cloudFormationPluginOptions{
-		InputParameters: nil,
-		AWSContext:      awsCtx,
+	out := make([]*options.AttributeRequirement, 0, len(reqs))
+	for _, r := range reqs {
+		if r == nil {
+			continue
+		}
+		out = append(out, &options.AttributeRequirement{ResourceType: r.ResourceType, Attributes: r.Attributes})
 	}
+	return out
 }
 
 type namingValidationSettings struct {