feat: persist plugin raw_options blob in the config file

autodetect/node.go

@@ -18,6 +18,9 @@ type Node struct {
 	Depth           int
 	Parent          *Node
 	DependencyPaths []string
+	// RawOptions is the directory-level seed blob returned by the plugin's IdentifyProjects,
+	// threaded into IdentifyEnvironments. Empty when the plugin does not emit one.
+	RawOptions []byte
 }
 
 func (n *Node) IsRoot() bool {

autodetect/project.go

@@ -10,6 +10,11 @@ type Project struct {
 	Env               string
 	Type              types.ProjectType
 	Metadata          map[string]string
+	// RawOptions is the plugin-specific, opaque parse-options blob (JSON) for this project. It is
+	// either what the plugin returned from IdentifyEnvironments/IdentifyProjects, or - for plugins
+	// that don't yet emit one - a value backfilled from the data config derived itself (see
+	// rawoptions_backfill.go).
+	RawOptions []byte
 }
 
 type RootModule struct {

autodetect/rawoptions_backfill.go

@@ -0,0 +1,59 @@
+package autodetect
+
+import (
+	"encoding/json"
+
+	"github.com/infracost/config/types"
+)
+
+// BACKWARDS-COMPATIBILITY SHIM.
+//
+// This whole file exists to keep plugins that predate the raw_options blob working. Once every
+// parser plugin emits its own raw_options from IdentifyProjects/IdentifyEnvironments, delete this
+// file and the calls to resolveRawOptions in search.go.
+//
+// When a plugin does not return a raw_options blob, we synthesise one from the data config
+// derived itself (the attributed var files and the resolved environment) so the persisted blob
+// matches what the callers historically built from the config file. That deliberately duplicates
+// a subset of the plugin's option schema here; it is isolated in this file so it can be removed
+// wholesale later.
+
+// terraformRawOptions mirrors the subset of the terraform-family plugin options that config can
+// derive on its own. The JSON tags MUST match the plugin's options.Options wire shape.
+type terraformRawOptions struct {
+	TfVarsFiles []string `json:"tfVarsFiles,omitempty"`
+	Workspace   string   `json:"workspace,omitempty"`
+}
+
+// resolveRawOptions returns the blob to persist for a project: the plugin's own blob when it
+// provided one (either per-environment or the directory-level seed), otherwise a backfilled blob
+// derived from config's own attribution.
+func resolveRawOptions(pluginRawOptions, seedRawOptions []byte, projectType types.ProjectType, varFiles []string, workspace string) []byte {
+	if len(pluginRawOptions) > 0 {
+		return pluginRawOptions
+	}
+	if len(seedRawOptions) > 0 {
+		return seedRawOptions
+	}
+	return backfillRawOptions(projectType, varFiles, workspace)
+}
+
+// backfillRawOptions builds a raw_options blob from config-derived data for plugins that don't
+// emit one. Only the terraform family consumes the shape it produces; other types get no blob.
+func backfillRawOptions(projectType types.ProjectType, varFiles []string, workspace string) []byte {
+	switch projectType {
+	case types.ProjectTypeTerraform, types.ProjectTypeTerragrunt, types.ProjectTypeCiscoStacks:
+	default:
+		return nil
+	}
+
+	if len(varFiles) == 0 && workspace == "" {
+		return nil
+	}
+
+	blob, err := json.Marshal(terraformRawOptions{TfVarsFiles: varFiles, Workspace: workspace})
+	if err != nil {
+		return nil
+	}
+	return blob
+}

autodetect/search.go

@@ -340,7 +340,7 @@ func expandProjects(ctx context.Context, identifier *plugin.Identifier, projectN
 		var authoritative bool
 		if identifier != nil {
 			var err error
-			pluginEnvironments, authoritative, err = identifier.IdentifyEnvironments(ctx, project.AbsolutePath, projectType, attributedFiles)
+			pluginEnvironments, authoritative, err = identifier.IdentifyEnvironments(ctx, project.AbsolutePath, projectType, attributedFiles, project.RawOptions)
 			if err != nil {
 				return nil, nil, err
 			}
@@ -384,6 +384,7 @@ func expandProjects(ctx context.Context, identifier *plugin.Identifier, projectN
 					DependencyPaths:   escapeStringListForYAML(envDeps),
 					Env:               escapeStringForYAML(env.Name),
 					Type:              projectType,
+					RawOptions:        resolveRawOptions(env.RawOptions, project.RawOptions, projectType, varFiles, env.Name),
 				})
 
 				// record the directories this environment claims so they aren't also emitted as
@@ -408,6 +409,7 @@ func expandProjects(ctx context.Context, identifier *plugin.Identifier, projectN
 				DependencyPaths:   escapeStringListForYAML(deps),
 				Env:               "", // deliberately empty
 				Type:              projectType,
+				RawOptions:        resolveRawOptions(nil, project.RawOptions, projectType, globalFiles, ""),
 			})
 
 		case len(envFiles) > 0 && (project.IsTerraform() || (project.IsTerragrunt() && project.Terragrunt.LinkTFVars)):
@@ -455,6 +457,7 @@ func expandProjects(ctx context.Context, identifier *plugin.Identifier, projectN
 					DependencyPaths:   escapeStringListForYAML(deps),
 					Env:               escapeStringForYAML(envName),
 					Type:              projectType,
+					RawOptions:        resolveRawOptions(nil, project.RawOptions, projectType, tfvarFiles, envName),
 				})
 			}
 
@@ -466,6 +469,7 @@ func expandProjects(ctx context.Context, identifier *plugin.Identifier, projectN
 				DependencyPaths:   escapeStringListForYAML(deps),
 				Env:               "", // deliberately empty
 				Type:              projectType,
+				RawOptions:        resolveRawOptions(nil, project.RawOptions, projectType, globalFiles, ""),
 			})
 		}
 

autodetect/tree.go

@@ -182,6 +182,7 @@ func (b *treeBuilder) buildSubtree(ctx context.Context, path string, depth int,
 			}
 
 			node.ProjectType = pt
+			node.RawOptions = idResult.RawOptions
 			switch pt {
 			case types.ProjectTypeTerragrunt:
 				node.Terragrunt.HasFiles = true
@@ -204,6 +205,7 @@ func (b *treeBuilder) buildSubtree(ctx context.Context, path string, depth int,
 					Parent:       node,
 					Depth:        depth + 1,
 					ProjectType:  fileType,
+					RawOptions:   idResult.RawOptions,
 				})
 			}
 		}

config.go

@@ -3,6 +3,7 @@ package config
 import (
 	"crypto/sha1" // nolint:gosec
 	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"sort"
 	"strings"
@@ -50,6 +51,10 @@ type Config struct {
 	Terraform     Terraform  `yaml:"terraform,omitempty"`
 	Projects      []*Project `yaml:"projects"`
 	CDK           cdk.Config `yaml:"cdk,omitempty"`
+	// Plugins holds repo-level, plugin-specific defaults keyed by plugin name. They are
+	// deep-merged into each project's matching Plugins entry during normalization (per-project
+	// values win). Opaque to config - the plugin that matches the key owns the schema.
+	Plugins map[string]map[string]any `yaml:"plugins,omitempty"`
 }
 
 type ConfigWithAutodetect struct {
@@ -127,6 +132,10 @@ type Project struct {
 	DependencyPaths []string          `yaml:"dependency_paths,omitempty"`
 	AWS             ProjectAWSConfig  `yaml:"aws,omitempty"`
 	CDKSynthError   string            `yaml:"cdk_synth_error,omitempty"`
+	// Plugins holds plugin-specific parse options keyed by the consuming plugin name (e.g.
+	// "terraform"). It is the persisted raw_options blob - autogenerated during identification
+	// and user-editable. Opaque to config; the plugin that matches the key owns the schema.
+	Plugins map[string]map[string]any `yaml:"plugins,omitempty"`
 }
 
 // ConfigSHA computes a deterministic SHA for the project config based on its
@@ -161,6 +170,17 @@ func (p *Project) ConfigSHA() string {
 	sort.Strings(orderEnv)
 	inputs = append(inputs, orderEnv...)
 
+	// Include the plugin blobs so the SHA reflects options that live only in plugins.<name>.
+	// The deprecated terraform.* fields hashed above are no longer emitted by generation, so
+	// without this the SHA would miss var-file/workspace changes for generated configs. json.Marshal
+	// sorts map keys, so this is deterministic. (Once the deprecated fields and the compat shim are
+	// removed, the terraform.* inputs above can go and this becomes the sole source.)
+	if len(p.Plugins) > 0 {
+		if b, err := json.Marshal(p.Plugins); err == nil {
+			inputs = append(inputs, string(b))
+		}
+	}
+
 	// nolint:gosec
 	gen := sha1.New()
 	_, _ = gen.Write([]byte(strings.Join(inputs, "|")))

config_file.go

@@ -111,6 +111,28 @@ func (c *Config) normalize() error {
 		if project.Terraform.Spacelift.APIKey.Secret == "" && c.Terraform.Defaults.Spacelift.APIKey.Secret != "" {
 			project.Terraform.Spacelift.APIKey.Secret = c.Terraform.Defaults.Spacelift.APIKey.Secret
 		}
+
+		// Merge repo-level plugin defaults (c.Plugins) into the project's matching plugin blob.
+		// This is a generic, schema-agnostic deep merge over the raw YAML types: nested maps
+		// merge recursively, and every other value (scalars and lists alike) is atomic with the
+		// per-project value winning over the global default. Config never needs to understand the shape.
+		for pluginName, globalOpts := range c.Plugins {
+			if len(globalOpts) == 0 {
+				continue
+			}
+			merged := mergePluginBlobs(globalOpts, project.Plugins[pluginName])
+			if len(merged) > 0 {
+				if project.Plugins == nil {
+					project.Plugins = make(map[string]map[string]any, len(c.Plugins))
+				}
+				project.Plugins[pluginName] = merged
+			}
+		}
+
+		// Fold the deprecated structured terraform.* / aws.* fields into the plugins blob so that
+		// consumers can read only the new plugins.<name> style. Structured values win over the
+		// blob (they were either there before or added by the user). See config_plugins_compat.go.
+		project.foldDeprecatedFieldsIntoPlugins(c.Terraform.SourceMap)
 	}
 
 	// first sort by path + env to ensure duplicate name resolution uses the same path for each iteration
@@ -148,6 +170,72 @@ func (c *Config) normalize() error {
 
 }
 
+// mergePluginBlobs deep-merges a repo-level plugin defaults map (global) with a per-project plugin
+// map (project) and returns the result. It is schema-agnostic: nested maps merge recursively, and
+// every other value (scalars and lists alike) is atomic with the per-project value winning.
+// Neither input is mutated.
+func mergePluginBlobs(global, project map[string]any) map[string]any {
+	merged := deepMergeAny(global, project)
+	if m, ok := merged.(map[string]any); ok {
+		return m
+	}
+	return nil
+}
+
+// deepMergeAny merges project into global following the plugin-merge convention (see
+// mergePluginBlobs). Only maps merge recursively; every other value - scalars AND lists - is
+// treated atomically, with the per-project value winning when it is set.
+func deepMergeAny(global, project any) any {
+	gm, gok := global.(map[string]any)
+	pm, pok := project.(map[string]any)
+	if gok && pok {
+		out := make(map[string]any, len(gm)+len(pm))
+		for k, gv := range gm {
+			// deep-copy global branches. global is the shared repo-level defaults map reused for
+			// every project, so aliasing its nested maps/slices into a project's result would let
+			// a later mutation of one project's blob corrupt the global and every sibling project.
+			out[k] = deepCopyAny(gv)
+		}
+		for k, pv := range pm {
+			if gv, ok := gm[k]; ok {
+				out[k] = deepMergeAny(gv, pv)
+			} else {
+				out[k] = pv
+			}
+		}
+		return out
+	}
+
+	// Non-map values (scalars and lists) are atomic: the per-project value wins, unless the
+	// project didn't set anything, in which case fall back to a copy of the global default. Lists
+	// are deliberately NOT concatenated - a project's list replaces the global one.
+	if project == nil {
+		return deepCopyAny(global)
+	}
+	return project
+}
+
+// deepCopyAny returns a deep copy of a decoded YAML/JSON value (nested maps, slices, scalars) so
+// that a merged result never aliases the shared global-defaults map.
+func deepCopyAny(v any) any {
+	switch t := v.(type) {
+	case map[string]any:
+		out := make(map[string]any, len(t))
+		for k, val := range t {
+			out[k] = deepCopyAny(val)
+		}
+		return out
+	case []any:
+		out := make([]any, len(t))
+		for i, val := range t {
+			out[i] = deepCopyAny(val)
+		}
+		return out
+	default:
+		return v
+	}
+}
+
 func fileExists(path string) bool {
 	info, err := os.Stat(path)
 	if os.IsNotExist(err) {