feat: plugin-keyed config options (typed/blob split, file-syntax layer)

autodetect/project.go

@@ -10,6 +10,11 @@ type Project struct {
 	Env               string
 	Type              types.ProjectType
 	Metadata          map[string]string
+	// RawOptions is the plugin-authored parse-options blob (always JSON) for this project, carried
+	// from the IdentifyEnvironments RPC. It is only set when a plugin returned environments
+	// authoritatively; otherwise it is nil and generation sideloads the locally-derived var files
+	// into the plugins blob instead.
+	RawOptions []byte
 }
 
 type RootModule struct {

autodetect/search.go

@@ -384,6 +384,11 @@ func expandProjects(ctx context.Context, identifier *plugin.Identifier, projectN
 					DependencyPaths:   escapeStringListForYAML(envDeps),
 					Env:               escapeStringForYAML(env.Name),
 					Type:              projectType,
+					// the plugin authored this environment's parse-options blob (var files, workspace,
+					// etc.); carry it through so generation persists it verbatim under plugins.<name>.
+					// Only this authoritative-with-environments branch has a plugin blob - the others
+					// sideload config's locally-derived var files at generation time instead.
+					RawOptions: env.RawOptions,
 				})
 
 				// record the directories this environment claims so they aren't also emitted as

config.go

@@ -3,61 +3,39 @@ package config
 import (
 	"crypto/sha1" // nolint:gosec
 	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"sort"
 	"strings"
 
 	"github.com/infracost/config/cdk"
 	"github.com/infracost/config/types"
-	"gopkg.in/yaml.v3"
 )
 
 const CurrentVersion = "0.3"
 
-var legacyVersions = []string{
-	"0.1", // The 0.1 version of the config file was used in the original CLI.
-	"0.2", // The 0.2 version of the config file was used in the original version of the Infracost Cloud Platform.
-}
-
 type ConfigBase struct {
 	Version string `yaml:"version"`
 }
 
-// this handles the parameters that were supported in 0.1 and 0.2 but are not supported in 0.3. This allows us to parse legacy config files.
-type ConfigWithLegacySupport struct {
-	ConfigBase              `yaml:",inline"`
-	Currency                string                      `yaml:"currency,omitempty"`
-	UsageFilePath           string                      `yaml:"usage_file,omitempty"`
-	TerraformRegexSourceMap []TerraformRegexSource      `yaml:"terraform_source_map,omitempty"`
-	TerraformCloudHost      string                      `yaml:"terraform_cloud_host,omitempty"`
-	TerraformCloudOrg       string                      `yaml:"terraform_cloud_org,omitempty"`
-	TerraformCloudWorkspace string                      `yaml:"terraform_cloud_workspace,omitempty"`
-	TerraformCloudToken     string                      `yaml:"terraform_cloud_token,omitempty"`
-	SpaceliftAPIKeyEndpoint string                      `yaml:"spacelift_api_key_endpoint,omitempty"`
-	SpaceliftAPIKeyID       string                      `yaml:"spacelift_api_key_id,omitempty"`
-	SpaceliftAPIKeySecret   string                      `yaml:"spacelift_api_key_secret,omitempty"`
-	TerraformWorkspace      string                      `yaml:"terraform_workspace,omitempty"`
-	Projects                []*ProjectWithLegacySupport `yaml:"projects"`
-	Autodetect              yaml.Node                   `yaml:"autodetect,omitempty"`
-	CDK                     []*cdk.ConfigEntry          `yaml:"cdk,omitempty"`
-	CDKDefaults             cdk.Defaults                `yaml:"cdk_defaults,omitempty"`
-}
-
 type Config struct {
 	ConfigBase    `yaml:",inline"`
 	Currency      string     `yaml:"currency,omitempty"`
 	UsageFilePath string     `yaml:"usage_file,omitempty"`
 	Terraform     Terraform  `yaml:"terraform,omitempty"`
 	Projects      []*Project `yaml:"projects"`
 	CDK           cdk.Config `yaml:"cdk,omitempty"`
-}
-
-type ConfigWithAutodetect struct {
-	*Config    `yaml:",inline"`
-	Autodetect yaml.Node `yaml:"autodetect,omitempty"`
+	// Plugins holds repo-level, plugin-specific defaults keyed by the consuming plugin name (e.g.
+	// "terraform"). Opaque to config - the plugin that matches the key owns the schema. It is NOT
+	// serialized directly (yaml:"-"): (Un)MarshalYAML route through the configfile layer so each blob
+	// lives under its own heading rather than a `plugins:` block.
+	Plugins map[string]map[string]any `yaml:"-"`
 }
 
 type Terraform struct {
+	// Deprecated: set the regex source map per-project under plugins.terraform.regexSourceMap instead.
+	// Still read for backwards compatibility (folded into each terraform project's plugins blob on
+	// load); no longer written by config generation.
 	SourceMap []TerraformRegexSource `yaml:"source_map,omitempty"`
 	Defaults  TerraformDefaults      `yaml:"defaults,omitempty"`
 }
@@ -68,6 +46,11 @@ type TerraformDefaults struct {
 	Workspace string         `yaml:"workspace,omitempty"`
 }
 
+// TerraformCloud is the authoring surface for Terraform Cloud / Enterprise. It is NOT deprecated and
+// NOT folded into the plugins blob: Host/Org/Workspace are caller-sourced and read outside the plugin
+// (the caller passes them to the plugin via GenericOptions.TerraformCloudConfiguration, and reads the
+// host to build the terraform-cloud credential set). The token is a credential supplied here or via
+// INFRACOST_TERRAFORM_CLOUD_TOKEN and travels via GenericOptions.CredentialSets.
 type TerraformCloud struct {
 	Host      string `yaml:"host,omitempty"`
 	Org       string `yaml:"org,omitempty"`
@@ -85,29 +68,22 @@ type SpaceliftAPIKey struct {
 	Secret   string `yaml:"secret,omitempty"`
 }
 
-type ProjectWithLegacySupport struct {
-	Project                 `yaml:",inline"`
-	TerraformVars           map[string]any `yaml:"terraform_vars,omitempty"`
-	TerraformWorkspace      string         `yaml:"terraform_workspace,omitempty"`
-	TerraformCloudHost      string         `yaml:"terraform_cloud_host,omitempty"`
-	TerraformCloudOrg       string         `yaml:"terraform_cloud_org,omitempty"`
-	TerraformCloudWorkspace string         `yaml:"terraform_cloud_workspace,omitempty"`
-	TerraformCloudToken     string         `yaml:"terraform_cloud_token,omitempty"`
-	SpaceliftAPIKeyEndpoint string         `yaml:"spacelift_api_key_endpoint,omitempty"`
-	SpaceliftAPIKeyID       string         `yaml:"spacelift_api_key_id,omitempty"`
-	SpaceliftAPIKeySecret   string         `yaml:"spacelift_api_key_secret,omitempty"`
-	TerraformVarFiles       []string       `yaml:"terraform_var_files,omitempty"`
-	ProjectType             ProjectType    `yaml:"project_type,omitempty"`      // terraform, terragrunt
-	SkipAutodetect          bool           `yaml:"skip_autodetect,omitempty"`   // deprecated, ignored
-	IncludeAllPaths         bool           `yaml:"include_all_paths,omitempty"` // deprecated, ignored
-}
-
 type ProjectTerraform struct {
 	Cloud     TerraformCloud `yaml:"cloud,omitempty"`
 	Spacelift Spacelift      `yaml:"spacelift,omitempty"`
-	Vars      map[string]any `yaml:"vars,omitempty"`
-	VarFiles  []string       `yaml:"var_files,omitempty"`
-	Workspace string         `yaml:"workspace,omitempty"`
+	// Deprecated: read the vars from the plugins blob (Project.Plugins["terraform"]["vars"]) instead.
+	// Populated on load as a backwards-compat mirror of that blob (the canonical source, and the only
+	// representation serialized); not read by config itself.
+	Vars map[string]any `yaml:"vars,omitempty"`
+	// Deprecated: read the var files from the plugins blob (Project.Plugins["terraform"]["var_files"])
+	// instead. Populated on load as a backwards-compat mirror of that blob (the canonical source, and
+	// the only representation serialized); not read by config itself.
+	VarFiles []string `yaml:"var_files,omitempty"`
+	// Workspace is the terraform workspace to select. It is NOT part of the plugins blob: it is a
+	// caller-sourced runtime option passed to the plugin via GenericOptions.Workspace, and is also read
+	// outside the plugin (the project's workspace in the cost output / provider input). It stays a
+	// top-level field and is still written by generation.
+	Workspace string `yaml:"workspace,omitempty"`
 }
 
 // Project defines a specific terraform project config. This can be used
@@ -127,6 +103,12 @@ type Project struct {
 	DependencyPaths []string          `yaml:"dependency_paths,omitempty"`
 	AWS             ProjectAWSConfig  `yaml:"aws,omitempty"`
 	CDKSynthError   string            `yaml:"cdk_synth_error,omitempty"`
+	// Plugins holds plugin-specific parse options keyed by the consuming plugin name (e.g.
+	// "terraform"). It is the persisted raw_options blob - autogenerated during identification and
+	// user-editable. Opaque to config; the plugin that matches the key owns the schema. NOT serialized
+	// directly (yaml:"-"): the configfile layer splits/merges it under the project's headings
+	// (terraform:, aws:, ...) rather than a `plugins:` block.
+	Plugins map[string]map[string]any `yaml:"-"`
 }
 
 // ConfigSHA computes a deterministic SHA for the project config based on its
@@ -138,13 +120,8 @@ func (p *Project) ConfigSHA() string {
 	inputs = append(inputs, p.Path)
 	inputs = append(inputs, p.ExcludePaths...)
 	inputs = append(inputs, p.DependencyPaths...)
-	orderVars := make([]string, 0, len(p.Terraform.Vars))
-	for k, v := range p.Terraform.Vars {
-		orderVars = append(orderVars, fmt.Sprintf("%s=%s", k, v))
-	}
-	sort.Strings(orderVars)
-	inputs = append(inputs, orderVars...)
-	inputs = append(inputs, p.Terraform.VarFiles...)
+	// Terraform.Vars/VarFiles are backwards-compat mirrors of the Plugins blob (hashed below), so
+	// they are deliberately NOT hashed here - that would double-count them.
 	inputs = append(inputs, p.Terraform.Workspace)
 	inputs = append(inputs, p.Terraform.Cloud.Host)
 	inputs = append(inputs, p.Terraform.Cloud.Org)
@@ -161,12 +138,26 @@ func (p *Project) ConfigSHA() string {
 	sort.Strings(orderEnv)
 	inputs = append(inputs, orderEnv...)
 
+	// Include the plugin blobs so the sha reflects options that now live only under plugins.<name>:
+	// generated configs no longer write the deprecated terraform.* fields, so without this the sha
+	// would miss var-file/workspace changes for them. json.Marshal sorts map keys, so this is
+	// deterministic.
+	if len(p.Plugins) > 0 {
+		if b, err := json.Marshal(p.Plugins); err == nil {
+			inputs = append(inputs, string(b))
+		}
+	}
+
 	// nolint:gosec
 	gen := sha1.New()
 	_, _ = gen.Write([]byte(strings.Join(inputs, "|")))
 	return hex.EncodeToString(gen.Sum(nil))
 }
 
+// ProjectAWSConfig is the deprecated typed view of the cloudformation aws context. The canonical
+// representation is the plugins blob Project.Plugins["cloudformation"] (flat snake_case keys: region,
+// account_id, stack_id, stack_name). These fields are populated on load as a backwards-compat mirror
+// of that blob; prefer reading the blob. Config itself does not read them.
 type ProjectAWSConfig struct {
 	Region    string `yaml:"region,omitempty"`
 	StackID   string `yaml:"stack_id,omitempty"`

config_file.go

@@ -1,7 +1,6 @@
 package config
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
 	"os"
@@ -110,6 +109,10 @@ func (c *Config) normalize() error {
 		}
 	}
 
+	// fold repo-level plugin defaults (incl. the terraform.source_map compat shim) into each project's
+	// blob, per-project winning.
+	c.foldRepoPluginDefaults()
+
 	// first sort by path + env to ensure duplicate name resolution uses the same path for each iteration
 	sort.Slice(c.Projects, func(i, j int) bool {
 		if c.Projects[i].Path == c.Projects[j].Path {
@@ -254,15 +257,11 @@ func parseWithAutodetectAllowed(content []byte, target *Config, baseDir string)
 				return nil, fmt.Errorf("%w: %s", ErrInvalidConfigYAML, err)
 			}
 		case version == CurrentVersion:
-			config := ConfigWithAutodetect{
-				Config: target,
-			}
-
-			decoder := yaml.NewDecoder(bytes.NewReader(content))
-			decoder.KnownFields(true)
-
-			if err := decoder.Decode(&config); err != nil {
-				return nil, fmt.Errorf("%w: %s", ErrInvalidConfigYAML, err)
+			// Config.UnmarshalYAML routes through the configfile layer, which also captures the
+			// autodetect: node (ignored here; the autodetect config is parsed separately from the raw
+			// bytes), so no dedicated wrapper is needed to tolerate it.
+			if err := parseCurrentVersion(content, target); err != nil {
+				return nil, err
 			}
 		default:
 			return nil, fmt.Errorf("%w: unsupported config file version: %s", ErrInvalidConfigYAML, version)
@@ -281,10 +280,9 @@ func parseWithAutodetectAllowed(content []byte, target *Config, baseDir string)
 }
 
 func parseCurrentVersion(content []byte, config *Config) error {
-	decoder := yaml.NewDecoder(bytes.NewReader(content))
-	decoder.KnownFields(true)
-
-	if err := decoder.Decode(&config); err != nil {
+	// yaml.Unmarshal invokes Config.UnmarshalYAML, which decodes via the configfile layer and applies
+	// the typed/blob split. config keeps any defaults it was seeded with for keys the file omits.
+	if err := yaml.Unmarshal(content, config); err != nil {
 		return fmt.Errorf("%w: %s", ErrInvalidConfigYAML, simplifyYAMLError(err))
 	}
 
@@ -316,99 +314,6 @@ func simplifyErrorLine(line string) string {
 	return simple
 }
 
-func parseLegacyVersion(content []byte, config *Config) error {
-	var intermediary ConfigWithLegacySupport
-	if len(config.Projects) > 0 {
-		// if the config came with projectsd set (e.g. default main) we need to set it here to see if it gets overridden by legacy projects
-		intermediary.Projects = make([]*ProjectWithLegacySupport, 0, len(config.Projects))
-		for _, project := range config.Projects {
-			intermediary.Projects = append(intermediary.Projects, &ProjectWithLegacySupport{
-				Project: *project,
-			})
-		}
-	}
-
-	decoder := yaml.NewDecoder(bytes.NewReader(content))
-	decoder.KnownFields(true)
-
-	if err := decoder.Decode(&intermediary); err != nil {
-		return fmt.Errorf("%w: %s", ErrInvalidConfigYAML, simplifyYAMLError(err))
-	}
-
-	// copy across fields that exist in both
-	config.ConfigBase = intermediary.ConfigBase
-	config.Version = CurrentVersion // force version to latest after converting
-	config.Currency = intermediary.Currency
-	config.UsageFilePath = intermediary.UsageFilePath
-	config.CDK.Projects = intermediary.CDK
-	config.CDK.Defaults = intermediary.CDKDefaults
-
-	// copy legacy fields to their new locations
-	config.Terraform.SourceMap = intermediary.TerraformRegexSourceMap
-	config.Terraform.Defaults.Cloud.Host = intermediary.TerraformCloudHost
-	config.Terraform.Defaults.Cloud.Org = intermediary.TerraformCloudOrg
-	config.Terraform.Defaults.Cloud.Workspace = intermediary.TerraformCloudWorkspace
-	config.Terraform.Defaults.Cloud.Token = intermediary.TerraformCloudToken
-	config.Terraform.Defaults.Spacelift.APIKey.Endpoint = intermediary.SpaceliftAPIKeyEndpoint
-	config.Terraform.Defaults.Spacelift.APIKey.ID = intermediary.SpaceliftAPIKeyID
-	config.Terraform.Defaults.Spacelift.APIKey.Secret = intermediary.SpaceliftAPIKeySecret
-	config.Terraform.Defaults.Workspace = intermediary.TerraformWorkspace
-
-	// remove default projects and take whatever the decode gace us - if the user didn't specify the projects key, we'll get the defaults preserved anyway
-	config.Projects = nil
-
-	// convert legacy projects to new ones
-	// this is deliberately a nil check rather than checking length, as we want to preserve an empty projects section if it was explicitly set to empty in the legacy config
-	if len(intermediary.Projects) > 0 {
-		config.Projects = make([]*Project, 0, len(intermediary.Projects))
-
-		for _, legacyProject := range intermediary.Projects {
-			project := legacyProject.Project
-			if len(legacyProject.TerraformVars) > 0 {
-				if project.Terraform.Vars == nil {
-					project.Terraform.Vars = make(map[string]any)
-				}
-				for k, v := range legacyProject.TerraformVars {
-					project.Terraform.Vars[k] = v
-				}
-			}
-			if legacyProject.TerraformWorkspace != "" {
-				project.Terraform.Workspace = legacyProject.TerraformWorkspace
-			}
-			if legacyProject.TerraformCloudHost != "" {
-				project.Terraform.Cloud.Host = legacyProject.TerraformCloudHost
-			}
-			if legacyProject.TerraformCloudOrg != "" {
-				project.Terraform.Cloud.Org = legacyProject.TerraformCloudOrg
-			}
-			if legacyProject.TerraformCloudWorkspace != "" {
-				project.Terraform.Cloud.Workspace = legacyProject.TerraformCloudWorkspace
-			}
-			if legacyProject.TerraformCloudToken != "" {
-				project.Terraform.Cloud.Token = legacyProject.TerraformCloudToken
-			}
-			if legacyProject.SpaceliftAPIKeyEndpoint != "" {
-				project.Terraform.Spacelift.APIKey.Endpoint = legacyProject.SpaceliftAPIKeyEndpoint
-			}
-			if legacyProject.SpaceliftAPIKeyID != "" {
-				project.Terraform.Spacelift.APIKey.ID = legacyProject.SpaceliftAPIKeyID
-			}
-			if legacyProject.SpaceliftAPIKeySecret != "" {
-				project.Terraform.Spacelift.APIKey.Secret = legacyProject.SpaceliftAPIKeySecret
-			}
-			if len(legacyProject.TerraformVarFiles) > 0 {
-				project.Terraform.VarFiles = legacyProject.TerraformVarFiles
-			}
-			if legacyProject.ProjectType != "" {
-				project.Type = legacyProject.ProjectType
-			}
-			config.Projects = append(config.Projects, &project)
-		}
-	}
-
-	return nil
-}
-
 func finalizeCDKConfig(repoPath string, cfg *Config, ignorePermissionErrors, ignoreHiddenDirs bool) error {
 	// If no CDK entries and no defaults, nothing to do
 	if len(cfg.CDK.Projects) == 0 && len(cfg.CDK.Defaults.Context) == 0 && len(cfg.CDK.Defaults.Env) == 0 {