diff --git a/autodetect/project.go b/autodetect/project.go index e512727..192ceb6 100644 --- a/autodetect/project.go +++ b/autodetect/project.go @@ -10,6 +10,11 @@ type Project struct { Env string Type types.ProjectType Metadata map[string]string + // RawOptions is the plugin-authored parse-options blob (always JSON) for this project, carried + // from the IdentifyEnvironments RPC. It is only set when a plugin returned environments + // authoritatively; otherwise it is nil and generation sideloads the locally-derived var files + // into the plugins blob instead. + RawOptions []byte } type RootModule struct { diff --git a/autodetect/search.go b/autodetect/search.go index dd266ca..70bf5f6 100644 --- a/autodetect/search.go +++ b/autodetect/search.go @@ -384,6 +384,11 @@ func expandProjects(ctx context.Context, identifier *plugin.Identifier, projectN DependencyPaths: escapeStringListForYAML(envDeps), Env: escapeStringForYAML(env.Name), Type: projectType, + // the plugin authored this environment's parse-options blob (var files, workspace, + // etc.); carry it through so generation persists it verbatim under plugins.. + // Only this authoritative-with-environments branch has a plugin blob - the others + // sideload config's locally-derived var files at generation time instead. + RawOptions: env.RawOptions, }) // record the directories this environment claims so they aren't also emitted as diff --git a/config.go b/config.go index 3cf5797..b284572 100644 --- a/config.go +++ b/config.go @@ -3,46 +3,21 @@ package config import ( "crypto/sha1" // nolint:gosec "encoding/hex" + "encoding/json" "fmt" "sort" "strings" "github.com/infracost/config/cdk" "github.com/infracost/config/types" - "gopkg.in/yaml.v3" ) const CurrentVersion = "0.3" -var legacyVersions = []string{ - "0.1", // The 0.1 version of the config file was used in the original CLI. - "0.2", // The 0.2 version of the config file was used in the original version of the Infracost Cloud Platform. -} - type ConfigBase struct { Version string `yaml:"version"` } -// this handles the parameters that were supported in 0.1 and 0.2 but are not supported in 0.3. This allows us to parse legacy config files. -type ConfigWithLegacySupport struct { - ConfigBase `yaml:",inline"` - Currency string `yaml:"currency,omitempty"` - UsageFilePath string `yaml:"usage_file,omitempty"` - TerraformRegexSourceMap []TerraformRegexSource `yaml:"terraform_source_map,omitempty"` - TerraformCloudHost string `yaml:"terraform_cloud_host,omitempty"` - TerraformCloudOrg string `yaml:"terraform_cloud_org,omitempty"` - TerraformCloudWorkspace string `yaml:"terraform_cloud_workspace,omitempty"` - TerraformCloudToken string `yaml:"terraform_cloud_token,omitempty"` - SpaceliftAPIKeyEndpoint string `yaml:"spacelift_api_key_endpoint,omitempty"` - SpaceliftAPIKeyID string `yaml:"spacelift_api_key_id,omitempty"` - SpaceliftAPIKeySecret string `yaml:"spacelift_api_key_secret,omitempty"` - TerraformWorkspace string `yaml:"terraform_workspace,omitempty"` - Projects []*ProjectWithLegacySupport `yaml:"projects"` - Autodetect yaml.Node `yaml:"autodetect,omitempty"` - CDK []*cdk.ConfigEntry `yaml:"cdk,omitempty"` - CDKDefaults cdk.Defaults `yaml:"cdk_defaults,omitempty"` -} - type Config struct { ConfigBase `yaml:",inline"` Currency string `yaml:"currency,omitempty"` @@ -50,14 +25,17 @@ type Config struct { Terraform Terraform `yaml:"terraform,omitempty"` Projects []*Project `yaml:"projects"` CDK cdk.Config `yaml:"cdk,omitempty"` -} - -type ConfigWithAutodetect struct { - *Config `yaml:",inline"` - Autodetect yaml.Node `yaml:"autodetect,omitempty"` + // Plugins holds repo-level, plugin-specific defaults keyed by the consuming plugin name (e.g. + // "terraform"). Opaque to config - the plugin that matches the key owns the schema. It is NOT + // serialized directly (yaml:"-"): (Un)MarshalYAML route through the configfile layer so each blob + // lives under its own heading rather than a `plugins:` block. + Plugins map[string]map[string]any `yaml:"-"` } type Terraform struct { + // Deprecated: set the regex source map per-project under plugins.terraform.regexSourceMap instead. + // Still read for backwards compatibility (folded into each terraform project's plugins blob on + // load); no longer written by config generation. SourceMap []TerraformRegexSource `yaml:"source_map,omitempty"` Defaults TerraformDefaults `yaml:"defaults,omitempty"` } @@ -68,6 +46,11 @@ type TerraformDefaults struct { Workspace string `yaml:"workspace,omitempty"` } +// TerraformCloud is the authoring surface for Terraform Cloud / Enterprise. It is NOT deprecated and +// NOT folded into the plugins blob: Host/Org/Workspace are caller-sourced and read outside the plugin +// (the caller passes them to the plugin via GenericOptions.TerraformCloudConfiguration, and reads the +// host to build the terraform-cloud credential set). The token is a credential supplied here or via +// INFRACOST_TERRAFORM_CLOUD_TOKEN and travels via GenericOptions.CredentialSets. type TerraformCloud struct { Host string `yaml:"host,omitempty"` Org string `yaml:"org,omitempty"` @@ -85,29 +68,22 @@ type SpaceliftAPIKey struct { Secret string `yaml:"secret,omitempty"` } -type ProjectWithLegacySupport struct { - Project `yaml:",inline"` - TerraformVars map[string]any `yaml:"terraform_vars,omitempty"` - TerraformWorkspace string `yaml:"terraform_workspace,omitempty"` - TerraformCloudHost string `yaml:"terraform_cloud_host,omitempty"` - TerraformCloudOrg string `yaml:"terraform_cloud_org,omitempty"` - TerraformCloudWorkspace string `yaml:"terraform_cloud_workspace,omitempty"` - TerraformCloudToken string `yaml:"terraform_cloud_token,omitempty"` - SpaceliftAPIKeyEndpoint string `yaml:"spacelift_api_key_endpoint,omitempty"` - SpaceliftAPIKeyID string `yaml:"spacelift_api_key_id,omitempty"` - SpaceliftAPIKeySecret string `yaml:"spacelift_api_key_secret,omitempty"` - TerraformVarFiles []string `yaml:"terraform_var_files,omitempty"` - ProjectType ProjectType `yaml:"project_type,omitempty"` // terraform, terragrunt - SkipAutodetect bool `yaml:"skip_autodetect,omitempty"` // deprecated, ignored - IncludeAllPaths bool `yaml:"include_all_paths,omitempty"` // deprecated, ignored -} - type ProjectTerraform struct { Cloud TerraformCloud `yaml:"cloud,omitempty"` Spacelift Spacelift `yaml:"spacelift,omitempty"` - Vars map[string]any `yaml:"vars,omitempty"` - VarFiles []string `yaml:"var_files,omitempty"` - Workspace string `yaml:"workspace,omitempty"` + // Deprecated: read the vars from the plugins blob (Project.Plugins["terraform"]["vars"]) instead. + // Populated on load as a backwards-compat mirror of that blob (the canonical source, and the only + // representation serialized); not read by config itself. + Vars map[string]any `yaml:"vars,omitempty"` + // Deprecated: read the var files from the plugins blob (Project.Plugins["terraform"]["var_files"]) + // instead. Populated on load as a backwards-compat mirror of that blob (the canonical source, and + // the only representation serialized); not read by config itself. + VarFiles []string `yaml:"var_files,omitempty"` + // Workspace is the terraform workspace to select. It is NOT part of the plugins blob: it is a + // caller-sourced runtime option passed to the plugin via GenericOptions.Workspace, and is also read + // outside the plugin (the project's workspace in the cost output / provider input). It stays a + // top-level field and is still written by generation. + Workspace string `yaml:"workspace,omitempty"` } // Project defines a specific terraform project config. This can be used @@ -127,6 +103,12 @@ type Project struct { DependencyPaths []string `yaml:"dependency_paths,omitempty"` AWS ProjectAWSConfig `yaml:"aws,omitempty"` CDKSynthError string `yaml:"cdk_synth_error,omitempty"` + // Plugins holds plugin-specific parse options keyed by the consuming plugin name (e.g. + // "terraform"). It is the persisted raw_options blob - autogenerated during identification and + // user-editable. Opaque to config; the plugin that matches the key owns the schema. NOT serialized + // directly (yaml:"-"): the configfile layer splits/merges it under the project's headings + // (terraform:, aws:, ...) rather than a `plugins:` block. + Plugins map[string]map[string]any `yaml:"-"` } // ConfigSHA computes a deterministic SHA for the project config based on its @@ -138,13 +120,8 @@ func (p *Project) ConfigSHA() string { inputs = append(inputs, p.Path) inputs = append(inputs, p.ExcludePaths...) inputs = append(inputs, p.DependencyPaths...) - orderVars := make([]string, 0, len(p.Terraform.Vars)) - for k, v := range p.Terraform.Vars { - orderVars = append(orderVars, fmt.Sprintf("%s=%s", k, v)) - } - sort.Strings(orderVars) - inputs = append(inputs, orderVars...) - inputs = append(inputs, p.Terraform.VarFiles...) + // Terraform.Vars/VarFiles are backwards-compat mirrors of the Plugins blob (hashed below), so + // they are deliberately NOT hashed here - that would double-count them. inputs = append(inputs, p.Terraform.Workspace) inputs = append(inputs, p.Terraform.Cloud.Host) inputs = append(inputs, p.Terraform.Cloud.Org) @@ -161,12 +138,26 @@ func (p *Project) ConfigSHA() string { sort.Strings(orderEnv) inputs = append(inputs, orderEnv...) + // Include the plugin blobs so the sha reflects options that now live only under plugins.: + // generated configs no longer write the deprecated terraform.* fields, so without this the sha + // would miss var-file/workspace changes for them. json.Marshal sorts map keys, so this is + // deterministic. + if len(p.Plugins) > 0 { + if b, err := json.Marshal(p.Plugins); err == nil { + inputs = append(inputs, string(b)) + } + } + // nolint:gosec gen := sha1.New() _, _ = gen.Write([]byte(strings.Join(inputs, "|"))) return hex.EncodeToString(gen.Sum(nil)) } +// ProjectAWSConfig is the deprecated typed view of the cloudformation aws context. The canonical +// representation is the plugins blob Project.Plugins["cloudformation"] (flat snake_case keys: region, +// account_id, stack_id, stack_name). These fields are populated on load as a backwards-compat mirror +// of that blob; prefer reading the blob. Config itself does not read them. type ProjectAWSConfig struct { Region string `yaml:"region,omitempty"` StackID string `yaml:"stack_id,omitempty"` diff --git a/config_file.go b/config_file.go index e6756b1..1d7ebdb 100644 --- a/config_file.go +++ b/config_file.go @@ -1,7 +1,6 @@ package config import ( - "bytes" "errors" "fmt" "os" @@ -110,6 +109,10 @@ func (c *Config) normalize() error { } } + // fold repo-level plugin defaults (incl. the terraform.source_map compat shim) into each project's + // blob, per-project winning. + c.foldRepoPluginDefaults() + // first sort by path + env to ensure duplicate name resolution uses the same path for each iteration sort.Slice(c.Projects, func(i, j int) bool { if c.Projects[i].Path == c.Projects[j].Path { @@ -254,15 +257,11 @@ func parseWithAutodetectAllowed(content []byte, target *Config, baseDir string) return nil, fmt.Errorf("%w: %s", ErrInvalidConfigYAML, err) } case version == CurrentVersion: - config := ConfigWithAutodetect{ - Config: target, - } - - decoder := yaml.NewDecoder(bytes.NewReader(content)) - decoder.KnownFields(true) - - if err := decoder.Decode(&config); err != nil { - return nil, fmt.Errorf("%w: %s", ErrInvalidConfigYAML, err) + // Config.UnmarshalYAML routes through the configfile layer, which also captures the + // autodetect: node (ignored here; the autodetect config is parsed separately from the raw + // bytes), so no dedicated wrapper is needed to tolerate it. + if err := parseCurrentVersion(content, target); err != nil { + return nil, err } default: return nil, fmt.Errorf("%w: unsupported config file version: %s", ErrInvalidConfigYAML, version) @@ -281,10 +280,9 @@ func parseWithAutodetectAllowed(content []byte, target *Config, baseDir string) } func parseCurrentVersion(content []byte, config *Config) error { - decoder := yaml.NewDecoder(bytes.NewReader(content)) - decoder.KnownFields(true) - - if err := decoder.Decode(&config); err != nil { + // yaml.Unmarshal invokes Config.UnmarshalYAML, which decodes via the configfile layer and applies + // the typed/blob split. config keeps any defaults it was seeded with for keys the file omits. + if err := yaml.Unmarshal(content, config); err != nil { return fmt.Errorf("%w: %s", ErrInvalidConfigYAML, simplifyYAMLError(err)) } @@ -316,99 +314,6 @@ func simplifyErrorLine(line string) string { return simple } -func parseLegacyVersion(content []byte, config *Config) error { - var intermediary ConfigWithLegacySupport - if len(config.Projects) > 0 { - // if the config came with projectsd set (e.g. default main) we need to set it here to see if it gets overridden by legacy projects - intermediary.Projects = make([]*ProjectWithLegacySupport, 0, len(config.Projects)) - for _, project := range config.Projects { - intermediary.Projects = append(intermediary.Projects, &ProjectWithLegacySupport{ - Project: *project, - }) - } - } - - decoder := yaml.NewDecoder(bytes.NewReader(content)) - decoder.KnownFields(true) - - if err := decoder.Decode(&intermediary); err != nil { - return fmt.Errorf("%w: %s", ErrInvalidConfigYAML, simplifyYAMLError(err)) - } - - // copy across fields that exist in both - config.ConfigBase = intermediary.ConfigBase - config.Version = CurrentVersion // force version to latest after converting - config.Currency = intermediary.Currency - config.UsageFilePath = intermediary.UsageFilePath - config.CDK.Projects = intermediary.CDK - config.CDK.Defaults = intermediary.CDKDefaults - - // copy legacy fields to their new locations - config.Terraform.SourceMap = intermediary.TerraformRegexSourceMap - config.Terraform.Defaults.Cloud.Host = intermediary.TerraformCloudHost - config.Terraform.Defaults.Cloud.Org = intermediary.TerraformCloudOrg - config.Terraform.Defaults.Cloud.Workspace = intermediary.TerraformCloudWorkspace - config.Terraform.Defaults.Cloud.Token = intermediary.TerraformCloudToken - config.Terraform.Defaults.Spacelift.APIKey.Endpoint = intermediary.SpaceliftAPIKeyEndpoint - config.Terraform.Defaults.Spacelift.APIKey.ID = intermediary.SpaceliftAPIKeyID - config.Terraform.Defaults.Spacelift.APIKey.Secret = intermediary.SpaceliftAPIKeySecret - config.Terraform.Defaults.Workspace = intermediary.TerraformWorkspace - - // remove default projects and take whatever the decode gace us - if the user didn't specify the projects key, we'll get the defaults preserved anyway - config.Projects = nil - - // convert legacy projects to new ones - // this is deliberately a nil check rather than checking length, as we want to preserve an empty projects section if it was explicitly set to empty in the legacy config - if len(intermediary.Projects) > 0 { - config.Projects = make([]*Project, 0, len(intermediary.Projects)) - - for _, legacyProject := range intermediary.Projects { - project := legacyProject.Project - if len(legacyProject.TerraformVars) > 0 { - if project.Terraform.Vars == nil { - project.Terraform.Vars = make(map[string]any) - } - for k, v := range legacyProject.TerraformVars { - project.Terraform.Vars[k] = v - } - } - if legacyProject.TerraformWorkspace != "" { - project.Terraform.Workspace = legacyProject.TerraformWorkspace - } - if legacyProject.TerraformCloudHost != "" { - project.Terraform.Cloud.Host = legacyProject.TerraformCloudHost - } - if legacyProject.TerraformCloudOrg != "" { - project.Terraform.Cloud.Org = legacyProject.TerraformCloudOrg - } - if legacyProject.TerraformCloudWorkspace != "" { - project.Terraform.Cloud.Workspace = legacyProject.TerraformCloudWorkspace - } - if legacyProject.TerraformCloudToken != "" { - project.Terraform.Cloud.Token = legacyProject.TerraformCloudToken - } - if legacyProject.SpaceliftAPIKeyEndpoint != "" { - project.Terraform.Spacelift.APIKey.Endpoint = legacyProject.SpaceliftAPIKeyEndpoint - } - if legacyProject.SpaceliftAPIKeyID != "" { - project.Terraform.Spacelift.APIKey.ID = legacyProject.SpaceliftAPIKeyID - } - if legacyProject.SpaceliftAPIKeySecret != "" { - project.Terraform.Spacelift.APIKey.Secret = legacyProject.SpaceliftAPIKeySecret - } - if len(legacyProject.TerraformVarFiles) > 0 { - project.Terraform.VarFiles = legacyProject.TerraformVarFiles - } - if legacyProject.ProjectType != "" { - project.Type = legacyProject.ProjectType - } - config.Projects = append(config.Projects, &project) - } - } - - return nil -} - func finalizeCDKConfig(repoPath string, cfg *Config, ignorePermissionErrors, ignoreHiddenDirs bool) error { // If no CDK entries and no defaults, nothing to do if len(cfg.CDK.Projects) == 0 && len(cfg.CDK.Defaults.Context) == 0 && len(cfg.CDK.Defaults.Env) == 0 { diff --git a/config_plugins.go b/config_plugins.go new file mode 100644 index 0000000..1ee70ac --- /dev/null +++ b/config_plugins.go @@ -0,0 +1,233 @@ +package config + +// This file holds the plumbing for the generic, plugin-keyed `plugins.` config section: the +// map from a project type to the plugin that consumes its options. It does not understand a plugin's +// option schema - that stays opaque to config. + +// PluginKeyForType returns the config-file plugins. key for a project type: the name of the +// plugin that consumes that project's parse options. It is the identity for every type except the +// CDK variants, which are parsed by the cloudformation plugin and so share its options blob. +// +// Keying by consuming plugin (rather than project type) is what lets generation and the deprecated +// fold agree on where a project's blob lives - the two only diverge for CDK. +func PluginKeyForType(t ProjectType) string { + switch t { + case ProjectTypeCDKTypeScript, ProjectTypeCDKJavaScript, ProjectTypeCDKPython: + return string(ProjectTypeCloudFormation) + default: + return string(t) + } +} + +// IsTerraformFamily reports whether a project type is parsed with the terraform options schema +// (terraform, terragrunt and cisco stacks all share it), and so folds/sideloads into a +// terraform-shaped blob. +func IsTerraformFamily(t ProjectType) bool { + switch t { + case ProjectTypeTerraform, ProjectTypeTerragrunt, ProjectTypeCiscoStacks: + return true + default: + return false + } +} + +// The config file collapses a few project families onto a single heading (terraform-family -> the +// `terraform:` heading; cloudformation/CDK -> the historical `aws:` heading), while the in-memory +// Plugins map is keyed by the exact project type (what the CLI/runner look up). blobKeyForHeading and +// headingForBlobKey are the two directions of that mapping, so read and write agree. + +// blobKeyForHeading returns the Plugins key that a given file heading maps to for a project of type t. +// It is the read direction: file heading -> in-memory Plugins key. +func blobKeyForHeading(heading string, t ProjectType) string { + switch heading { + case "terraform": + return terraformBlobKey(t) + case "aws": + return string(ProjectTypeCloudFormation) + default: + return heading + } +} + +// terraformBlobKey returns the Plugins key for a terraform-family project's blob: the exact type for +// terraform/terragrunt/cisco, and terraform for an untyped project carrying terraform options. +func terraformBlobKey(t ProjectType) string { + if IsTerraformFamily(t) { + return string(t) + } + return string(ProjectTypeTerraform) +} + +// headingForBlobKey returns the file heading a Plugins key is written under. It is the write direction +// and the inverse of blobKeyForHeading: the terraform family collapses to `terraform:`, cloudformation +// (which cdk_* keys resolve to) to `aws:`, and everything else keeps its own name. +func headingForBlobKey(key string) string { + switch ProjectType(key) { + case ProjectTypeTerraform, ProjectTypeTerragrunt, ProjectTypeCiscoStacks: + return "terraform" + case ProjectTypeCloudFormation: + return "aws" + default: + return key + } +} + +// terraformTypedKeys are the keys under a project's terraform: heading that config keeps as typed +// fields (read outside the plugin) rather than forwarding in the opaque blob. Everything else under +// terraform: is blob. +var terraformTypedKeys = map[string]bool{ + "cloud": true, + "spacelift": true, + "workspace": true, +} + +// setPluginOption stores a single option on p.Plugins[key], allocating the maps as needed. +func (p *Project) setPluginOption(key, option string, value any) { + if p.Plugins == nil { + p.Plugins = map[string]map[string]any{} + } + if p.Plugins[key] == nil { + p.Plugins[key] = map[string]any{} + } + p.Plugins[key][option] = value +} + +// isCDKType reports whether a project type is one of the CDK variants (parsed by the cloudformation +// plugin). +func isCDKType(t ProjectType) bool { + switch t { + case ProjectTypeCDKTypeScript, ProjectTypeCDKJavaScript, ProjectTypeCDKPython: + return true + default: + return false + } +} + +// foldRepoPluginDefaults folds repo-level config into each project's plugin blob, then re-syncs the +// deprecated typed mirrors. Two things fold in: +// - the legacy repo-level terraform.source_map (a list of {match,replace}) is transformed into the +// blob's regex_source_map (a map) under the terraform repo defaults - a compat shim, so it then +// flows through the generic merge like any other repo-level terraform option; and +// - repo-level plugin defaults (c.Plugins) are deep-merged into each project whose type they apply +// to, with the per-project value winning. +func (c *Config) foldRepoPluginDefaults() { + for _, project := range c.Projects { + for repoKey, defaults := range c.Plugins { + if len(defaults) == 0 { + continue + } + target, ok := repoDefaultAppliesTo(repoKey, project.Type) + if !ok { + continue + } + if merged := mergePluginBlobs(defaults, project.Plugins[target]); len(merged) > 0 { + setProjectPlugin(project, target, merged) + } + } + + c.foldSourceMapInto(project) + + // re-sync the deprecated typed mirrors, which may now include merged-in repo defaults. + mirrorBlobToDeprecated(project) + } +} + +// foldSourceMapInto is the compat shim for the deprecated repo-level terraform.source_map: it folds +// it into each terraform-family project's blob as regex_source_map (transforming the list of +// {match,replace} into a map), unless the project already sets one (per-project wins). It stays on +// c.Terraform.SourceMap (the deprecated typed field / what's serialized) rather than being moved to a +// repo blob, so it survives the env-var marshal/re-parse round-trip which runs before normalize. +func (c *Config) foldSourceMapInto(project *Project) { + if len(c.Terraform.SourceMap) == 0 { + return + } + if !IsTerraformFamily(project.Type) && project.Type != ProjectTypeUnknown { + return + } + key := terraformBlobKey(project.Type) + if project.Plugins[key]["regex_source_map"] != nil { + return + } + m := make(map[string]any, len(c.Terraform.SourceMap)) + for _, s := range c.Terraform.SourceMap { + m[s.Match] = s.Replace + } + project.setPluginOption(key, "regex_source_map", m) +} + +// repoDefaultAppliesTo reports whether a repo-level plugin default (keyed by repoKey) applies to a +// project of type t and, if so, which of the project's blob keys it merges into. The terraform family +// shares the "terraform" repo default (merged into each member's exact-type blob); cloudformation +// covers the CDK variants. +func repoDefaultAppliesTo(repoKey string, t ProjectType) (string, bool) { + switch repoKey { + case string(ProjectTypeTerraform): + if IsTerraformFamily(t) || t == ProjectTypeUnknown { + return terraformBlobKey(t), true + } + case string(ProjectTypeCloudFormation): + if t == ProjectTypeCloudFormation || isCDKType(t) { + return string(ProjectTypeCloudFormation), true + } + default: + if string(t) == repoKey { + return repoKey, true + } + } + return "", false +} + +// mergePluginBlobs deep-merges repo-level defaults under a per-project blob and returns the result; +// the per-project value wins. Nested maps merge recursively; every other value (scalars and lists) is +// atomic. Neither input is mutated. +func mergePluginBlobs(defaults, project map[string]any) map[string]any { + if m, ok := deepMergeAny(defaults, project).(map[string]any); ok { + return m + } + return nil +} + +// deepMergeAny merges override onto base: only maps merge recursively; every other value (scalars AND +// lists) is atomic, with override winning when set. base branches are deep-copied so the shared +// repo-defaults map is never aliased into a project's result. +func deepMergeAny(base, override any) any { + bm, bok := base.(map[string]any) + om, ook := override.(map[string]any) + if bok && ook { + out := make(map[string]any, len(bm)+len(om)) + for k, v := range bm { + out[k] = deepCopyAny(v) + } + for k, v := range om { + if bv, ok := bm[k]; ok { + out[k] = deepMergeAny(bv, v) + } else { + out[k] = v + } + } + return out + } + if override == nil { + return deepCopyAny(base) + } + return override +} + +func deepCopyAny(v any) any { + switch t := v.(type) { + case map[string]any: + out := make(map[string]any, len(t)) + for k, val := range t { + out[k] = deepCopyAny(val) + } + return out + case []any: + out := make([]any, len(t)) + for i, val := range t { + out[i] = deepCopyAny(val) + } + return out + default: + return v + } +} diff --git a/config_test.go b/config_test.go index 290db51..aadd353 100644 --- a/config_test.go +++ b/config_test.go @@ -269,6 +269,15 @@ projects: { Name: "main", Path: "path/to/my_terraform", + // the repo-level terraform source map is folded into the project's plugin blob as + // regex_source_map (a map), which is what the plugin reads. + Plugins: map[string]map[string]any{ + "terraform": { + "regex_source_map": map[string]any{ + "^ANOTHER_MODULE$": "github.com/CentricaDevOps/networks-aws-modules//modules/another?ref=another_v1.0.0", + }, + }, + }, }, }, }, diff --git a/configfile/configfile.go b/configfile/configfile.go new file mode 100644 index 0000000..2590724 --- /dev/null +++ b/configfile/configfile.go @@ -0,0 +1,131 @@ +// Package configfile is the file-syntax layer for the Infracost config file. It models the on-disk +// shape only: a project (or the repo root) is a set of structural fields plus an arbitrary set of +// named "headings" (terraform:, aws:, kubernetes:, ...) captured verbatim as opaque nodes. +// +// It deliberately does NOT understand what any heading means - the typed/blob split, the +// aws->cloudformation mapping, the terraform-family collapse, etc. all live in the semantic layer +// (package config), which imports this package and converts between configfile.* and the config API +// types. Keeping the semantics out of here is what lets this package stay a pure, dependency-light +// representation of the file and avoids an import cycle with package config. +// +// The file is split by type: this file holds the whole-file (Config) representation, configproject.go +// holds a single project entry. +package configfile + +import ( + "fmt" + "sort" + + "gopkg.in/yaml.v3" +) + +// captureHeadings stores every non-structural top-level key from all (keyed by structural) into +// headings. A heading is a plugin blob, which is always a mapping (raw_options is a JSON object), so +// a non-mapping value under an unknown key is a typo of a structural field, not a blob, and is +// rejected with a clear error rather than silently captured. An explicitly empty/null value is +// allowed (an empty heading). +func captureHeadings(all map[string]yaml.Node, structural map[string]bool, headings *map[string]yaml.Node) error { + for key, n := range all { + if structural[key] { + continue + } + if !isHeadingNode(n) { + return fmt.Errorf("unknown field %q", key) + } + if *headings == nil { + *headings = map[string]yaml.Node{} + } + (*headings)[key] = n + } + return nil +} + +// isHeadingNode reports whether a captured value can be a plugin blob: a mapping, or an explicitly +// empty/null value. Scalars and sequences cannot be a raw_options blob. +func isHeadingNode(n yaml.Node) bool { + switch n.Kind { + case yaml.MappingNode: + return true + case yaml.ScalarNode: + return n.Tag == "!!null" || n.Value == "" + default: + return false + } +} + +// Config is the on-disk representation of the whole config file. The structural fields are the keys +// config passes through untouched; every other top-level key is an opaque heading captured in +// Headings (e.g. terraform for repo-level defaults, cdk, or repo-level plugin defaults). The semantic +// layer (package config) decides what those headings mean. +type Config struct { + Version string `yaml:"version"` + Currency string `yaml:"currency,omitempty"` + UsageFile string `yaml:"usage_file,omitempty"` + Projects []Project `yaml:"projects"` + Autodetect yaml.Node `yaml:"autodetect,omitempty"` + + // Headings holds every top-level key that is not one of the structural fields above (terraform, + // cdk, and future repo-level plugin defaults), captured raw. Never serialized directly + // (yaml:"-"); MarshalYAML re-emits its entries as top-level keys. + Headings map[string]yaml.Node `yaml:"-"` +} + +// structuralConfigKeys is the set of top-level file keys that map to the structural fields above. Any +// other key is captured as a heading. Kept in sync with the yaml tags on Config by the test +// TestConfig_StructuralKeysMatchTags. +var structuralConfigKeys = map[string]bool{ + "version": true, + "currency": true, + "usage_file": true, + "projects": true, + "autodetect": true, +} + +// UnmarshalYAML decodes the structural fields normally, then captures every remaining top-level key +// as an opaque heading. A local alias type avoids recursing back into this method. +func (c *Config) UnmarshalYAML(node *yaml.Node) error { + type structural Config + var s structural + if err := node.Decode(&s); err != nil { + return err + } + *c = Config(s) + + var all map[string]yaml.Node + if err := node.Decode(&all); err != nil { + return err + } + return captureHeadings(all, structuralConfigKeys, &c.Headings) +} + +// MarshalYAML emits the structural fields followed by the captured headings as top-level keys (sorted +// for deterministic output), so the file round-trips to one heading per plugin with no `plugins:` +// block. +func (c Config) MarshalYAML() (any, error) { + type structural Config + var out yaml.Node + if err := out.Encode(structural(c)); err != nil { + return nil, err + } + + for _, key := range sortedKeys(c.Headings) { + n := c.Headings[key] + out.Content = append(out.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: key}, + &n, + ) + } + + return &out, nil +} + +// sortedKeys returns the keys of a heading map in deterministic (sorted) order, used when re-emitting +// captured headings so marshalled output is stable. +func sortedKeys(m map[string]yaml.Node) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/configfile/configfile_test.go b/configfile/configfile_test.go new file mode 100644 index 0000000..c9252a3 --- /dev/null +++ b/configfile/configfile_test.go @@ -0,0 +1,133 @@ +package configfile + +import ( + "reflect" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func decodeConfig(t *testing.T, src string) Config { + t.Helper() + var c Config + require.NoError(t, yaml.Unmarshal([]byte(src), &c)) + return c +} + +// Structural top-level fields decode into their fields; repo-level headings (terraform defaults, cdk, +// unknown plugin defaults) are captured opaquely and don't leak the structural keys. +func TestConfig_UnmarshalSplitsStructuralFromHeadings(t *testing.T) { + c := decodeConfig(t, ` +version: "0.3" +currency: USD +usage_file: usage.yml +terraform: + source_map: + - match: a + replace: b +cdk: + defaults: + context: + key: val +kubernetes: + some_default: true +projects: + - name: main + type: terraform + path: . + terraform: + var_files: [prod.tfvars] +`) + + assert.Equal(t, "0.3", c.Version) + assert.Equal(t, "USD", c.Currency) + assert.Equal(t, "usage.yml", c.UsageFile) + require.Len(t, c.Projects, 1) + assert.Equal(t, "main", c.Projects[0].Name) + assert.Contains(t, c.Projects[0].Headings, "terraform") + + // repo-level headings captured + assert.Contains(t, c.Headings, "terraform") + assert.Contains(t, c.Headings, "cdk") + assert.Contains(t, c.Headings, "kubernetes") + + // structural keys must NOT be captured as repo-level headings + for _, k := range []string{"version", "currency", "usage_file", "projects"} { + _, leaked := c.Headings[k] + assert.Falsef(t, leaked, "structural key %q leaked into repo Headings", k) + } +} + +// Round-trip: decode -> encode -> decode preserves structural fields, projects, and repo headings, +// and never emits a `plugins:` block. +func TestConfig_RoundTrip(t *testing.T) { + src := ` +version: "0.3" +currency: USD +terraform: + source_map: + - match: a + replace: b +cdk: + defaults: + context: + key: val +projects: + - name: main + type: terraform + path: . + terraform: + var_files: + - prod.tfvars + kubernetes: + opt: true +` + c := decodeConfig(t, src) + + out, err := yaml.Marshal(c) + require.NoError(t, err) + rendered := string(out) + t.Logf("\n--- MARSHALLED ---\n%s", rendered) + assert.NotContains(t, rendered, "plugins:", "must not emit a plugins: block") + assert.Contains(t, rendered, "terraform:") + assert.Contains(t, rendered, "cdk:") + + c2 := decodeConfig(t, rendered) + assert.Equal(t, c.Version, c2.Version) + assert.Equal(t, c.Currency, c2.Currency) + require.Len(t, c2.Projects, 1) + assert.Equal(t, "main", c2.Projects[0].Name) + assert.Contains(t, c2.Projects[0].Headings, "terraform") + assert.Contains(t, c2.Projects[0].Headings, "kubernetes") + + assert.Equal(t, decodeRepoHeading(t, c, "terraform"), decodeRepoHeading(t, c2, "terraform")) + assert.Equal(t, decodeRepoHeading(t, c, "cdk"), decodeRepoHeading(t, c2, "cdk")) +} + +func decodeRepoHeading(t *testing.T, c Config, name string) map[string]any { + t.Helper() + node := c.Headings[name] + var out map[string]any + require.NoError(t, node.Decode(&out)) + return out +} + +func TestConfig_StructuralKeysMatchTags(t *testing.T) { + tagged := map[string]bool{} + rt := reflect.TypeFor[Config]() + for i := 0; i < rt.NumField(); i++ { + tag := rt.Field(i).Tag.Get("yaml") + if tag == "" || tag == "-" { + continue + } + name := strings.Split(tag, ",")[0] + if name == "" { + continue + } + tagged[name] = true + } + assert.Equal(t, tagged, structuralConfigKeys, "structuralConfigKeys is out of sync with Config's yaml tags") +} diff --git a/configfile/configproject.go b/configfile/configproject.go new file mode 100644 index 0000000..1a71281 --- /dev/null +++ b/configfile/configproject.go @@ -0,0 +1,83 @@ +package configfile + +import "gopkg.in/yaml.v3" + +// Project is the on-disk representation of a single entry in the config file's projects list. The +// structural fields are the keys config passes through untouched; every other top-level key is an +// opaque heading captured in Headings (e.g. terraform, aws, kubernetes). The semantic layer decides +// what those headings mean. +type Project struct { + Name string `yaml:"name,omitempty"` + Type string `yaml:"type,omitempty"` + Path string `yaml:"path"` + Env map[string]string `yaml:"env,omitempty"` + Metadata map[string]string `yaml:"metadata,omitempty"` + EnvName string `yaml:"env_name,omitempty"` + UsageFile string `yaml:"usage_file,omitempty"` + YorConfigPath string `yaml:"yor_config_path,omitempty"` + ExcludePaths []string `yaml:"exclude_paths,omitempty"` + DependencyPaths []string `yaml:"dependency_paths,omitempty"` + CDKSynthError string `yaml:"cdk_synth_error,omitempty"` + + // Headings holds every top-level key that is not one of the structural fields above, keyed by the + // heading name (terraform, aws, kubernetes, ...) and captured as a raw node so the semantic layer + // can interpret or forward it without this package understanding its contents. Never serialized + // directly (yaml:"-"); MarshalYAML re-emits its entries as top-level keys. + Headings map[string]yaml.Node `yaml:"-"` +} + +// structuralProjectKeys is the set of top-level project keys that map to the structural fields above. +// Any other key is captured as a heading. Kept in sync with the yaml tags on Project by the test +// TestProject_StructuralKeysMatchTags. +var structuralProjectKeys = map[string]bool{ + "name": true, + "type": true, + "path": true, + "env": true, + "metadata": true, + "env_name": true, + "usage_file": true, + "yor_config_path": true, + "exclude_paths": true, + "dependency_paths": true, + "cdk_synth_error": true, +} + +// UnmarshalYAML decodes the structural fields normally, then captures every remaining top-level key +// as an opaque heading. Using a local alias type for the structural decode avoids recursing back into +// this method. +func (p *Project) UnmarshalYAML(node *yaml.Node) error { + type structural Project + var s structural + if err := node.Decode(&s); err != nil { + return err + } + *p = Project(s) + + var all map[string]yaml.Node + if err := node.Decode(&all); err != nil { + return err + } + return captureHeadings(all, structuralProjectKeys, &p.Headings) +} + +// MarshalYAML emits the structural fields followed by the captured headings as top-level keys, so a +// project round-trips to one heading per plugin with no separate `plugins:` block. Heading keys are +// emitted in sorted order for deterministic output. +func (p Project) MarshalYAML() (any, error) { + type structural Project + var out yaml.Node + if err := out.Encode(structural(p)); err != nil { + return nil, err + } + + for _, key := range sortedKeys(p.Headings) { + n := p.Headings[key] + out.Content = append(out.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: key}, + &n, + ) + } + + return &out, nil +} diff --git a/configfile/configproject_test.go b/configfile/configproject_test.go new file mode 100644 index 0000000..e374212 --- /dev/null +++ b/configfile/configproject_test.go @@ -0,0 +1,202 @@ +package configfile + +import ( + "reflect" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func decodeProject(t *testing.T, src string) Project { + t.Helper() + var p Project + require.NoError(t, yaml.Unmarshal([]byte(src), &p)) + return p +} + +// Structural fields decode into their fields; every other top-level key is captured as an opaque +// heading, and the structural keys never leak into Headings. +func TestProject_UnmarshalSplitsStructuralFromHeadings(t *testing.T) { + p := decodeProject(t, ` +name: main +type: terragrunt +path: app +env_name: prod +usage_file: usage.yml +exclude_paths: [vendor] +terraform: + workspace: prod + cloud: + host: app.terraform.io + org: acme + var_files: [prod.tfvars] + vars: + region: us-east-1 +aws: + region: eu-west-1 + stack_id: s-123 +kubernetes: + some_option: true +`) + + assert.Equal(t, "main", p.Name) + assert.Equal(t, "terragrunt", p.Type) + assert.Equal(t, "app", p.Path) + assert.Equal(t, "prod", p.EnvName) + assert.Equal(t, "usage.yml", p.UsageFile) + assert.Equal(t, []string{"vendor"}, p.ExcludePaths) + + // headings captured + assert.Contains(t, p.Headings, "terraform") + assert.Contains(t, p.Headings, "aws") + assert.Contains(t, p.Headings, "kubernetes") + + // structural keys must NOT be captured as headings + for _, k := range []string{"name", "type", "path", "env_name", "usage_file", "exclude_paths"} { + _, leaked := p.Headings[k] + assert.Falsef(t, leaked, "structural key %q leaked into Headings", k) + } + + // heading contents are preserved as raw nodes we can decode + tf := decodeHeading(t, p, "terraform") + assert.Equal(t, "prod", tf["workspace"]) + assert.Equal(t, []any{"prod.tfvars"}, tf["var_files"]) + assert.Equal(t, map[string]any{"host": "app.terraform.io", "org": "acme"}, tf["cloud"]) +} + +// An unknown key with a non-mapping value is a typo of a structural field (plugin blobs are always +// mappings), so it errors rather than being silently captured as a heading. Mapping and empty/null +// headings are accepted. +func TestProject_RejectsNonMappingUnknownKeys(t *testing.T) { + for _, tc := range []struct { + name string + src string + wantErr string + }{ + {"scalar typo", "path: app\npathh: x", `unknown field "pathh"`}, + {"sequence heading", "path: app\nterraform: [foo]", `unknown field "terraform"`}, + {"mapping heading", "path: app\nkubernetes: {ns: x}", ""}, + {"empty heading", "path: app\nkubernetes:", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + var p Project + err := yaml.Unmarshal([]byte(tc.src), &p) + if tc.wantErr == "" { + assert.NoError(t, err) + } else { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantErr) + } + }) + } +} + +// A project with no headings has a nil/empty Headings map (not spurious captures). +func TestProject_NoHeadings(t *testing.T) { + p := decodeProject(t, ` +name: main +type: terraform +path: . +`) + assert.Empty(t, p.Headings) +} + +// Marshalling emits one heading per plugin (no `plugins:` block) and round-trips: decode -> encode -> +// decode preserves structural fields and heading contents. +func TestProject_RoundTrip(t *testing.T) { + src := ` +name: main +type: terragrunt +path: app +terraform: + cloud: + host: app.terraform.io + org: acme + workspace: prod + var_files: + - prod.tfvars +aws: + region: eu-west-1 +kubernetes: + some_option: true +` + p := decodeProject(t, src) + + out, err := yaml.Marshal(p) + require.NoError(t, err) + + rendered := string(out) + t.Logf("\n--- MARSHALLED ---\n%s", rendered) + assert.NotContains(t, rendered, "plugins:", "must not emit a plugins: block") + // one heading per plugin, at project level + assert.Contains(t, rendered, "terraform:") + assert.Contains(t, rendered, "aws:") + assert.Contains(t, rendered, "kubernetes:") + + p2 := decodeProject(t, rendered) + assert.Equal(t, p.Name, p2.Name) + assert.Equal(t, p.Type, p2.Type) + assert.Equal(t, p.Path, p2.Path) + + assert.Equal(t, decodeHeading(t, p, "terraform"), decodeHeading(t, p2, "terraform"), "terraform heading must survive round-trip") + assert.Equal(t, decodeHeading(t, p, "aws"), decodeHeading(t, p2, "aws")) +} + +// decodeHeading decodes a captured heading node into a generic map (map values aren't addressable, so +// the node must be copied to a local before calling the pointer-receiver Decode). +func decodeHeading(t *testing.T, p Project, name string) map[string]any { + t.Helper() + node := p.Headings[name] + var out map[string]any + require.NoError(t, node.Decode(&out)) + return out +} + +// A project decoded as an element of a list (the real config shape) still splits correctly - i.e. the +// custom (un)marshal works in the position it's actually used. +func TestProject_InProjectsList(t *testing.T) { + var doc struct { + Projects []Project `yaml:"projects"` + } + require.NoError(t, yaml.Unmarshal([]byte(` +projects: + - name: a + type: terraform + path: a + terraform: + var_files: [a.tfvars] + - name: b + type: cloudformation + path: b + aws: + region: us-east-1 +`), &doc)) + + require.Len(t, doc.Projects, 2) + assert.Contains(t, doc.Projects[0].Headings, "terraform") + assert.NotContains(t, doc.Projects[0].Headings, "aws") + assert.Contains(t, doc.Projects[1].Headings, "aws") +} + +// structuralProjectKeys must exactly match the yaml tags of the non-Headings fields, so nothing config +// treats as structural is accidentally captured as a heading (or vice versa). +func TestProject_StructuralKeysMatchTags(t *testing.T) { + tagged := map[string]bool{} + rt := reflect.TypeFor[Project]() + for i := 0; i < rt.NumField(); i++ { + f := rt.Field(i) + tag := f.Tag.Get("yaml") + if tag == "" || tag == "-" { + continue + } + name := strings.Split(tag, ",")[0] + if name == "" { + continue + } + tagged[name] = true + } + assert.Equal(t, tagged, structuralProjectKeys, "structuralProjectKeys is out of sync with Project's yaml tags") +} diff --git a/configfile_convert.go b/configfile_convert.go new file mode 100644 index 0000000..244f658 --- /dev/null +++ b/configfile_convert.go @@ -0,0 +1,451 @@ +package config + +import ( + "fmt" + + "github.com/infracost/config/configfile" + "gopkg.in/yaml.v3" +) + +// This file is the semantic bridge between the file-syntax layer (package configfile) and the config +// API types. It interprets the opaque headings the file layer captures. +// +// NOTE: for now headings map to the current typed fields (terraform -> Terraform, aws -> +// ProjectAWSConfig, cdk -> CDK) so behaviour is unchanged. Routing the opaque remainder into the +// Plugins blob (the typed/blob split) lands with the read-path work; this bridge is the seam where +// that change will happen. + +// fromConfigFile populates target from the neutral file representation, interpreting each heading. +// Structural fields copy across directly; known headings decode into their typed API fields; any +// other heading is an opaque plugin blob. +// MarshalYAML renders the API Config in the on-disk heading shape (one heading per plugin, no +// `plugins:` block) by going through the file layer. Every yaml.Marshal of a Config - generation's +// re-parse, env-var replacement, callers saving the file - therefore emits the file-syntax form. +func (c Config) MarshalYAML() (any, error) { + return toConfigFile(&c) +} + +// UnmarshalYAML decodes the on-disk heading shape into the API Config via the file layer, so every +// yaml.Unmarshal into a Config interprets headings (typed/blob split) consistently. It decodes into +// the receiver, preserving any defaults it was seeded with (see fromConfigFile). +func (c *Config) UnmarshalYAML(node *yaml.Node) error { + var fc configfile.Config + if err := node.Decode(&fc); err != nil { + return err + } + return fromConfigFile(&fc, c) +} + +func fromConfigFile(fc *configfile.Config, target *Config) error { + if fc.Version != "" { + target.Version = fc.Version + } + // Only overwrite scalars the file actually set, so a Config seeded with defaults (e.g. currency + // USD) keeps them when the file omits the key. + if fc.Currency != "" { + target.Currency = fc.Currency + } + if fc.UsageFile != "" { + target.UsageFilePath = fc.UsageFile + } + + for name, node := range fc.Headings { + switch name { + case "terraform": + // repo-level terraform is the typed {source_map, defaults} section. Its defaults: sub-block + // splits like a project's terraform: heading - the typed keys (cloud/spacelift/workspace) + // decode into TerraformDefaults, and everything else is a repo-level terraform blob default + // forwarded to the plugin. Because the split extracts the typed keys, the blob can never hold + // them, so there is no collision when they recombine under defaults: on write. + if err := decodeInto(node, &target.Terraform); err != nil { + return fmt.Errorf("%w: terraform: %s", ErrInvalidConfigYAML, err) + } + if defaultsNode, ok := mappingChild(node, "defaults"); ok { + _, blob, err := splitMapping(defaultsNode, terraformTypedKeys) + if err != nil { + return fmt.Errorf("%w: terraform defaults: %s", ErrInvalidConfigYAML, err) + } + if len(blob) > 0 { + if target.Plugins == nil { + target.Plugins = map[string]map[string]any{} + } + target.Plugins[string(ProjectTypeTerraform)] = blob + } + } + case "cdk": + // cdk is a typed section, not a plugin blob. + if err := decodeInto(node, &target.CDK); err != nil { + return fmt.Errorf("%w: cdk: %s", ErrInvalidConfigYAML, err) + } + default: + // Repo-level plugin defaults live under a defaults: sub-key (consistent with + // terraform.defaults). Only that sub-block is a repo default merged into projects; any other + // direct key is ignored - config is plugin-agnostic and only understands the defaults: + // convention it owns. Key like a project blob (aws -> cloudformation) so the fold in + // normalize matches it to the projects it applies to. + blob, err := decodeBlob(node) + if err != nil { + return fmt.Errorf("%w: heading %q: %s", ErrInvalidConfigYAML, name, err) + } + defaults, ok := blob["defaults"].(map[string]any) + if !ok || len(defaults) == 0 { + continue + } + if target.Plugins == nil { + target.Plugins = map[string]map[string]any{} + } + target.Plugins[blobKeyForHeading(name, ProjectTypeUnknown)] = defaults + } + } + + // Only replace the target's projects when the file actually specified them (nil => key omitted, + // so keep whatever defaults the caller seeded; non-nil => honour it, even if empty). + if fc.Projects != nil { + projects := make([]*Project, 0, len(fc.Projects)) + for i := range fc.Projects { + p, err := projectFromConfigFile(&fc.Projects[i]) + if err != nil { + return err + } + projects = append(projects, p) + } + target.Projects = projects + } + + return nil +} + +// projectFromConfigFile converts a single file-layer project into the API Project. +func projectFromConfigFile(fp *configfile.Project) (*Project, error) { + p := &Project{ + Name: fp.Name, + Type: ProjectType(fp.Type), + Path: fp.Path, + Env: fp.Env, + Metadata: fp.Metadata, + EnvName: fp.EnvName, + UsageFile: fp.UsageFile, + YorConfigPath: fp.YorConfigPath, + ExcludePaths: fp.ExcludePaths, + DependencyPaths: fp.DependencyPaths, + CDKSynthError: fp.CDKSynthError, + } + + for name, node := range fp.Headings { + switch name { + case "terraform": + // split: the typed keys config reads outside the plugin stay on ProjectTerraform; the rest + // (var_files, vars, and anything unrecognised) is the opaque blob forwarded to the plugin. + typedNode, blob, err := splitMapping(node, terraformTypedKeys) + if err != nil { + return nil, fmt.Errorf("%w: project %q terraform: %s", ErrInvalidConfigYAML, p.Path, err) + } + if len(typedNode.Content) > 0 { + if err := typedNode.Decode(&p.Terraform); err != nil { + return nil, fmt.Errorf("%w: project %q terraform: %s", ErrInvalidConfigYAML, p.Path, err) + } + } + setProjectPlugin(p, blobKeyForHeading("terraform", p.Type), blob) + default: + // aws and any other heading are opaque blobs; aws keys under cloudformation. + blob, err := decodeBlob(node) + if err != nil { + return nil, fmt.Errorf("%w: project %q heading %q: %s", ErrInvalidConfigYAML, p.Path, name, err) + } + setProjectPlugin(p, blobKeyForHeading(name, p.Type), blob) + } + } + + mirrorBlobToDeprecated(p) + + return p, nil +} + +// mirrorBlobToDeprecated populates the deprecated typed fields (Terraform.VarFiles/.Vars, AWS) from +// the canonical Plugins blob, so consumers still reading them get real data during the deprecation +// window. The blob remains the source of truth and the only representation serialized. +func mirrorBlobToDeprecated(p *Project) { + if blob := p.Plugins[terraformBlobKey(p.Type)]; blob != nil { + if v, ok := blob["var_files"]; ok { + p.Terraform.VarFiles = anyToStringSlice(v) + } + if v, ok := blob["vars"].(map[string]any); ok { + // deep-copy so the deprecated mirror never aliases the canonical blob's map. + if cp, ok := deepCopyAny(v).(map[string]any); ok { + p.Terraform.Vars = cp + } + } + } + if blob := p.Plugins[string(ProjectTypeCloudFormation)]; blob != nil { + p.AWS = ProjectAWSConfig{ + Region: anyToString(blob["region"]), + AccountID: anyToString(blob["account_id"]), + StackID: anyToString(blob["stack_id"]), + StackName: anyToString(blob["stack_name"]), + } + } +} + +// anyToStringSlice coerces a decoded blob value into []string. Values that have been through a YAML +// round-trip are []any of strings; ones built in-process may be []string already. +func anyToStringSlice(v any) []string { + switch t := v.(type) { + case []string: + return t + case []any: + out := make([]string, 0, len(t)) + for _, e := range t { + if s, ok := e.(string); ok { + out = append(out, s) + } + } + return out + default: + return nil + } +} + +func anyToString(v any) string { + s, _ := v.(string) + return s +} + +// setProjectPlugin stores a non-empty blob under p.Plugins[key], allocating the maps as needed. +func setProjectPlugin(p *Project, key string, blob map[string]any) { + if len(blob) == 0 { + return + } + if p.Plugins == nil { + p.Plugins = map[string]map[string]any{} + } + p.Plugins[key] = blob +} + +// decodeInto decodes a captured heading node into out. The node is copied to a local first because +// yaml.Node.Decode has a pointer receiver and map-indexed values aren't addressable. +func decodeInto(node yaml.Node, out any) error { + n := node + return n.Decode(out) +} + +func decodeBlob(node yaml.Node) (map[string]any, error) { + var blob map[string]any + if err := decodeInto(node, &blob); err != nil { + return nil, err + } + return blob, nil +} + +// toConfigFile is the inverse of fromConfigFile: it renders the API Config back into the neutral file +// representation, emitting each typed section as its heading. It mirrors fromConfigFile so a +// Config -> configfile -> Config round-trip is an identity (which is what the generate marshal / +// re-parse cycle relies on). +func toConfigFile(c *Config) (*configfile.Config, error) { + fc := &configfile.Config{ + Version: c.Version, + Currency: c.Currency, + UsageFile: c.UsageFilePath, + } + + // Repo-level plugin defaults render under their heading (cloudformation -> aws) wrapped in a + // defaults: sub-key - the convention every plugin uses for repo-level defaults, matching + // terraform.defaults. (terraform's own defaults live in its typed section below.) + for key, blob := range c.Plugins { + if len(blob) == 0 || key == string(ProjectTypeTerraform) { + // terraform's repo blob defaults are merged into its typed section below. + continue + } + if err := setHeading(&fc.Headings, headingForBlobKey(key), map[string]any{"defaults": blob}); err != nil { + return nil, err + } + } + + // Repo-level typed terraform ({source_map, defaults}), with any repo-level terraform blob defaults + // merged under defaults: alongside the typed ones (their keys are disjoint by construction). + // source_map stays here as the deprecated typed field; it is folded into each project as + // regex_source_map during normalize. + var tfNode yaml.Node + if err := tfNode.Encode(c.Terraform); err != nil { + return nil, fmt.Errorf("%w: encode terraform: %s", ErrInvalidConfigYAML, err) + } + if blob := c.Plugins[string(ProjectTypeTerraform)]; len(blob) > 0 { + if err := mergeUnderDefaults(&tfNode, blob); err != nil { + return nil, fmt.Errorf("%w: encode terraform defaults: %s", ErrInvalidConfigYAML, err) + } + } + if len(tfNode.Content) > 0 { + if fc.Headings == nil { + fc.Headings = map[string]yaml.Node{} + } + fc.Headings["terraform"] = tfNode + } + + if err := setHeading(&fc.Headings, "cdk", c.CDK); err != nil { + return nil, err + } + + if c.Projects != nil { + fc.Projects = make([]configfile.Project, 0, len(c.Projects)) + for _, p := range c.Projects { + fp, err := toConfigFileProject(p) + if err != nil { + return nil, err + } + fc.Projects = append(fc.Projects, *fp) + } + } + + return fc, nil +} + +func toConfigFileProject(p *Project) (*configfile.Project, error) { + fp := &configfile.Project{ + Name: p.Name, + Type: string(p.Type), + Path: p.Path, + Env: p.Env, + Metadata: p.Metadata, + EnvName: p.EnvName, + UsageFile: p.UsageFile, + YorConfigPath: p.YorConfigPath, + ExcludePaths: p.ExcludePaths, + DependencyPaths: p.DependencyPaths, + CDKSynthError: p.CDKSynthError, + } + + // Each plugin blob renders under its heading (terraform-family -> terraform:, cloudformation -> + // aws:, others keep their name). + for key, blob := range p.Plugins { + if len(blob) == 0 { + continue + } + var n yaml.Node + if err := n.Encode(blob); err != nil { + return nil, fmt.Errorf("%w: encode blob %q: %s", ErrInvalidConfigYAML, key, err) + } + mergeHeading(&fp.Headings, headingForBlobKey(key), n) + } + + // The typed terraform fields (workspace/cloud/spacelift) merge into the terraform: heading ahead of + // the blob keys, so they read first. + typedNode, err := filterEncode(p.Terraform, terraformTypedKeys) + if err != nil { + return nil, fmt.Errorf("%w: encode terraform: %s", ErrInvalidConfigYAML, err) + } + if len(typedNode.Content) > 0 { + if existing, ok := fp.Headings["terraform"]; ok { + fp.Headings["terraform"] = concatMappings(typedNode, existing) + } else { + if fp.Headings == nil { + fp.Headings = map[string]yaml.Node{} + } + fp.Headings["terraform"] = typedNode + } + } + + return fp, nil +} + +// mergeHeading stores node under headings[name], concatenating if that heading already has content +// (so multiple plugin keys that map to the same heading don't clobber each other). +func mergeHeading(headings *map[string]yaml.Node, name string, node yaml.Node) { + if *headings == nil { + *headings = map[string]yaml.Node{} + } + if existing, ok := (*headings)[name]; ok { + (*headings)[name] = concatMappings(existing, node) + return + } + (*headings)[name] = node +} + +// splitMapping partitions a mapping node into a node holding the given keys and a generic map holding +// the rest. +func splitMapping(node yaml.Node, keys map[string]bool) (yaml.Node, map[string]any, error) { + kept := yaml.Node{Kind: yaml.MappingNode} + rest := map[string]any{} + for i := 0; i+1 < len(node.Content); i += 2 { + k, v := node.Content[i], node.Content[i+1] + if keys[k.Value] { + kept.Content = append(kept.Content, k, v) + continue + } + var val any + if err := v.Decode(&val); err != nil { + return kept, nil, err + } + rest[k.Value] = val + } + return kept, rest, nil +} + +// filterEncode encodes v to a mapping node and keeps only the given keys. +func filterEncode(v any, keys map[string]bool) (yaml.Node, error) { + var full yaml.Node + if err := full.Encode(v); err != nil { + return full, err + } + out := yaml.Node{Kind: yaml.MappingNode} + for i := 0; i+1 < len(full.Content); i += 2 { + if keys[full.Content[i].Value] { + out.Content = append(out.Content, full.Content[i], full.Content[i+1]) + } + } + return out, nil +} + +// concatMappings returns a mapping node with a's key/value pairs followed by b's. +func concatMappings(a, b yaml.Node) yaml.Node { + out := yaml.Node{Kind: yaml.MappingNode} + out.Content = append(out.Content, a.Content...) + out.Content = append(out.Content, b.Content...) + return out +} + +// mappingChild returns the value node for key in a mapping node. +func mappingChild(node yaml.Node, key string) (yaml.Node, bool) { + for i := 0; i+1 < len(node.Content); i += 2 { + if node.Content[i].Value == key { + return *node.Content[i+1], true + } + } + return yaml.Node{}, false +} + +// mergeUnderDefaults appends blob's key/value pairs to the defaults: child of a terraform heading +// node (creating defaults: if absent). Callers guarantee the blob's keys are disjoint from the typed +// defaults (the read split extracts the typed keys), so no key is emitted twice. +func mergeUnderDefaults(node *yaml.Node, blob map[string]any) error { + var blobNode yaml.Node + if err := blobNode.Encode(blob); err != nil { + return err + } + for i := 0; i+1 < len(node.Content); i += 2 { + if node.Content[i].Value == "defaults" { + node.Content[i+1].Content = append(node.Content[i+1].Content, blobNode.Content...) + return nil + } + } + node.Content = append(node.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: "defaults"}, + &blobNode, + ) + return nil +} + +// setHeading encodes a typed value into a heading node and stores it, skipping values that encode to +// an empty mapping so we never emit a bare `heading: {}`. +func setHeading(headings *map[string]yaml.Node, name string, v any) error { + var n yaml.Node + if err := n.Encode(v); err != nil { + return fmt.Errorf("%w: encode heading %q: %s", ErrInvalidConfigYAML, name, err) + } + if len(n.Content) == 0 { + return nil + } + if *headings == nil { + *headings = map[string]yaml.Node{} + } + (*headings)[name] = n + return nil +} diff --git a/configfile_convert_test.go b/configfile_convert_test.go new file mode 100644 index 0000000..df24caa --- /dev/null +++ b/configfile_convert_test.go @@ -0,0 +1,215 @@ +package config + +import ( + "testing" + + "github.com/infracost/config/configfile" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func headingNode(t *testing.T, yamlBody string) yaml.Node { + t.Helper() + var n yaml.Node + require.NoError(t, yaml.Unmarshal([]byte(yamlBody), &n)) + // unwrap the document node to the mapping node + return *n.Content[0] +} + +// fromConfigFile maps the known headings (terraform, aws, cdk) into their typed API fields and routes +// any other heading into the opaque Plugins blob. +func TestFromConfigFile_MapsHeadingsToTypedFields(t *testing.T) { + fc := &configfile.Config{ + Version: "0.3", + Currency: "USD", + UsageFile: "usage.yml", + Headings: map[string]yaml.Node{ + "terraform": headingNode(t, ` +source_map: + - match: "^MOD$" + replace: "github.com/x//mod" +defaults: + workspace: prod +`), + "cdk": headingNode(t, ` +defaults: + context: + key: val +`), + // repo-level plugin defaults live under a defaults: sub-key (consistent with terraform.defaults) + "kubernetes": headingNode(t, ` +defaults: + some_default: true +`), + }, + Projects: []configfile.Project{ + { + Name: "main", + Type: "terragrunt", + Path: "app", + Headings: map[string]yaml.Node{ + "terraform": headingNode(t, ` +workspace: prod +cloud: + host: app.terraform.io + org: acme +var_files: + - prod.tfvars +vars: + region: us-east-1 +`), + }, + }, + { + Name: "stack", + Type: "cloudformation", + Path: "stack", + Headings: map[string]yaml.Node{ + "aws": headingNode(t, ` +region: us-east-1 +stack_name: my-stack +`), + "kubernetes": headingNode(t, ` +opt: true +`), + }, + }, + }, + } + + var target Config + require.NoError(t, fromConfigFile(fc, &target)) + + // structural + assert.Equal(t, "0.3", target.Version) + assert.Equal(t, "USD", target.Currency) + assert.Equal(t, "usage.yml", target.UsageFilePath) + + // repo-level terraform heading -> typed Terraform + require.Len(t, target.Terraform.SourceMap, 1) + assert.Equal(t, "^MOD$", target.Terraform.SourceMap[0].Match) + assert.Equal(t, "prod", target.Terraform.Defaults.Workspace) + + // repo-level cdk heading -> typed CDK (not a plugin blob) + assert.Equal(t, "val", target.CDK.Defaults.Context["key"]) + assert.NotContains(t, target.Plugins, "cdk") + + // repo-level unknown heading -> repo Plugins blob + require.Contains(t, target.Plugins, "kubernetes") + assert.Equal(t, true, target.Plugins["kubernetes"]["some_default"]) + + // project terraform heading splits: workspace/cloud stay typed; var_files/vars go to the blob + // keyed by the exact project type (terragrunt). + require.Len(t, target.Projects, 2) + tg := target.Projects[0] + assert.Equal(t, "main", tg.Name) + assert.Equal(t, ProjectTypeTerragrunt, tg.Type) + assert.Equal(t, "prod", tg.Terraform.Workspace) + assert.Equal(t, "app.terraform.io", tg.Terraform.Cloud.Host) + assert.Equal(t, []any{"prod.tfvars"}, tg.Plugins["terragrunt"]["var_files"], "canonical source is the blob") + assert.Equal(t, map[string]any{"region": "us-east-1"}, tg.Plugins["terragrunt"]["vars"]) + // the deprecated typed fields are populated as a backwards-compat mirror of the blob + assert.Equal(t, []string{"prod.tfvars"}, tg.Terraform.VarFiles, "mirrored from the blob") + assert.Equal(t, map[string]any{"region": "us-east-1"}, tg.Terraform.Vars, "mirrored from the blob") + + // project aws heading -> cloudformation blob (canonical); the deprecated typed AWS mirrors it. + cfn := target.Projects[1] + assert.Equal(t, "us-east-1", cfn.Plugins["cloudformation"]["region"]) + assert.Equal(t, "us-east-1", cfn.AWS.Region, "mirrored from the cloudformation blob") + assert.Equal(t, "my-stack", cfn.AWS.StackName, "mirrored from the cloudformation blob") + assert.Equal(t, "my-stack", cfn.Plugins["cloudformation"]["stack_name"]) + require.Contains(t, cfn.Plugins, "kubernetes") + assert.Equal(t, true, cfn.Plugins["kubernetes"]["opt"]) +} + +// toConfigFile is the inverse of fromConfigFile: an API Config -> configfile -> API Config round-trip +// preserves the typed sections and plugin blobs. +func TestToConfigFile_RoundTripsThroughAPI(t *testing.T) { + original := &Config{ + ConfigBase: ConfigBase{Version: "0.3"}, + Currency: "USD", + UsageFilePath: "usage.yml", + Terraform: Terraform{ + SourceMap: []TerraformRegexSource{{Match: "^MOD$", Replace: "github.com/x//mod"}}, + Defaults: TerraformDefaults{Workspace: "prod"}, + }, + Plugins: map[string]map[string]any{ + "kubernetes": {"some_default": true}, + }, + Projects: []*Project{ + { + Name: "main", + Type: ProjectTypeTerragrunt, + Path: "app", + // post-split API shape: workspace/cloud typed, var_files/vars in the blob keyed by type + Terraform: ProjectTerraform{ + Workspace: "prod", + Cloud: TerraformCloud{Host: "app.terraform.io", Org: "acme"}, + }, + Plugins: map[string]map[string]any{ + "terragrunt": { + "var_files": []any{"prod.tfvars"}, + "vars": map[string]any{"region": "us-east-1"}, + }, + }, + }, + { + Name: "stack", + Type: ProjectTypeCloudFormation, + Path: "stack", + Plugins: map[string]map[string]any{ + "cloudformation": {"region": "us-east-1", "stack_name": "my-stack"}, + "kubernetes": {"opt": true}, + }, + }, + }, + } + + fc, err := toConfigFile(original) + require.NoError(t, err) + + // the file layer must carry one heading per plugin - no `plugins:` block anywhere + out, err := yaml.Marshal(fc) + require.NoError(t, err) + rendered := string(out) + t.Logf("\n--- MARSHALLED ---\n%s", rendered) + assert.NotContains(t, rendered, "plugins:") + // the cloudformation blob renders under the historical aws: heading + assert.Contains(t, rendered, "aws:") + + var back Config + require.NoError(t, fromConfigFile(fc, &back)) + + assert.Equal(t, original.Version, back.Version) + assert.Equal(t, original.Currency, back.Currency) + assert.Equal(t, original.UsageFilePath, back.UsageFilePath) + assert.Equal(t, original.Terraform, back.Terraform) + assert.Equal(t, original.Plugins, back.Plugins) + require.Len(t, back.Projects, 2) + // the canonical blob and the typed (non-mirrored) fields round-trip exactly; back also gains the + // deprecated mirror (VarFiles/Vars/AWS), which is derived on load, so compare those explicitly + // rather than the whole Terraform struct. + assert.Equal(t, original.Projects[0].Plugins, back.Projects[0].Plugins) + assert.Equal(t, original.Projects[0].Terraform.Workspace, back.Projects[0].Terraform.Workspace) + assert.Equal(t, original.Projects[0].Terraform.Cloud, back.Projects[0].Terraform.Cloud) + assert.Equal(t, original.Projects[0].Name, back.Projects[0].Name) + assert.Equal(t, original.Projects[0].Type, back.Projects[0].Type) + assert.Equal(t, original.Projects[1].Plugins, back.Projects[1].Plugins) +} + +// A nil Projects slice (key omitted) leaves the target's existing projects untouched; a non-nil slice +// replaces them. +func TestFromConfigFile_ProjectsSeeding(t *testing.T) { + seeded := []*Project{{Path: "."}} + + // omitted -> keep seeded + target := Config{Projects: seeded} + require.NoError(t, fromConfigFile(&configfile.Config{Version: "0.3"}, &target)) + assert.Equal(t, seeded, target.Projects, "omitted projects must keep the seeded defaults") + + // explicit empty -> replace with empty + target = Config{Projects: seeded} + require.NoError(t, fromConfigFile(&configfile.Config{Version: "0.3", Projects: []configfile.Project{}}, &target)) + assert.Empty(t, target.Projects, "explicit empty projects must clear the defaults") +} diff --git a/configlegacy.go b/configlegacy.go new file mode 100644 index 0000000..7e50d45 --- /dev/null +++ b/configlegacy.go @@ -0,0 +1,195 @@ +package config + +import ( + "bytes" + "fmt" + + "github.com/infracost/config/cdk" + "gopkg.in/yaml.v3" +) + +// This file isolates the deprecated 0.1/0.2 config-file support: the intermediary types that mirror +// the old flat field names and the parser that migrates them into the current API Config. It is a +// verbatim relocation of what used to live in config.go / config_file.go, kept together so it can be +// deleted wholesale once 0.1/0.2 support is dropped. It deliberately produces a *Config directly (it +// is same-package), so it never touches the configfile file-syntax layer used by the current format. + +var legacyVersions = []string{ + "0.1", // The 0.1 version of the config file was used in the original CLI. + "0.2", // The 0.2 version of the config file was used in the original version of the Infracost Cloud Platform. +} + +// ConfigWithLegacySupport handles the parameters that were supported in 0.1 and 0.2 but are not +// supported in 0.3. This allows us to parse legacy config files. +type ConfigWithLegacySupport struct { + ConfigBase `yaml:",inline"` + Currency string `yaml:"currency,omitempty"` + UsageFilePath string `yaml:"usage_file,omitempty"` + TerraformRegexSourceMap []TerraformRegexSource `yaml:"terraform_source_map,omitempty"` + TerraformCloudHost string `yaml:"terraform_cloud_host,omitempty"` + TerraformCloudOrg string `yaml:"terraform_cloud_org,omitempty"` + TerraformCloudWorkspace string `yaml:"terraform_cloud_workspace,omitempty"` + TerraformCloudToken string `yaml:"terraform_cloud_token,omitempty"` + SpaceliftAPIKeyEndpoint string `yaml:"spacelift_api_key_endpoint,omitempty"` + SpaceliftAPIKeyID string `yaml:"spacelift_api_key_id,omitempty"` + SpaceliftAPIKeySecret string `yaml:"spacelift_api_key_secret,omitempty"` + TerraformWorkspace string `yaml:"terraform_workspace,omitempty"` + Projects []*ProjectWithLegacySupport `yaml:"projects"` + Autodetect yaml.Node `yaml:"autodetect,omitempty"` + CDK []*cdk.ConfigEntry `yaml:"cdk,omitempty"` + CDKDefaults cdk.Defaults `yaml:"cdk_defaults,omitempty"` +} + +type ProjectWithLegacySupport struct { + Project `yaml:",inline"` + TerraformVars map[string]any `yaml:"terraform_vars,omitempty"` + TerraformWorkspace string `yaml:"terraform_workspace,omitempty"` + TerraformCloudHost string `yaml:"terraform_cloud_host,omitempty"` + TerraformCloudOrg string `yaml:"terraform_cloud_org,omitempty"` + TerraformCloudWorkspace string `yaml:"terraform_cloud_workspace,omitempty"` + TerraformCloudToken string `yaml:"terraform_cloud_token,omitempty"` + SpaceliftAPIKeyEndpoint string `yaml:"spacelift_api_key_endpoint,omitempty"` + SpaceliftAPIKeyID string `yaml:"spacelift_api_key_id,omitempty"` + SpaceliftAPIKeySecret string `yaml:"spacelift_api_key_secret,omitempty"` + TerraformVarFiles []string `yaml:"terraform_var_files,omitempty"` + ProjectType ProjectType `yaml:"project_type,omitempty"` // terraform, terragrunt + SkipAutodetect bool `yaml:"skip_autodetect,omitempty"` // deprecated, ignored + IncludeAllPaths bool `yaml:"include_all_paths,omitempty"` // deprecated, ignored +} + +func parseLegacyVersion(content []byte, config *Config) error { + + var intermediary ConfigWithLegacySupport + if len(config.Projects) > 0 { + // if the config came with projectsd set (e.g. default main) we need to set it here to see if it gets overridden by legacy projects + intermediary.Projects = make([]*ProjectWithLegacySupport, 0, len(config.Projects)) + for _, project := range config.Projects { + intermediary.Projects = append(intermediary.Projects, &ProjectWithLegacySupport{ + Project: *project, + }) + } + } + + decoder := yaml.NewDecoder(bytes.NewReader(content)) + decoder.KnownFields(true) + + if err := decoder.Decode(&intermediary); err != nil { + return fmt.Errorf("%w: %s", ErrInvalidConfigYAML, simplifyYAMLError(err)) + } + + // copy across fields that exist in both + config.ConfigBase = intermediary.ConfigBase + config.Version = CurrentVersion // force version to latest after converting + config.Currency = intermediary.Currency + config.UsageFilePath = intermediary.UsageFilePath + config.CDK.Projects = intermediary.CDK + config.CDK.Defaults = intermediary.CDKDefaults + + // copy legacy fields to their new locations + config.Terraform.SourceMap = intermediary.TerraformRegexSourceMap + config.Terraform.Defaults.Cloud.Host = intermediary.TerraformCloudHost + config.Terraform.Defaults.Cloud.Org = intermediary.TerraformCloudOrg + config.Terraform.Defaults.Cloud.Workspace = intermediary.TerraformCloudWorkspace + config.Terraform.Defaults.Cloud.Token = intermediary.TerraformCloudToken + config.Terraform.Defaults.Spacelift.APIKey.Endpoint = intermediary.SpaceliftAPIKeyEndpoint + config.Terraform.Defaults.Spacelift.APIKey.ID = intermediary.SpaceliftAPIKeyID + config.Terraform.Defaults.Spacelift.APIKey.Secret = intermediary.SpaceliftAPIKeySecret + config.Terraform.Defaults.Workspace = intermediary.TerraformWorkspace + + // remove default projects and take whatever the decode gace us - if the user didn't specify the projects key, we'll get the defaults preserved anyway + config.Projects = nil + + // convert legacy projects to new ones + // this is deliberately a nil check rather than checking length, as we want to preserve an empty projects section if it was explicitly set to empty in the legacy config + if len(intermediary.Projects) > 0 { + config.Projects = make([]*Project, 0, len(intermediary.Projects)) + + for _, legacyProject := range intermediary.Projects { + project := legacyProject.Project + if len(legacyProject.TerraformVars) > 0 { + if project.Terraform.Vars == nil { + project.Terraform.Vars = make(map[string]any) + } + for k, v := range legacyProject.TerraformVars { + project.Terraform.Vars[k] = v + } + } + if legacyProject.TerraformWorkspace != "" { + project.Terraform.Workspace = legacyProject.TerraformWorkspace + } + if legacyProject.TerraformCloudHost != "" { + project.Terraform.Cloud.Host = legacyProject.TerraformCloudHost + } + if legacyProject.TerraformCloudOrg != "" { + project.Terraform.Cloud.Org = legacyProject.TerraformCloudOrg + } + if legacyProject.TerraformCloudWorkspace != "" { + project.Terraform.Cloud.Workspace = legacyProject.TerraformCloudWorkspace + } + if legacyProject.TerraformCloudToken != "" { + project.Terraform.Cloud.Token = legacyProject.TerraformCloudToken + } + if legacyProject.SpaceliftAPIKeyEndpoint != "" { + project.Terraform.Spacelift.APIKey.Endpoint = legacyProject.SpaceliftAPIKeyEndpoint + } + if legacyProject.SpaceliftAPIKeyID != "" { + project.Terraform.Spacelift.APIKey.ID = legacyProject.SpaceliftAPIKeyID + } + if legacyProject.SpaceliftAPIKeySecret != "" { + project.Terraform.Spacelift.APIKey.Secret = legacyProject.SpaceliftAPIKeySecret + } + if len(legacyProject.TerraformVarFiles) > 0 { + project.Terraform.VarFiles = legacyProject.TerraformVarFiles + } + if legacyProject.ProjectType != "" { + project.Type = legacyProject.ProjectType + } + foldDeprecatedIntoPlugins(&project) + config.Projects = append(config.Projects, &project) + } + } + + return nil +} + +// foldDeprecatedIntoPlugins copies a legacy project's typed terraform var files/vars and aws context +// into the opaque Plugins blob, keyed exactly as the 0.3 read path keys them. It does NOT clear the +// typed fields: the blob is the canonical source (serialized, consumer-preferred) and the typed fields +// remain as a backwards-compat mirror (matching mirrorBlobToDeprecated on the 0.3 path). configlegacy +// is read-only, so it maps straight to the final API shape - and the blob copy is what survives the +// env-var marshal/re-parse round-trip (which keeps only workspace/cloud/spacelift typed). +func foldDeprecatedIntoPlugins(p *Project) { + tfKey := terraformBlobKey(p.Type) + if len(p.Terraform.VarFiles) > 0 { + p.setPluginOption(tfKey, "var_files", p.Terraform.VarFiles) + } + if len(p.Terraform.Vars) > 0 { + p.setPluginOption(tfKey, "vars", p.Terraform.Vars) + } + + if aws := awsContextBlob(p.AWS); len(aws) > 0 { + if p.Plugins == nil { + p.Plugins = map[string]map[string]any{} + } + p.Plugins[string(ProjectTypeCloudFormation)] = aws + } +} + +// awsContextBlob renders the deprecated typed aws context as the flat snake_case blob the +// cloudformation plugin reads (matching the aws: heading keys). +func awsContextBlob(a ProjectAWSConfig) map[string]any { + blob := map[string]any{} + if a.Region != "" { + blob["region"] = a.Region + } + if a.AccountID != "" { + blob["account_id"] = a.AccountID + } + if a.StackID != "" { + blob["stack_id"] = a.StackID + } + if a.StackName != "" { + blob["stack_name"] = a.StackName + } + return blob +} diff --git a/generate.go b/generate.go index 83afd47..3601116 100644 --- a/generate.go +++ b/generate.go @@ -3,6 +3,7 @@ package config import ( "bytes" "context" + "encoding/json" "errors" "fmt" "os" @@ -248,18 +249,32 @@ func Generate( if !hasProjectsSection { output.Projects = make([]*Project, 0, len(projects)) for _, project := range projects { - output.Projects = append(output.Projects, &Project{ + p := &Project{ Path: project.Path, Name: project.Name, EnvName: project.Env, DependencyPaths: project.DependencyPaths, Metadata: project.Metadata, - Terraform: ProjectTerraform{ - VarFiles: project.TerraformVarFiles, - Workspace: project.Env, - }, - Type: ProjectType(project.Type), - }) + Type: ProjectType(project.Type), + // Workspace is a caller-sourced runtime option (passed to the plugin via + // GenericOptions.Workspace) and is also read outside the plugin, so it stays a top-level + // field rather than going into the plugins blob. + Terraform: ProjectTerraform{Workspace: project.Env}, + } + + // Persist the plugin's parse options under plugins., keyed by the consuming plugin, + // as a native YAML map so it stays readable and editable. This is now the ONLY place + // generated config carries these options - the deprecated terraform.* / aws.* fields are + // no longer emitted (they are folded into plugins. on read for hand-written configs). + opts, err := generatedPluginOptions(project) + if err != nil { + return nil, fmt.Errorf("%w: failed to build plugin options for project %q: %s", ErrInvalidConfigYAML, project.Path, err) + } + if len(opts) > 0 { + p.Plugins = map[string]map[string]any{PluginKeyForType(p.Type): opts} + } + + output.Projects = append(output.Projects, p) } } @@ -282,3 +297,37 @@ func Generate( return output, nil } + +// generatedPluginOptions returns the plugins. blob to persist for an autodetected project, as +// a native map ready to store under plugins.. +// +// When a plugin authored a raw_options blob during identification (only when it returned +// environments authoritatively) that blob is persisted verbatim. Otherwise - when the plugin +// returned no environments, was unimplemented, or no plugin was used - config sideloads the var +// files it attributed locally into a terraform-family blob. Those var files depend on the +// whole-directory (sibling/pibling) attribution config performs in its tree passes, which the +// plugins cannot yet reproduce, so config still owns them; this sideload is the one deliberate, +// minimal special case and is expected to go away once IdentifyAllProjects moves that attribution +// plugin-side. Non-terraform-family projects get no generated blob - their options (e.g. aws +// context) are user-authored and folded in on read. +func generatedPluginOptions(project autodetect.Project) (map[string]any, error) { + if len(project.RawOptions) > 0 { + var opts map[string]any + if err := json.Unmarshal(project.RawOptions, &opts); err != nil { + return nil, err + } + return opts, nil + } + + if !IsTerraformFamily(ProjectType(project.Type)) { + return nil, nil + } + + opts := map[string]any{} + if len(project.TerraformVarFiles) > 0 { + opts["var_files"] = project.TerraformVarFiles + } + // Workspace is not part of the blob - it is written to the top-level terraform.workspace field + // (see Generate) and passed to the plugin via GenericOptions.Workspace. + return opts, nil +} diff --git a/go.mod b/go.mod index f688a6c..b28112e 100644 --- a/go.mod +++ b/go.mod @@ -8,8 +8,8 @@ require ( github.com/hashicorp/go-hclog v1.6.3 github.com/hashicorp/go-plugin v1.8.0 github.com/hashicorp/hcl/v2 v2.24.0 - github.com/infracost/go-proto v0.1.2 - github.com/infracost/proto v1.154.0 + github.com/infracost/go-proto v1.26.0 + github.com/infracost/proto v1.159.0 github.com/json-iterator/go v1.1.12 github.com/soongo/path-to-regexp v1.6.4 github.com/stretchr/testify v1.11.1 diff --git a/go.sum b/go.sum index 19a02c3..1d966d5 100644 --- a/go.sum +++ b/go.sum @@ -36,12 +36,10 @@ github.com/hashicorp/hcl/v2 v2.24.0 h1:2QJdZ454DSsYGoaE6QheQZjtKZSUs9Nh2izTWiwQx github.com/hashicorp/hcl/v2 v2.24.0/go.mod h1:oGoO1FIQYfn/AgyOhlg9qLC6/nOJPX3qGbkZpYAcqfM= github.com/hashicorp/yamux v0.1.2 h1:XtB8kyFOyHXYVFnwT5C3+Bdo8gArse7j2AQ0DA0Uey8= github.com/hashicorp/yamux v0.1.2/go.mod h1:C+zze2n6e/7wshOZep2A70/aQU6QBRWJO/G6FT1wIns= -github.com/infracost/go-proto v0.1.2 h1:BF6QbKPOEZp06RgJvvMcw1cFX/WR25gGV5QCjopRU8c= -github.com/infracost/go-proto v0.1.2/go.mod h1:Rj+Od+a85tG7U9nDDvpsUh1gQRnSpZrTswvPEnAtaCw= -github.com/infracost/proto v1.153.1-0.20260701083223-7dd6cb640924 h1:uqcs0th6ApuTOai8/SK8/W7B2LIg/LY3DXWehLXw1D4= -github.com/infracost/proto v1.153.1-0.20260701083223-7dd6cb640924/go.mod h1:Z8vPWBWblwJlw+/ksO+BtsXwf9NiOcSTWx0WRWNbfUA= -github.com/infracost/proto v1.154.0 h1:BgiNeV0wq17FjFk7DRPaxTgYryJzj6Zfr6nUT3oqZb4= -github.com/infracost/proto v1.154.0/go.mod h1:Z8vPWBWblwJlw+/ksO+BtsXwf9NiOcSTWx0WRWNbfUA= +github.com/infracost/go-proto v1.26.0 h1:DkvnNA3jbYp/i6PALl59JHO552W5Hw+bEhfhg5JH7e0= +github.com/infracost/go-proto v1.26.0/go.mod h1:9m9zdHPl6mME24y3J0CdTBWXr+d03mfgXuy+9JzA/wU= +github.com/infracost/proto v1.159.0 h1:fcgE4AApNvK8M4cbumOpRpAq+HlbVXzGaCdqCPkl6kA= +github.com/infracost/proto v1.159.0/go.mod h1:Z8vPWBWblwJlw+/ksO+BtsXwf9NiOcSTWx0WRWNbfUA= github.com/jhump/protoreflect v1.17.0 h1:qOEr613fac2lOuTgWN4tPAtLL7fUSbuJL5X5XumQh94= github.com/jhump/protoreflect v1.17.0/go.mod h1:h9+vUUL38jiBzck8ck+6G/aeMX8Z4QUY/NiJPwPNi+8= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= diff --git a/plugin/identifier.go b/plugin/identifier.go index 873072d..3faf8c6 100644 --- a/plugin/identifier.go +++ b/plugin/identifier.go @@ -47,6 +47,10 @@ type Environment struct { Path string Files []string DependencyPaths []string + // RawOptions is the plugin-authored, opaque parse-options blob (always JSON) for this + // environment. The caller persists it under plugins. in the config file and forwards it + // verbatim into Parse; it is never interpreted here. + RawOptions []byte } // AttributedVarFile is the plugin-agnostic view of a var file the caller has already attributed @@ -172,6 +176,7 @@ func (i *Identifier) IdentifyEnvironments(ctx context.Context, dir string, proje Path: e.Path, Files: e.Files, DependencyPaths: e.DependencyPaths, + RawOptions: e.RawOptions, }) } return environments, true, nil diff --git a/plugins_test.go b/plugins_test.go new file mode 100644 index 0000000..0dd9291 --- /dev/null +++ b/plugins_test.go @@ -0,0 +1,86 @@ +package config_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/infracost/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Generation persists terraform var files under plugins..var_files (the key the terraform +// plugin reads) and no longer writes the deprecated top-level terraform.var_files field. +func TestPlugins_GenerationSideloadsVarFilesIntoBlob(t *testing.T) { + root := NewFilesystem(t) + root.AddTerraformFileWithProviderBlock("main.tf") + root.AddTFVarsFile("terraform.tfvars") + + cfg, err := config.Generate(t.Context(), root.Path()) + require.NoError(t, err) + require.Len(t, cfg.Projects, 1) + + p := cfg.Projects[0] + assert.Equal(t, config.ProjectTypeTerraform, p.Type) + // the canonical representation is the plugins blob... + assert.Contains(t, toStringSlice(p.Plugins["terraform"]["var_files"]), "terraform.tfvars") + // ...and the deprecated terraform.var_files field is populated as a backwards-compat mirror of it. + assert.Contains(t, p.Terraform.VarFiles, "terraform.tfvars") +} + +// Repo-level plugin defaults for every plugin live under a .defaults: sub-key (consistent with +// terraform.defaults) and deep-merge into each matching project's blob, with the per-project value +// winning. A repo-level key placed directly (not under defaults:) is ignored. +func TestPlugins_RepoDefaultsUnderDefaultsKey(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "infracost.yml") + require.NoError(t, os.WriteFile(path, []byte(`version: "0.3" +kubernetes: + defaults: + namespace: default + replicas: 1 +terraform: + ignored_direct_key: nope + defaults: + workspace: prod + vars: + env: staging +projects: + - path: k8s + type: kubernetes + kubernetes: + replicas: 3 + - path: other + type: kubernetes + - path: tf + type: terraform +`), 0o600)) + + cfg, err := config.LoadConfigFile(path, dir) + require.NoError(t, err) + + byPath := map[string]*config.Project{} + for _, p := range cfg.Projects { + byPath[p.Path] = p + } + + // per-project value wins over the repo default; unset default is inherited + assert.Equal(t, 3, byPath["k8s"].Plugins["kubernetes"]["replicas"], "per-project replicas wins") + assert.Equal(t, "default", byPath["k8s"].Plugins["kubernetes"]["namespace"], "repo default namespace inherited") + + // a project with no own blob inherits the repo defaults wholesale + assert.Equal(t, "default", byPath["other"].Plugins["kubernetes"]["namespace"]) + assert.Equal(t, 1, byPath["other"].Plugins["kubernetes"]["replicas"]) + + // terraform.defaults splits: workspace is a typed default (inherited on the typed field), while the + // rest (vars) is a blob default merged into the terraform project's blob. + assert.Equal(t, "prod", byPath["tf"].Terraform.Workspace, "typed default inherited") + assert.Equal(t, map[string]any{"env": "staging"}, byPath["tf"].Plugins["terraform"]["vars"], "blob default merged") + assert.NotContains(t, byPath["tf"].Plugins["terraform"], "workspace", "typed key never leaks into the blob") + + // a repo-level key placed directly (not under defaults:) is not applied as a default + for _, p := range cfg.Projects { + assert.NotContains(t, p.Plugins["terraform"], "ignored_direct_key") + } +} diff --git a/setup_test.go b/setup_test.go index f34cc66..359198b 100644 --- a/setup_test.go +++ b/setup_test.go @@ -231,11 +231,51 @@ func assertProjectMatches(t *testing.T, want, got *config.Project, prefix string assert.Equal(t, want.Name, got.Name, "%s: expected project name %q, got %q", prefix, want.Name, got.Name) assert.Equal(t, want.Path, got.Path, "%s: expected project path %q, got %q", prefix, want.Path, got.Path) assert.Equal(t, want.EnvName, got.EnvName, "%s: expected project env name %q, got %q", prefix, want.EnvName, got.EnvName) - if len(want.Terraform.VarFiles) > 0 || len(got.Terraform.VarFiles) > 0 { - assert.EqualValues(t, want.Terraform.VarFiles, got.Terraform.VarFiles, "%s: expected project terraform var files %q, got %q", prefix, want.Terraform.VarFiles, got.Terraform.VarFiles) + // Generated config now carries terraform var files under plugins..var_files rather than + // the deprecated top-level terraform.var_files field. These tests express the expectation with the + // old field for readability, so compare the effective var files from whichever representation each + // side uses. + wantVarFiles := effectiveVarFiles(want) + gotVarFiles := effectiveVarFiles(got) + if len(wantVarFiles) > 0 || len(gotVarFiles) > 0 { + assert.EqualValues(t, wantVarFiles, gotVarFiles, "%s: expected project terraform var files %q, got %q", prefix, wantVarFiles, gotVarFiles) } if len(want.DependencyPaths) > 0 || len(got.DependencyPaths) > 0 { assert.EqualValues(t, want.DependencyPaths, got.DependencyPaths, "%s: expected project dependency paths %q, got %q", prefix, want.DependencyPaths, got.DependencyPaths) } assert.Equal(t, want.Type, got.Type, "%s: expected project type %q, got %q", prefix, want.Type, got.Type) } + +// effectiveVarFiles returns a project's terraform var files from whichever representation is set: the +// new Plugins blob (keyed by the exact project type, or "terraform" for an untyped project - mirroring +// how the CLI/runner look it up via finalProjectType) or, as a fallback, the deprecated +// terraform.var_files field the older tests author their expectations with. +func effectiveVarFiles(p *config.Project) []string { + for _, key := range []string{string(p.Type), "terraform"} { + if blob, ok := p.Plugins[key]; ok { + if v, ok := blob["var_files"]; ok { + return toStringSlice(v) + } + } + } + return p.Terraform.VarFiles +} + +// toStringSlice coerces a decoded YAML/JSON value into a []string. Blobs that have been through a +// YAML round-trip hold []any of strings; ones built in-process may hold []string directly. +func toStringSlice(v any) []string { + switch t := v.(type) { + case []string: + return t + case []any: + out := make([]string, 0, len(t)) + for _, e := range t { + if s, ok := e.(string); ok { + out = append(out, s) + } + } + return out + default: + return nil + } +}