diff --git a/README.rst b/README.rst index aba7dea70..b32efdcb6 100644 --- a/README.rst +++ b/README.rst @@ -1217,15 +1217,28 @@ This command requires a ``.sops.yaml`` configuration file. Below is an example: vault_kv_version: 2 # default path_regex: vault/* omit_extensions: true + - aws_secrets_manager_secret_name: "my-secret" + aws_region: "us-west-2" + path_regex: aws-secrets/* + - aws_parameter_store_path: "/sops/" + aws_region: "us-west-2" + path_regex: aws-params/* The above configuration will place all files under ``s3/*`` into the S3 bucket ``sops-secrets``, -all files under ``gcs/*`` into the GCS bucket ``sops-secrets``, and the contents of all files under -``vault/*`` into Vault's KV store under the path ``secrets/sops/``. For the files that will be -published to S3 and GCS, it will decrypt them and re-encrypt them using the -``F69E4901EDBAD2D1753F8C67A64535C4163FB307`` pgp key. +all files under ``gcs/*`` into the GCS bucket ``sops-secrets``, the contents of all files under +``vault/*`` into Vault's KV store under the path ``secrets/sops/``, files under ``aws-secrets/*`` +into AWS Secrets Manager as JSON secrets, and files under ``aws-params/*`` into AWS Parameter Store +as SecureString parameters. For the files that will be published to S3 and GCS, it will decrypt them +and re-encrypt them using the ``F69E4901EDBAD2D1753F8C67A64535C4163FB307`` pgp key. Files published to Vault +will be decrypted and stored as JSON data encrypted by Vault. Files published to AWS Secrets Manager and AWS Parameter Store +will be decrypted and stored as JSON data encrypted by AWS KMS. You would deploy a file to S3 with a command like: ``sops publish s3/app.yaml`` +Similarly, you can publish to AWS Secrets Manager: ``sops publish aws-secrets/database-config.yaml`` + +Or to AWS Parameter Store: ``sops publish aws-params/app-config.yaml`` + To publish all files in selected directory recursively, you need to specify ``--recursive`` flag. If you don't want file extension to appear in destination secret path, use ``--omit-extensions`` @@ -1277,6 +1290,77 @@ Below is an example of publishing to Vault (using token auth with a local dev in example_number 42 example_string bar +Publishing to AWS Secrets Manager +********************************** + +AWS Secrets Manager is a service that helps you protect secrets needed to access your applications, +services, and IT resources. SOPS can publish decrypted data directly to AWS Secrets Manager as JSON secrets. + +There are a few settings for AWS Secrets Manager that you can place in your destination rules: + +* ``aws_secrets_manager_secret_name`` - The name of the secret in AWS Secrets Manager. This is required. +* ``aws_region`` - The AWS region where the secret should be stored. If not specified, SOPS will use the region from the AWS SDK's default credential chain (environment variables, AWS config file, or IAM role). + +SOPS uses the AWS SDK for Go v2, which automatically uses your configured AWS credentials from the AWS CLI, +environment variables, or IAM roles. + +If the destination secret already exists in AWS Secrets Manager and contains the same data as the source +file, it will be skipped to avoid creating unnecessary versions. + +Note: Recreation rules (re-encryption with different keys) are not supported for AWS Secrets Manager. +The data is decrypted from the source file and stored as plaintext JSON in the secret. + +Below is an example of publishing to AWS Secrets Manager: + +.. code:: sh + + $ export AWS_REGION=us-west-2 + $ sops decrypt aws-secrets/database-config.yaml + database: + host: db.example.com + port: 5432 + username: myuser + password: mypassword + $ sops publish aws-secrets/database-config.yaml + uploading /home/user/sops_directory/aws-secrets/database-config.yaml to AWS Secrets Manager secret database-config in us-west-2 ? (y/n): y + Successfully created secret database-config + +Publishing to AWS Parameter Store +********************************** + +AWS Systems Manager Parameter Store provides secure, hierarchical storage for configuration data +and secrets management. SOPS can publish decrypted data directly to Parameter Store as JSON parameters encrypted by AWS KMS. + +There are a few settings for AWS Parameter Store that you can place in your destination rules: + +* ``aws_parameter_store_path`` - The parameter path in AWS Parameter Store. This is required. If it ends with ``/``, the filename will be appended. +* ``aws_region`` - The AWS region where the parameter should be stored. If not specified, SOPS will use the region from the AWS SDK's default credential chain (environment variables, AWS config file, or IAM role). + +All parameters are stored as ``SecureString`` type for security, since SOPS files may contain sensitive data. + +SOPS uses the AWS SDK for Go v2, which automatically uses your configured AWS credentials from the AWS CLI, +environment variables, or IAM roles. + +If the destination parameter already exists in AWS Parameter Store and contains the same data as the source +file, it will be skipped to avoid creating unnecessary versions. + +Note: Recreation rules (re-encryption with different keys) are not supported for AWS Parameter Store. +The data is decrypted from the source file and stored as JSON in the SecureString parameter, encrypted by AWS KMS. + +Below is an example of publishing to AWS Parameter Store: + +.. code:: sh + + $ export AWS_REGION=us-west-2 + $ sops decrypt aws-params/app-config.yaml + app: + debug: false + database_url: postgres://user:pass@localhost/db + api_key: secret-api-key + $ sops publish aws-params/app-config.yaml + uploading /home/user/sops_directory/aws-params/app-config.yaml to AWS Parameter Store parameter /app-config in us-west-2 ? (y/n): y + Successfully created parameter /app-config + Important information on types ------------------------------ diff --git a/cmd/sops/subcommand/publish/publish.go b/cmd/sops/subcommand/publish/publish.go index aed1118de..fba541003 100644 --- a/cmd/sops/subcommand/publish/publish.go +++ b/cmd/sops/subcommand/publish/publish.go @@ -137,7 +137,7 @@ func Run(opts Opts) error { return fmt.Errorf("could not read file: %s", err) } } - case *publish.VaultDestination: + case *publish.VaultDestination, *publish.AWSSecretsManagerDestination, *publish.AWSParameterStoreDestination: _, err = common.DecryptTree(common.DecryptTreeOpts{ Cipher: opts.Cipher, IgnoreMac: false, @@ -177,7 +177,7 @@ func Run(opts Opts) error { switch dest := conf.Destination.(type) { case *publish.S3Destination, *publish.GCSDestination: err = dest.Upload(fileContents, destinationPath) - case *publish.VaultDestination: + case *publish.VaultDestination, *publish.AWSSecretsManagerDestination, *publish.AWSParameterStoreDestination: err = dest.UploadUnencrypted(data, destinationPath) } diff --git a/config/config.go b/config/config.go index 6a67e0619..61a6aaeac 100644 --- a/config/config.go +++ b/config/config.go @@ -156,17 +156,20 @@ type azureKVKey struct { } type destinationRule struct { - PathRegex string `yaml:"path_regex"` - S3Bucket string `yaml:"s3_bucket"` - S3Prefix string `yaml:"s3_prefix"` - GCSBucket string `yaml:"gcs_bucket"` - GCSPrefix string `yaml:"gcs_prefix"` - VaultPath string `yaml:"vault_path"` - VaultAddress string `yaml:"vault_address"` - VaultKVMountName string `yaml:"vault_kv_mount_name"` - VaultKVVersion int `yaml:"vault_kv_version"` - RecreationRule creationRule `yaml:"recreation_rule,omitempty"` - OmitExtensions bool `yaml:"omit_extensions"` + PathRegex string `yaml:"path_regex"` + S3Bucket string `yaml:"s3_bucket"` + S3Prefix string `yaml:"s3_prefix"` + GCSBucket string `yaml:"gcs_bucket"` + GCSPrefix string `yaml:"gcs_prefix"` + VaultPath string `yaml:"vault_path"` + VaultAddress string `yaml:"vault_address"` + VaultKVMountName string `yaml:"vault_kv_mount_name"` + VaultKVVersion int `yaml:"vault_kv_version"` + RecreationRule creationRule `yaml:"recreation_rule,omitempty"` + OmitExtensions bool `yaml:"omit_extensions"` + AWSRegion string `yaml:"aws_region"` + AWSSecretsManagerSecretName string `yaml:"aws_secrets_manager_secret_name"` + AWSParameterStorePath string `yaml:"aws_parameter_store_path"` } type creationRule struct { @@ -522,6 +525,12 @@ func parseDestinationRuleForFile(conf *configFile, filePath string, kmsEncryptio if dRule.VaultPath != "" { destinationCount++ } + if dRule.AWSSecretsManagerSecretName != "" { + destinationCount++ + } + if dRule.AWSParameterStorePath != "" { + destinationCount++ + } if destinationCount > 1 { return nil, fmt.Errorf("error loading config: more than one destinations were found in a single destination rule, you can only use one per rule") @@ -535,6 +544,12 @@ func parseDestinationRuleForFile(conf *configFile, filePath string, kmsEncryptio if dRule.VaultPath != "" { dest = publish.NewVaultDestination(dRule.VaultAddress, dRule.VaultPath, dRule.VaultKVMountName, dRule.VaultKVVersion) } + if dRule.AWSSecretsManagerSecretName != "" { + dest = publish.NewAWSSecretsManagerDestination(dRule.AWSRegion, dRule.AWSSecretsManagerSecretName) + } + if dRule.AWSParameterStorePath != "" { + dest = publish.NewAWSParameterStoreDestination(dRule.AWSRegion, dRule.AWSParameterStorePath) + } config, err := configFromRule(rule, kmsEncryptionContext) if err != nil { diff --git a/config/config_aws_test.go b/config/config_aws_test.go new file mode 100644 index 000000000..c761a8640 --- /dev/null +++ b/config/config_aws_test.go @@ -0,0 +1,122 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +var sampleConfigWithAWSSecretsManagerDestinationRules = []byte(` +creation_rules: + - path_regex: foobar* + kms: "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012" +destination_rules: + - aws_region: "us-east-1" + aws_secrets_manager_secret_name: "myapp/database" + path_regex: "^secrets/.*" + - aws_region: "us-west-2" + aws_secrets_manager_secret_name: "api" + path_regex: "^west-secrets/.*" +`) + +var sampleConfigWithAWSParameterStoreDestinationRules = []byte(` +creation_rules: + - path_regex: foobar* + kms: "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012" +destination_rules: + - aws_region: "us-east-1" + aws_parameter_store_path: "/myapp/config" + path_regex: "^parameters/.*" + - aws_region: "us-west-2" + aws_parameter_store_path: "/myapp/west/" + path_regex: "^west-parameters/.*" +`) + +var sampleConfigWithMixedAWSDestinationRules = []byte(` +creation_rules: + - path_regex: foobar* + kms: "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012" +destination_rules: + - aws_region: "us-east-1" + aws_secrets_manager_secret_name: "myapp/database" + path_regex: "^secrets/.*" + - aws_region: "us-east-1" + aws_parameter_store_path: "/myapp/config" + path_regex: "^parameters/.*" + - s3_bucket: "mybucket" + path_regex: "^s3/.*" +`) + +func TestLoadConfigFileWithAWSSecretsManagerDestinationRules(t *testing.T) { + conf, err := parseDestinationRuleForFile(parseConfigFile(sampleConfigWithAWSSecretsManagerDestinationRules, t), "secrets/database.yaml", nil) + assert.Nil(t, err) + assert.NotNil(t, conf.Destination) + path := conf.Destination.Path("database.yaml") + assert.Contains(t, path, "arn:aws:secretsmanager:us-east-1:*:secret:myapp/database") + + // Test second rule with different region + conf, err = parseDestinationRuleForFile(parseConfigFile(sampleConfigWithAWSSecretsManagerDestinationRules, t), "west-secrets/api.yaml", nil) + assert.Nil(t, err) + assert.NotNil(t, conf.Destination) + path = conf.Destination.Path("api.yaml") + assert.Contains(t, path, "arn:aws:secretsmanager:us-west-2:*:secret:api") +} + +func TestLoadConfigFileWithAWSParameterStoreDestinationRules(t *testing.T) { + conf, err := parseDestinationRuleForFile(parseConfigFile(sampleConfigWithAWSParameterStoreDestinationRules, t), "parameters/app.yaml", nil) + assert.Nil(t, err) + assert.NotNil(t, conf.Destination) + assert.Equal(t, "/myapp/config", conf.Destination.Path("app.yaml")) + + // Test with path ending with slash + conf, err = parseDestinationRuleForFile(parseConfigFile(sampleConfigWithAWSParameterStoreDestinationRules, t), "west-parameters/config.yaml", nil) + assert.Nil(t, err) + assert.NotNil(t, conf.Destination) + assert.Equal(t, "/myapp/west/config.yaml", conf.Destination.Path("config.yaml")) +} + +func TestLoadConfigFileWithMixedAWSDestinationRules(t *testing.T) { + // Test AWS Secrets Manager + conf, err := parseDestinationRuleForFile(parseConfigFile(sampleConfigWithMixedAWSDestinationRules, t), "secrets/database.yaml", nil) + assert.Nil(t, err) + assert.NotNil(t, conf.Destination) + assert.Contains(t, conf.Destination.Path("database.yaml"), "arn:aws:secretsmanager:us-east-1:*:secret:myapp/database") + + // Test AWS Parameter Store + conf, err = parseDestinationRuleForFile(parseConfigFile(sampleConfigWithMixedAWSDestinationRules, t), "parameters/config.yaml", nil) + assert.Nil(t, err) + assert.NotNil(t, conf.Destination) + assert.Equal(t, "/myapp/config", conf.Destination.Path("config.yaml")) + + // Test S3 + conf, err = parseDestinationRuleForFile(parseConfigFile(sampleConfigWithMixedAWSDestinationRules, t), "s3/backup.yaml", nil) + assert.Nil(t, err) + assert.NotNil(t, conf.Destination) + assert.Contains(t, conf.Destination.Path("backup.yaml"), "s3://mybucket/backup.yaml") +} + +func TestValidateMultipleDestinationsInRule(t *testing.T) { + invalidConfig := []byte(` +destination_rules: + - aws_secrets_manager_secret_name: "my-secret" + aws_parameter_store_path: "/my/path" + path_regex: "^invalid/.*" +`) + + _, err := parseDestinationRuleForFile(parseConfigFile(invalidConfig, t), "invalid/test.yaml", nil) + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "more than one destinations were found") +} + +func TestValidateConflictingAWSDestinations(t *testing.T) { + invalidConfig := []byte(` +destination_rules: + - aws_secrets_manager_secret_name: "my-secret" + s3_bucket: "mybucket" + path_regex: "^invalid/.*" +`) + + _, err := parseDestinationRuleForFile(parseConfigFile(invalidConfig, t), "invalid/test.yaml", nil) + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "more than one destinations were found") +} diff --git a/config/config_test.go b/config/config_test.go index 753f870b1..21465ee6e 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -879,3 +879,114 @@ destination_rules: assert.NotNil(t, conf.Destination) assert.Contains(t, conf.Destination.Path("secrets.yaml"), "https://vault.example.com/v1/secret/data/secret/sops/secrets.yaml") } + +func TestDestinationValidationAWSSecretsManagerConflicts(t *testing.T) { + testCases := []struct { + name string + config []byte + }{ + { + name: "AWS Secrets Manager + GCS conflict", + config: []byte(` +destination_rules: + - aws_secrets_manager_secret_name: "my-secret" + gcs_bucket: "my-gcs-bucket" + path_regex: "^test/.*" +`), + }, + { + name: "AWS Secrets Manager + Vault conflict", + config: []byte(` +destination_rules: + - aws_secrets_manager_secret_name: "my-secret" + vault_path: "secret/sops" + vault_address: "https://vault.example.com" + path_regex: "^test/.*" +`), + }, + { + name: "AWS Secrets Manager + AWS Parameter Store conflict", + config: []byte(` +destination_rules: + - aws_secrets_manager_secret_name: "my-secret" + aws_parameter_store_path: "/my/path" + path_regex: "^test/.*" +`), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _, err := parseDestinationRuleForFile(parseConfigFile(tc.config, t), "test/secrets.yaml", nil) + assert.NotNil(t, err, "Expected error for %s", tc.name) + if err != nil { + assert.Contains(t, err.Error(), "more than one destinations were found") + } + }) + } +} + +func TestDestinationValidationAWSParameterStoreConflicts(t *testing.T) { + testCases := []struct { + name string + config []byte + }{ + { + name: "AWS Parameter Store + S3 conflict", + config: []byte(` +destination_rules: + - aws_parameter_store_path: "/my/path" + s3_bucket: "my-s3-bucket" + path_regex: "^test/.*" +`), + }, + { + name: "AWS Parameter Store + GCS conflict", + config: []byte(` +destination_rules: + - aws_parameter_store_path: "/my/path" + gcs_bucket: "my-gcs-bucket" + path_regex: "^test/.*" +`), + }, + { + name: "AWS Parameter Store + Vault conflict", + config: []byte(` +destination_rules: + - aws_parameter_store_path: "/my/path" + vault_path: "secret/sops" + vault_address: "https://vault.example.com" + path_regex: "^test/.*" +`), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _, err := parseDestinationRuleForFile(parseConfigFile(tc.config, t), "test/secrets.yaml", nil) + assert.NotNil(t, err, "Expected error for %s", tc.name) + if err != nil { + assert.Contains(t, err.Error(), "more than one destinations were found") + } + }) + } +} + +func TestDestinationValidationAllFiveDestinationsConflict(t *testing.T) { + invalidConfig := []byte(` +destination_rules: + - aws_secrets_manager_secret_name: "my-secret" + aws_parameter_store_path: "/my/path" + s3_bucket: "my-s3-bucket" + gcs_bucket: "my-gcs-bucket" + vault_path: "secret/sops" + vault_address: "https://vault.example.com" + path_regex: "^test/.*" +`) + + _, err := parseDestinationRuleForFile(parseConfigFile(invalidConfig, t), "test/secrets.yaml", nil) + assert.NotNil(t, err, "Expected error when all five destinations are specified") + if err != nil { + assert.Contains(t, err.Error(), "more than one destinations were found") + } +} diff --git a/go.mod b/go.mod index e3c542667..06b381320 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,10 @@ require ( github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.13 github.com/aws/aws-sdk-go-v2/service/kms v1.49.2 github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.39.2 + github.com/aws/aws-sdk-go-v2/service/ssm v1.64.2 github.com/aws/aws-sdk-go-v2/service/sts v1.41.3 + github.com/aws/smithy-go v1.24.0 github.com/blang/semver v3.5.1+incompatible github.com/fatih/color v1.18.0 github.com/getsops/gopgagent v0.0.0-20241224165529-7044f28e491e @@ -79,7 +82,6 @@ require ( github.com/aws/aws-sdk-go-v2/service/signin v1.0.3 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.30.6 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11 // indirect - github.com/aws/smithy-go v1.24.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudflare/circl v1.6.1 // indirect diff --git a/go.sum b/go.sum index ffa61ba0a..3e0b47e9c 100644 --- a/go.sum +++ b/go.sum @@ -95,8 +95,12 @@ github.com/aws/aws-sdk-go-v2/service/kms v1.49.2 h1:eEKImXK7MTiTdphS/C68OOQ0mY5i github.com/aws/aws-sdk-go-v2/service/kms v1.49.2/go.mod h1:hVFBUDC37+DMEtyd4LyKnJDqrV1Y/GD2S6p8VT2PC6U= github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 h1:IrbE3B8O9pm3lsg96AXIN5MXX4pECEuExh/A0Du3AuI= github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0/go.mod h1:/sJLzHtiiZvs6C1RbxS/anSAFwZD6oC6M/kotQzOiLw= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.39.2 h1:QMayWWWmfWyQwP4nZf3qdIVS39Pm65Yi5waYj1euCzo= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.39.2/go.mod h1:4eAXC8WdO1rRt01ZKKq57z8oTzzLkkIo5IReQ+b8hEU= github.com/aws/aws-sdk-go-v2/service/signin v1.0.3 h1:d/6xOGIllc/XW1lzG9a4AUBMmpLA9PXcQnVPTuHHcik= github.com/aws/aws-sdk-go-v2/service/signin v1.0.3/go.mod h1:fQ7E7Qj9GiW8y0ClD7cUJk3Bz5Iw8wZkWDHsTe8vDKs= +github.com/aws/aws-sdk-go-v2/service/ssm v1.64.2 h1:6P4W42RUTZixRG6TgfRB8KlsqNzHtvBhs6sTbkVPZvk= +github.com/aws/aws-sdk-go-v2/service/ssm v1.64.2/go.mod h1:wtxdacy3oO5sHO03uOtk8HMGfgo1gBHKwuJdYM220i0= github.com/aws/aws-sdk-go-v2/service/sso v1.30.6 h1:8sTTiw+9yuNXcfWeqKF2x01GqCF49CpP4Z9nKrrk/ts= github.com/aws/aws-sdk-go-v2/service/sso v1.30.6/go.mod h1:8WYg+Y40Sn3X2hioaaWAAIngndR8n1XFdRPPX+7QBaM= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11 h1:E+KqWoVsSrj1tJ6I/fjDIu5xoS2Zacuu1zT+H7KtiIk= diff --git a/publish/aws_integration_test.go b/publish/aws_integration_test.go new file mode 100644 index 000000000..8b88ba24b --- /dev/null +++ b/publish/aws_integration_test.go @@ -0,0 +1,225 @@ +//go:build integration + +package publish + +import ( + "context" + "encoding/json" + "os" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "github.com/aws/aws-sdk-go-v2/service/ssm" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Integration tests for AWS Secrets Manager and Parameter Store publishing. +// These tests require real AWS credentials and resources. +// +// To run these tests: +// 1. Set up AWS credentials (AWS_PROFILE, AWS_ACCESS_KEY_ID, etc.) +// 2. Set required environment variables: +// - SOPS_TEST_AWS_REGION (default: us-east-1) +// - SOPS_TEST_AWS_SECRET_NAME (secret for testing) +// - SOPS_TEST_AWS_PARAMETER_NAME (parameter for testing) +// 3. Run with: go test -tags=integration ./publish -run TestAWS -v +// +// Prerequisites: +// - AWS credentials with Secrets Manager and Parameter Store permissions +// - Test secret and parameter resources should already exist or be creatable + +var ( + testAWSRegion = getEnvOrDefault("SOPS_TEST_AWS_REGION", "us-east-1") + testSecretName = os.Getenv("SOPS_TEST_AWS_SECRET_NAME") // e.g., "sops-test-secret" + testParameterName = os.Getenv("SOPS_TEST_AWS_PARAMETER_NAME") // e.g., "/sops-test/parameter" +) + +func getEnvOrDefault(key, defaultValue string) string { + if value := os.Getenv(key); value != "" { + return value + } + return defaultValue +} + +func TestAWSSecretsManagerDestination_PlainText_Integration(t *testing.T) { + if testSecretName == "" { + t.Skip("Skipping integration test: SOPS_TEST_AWS_SECRET_NAME not set") + } + + ctx := context.Background() + dest := NewAWSSecretsManagerDestination(testAWSRegion, testSecretName) + + // Test data with complex nested structure + // Note: This format stores as Plain Text JSON and does NOT enable key/value editor in AWS console + // For key/value format, see TestAWSSecretsManagerDestination_KeyValue_Integration + testData := map[string]interface{}{ + "database": map[string]interface{}{ + "host": "localhost", + "port": float64(5432), + "username": "testuser", + "password": "supersecret", + }, + "api_keys": map[string]interface{}{ + "stripe": "sk_test_123456", + "github": "ghp_987654321", + }, + } + + // Upload test data + err := dest.UploadUnencrypted(testData, "test-secret") + require.NoError(t, err, "Failed to upload secret to Secrets Manager") + + // Verify the secret was stored correctly + cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(testAWSRegion)) + require.NoError(t, err, "Failed to load AWS config") + + client := secretsmanager.NewFromConfig(cfg) + result, err := client.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{ + SecretId: aws.String(testSecretName), + }) + require.NoError(t, err, "Failed to retrieve secret from Secrets Manager") + + // Parse and verify the stored data + var storedData map[string]interface{} + err = json.Unmarshal([]byte(*result.SecretString), &storedData) + require.NoError(t, err, "Failed to parse stored secret JSON") + + assert.Equal(t, testData, storedData, "Stored data doesn't match original") + + // Test no-op behavior (upload same data again) + err = dest.UploadUnencrypted(testData, "test-secret") + assert.NoError(t, err, "No-op upload should succeed") +} + +func TestAWSSecretsManagerDestination_KeyValue_Integration(t *testing.T) { + if testSecretName == "" { + t.Skip("Skipping integration test: SOPS_TEST_AWS_SECRET_NAME not set") + } + + ctx := context.Background() + // Use a different secret name for key/value test to avoid conflicts + keyValueSecretName := testSecretName + "-keyvalue" + dest := NewAWSSecretsManagerDestination(testAWSRegion, keyValueSecretName) + + // Test data with simple key/value pairs (no nested objects) + // This format enables the key/value editor in AWS Secrets Manager console + testData := map[string]interface{}{ + "database_host": "db.example.com", + "database_port": "5432", + "database_username": "app_user", + "database_password": "super_secret_password", + "api_key_stripe": "sk_live_abcdef123456", + "api_key_github": "ghp_xyz789012345", + "debug_mode": "false", + "log_level": "info", + "max_connections": "100", + "timeout_seconds": "30", + } + + // Upload test data + err := dest.UploadUnencrypted(testData, "test-keyvalue-secret") + require.NoError(t, err, "Failed to upload key/value secret to Secrets Manager") + + // Verify the secret was stored correctly + cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(testAWSRegion)) + require.NoError(t, err, "Failed to load AWS config") + + client := secretsmanager.NewFromConfig(cfg) + result, err := client.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{ + SecretId: aws.String(keyValueSecretName), + }) + require.NoError(t, err, "Failed to retrieve key/value secret from Secrets Manager") + + // Parse and verify the stored data + var storedData map[string]interface{} + err = json.Unmarshal([]byte(*result.SecretString), &storedData) + require.NoError(t, err, "Failed to parse stored key/value secret JSON") + + assert.Equal(t, testData, storedData, "Stored key/value data doesn't match original") + + // Verify that all values are stored as strings (important for key/value format) + for key, value := range storedData { + assert.IsType(t, "", value, "Value for key %s should be a string for key/value format", key) + } + + // Test no-op behavior (upload same data again) + err = dest.UploadUnencrypted(testData, "test-keyvalue-secret") + assert.NoError(t, err, "No-op upload should succeed for key/value format") +} + +func TestAWSParameterStoreDestination_Integration(t *testing.T) { + if testParameterName == "" { + t.Skip("Skipping integration test: SOPS_TEST_AWS_PARAMETER_NAME not set") + } + + ctx := context.Background() + dest := NewAWSParameterStoreDestination(testAWSRegion, testParameterName) + + // Test data + testData := map[string]interface{}{ + "app_config": map[string]interface{}{ + "debug": false, + "log_level": "info", + "max_workers": float64(10), + "features": map[string]interface{}{ + "new_ui": true, + "beta_feature": false, + }, + }, + } + + // Upload test data (this is the method used by the publish command) + err := dest.UploadUnencrypted(testData, "test-config") + require.NoError(t, err, "Failed to upload parameter to Parameter Store") + + // Verify the parameter was stored correctly + cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(testAWSRegion)) + require.NoError(t, err, "Failed to load AWS config") + + client := ssm.NewFromConfig(cfg) + result, err := client.GetParameter(ctx, &ssm.GetParameterInput{ + Name: aws.String(testParameterName), + WithDecryption: aws.Bool(true), + }) + require.NoError(t, err, "Failed to retrieve parameter from Parameter Store") + + // Parse and verify the stored data + var storedData map[string]interface{} + err = json.Unmarshal([]byte(*result.Parameter.Value), &storedData) + require.NoError(t, err, "Failed to parse stored parameter JSON") + + assert.Equal(t, testData, storedData, "Stored data doesn't match original") + + // Verify parameter type is always SecureString + assert.Equal(t, "SecureString", string(result.Parameter.Type), "Parameter type should always be SecureString") + + // Test no-op behavior (upload same data again) + err = dest.UploadUnencrypted(testData, "test-config") + assert.NoError(t, err, "No-op upload should succeed") +} + +func TestAWSParameterStoreDestination_EncryptedFile_Integration(t *testing.T) { + if testParameterName == "" { + t.Skip("Skipping integration test: SOPS_TEST_AWS_PARAMETER_NAME not set") + } + + dest := NewAWSParameterStoreDestination(testAWSRegion, testParameterName+"-file") + + encryptedContent := []byte(`# SOPS encrypted file +database: + host: ENC[AES256_GCM,data:xyz123,type:str] + password: ENC[AES256_GCM,data:abc456,type:str] +sops: + kms: + - arn: arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012 + version: 3.8.1`) + + // Upload should return NotImplementedError + err := dest.Upload(encryptedContent, "encrypted-test") + require.NotNil(t, err, "Upload should return an error") + assert.IsType(t, &NotImplementedError{}, err, "Should return NotImplementedError") + assert.Contains(t, err.Error(), "AWS Parameter Store does not support uploading encrypted sops files directly") +} diff --git a/publish/aws_parameter_store.go b/publish/aws_parameter_store.go new file mode 100644 index 000000000..c3708d52e --- /dev/null +++ b/publish/aws_parameter_store.go @@ -0,0 +1,136 @@ +package publish + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/ssm" + "github.com/aws/aws-sdk-go-v2/service/ssm/types" + "github.com/aws/smithy-go" + "github.com/getsops/sops/v3/logging" + "github.com/sirupsen/logrus" +) + +var parameterLog *logrus.Logger + +func init() { + parameterLog = logging.NewLogger("PUBLISH") +} + +// AWSParameterStoreDestination is the AWS Parameter Store implementation of the Destination interface. +type AWSParameterStoreDestination struct { + region string + parameterPath string +} + +// NewAWSParameterStoreDestination creates a new AWS Parameter Store destination. +func NewAWSParameterStoreDestination(region, parameterPath string) *AWSParameterStoreDestination { + // Ensure parameter path starts with / + if parameterPath != "" && !strings.HasPrefix(parameterPath, "/") { + parameterPath = "/" + parameterPath + } + + return &AWSParameterStoreDestination{region, parameterPath} +} + +// Path returns the AWS Parameter Store path for the given fileName. +func (awspsd *AWSParameterStoreDestination) Path(fileName string) string { + if awspsd.parameterPath != "" { + // If path ends with /, append filename; otherwise use path as-is + if strings.HasSuffix(awspsd.parameterPath, "/") { + return awspsd.parameterPath + fileName + } + return awspsd.parameterPath + } + // Default: use filename as parameter name + if !strings.HasPrefix(fileName, "/") { + return "/" + fileName + } + return fileName +} + +// Upload returns NotImplementedError as AWS Parameter Store does not support uploading encrypted files directly. +func (awspsd *AWSParameterStoreDestination) Upload(fileContents []byte, fileName string) error { + return &NotImplementedError{"AWS Parameter Store does not support uploading encrypted sops files directly. Use UploadUnencrypted instead."} +} + +// UploadUnencrypted uploads unencrypted data to AWS Parameter Store as JSON. +func (awspsd *AWSParameterStoreDestination) UploadUnencrypted(data map[string]interface{}, fileName string) error { + ctx := context.TODO() + + // Load AWS config - use explicit region if provided, otherwise rely on SDK defaults + var cfg aws.Config + var err error + if awspsd.region != "" { + cfg, err = config.LoadDefaultConfig(ctx, config.WithRegion(awspsd.region)) + } else { + cfg, err = config.LoadDefaultConfig(ctx) + } + if err != nil { + return fmt.Errorf("unable to load AWS SDK config: %w", err) + } + + client := ssm.NewFromConfig(cfg) + parameterName := awspsd.Path(fileName) + + // Convert data to JSON string for storage + jsonData, err := json.Marshal(data) + if err != nil { + return fmt.Errorf("failed to marshal data to JSON: %w", err) + } + parameterValue := string(jsonData) + + // Check if parameter already exists and compare content + getParamOutput, err := client.GetParameter(ctx, &ssm.GetParameterInput{ + Name: aws.String(parameterName), + WithDecryption: aws.Bool(true), // Decrypt for comparison if it's a SecureString + }) + + parameterExists := true + if err != nil { + var apiErr smithy.APIError + if errors.As(err, &apiErr) && apiErr.ErrorCode() == "ParameterNotFound" { + parameterExists = false + parameterLog.Infof("Parameter %s does not exist, will create new parameter", parameterName) + } else { + parameterLog.Warnf("Cannot check if destination parameter already exists in %s. New version will be created even if the data has not been changed.", parameterName) + } + } + + // If parameter exists, check if content is identical + if parameterExists && getParamOutput.Parameter.Value != nil { + if *getParamOutput.Parameter.Value == parameterValue { + parameterLog.Infof("Parameter %s is already up-to-date.", parameterName) + return nil + } + } + + // Always use SecureString for security - SOPS files may contain secrets + paramType := types.ParameterTypeSecureString + + // Put parameter (creates or updates) + _, err = client.PutParameter(ctx, &ssm.PutParameterInput{ + Name: aws.String(parameterName), + Value: aws.String(parameterValue), + Type: paramType, + Overwrite: aws.Bool(true), + Description: aws.String("Parameter created/updated by SOPS publish command"), + }) + + if err != nil { + return fmt.Errorf("failed to put parameter %s: %w", parameterName, err) + } + + if parameterExists { + parameterLog.Infof("Successfully updated parameter %s", parameterName) + } else { + parameterLog.Infof("Successfully created parameter %s", parameterName) + } + + return nil +} diff --git a/publish/aws_parameter_store_test.go b/publish/aws_parameter_store_test.go new file mode 100644 index 000000000..606cd04e2 --- /dev/null +++ b/publish/aws_parameter_store_test.go @@ -0,0 +1,57 @@ +package publish + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewAWSParameterStoreDestination(t *testing.T) { + dest := NewAWSParameterStoreDestination("us-east-1", "/myapp/config") + assert.NotNil(t, dest) + assert.Equal(t, "us-east-1", dest.region) + assert.Equal(t, "/myapp/config", dest.parameterPath) + + // Test path normalization (should add leading slash) + dest = NewAWSParameterStoreDestination("us-east-1", "myapp/config") + assert.Equal(t, "/myapp/config", dest.parameterPath) +} + +func TestAWSParameterStoreDestination_Path(t *testing.T) { + // Test with specific parameter path (no trailing slash) + dest := NewAWSParameterStoreDestination("us-east-1", "/myapp/database") + path := dest.Path("config.yaml") + assert.Equal(t, "/myapp/database", path) + + // Test with parameter path ending with slash + dest = NewAWSParameterStoreDestination("us-east-1", "/myapp/configs/") + path = dest.Path("api.yaml") + assert.Equal(t, "/myapp/configs/api.yaml", path) + + // Test with empty parameter path (uses filename) + dest = NewAWSParameterStoreDestination("us-east-1", "") + path = dest.Path("standalone.yaml") + assert.Equal(t, "/standalone.yaml", path) + + // Test with filename that already has leading slash + dest = NewAWSParameterStoreDestination("us-east-1", "") + path = dest.Path("/already-prefixed.yaml") + assert.Equal(t, "/already-prefixed.yaml", path) +} + +func TestAWSParameterStoreDestination_Upload(t *testing.T) { + dest := NewAWSParameterStoreDestination("us-east-1", "/test-parameter") + err := dest.Upload([]byte("test content"), "test.yaml") + + assert.NotNil(t, err) + assert.IsType(t, &NotImplementedError{}, err) + assert.Contains(t, err.Error(), "AWS Parameter Store does not support uploading encrypted sops files directly") +} + +func TestNewAWSParameterStoreDestination_EmptyRegion(t *testing.T) { + // Test that empty region is allowed (will use SDK defaults) + dest := NewAWSParameterStoreDestination("", "/myapp/config") + assert.NotNil(t, dest) + assert.Equal(t, "", dest.region) + assert.Equal(t, "/myapp/config", dest.parameterPath) +} diff --git a/publish/aws_secrets_manager.go b/publish/aws_secrets_manager.go new file mode 100644 index 000000000..afe6dd6e5 --- /dev/null +++ b/publish/aws_secrets_manager.go @@ -0,0 +1,147 @@ +package publish + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "github.com/aws/smithy-go" + "github.com/getsops/sops/v3/logging" + "github.com/sirupsen/logrus" +) + +var awsSecretsLog *logrus.Logger + +func init() { + awsSecretsLog = logging.NewLogger("PUBLISH") +} + +// AWSSecretsManagerDestination is the AWS Secrets Manager implementation of the Destination interface +type AWSSecretsManagerDestination struct { + region string + secretName string +} + +// NewAWSSecretsManagerDestination is the constructor for an AWS Secrets Manager Destination +func NewAWSSecretsManagerDestination(region, secretName string) *AWSSecretsManagerDestination { + return &AWSSecretsManagerDestination{region, secretName} +} + +// Path returns the AWS Secrets Manager path/ARN of a secret +func (awssmsd *AWSSecretsManagerDestination) Path(fileName string) string { + if awssmsd.secretName != "" { + return fmt.Sprintf("arn:aws:secretsmanager:%s:*:secret:%s", awssmsd.region, awssmsd.secretName) + } + return fmt.Sprintf("arn:aws:secretsmanager:%s:*:secret:%s", awssmsd.region, fileName) +} + +// Returns NotImplementedError +func (awssmsd *AWSSecretsManagerDestination) Upload(fileContents []byte, fileName string) error { + return &NotImplementedError{"AWS Secrets Manager does not support uploading encrypted sops files directly. Use UploadUnencrypted instead."} +} + +// UploadUnencrypted uploads unencrypted data to AWS Secrets Manager as JSON +func (awssmsd *AWSSecretsManagerDestination) UploadUnencrypted(data map[string]interface{}, fileName string) error { + ctx := context.TODO() + + // Load AWS config - use explicit region if provided, otherwise rely on SDK defaults + var cfg aws.Config + var err error + if awssmsd.region != "" { + cfg, err = config.LoadDefaultConfig(ctx, config.WithRegion(awssmsd.region)) + } else { + cfg, err = config.LoadDefaultConfig(ctx) + } + if err != nil { + return fmt.Errorf("unable to load AWS SDK config: %w", err) + } + + client := secretsmanager.NewFromConfig(cfg) + + // Determine secret name - use configured name or derive from filename + secretName := awssmsd.secretName + if secretName == "" { + secretName = fileName + } + + // Convert data to JSON string for storage + jsonData, err := json.Marshal(data) + if err != nil { + return fmt.Errorf("failed to marshal data to JSON: %w", err) + } + secretString := string(jsonData) + + // Check if secret metadata exists first + _, err = client.DescribeSecret(ctx, &secretsmanager.DescribeSecretInput{ + SecretId: aws.String(secretName), + }) + + secretExists := true + hasValue := false + var getSecretOutput *secretsmanager.GetSecretValueOutput + + if err != nil { + var apiErr smithy.APIError + if errors.As(err, &apiErr) && apiErr.ErrorCode() == "ResourceNotFoundException" { + secretExists = false + awsSecretsLog.Infof("Secret %s does not exist, will create new secret", secretName) + } else { + awsSecretsLog.Warnf("Cannot check if destination secret already exists in %s. New version will be created even if the data has not been changed.", secretName) + } + } else { + // Secret exists, now check if it has a value + getSecretOutput, err = client.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{ + SecretId: aws.String(secretName), + }) + if err != nil { + var apiErr smithy.APIError + if errors.As(err, &apiErr) && apiErr.ErrorCode() == "ResourceNotFoundException" { + hasValue = false + awsSecretsLog.Infof("Secret %s exists but has no value, will add initial value", secretName) + } else { + awsSecretsLog.Warnf("Cannot retrieve current value of secret %s: %v", secretName, err) + hasValue = false + } + } else { + hasValue = true + } + } + + // If secret exists and has value, check if content is identical + if secretExists && hasValue && getSecretOutput.SecretString != nil { + if *getSecretOutput.SecretString == secretString { + awsSecretsLog.Infof("Secret %s is already up-to-date.", secretName) + return nil + } + } + + // Create or update secret + if secretExists { + // Update existing secret + _, err = client.PutSecretValue(ctx, &secretsmanager.PutSecretValueInput{ + SecretId: aws.String(secretName), + SecretString: aws.String(secretString), + }) + if err != nil { + return fmt.Errorf("failed to update secret %s: %w", secretName, err) + } + awsSecretsLog.Infof("Successfully updated secret %s", secretName) + } else { + // Create new secret + _, err = client.CreateSecret(ctx, &secretsmanager.CreateSecretInput{ + Name: aws.String(secretName), + SecretString: aws.String(secretString), + Description: aws.String("Secret created by SOPS publish command"), + }) + if err != nil { + return fmt.Errorf("failed to create secret %s: %w", secretName, err) + } + awsSecretsLog.Infof("Successfully created secret %s", secretName) + } + + return nil +} diff --git a/publish/aws_secrets_manager_test.go b/publish/aws_secrets_manager_test.go new file mode 100644 index 000000000..a6105ffa4 --- /dev/null +++ b/publish/aws_secrets_manager_test.go @@ -0,0 +1,46 @@ +package publish + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewAWSSecretsManagerDestination(t *testing.T) { + dest := NewAWSSecretsManagerDestination("us-east-1", "myapp/database") + assert.NotNil(t, dest) + assert.Equal(t, "us-east-1", dest.region) + assert.Equal(t, "myapp/database", dest.secretName) +} + +func TestAWSSecretsManagerDestination_Path(t *testing.T) { + // Test with specified secret name + dest := NewAWSSecretsManagerDestination("us-east-1", "myapp/database") + path := dest.Path("config.yaml") + expected := "arn:aws:secretsmanager:us-east-1:*:secret:myapp/database" + assert.Equal(t, expected, path) + + // Test without specified secret name (uses filename) + dest = NewAWSSecretsManagerDestination("us-west-2", "") + path = dest.Path("api-keys.yaml") + expected = "arn:aws:secretsmanager:us-west-2:*:secret:api-keys.yaml" + assert.Equal(t, expected, path) +} + +func TestAWSSecretsManagerDestination_Upload(t *testing.T) { + dest := NewAWSSecretsManagerDestination("us-east-1", "test-secret") + err := dest.Upload([]byte("test content"), "test.yaml") + + // Should return NotImplementedError + assert.NotNil(t, err) + assert.IsType(t, &NotImplementedError{}, err) + assert.Contains(t, err.Error(), "AWS Secrets Manager does not support uploading encrypted sops files directly") +} + +func TestNewAWSSecretsManagerDestination_EmptyRegion(t *testing.T) { + // Test that empty region is allowed (will use SDK defaults) + dest := NewAWSSecretsManagerDestination("", "myapp/database") + assert.NotNil(t, dest) + assert.Equal(t, "", dest.region) + assert.Equal(t, "myapp/database", dest.secretName) +}