Add parameter_files and parameters configuration options to stack definition

CHANGELOG.md

@@ -10,6 +10,15 @@ The format is based on [Keep a Changelog], and this project adheres to
 
 ## [Unreleased]
 
+### Added
+
+- `parameter_dir` is now configurable to match the existing `template_dir`.
+- A `parameter_files` array allows configuring an array of *explicit* parameter
+  files relative to `parameter_dir`. If this option is specified, automatic
+  parameter files based on region and the stack name will be be used.
+- A `parameters` hash key allows defining parameters directly on the stack
+  definition rather than requiring an external parameter file.
+
 ### Fixed
 
 - JSON template bodies with whitespace on leading lines would incorrectly be

README.md

@@ -123,6 +123,7 @@ stack_defaults:
 ```
 
 Additional files can be configured to be uploaded to S3 alongside the templates:
+
 ```yaml
 stacks:
   production:
@@ -131,6 +132,7 @@ stacks:
       files:
         - userdata.sh
 ```
+
 ## Directories
 
 - `templates` - CloudFormation, SparkleFormation or CfnDsl templates.

features/apply_with_explicit_parameter_files.feature

@@ -0,0 +1,65 @@
+Feature: Apply command with explicit parameter files
+
+  Background:
+    Given a file named "stack_master.yml" with:
+      """
+      stack_defaults:
+        tags:
+          Application: myapp
+      stacks:
+        us-east-1:
+          myapp-web:
+            template: myapp.rb
+            parameter_files:
+            - myapp-web-parameters.yml
+      """
+    And a file named "parameters/us-east-1/myapp-web.yml" with:
+      """
+      Color: blue
+      """
+    And a file named "parameters/myapp-web-parameters.yml" with:
+      """
+      KeyName: my-key
+      """
+    And a directory named "templates"
+    And a file named "templates/myapp.rb" with:
+      """
+      SparkleFormation.new(:myapp) do
+        description "Test template"
+
+        parameters.key_name do
+          description 'Key name'
+          type 'String'
+        end
+
+        parameters.color do
+          description 'Color'
+          type 'String'
+          default 'red'
+        end
+
+        resources.instance do
+          type 'AWS::EC2::Instance'
+          properties do
+            image_id 'ami-0080e4c5bc078760e'
+            instance_type 't2.micro'
+          end
+        end
+      end
+      """
+
+  Scenario: Run apply and create stack with explicit parameter files
+    Given I stub the following stack events:
+      | stack_id | event_id | stack_name | logical_resource_id | resource_status | resource_type              | timestamp           |
+      | 1        | 1        | myapp-web  | myapp-web           | CREATE_COMPLETE | AWS::CloudFormation::Stack | 2020-10-29 00:00:00 |
+    When I run `stack_master apply us-east-1 myapp-web --trace`
+    Then the output should match /2020-10-29 00:00:00 (\+|\-)[0-9]{4} myapp-web AWS::CloudFormation::Stack CREATE_COMPLETE/
+    And the output should contain all of these lines:
+      | Stack diff:          |
+      | +    "Instance": {   |
+      | Parameters diff:     |
+      | KeyName: my-key      |
+      | Proposed change set: |
+    And the output should not contain "Color: blue"
+    And the output should contain "Color: red"
+    And the exit status should be 0

features/apply_with_s3.feature

@@ -69,7 +69,7 @@ Feature: Apply command
       | Parameters diff:                                                               |
       | KeyName: my-key                                                                |
     And the output should match /2020-10-29 00:00:00 (\+|\-)[0-9]{4} myapp-vpc AWS::CloudFormation::Stack CREATE_COMPLETE/
-    And an S3 file in bucket "my-bucket" with key "cfn_templates/my-app/myapp_vpc.json" exists with content:
+    And an S3 file in bucket "my-bucket" with key "cfn_templates/my-app/myapp_vpc.json" exists with JSON content:
       """
       {
         "Description": "Test template",

features/apply_with_stack_definition_parameters.feature

@@ -0,0 +1,46 @@
+Feature: Apply command with stack definition parameters
+
+  Background:
+    Given a file named "stack_master.yml" with:
+      """
+      stacks:
+        us-east-1:
+          myapp_web:
+            template: myapp.rb
+            parameters:
+              KeyName: my-key
+      """
+    And a directory named "templates"
+    And a file named "templates/myapp.rb" with:
+      """
+      SparkleFormation.new(:myapp) do
+        description "Test template"
+
+        parameters.key_name do
+          description 'Key name'
+          type 'String'
+        end
+
+        resources.instance do
+          type 'AWS::EC2::Instance'
+          properties do
+            image_id 'ami-0080e4c5bc078760e'
+            instance_type 't2.micro'
+          end
+        end
+      end
+      """
+
+  Scenario: Run apply with parameters contained in
+    Given I stub the following stack events:
+      | stack_id | event_id | stack_name | logical_resource_id | resource_status | resource_type              | timestamp           |
+      | 1        | 1        | myapp-web  | myapp-web           | CREATE_COMPLETE | AWS::CloudFormation::Stack | 2020-10-29 00:00:00 |
+    When I run `stack_master apply us-east-1 myapp-web --trace`
+    Then the output should match /2020-10-29 00:00:00 (\+|\-)[0-9]{4} myapp-web AWS::CloudFormation::Stack CREATE_COMPLETE/
+    And the output should contain all of these lines:
+      | Stack diff:          |
+      | +    "Instance": {   |
+      | Parameters diff:     |
+      | KeyName: my-key      |
+      | Proposed change set: |
+    And the exit status should be 0

features/step_definitions/stack_steps.rb

@@ -73,3 +73,9 @@ def extract_hash_from_kv_string(string)
   file = StackMaster.s3_driver.find_file(bucket: bucket, object_key: key)
   expect(file).to eq body
 end
+
+When(/^an S3 file in bucket "([^"]*)" with key "([^"]*)" exists with JSON content:$/) do |bucket, key, body|
+  file = StackMaster.s3_driver.find_file(bucket: bucket, object_key: key)
+  parsed_file = JSON.parse(file)
+  expect(parsed_file).to eq JSON.parse(body)
+end

lib/stack_master/commands/tidy.rb

@@ -12,7 +12,7 @@ def perform
         parameter_files = Set.new(find_parameter_files())
 
         status = @config.stacks.each do |stack_definition|
-          parameter_files.subtract(stack_definition.parameter_files)
+          parameter_files.subtract(stack_definition.parameter_files_from_globs)
           template = File.absolute_path(stack_definition.template_file_path)
 
           if template

lib/stack_master/config.rb

@@ -17,6 +17,7 @@ def self.load!(config_file = 'stack_master.yml')
     attr_accessor :stacks,
                   :base_dir,
                   :template_dir,
+                  :parameters_dir,
                   :stack_defaults,
                   :region_defaults,
                   :region_aliases,
@@ -39,6 +40,7 @@ def initialize(config, base_dir)
       @config = config
       @base_dir = base_dir
       @template_dir = config.fetch('template_dir', nil)
+      @parameters_dir = config.fetch('parameters_dir', nil)
       @stack_defaults = config.fetch('stack_defaults', {})
       @region_aliases = Utils.underscore_keys_to_hyphen(config.fetch('region_aliases', {}))
       @region_to_aliases = @region_aliases.inject({}) do |hash, (key, value)|
@@ -115,6 +117,7 @@ def load_stacks(stacks)
             'stack_name' => stack_name,
             'base_dir' => @base_dir,
             'template_dir' => @template_dir,
+            'parameters_dir' => @parameters_dir,
             'additional_parameter_lookup_dirs' => @region_to_aliases[region])
           stack_attributes['allowed_accounts'] = attributes['allowed_accounts'] if attributes['allowed_accounts']
           @stacks << StackDefinition.new(stack_attributes)

lib/stack_master/parameter_loader.rb

@@ -5,18 +5,17 @@ class ParameterLoader
 
     COMPILE_TIME_PARAMETERS_KEY = 'compile_time_parameters'
 
-    def self.load(parameter_files)
+    def self.load(parameter_files: [], parameters: {})
       StackMaster.debug 'Searching for parameter files...'
-      parameter_files.reduce({template_parameters: {}, compile_time_parameters: {}}) do |hash, file_name|
-        parameters = load_parameters(file_name)
+      all_parameters = parameter_files.map { |file_name| load_parameters(file_name) } + [parameters]
+      all_parameters.reduce({template_parameters: {}, compile_time_parameters: {}}) do |hash, parameters|
         template_parameters = create_template_parameters(parameters)
         compile_time_parameters = create_compile_time_parameters(parameters)
 
         merge_and_camelize(hash[:template_parameters], template_parameters)
         merge_and_camelize(hash[:compile_time_parameters], compile_time_parameters)
         hash
       end
-
     end
 
     private

lib/stack_master/stack.rb

@@ -56,7 +56,7 @@ def self.find(region, stack_name)
     end
 
     def self.generate(stack_definition, config)
-      parameter_hash = ParameterLoader.load(stack_definition.parameter_files)
+      parameter_hash = ParameterLoader.load(parameter_files: stack_definition.all_parameter_files, parameters: stack_definition.parameters)
       template_parameters = ParameterResolver.resolve(config, stack_definition, parameter_hash[:template_parameters])
       compile_time_parameters = ParameterResolver.resolve(config, stack_definition, parameter_hash[:compile_time_parameters])
       template_body = TemplateCompiler.compile(config, stack_definition.compiler, stack_definition.template_dir, stack_definition.template, compile_time_parameters, stack_definition.compiler_options)
@@ -76,7 +76,7 @@ def self.generate(stack_definition, config)
     end
 
     def self.generate_without_parameters(stack_definition, config)
-      parameter_hash = ParameterLoader.load(stack_definition.parameter_files)
+      parameter_hash = ParameterLoader.load(parameter_files: stack_definition.all_parameter_files, parameters: stack_definition.parameters)
       compile_time_parameters = ParameterResolver.resolve(config, stack_definition, parameter_hash[:compile_time_parameters])
       template_body = TemplateCompiler.compile(config, stack_definition.compiler, stack_definition.template_dir, stack_definition.template, compile_time_parameters, stack_definition.compiler_options)
       template_format = TemplateUtils.identify_template_format(template_body)

lib/stack_master/stack_definition.rb

@@ -16,7 +16,10 @@ class StackDefinition
                   :additional_parameter_lookup_dirs,
                   :s3,
                   :files,
-                  :compiler_options
+                  :compiler_options,
+                  :parameters_dir,
+                  :parameters,
+                  :parameter_files
 
     attr_reader :compiler
 
@@ -32,8 +35,12 @@ def initialize(attributes = {})
       @compiler = nil
       super
       @additional_parameter_lookup_dirs ||= []
+      @base_dir ||= ""
       @template_dir ||= File.join(@base_dir, 'templates')
+      @parameters_dir ||= File.join(@base_dir, 'parameters')
       @allowed_accounts = Array(@allowed_accounts)
+      @parameters ||= {}
+      @parameter_files ||= []
     end
 
     def ==(other)
@@ -57,7 +64,7 @@ def ==(other)
     end
 
     def compiler=(compiler)
-      @compiler = compiler.&to_sym
+      @compiler = compiler&.to_sym
     end
 
     def template_file_path
@@ -85,7 +92,15 @@ def s3_template_file_name
       Utils.change_extension(template, 'json')
     end
 
-    def parameter_files
+    def all_parameter_files
+      if parameter_files.empty?
+        parameter_files_from_globs
+      else
+        parameter_files
+      end
+    end
+
+    def parameter_files_from_globs
       parameter_file_globs.map(&Dir.method(:glob)).flatten
     end
 
@@ -101,20 +116,26 @@ def s3_configured?
       !s3.nil?
     end
 
+    def parameter_files
+      Array(@parameter_files).map do |file|
+        File.expand_path(File.join(parameters_dir, file))
+      end
+    end
+
     private
 
     def additional_parameter_lookup_globs
       additional_parameter_lookup_dirs.map do |a|
-        File.join(base_dir, 'parameters', a, "#{stack_name_glob}.y*ml")
+        File.join(parameters_dir, a, "#{stack_name_glob}.y*ml")
       end
     end
 
     def region_parameter_glob
-      File.join(base_dir, 'parameters', "#{region}", "#{stack_name_glob}.y*ml")
+      File.join(parameters_dir, "#{region}", "#{stack_name_glob}.y*ml")
     end
 
     def default_parameter_glob
-      File.join(base_dir, 'parameters', "#{stack_name_glob}.y*ml")
+      File.join(parameters_dir, "#{stack_name_glob}.y*ml")
     end
 
     def stack_name_glob

spec/stack_master/parameter_loader_spec.rb

@@ -2,7 +2,7 @@
   let(:stack_file_name) { '/base_dir/parameters/stack_name.yml' }
   let(:region_file_name) { '/base_dir/parameters/us-east-1/stack_name.yml' }
 
-  subject(:parameters) { StackMaster::ParameterLoader.load([stack_file_name, region_file_name]) }
+  subject(:parameters) { StackMaster::ParameterLoader.load(parameter_files: [stack_file_name, region_file_name]) }
 
   before do
     file_mock(stack_file_name, **stack_file_returns)
@@ -60,7 +60,7 @@
     let(:region_yaml_file_returns) { {exists: true, read: "Param1: value1\nParam2: valueX"} }
     let(:region_yaml_file_name) { "/base_dir/parameters/us-east-1/stack_name.yaml" }
 
-    subject(:parameters) { StackMaster::ParameterLoader.load([stack_file_name, region_yaml_file_name, region_file_name]) }
+    subject(:parameters) { StackMaster::ParameterLoader.load(parameter_files: [stack_file_name, region_yaml_file_name, region_file_name]) }
 
     before do
       file_mock(region_yaml_file_name, **region_yaml_file_returns)

spec/stack_master/stack_definition_spec.rb

@@ -5,14 +5,16 @@
       stack_name: stack_name,
       template: template,
       tags: tags,
-      base_dir: base_dir)
+      base_dir: base_dir,
+      parameter_files: parameter_files)
   end
 
   let(:region) { 'us-east-1' }
   let(:stack_name) { 'stack_name' }
   let(:template) { 'template.json' }
   let(:tags) { {'environment' => 'production'} }
   let(:base_dir) { '/base_dir' }
+  let(:parameter_files) { nil }
 
   before do
     allow(Dir).to receive(:glob).with(
@@ -35,7 +37,7 @@
   end
 
   it 'has default and region specific parameter file locations' do
-    expect(stack_definition.parameter_files).to eq([
+    expect(stack_definition.all_parameter_files).to eq([
       "/base_dir/parameters/#{stack_name}.yaml",
       "/base_dir/parameters/#{stack_name}.yml",
       "/base_dir/parameters/#{region}/#{stack_name}.yaml",
@@ -75,7 +77,7 @@
     end
 
     it 'includes a parameter lookup dir for it' do
-      expect(stack_definition.parameter_files).to eq([
+      expect(stack_definition.all_parameter_files).to eq([
         "/base_dir/parameters/#{stack_name}.yaml",
         "/base_dir/parameters/#{stack_name}.yml",
         "/base_dir/parameters/#{region}/#{stack_name}.yaml",
@@ -109,4 +111,12 @@
   it 'defaults ejson_file_kms to true' do
     expect(stack_definition.ejson_file_kms).to eq true
   end
+
+  context "with explicit parameter_files" do
+    let(:parameter_files) { ["my-stack.yml", "../my-stack.yml"] }
+
+    it "resolves them relative to parameters_dir" do
+      expect(stack_definition.parameter_files).to eq ["/base_dir/parameters/my-stack.yml", "/base_dir/my-stack.yml"]
+    end
+  end
 end

spec/stack_master/template_compiler_spec.rb

@@ -13,7 +13,7 @@ def self.compile(template_dir, template, compile_time_parameters, compile_option
     context 'when a template compiler is explicitly specified' do
       it 'uses it' do
         expect(StackMaster::TemplateCompilers::SparkleFormation).to receive(:compile).with('/base_dir/templates', 'template', compile_time_parameters, anything)
-        StackMaster::TemplateCompiler.compile(config, :sparkle_formation, '/base_dir/templates', 'template', compile_time_parameters, compile_time_parameters)
+        StackMaster::TemplateCompiler.compile(config, :sparkle_formation, '/base_dir/templates', 'template', compile_time_parameters, {})
       end
     end