From 02e2cb07205af7c219e7dcb631460ff37ac745db Mon Sep 17 00:00:00 2001 From: Steve Hodgkiss Date: Sat, 18 Apr 2020 14:26:31 +0000 Subject: [PATCH 01/17] Make it possible to define parameters in the stack yml --- ...y_with_stack_definition_parameters.feature | 46 +++++++++++++++++++ lib/stack_master/parameter_loader.rb | 7 ++- lib/stack_master/stack.rb | 2 +- lib/stack_master/stack_definition.rb | 4 +- spec/stack_master/parameter_loader_spec.rb | 4 +- spec/stack_master/template_compiler_spec.rb | 2 +- 6 files changed, 56 insertions(+), 9 deletions(-) create mode 100644 features/apply_with_stack_definition_parameters.feature diff --git a/features/apply_with_stack_definition_parameters.feature b/features/apply_with_stack_definition_parameters.feature new file mode 100644 index 00000000..32d9f9e8 --- /dev/null +++ b/features/apply_with_stack_definition_parameters.feature @@ -0,0 +1,46 @@ +Feature: Apply command with stack definition parameters + + Background: + Given a file named "stack_master.yml" with: + """ + stacks: + us-east-1: + myapp_web: + template: myapp.rb + parameters: + KeyName: my-key + """ + And a directory named "templates" + And a file named "templates/myapp.rb" with: + """ + SparkleFormation.new(:myapp) do + description "Test template" + + parameters.key_name do + description 'Key name' + type 'String' + end + + resources.instance do + type 'AWS::EC2::Instance' + properties do + image_id 'ami-0080e4c5bc078760e' + instance_type 't2.micro' + end + end + end + """ + + Scenario: Run apply with parameters contained in + Given I stub the following stack events: + | stack_id | event_id | stack_name | logical_resource_id | resource_status | resource_type | timestamp | + | 1 | 1 | myapp-web | myapp-web | CREATE_COMPLETE | AWS::CloudFormation::Stack | 2020-10-29 00:00:00 | + When I run `stack_master apply us-east-1 myapp-web --trace` + Then the output should match /2020-10-29 00:00:00 (\+|\-)[0-9]{4} myapp-web AWS::CloudFormation::Stack CREATE_COMPLETE/ + And the output should contain all of these lines: + | Stack diff: | + | + "Instance": { | + | Parameters diff: | + | KeyName: my-key | + | Proposed change set: | + And the exit status should be 0 diff --git a/lib/stack_master/parameter_loader.rb b/lib/stack_master/parameter_loader.rb index 452be95a..64dacfce 100644 --- a/lib/stack_master/parameter_loader.rb +++ b/lib/stack_master/parameter_loader.rb @@ -5,10 +5,10 @@ class ParameterLoader COMPILE_TIME_PARAMETERS_KEY = 'compile_time_parameters' - def self.load(parameter_files) + def self.load(parameter_files: [], parameters: {}) StackMaster.debug 'Searching for parameter files...' - parameter_files.reduce({template_parameters: {}, compile_time_parameters: {}}) do |hash, file_name| - parameters = load_parameters(file_name) + all_parameters = parameter_files.map { |file_name| load_parameters(file_name) } + [parameters] + all_parameters.reduce({template_parameters: {}, compile_time_parameters: {}}) do |hash, parameters| template_parameters = create_template_parameters(parameters) compile_time_parameters = create_compile_time_parameters(parameters) @@ -16,7 +16,6 @@ def self.load(parameter_files) merge_and_camelize(hash[:compile_time_parameters], compile_time_parameters) hash end - end private diff --git a/lib/stack_master/stack.rb b/lib/stack_master/stack.rb index 29f12025..09ab4bbf 100644 --- a/lib/stack_master/stack.rb +++ b/lib/stack_master/stack.rb @@ -56,7 +56,7 @@ def self.find(region, stack_name) end def self.generate(stack_definition, config) - parameter_hash = ParameterLoader.load(stack_definition.parameter_files) + parameter_hash = ParameterLoader.load(parameter_files: stack_definition.parameter_files, parameters: stack_definition.parameters) template_parameters = ParameterResolver.resolve(config, stack_definition, parameter_hash[:template_parameters]) compile_time_parameters = ParameterResolver.resolve(config, stack_definition, parameter_hash[:compile_time_parameters]) template_body = TemplateCompiler.compile(config, stack_definition.compiler, stack_definition.template_dir, stack_definition.template, compile_time_parameters, stack_definition.compiler_options) diff --git a/lib/stack_master/stack_definition.rb b/lib/stack_master/stack_definition.rb index 587afb77..f8282e4f 100644 --- a/lib/stack_master/stack_definition.rb +++ b/lib/stack_master/stack_definition.rb @@ -16,7 +16,8 @@ class StackDefinition :additional_parameter_lookup_dirs, :s3, :files, - :compiler_options + :compiler_options, + :parameters attr_reader :compiler @@ -34,6 +35,7 @@ def initialize(attributes = {}) @additional_parameter_lookup_dirs ||= [] @template_dir ||= File.join(@base_dir, 'templates') @allowed_accounts = Array(@allowed_accounts) + @parameters ||= {} end def ==(other) diff --git a/spec/stack_master/parameter_loader_spec.rb b/spec/stack_master/parameter_loader_spec.rb index 9d6015b1..65739e9e 100644 --- a/spec/stack_master/parameter_loader_spec.rb +++ b/spec/stack_master/parameter_loader_spec.rb @@ -2,7 +2,7 @@ let(:stack_file_name) { '/base_dir/parameters/stack_name.yml' } let(:region_file_name) { '/base_dir/parameters/us-east-1/stack_name.yml' } - subject(:parameters) { StackMaster::ParameterLoader.load([stack_file_name, region_file_name]) } + subject(:parameters) { StackMaster::ParameterLoader.load(parameter_files: [stack_file_name, region_file_name]) } before do file_mock(stack_file_name, **stack_file_returns) @@ -60,7 +60,7 @@ let(:region_yaml_file_returns) { {exists: true, read: "Param1: value1\nParam2: valueX"} } let(:region_yaml_file_name) { "/base_dir/parameters/us-east-1/stack_name.yaml" } - subject(:parameters) { StackMaster::ParameterLoader.load([stack_file_name, region_yaml_file_name, region_file_name]) } + subject(:parameters) { StackMaster::ParameterLoader.load(parameter_files: [stack_file_name, region_yaml_file_name, region_file_name]) } before do file_mock(region_yaml_file_name, **region_yaml_file_returns) diff --git a/spec/stack_master/template_compiler_spec.rb b/spec/stack_master/template_compiler_spec.rb index 403de7f7..5b018c9b 100644 --- a/spec/stack_master/template_compiler_spec.rb +++ b/spec/stack_master/template_compiler_spec.rb @@ -13,7 +13,7 @@ def self.compile(template_dir, template, compile_time_parameters, compile_option context 'when a template compiler is explicitly specified' do it 'uses it' do expect(StackMaster::TemplateCompilers::SparkleFormation).to receive(:compile).with('/base_dir/templates', 'template', compile_time_parameters, anything) - StackMaster::TemplateCompiler.compile(config, :sparkle_formation, '/base_dir/templates', 'template', compile_time_parameters, compile_time_parameters) + StackMaster::TemplateCompiler.compile(config, :sparkle_formation, '/base_dir/templates', 'template', compile_time_parameters, {}) end end From 3c5df43381438048e43555c73f81a46ca33ec69b Mon Sep 17 00:00:00 2001 From: Steve Hodgkiss Date: Mon, 20 Apr 2020 04:58:37 +0000 Subject: [PATCH 02/17] Allow parameter_files to be explicitly specified in stack definitions --- README.md | 2 + ...pply_with_explicit_parameter_files.feature | 53 +++++++++++++++++++ lib/stack_master/commands/tidy.rb | 2 +- lib/stack_master/config.rb | 3 ++ lib/stack_master/stack.rb | 4 +- lib/stack_master/stack_definition.rb | 27 +++++++--- spec/stack_master/stack_definition_spec.rb | 16 ++++-- 7 files changed, 95 insertions(+), 12 deletions(-) create mode 100644 features/apply_with_explicit_parameter_files.feature diff --git a/README.md b/README.md index e451d2c0..1649874f 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,7 @@ stack_defaults: ``` Additional files can be configured to be uploaded to S3 alongside the templates: + ```yaml stacks: production: @@ -131,6 +132,7 @@ stacks: files: - userdata.sh ``` + ## Directories - `templates` - CloudFormation, SparkleFormation or CfnDsl templates. diff --git a/features/apply_with_explicit_parameter_files.feature b/features/apply_with_explicit_parameter_files.feature new file mode 100644 index 00000000..c2506606 --- /dev/null +++ b/features/apply_with_explicit_parameter_files.feature @@ -0,0 +1,53 @@ +Feature: Apply command with explicit parameter files + + Background: + Given a file named "stack_master.yml" with: + """ + stack_defaults: + tags: + Application: myapp + stacks: + us-east-1: + myapp-web: + template: myapp.rb + parameter_files: + - myapp-web-parameters.yml + """ + And a file named "parameters/myapp-web-parameters.yml" with: + """ + KeyName: my-key + """ + And a directory named "templates" + And a file named "templates/myapp.rb" with: + """ + SparkleFormation.new(:myapp) do + description "Test template" + + parameters.key_name do + description 'Key name' + type 'String' + end + + resources.instance do + type 'AWS::EC2::Instance' + properties do + image_id 'ami-0080e4c5bc078760e' + instance_type 't2.micro' + end + end + end + """ + + Scenario: Run apply and create stack with explicit parameter files + Given I stub the following stack events: + | stack_id | event_id | stack_name | logical_resource_id | resource_status | resource_type | timestamp | + | 1 | 1 | myapp-web | myapp-web | CREATE_COMPLETE | AWS::CloudFormation::Stack | 2020-10-29 00:00:00 | + When I run `stack_master apply us-east-1 myapp-web --trace` + Then the output should match /2020-10-29 00:00:00 (\+|\-)[0-9]{4} myapp-web AWS::CloudFormation::Stack CREATE_COMPLETE/ + And the output should contain all of these lines: + | Stack diff: | + | + "Instance": { | + | Parameters diff: | + | KeyName: my-key | + | Proposed change set: | + And the exit status should be 0 diff --git a/lib/stack_master/commands/tidy.rb b/lib/stack_master/commands/tidy.rb index d7fc63b1..24b440e8 100644 --- a/lib/stack_master/commands/tidy.rb +++ b/lib/stack_master/commands/tidy.rb @@ -12,7 +12,7 @@ def perform parameter_files = Set.new(find_parameter_files()) status = @config.stacks.each do |stack_definition| - parameter_files.subtract(stack_definition.parameter_files) + parameter_files.subtract(stack_definition.parameter_files_from_globs) template = File.absolute_path(stack_definition.template_file_path) if template diff --git a/lib/stack_master/config.rb b/lib/stack_master/config.rb index f85e797f..3426ed52 100644 --- a/lib/stack_master/config.rb +++ b/lib/stack_master/config.rb @@ -17,6 +17,7 @@ def self.load!(config_file = 'stack_master.yml') attr_accessor :stacks, :base_dir, :template_dir, + :parameters_dir, :stack_defaults, :region_defaults, :region_aliases, @@ -39,6 +40,7 @@ def initialize(config, base_dir) @config = config @base_dir = base_dir @template_dir = config.fetch('template_dir', nil) + @parameters_dir = config.fetch('parameters_dir', nil) @stack_defaults = config.fetch('stack_defaults', {}) @region_aliases = Utils.underscore_keys_to_hyphen(config.fetch('region_aliases', {})) @region_to_aliases = @region_aliases.inject({}) do |hash, (key, value)| @@ -115,6 +117,7 @@ def load_stacks(stacks) 'stack_name' => stack_name, 'base_dir' => @base_dir, 'template_dir' => @template_dir, + 'parameters_dir' => @parameters_dir, 'additional_parameter_lookup_dirs' => @region_to_aliases[region]) stack_attributes['allowed_accounts'] = attributes['allowed_accounts'] if attributes['allowed_accounts'] @stacks << StackDefinition.new(stack_attributes) diff --git a/lib/stack_master/stack.rb b/lib/stack_master/stack.rb index 09ab4bbf..99357dad 100644 --- a/lib/stack_master/stack.rb +++ b/lib/stack_master/stack.rb @@ -56,7 +56,7 @@ def self.find(region, stack_name) end def self.generate(stack_definition, config) - parameter_hash = ParameterLoader.load(parameter_files: stack_definition.parameter_files, parameters: stack_definition.parameters) + parameter_hash = ParameterLoader.load(parameter_files: stack_definition.all_parameter_files, parameters: stack_definition.parameters) template_parameters = ParameterResolver.resolve(config, stack_definition, parameter_hash[:template_parameters]) compile_time_parameters = ParameterResolver.resolve(config, stack_definition, parameter_hash[:compile_time_parameters]) template_body = TemplateCompiler.compile(config, stack_definition.compiler, stack_definition.template_dir, stack_definition.template, compile_time_parameters, stack_definition.compiler_options) @@ -76,7 +76,7 @@ def self.generate(stack_definition, config) end def self.generate_without_parameters(stack_definition, config) - parameter_hash = ParameterLoader.load(stack_definition.parameter_files) + parameter_hash = ParameterLoader.load(parameter_files: stack_definition.all_parameter_files, parameters: stack_definition.parameters) compile_time_parameters = ParameterResolver.resolve(config, stack_definition, parameter_hash[:compile_time_parameters]) template_body = TemplateCompiler.compile(config, stack_definition.compiler, stack_definition.template_dir, stack_definition.template, compile_time_parameters, stack_definition.compiler_options) template_format = TemplateUtils.identify_template_format(template_body) diff --git a/lib/stack_master/stack_definition.rb b/lib/stack_master/stack_definition.rb index f8282e4f..ed53c97e 100644 --- a/lib/stack_master/stack_definition.rb +++ b/lib/stack_master/stack_definition.rb @@ -17,7 +17,9 @@ class StackDefinition :s3, :files, :compiler_options, - :parameters + :parameters_dir, + :parameters, + :parameter_files attr_reader :compiler @@ -33,9 +35,12 @@ def initialize(attributes = {}) @compiler = nil super @additional_parameter_lookup_dirs ||= [] + @base_dir ||= "" @template_dir ||= File.join(@base_dir, 'templates') + @parameters_dir ||= File.join(@base_dir, 'parameters') @allowed_accounts = Array(@allowed_accounts) @parameters ||= {} + @parameter_files ||= [] end def ==(other) @@ -59,7 +64,7 @@ def ==(other) end def compiler=(compiler) - @compiler = compiler.&to_sym + @compiler = compiler&.to_sym end def template_file_path @@ -87,7 +92,11 @@ def s3_template_file_name Utils.change_extension(template, 'json') end - def parameter_files + def all_parameter_files + parameter_files_from_globs + parameter_files + end + + def parameter_files_from_globs parameter_file_globs.map(&Dir.method(:glob)).flatten end @@ -103,20 +112,26 @@ def s3_configured? !s3.nil? end + def parameter_files + Array(@parameter_files).map do |file| + File.expand_path(File.join(parameters_dir, file)) + end + end + private def additional_parameter_lookup_globs additional_parameter_lookup_dirs.map do |a| - File.join(base_dir, 'parameters', a, "#{stack_name_glob}.y*ml") + File.join(parameters_dir, a, "#{stack_name_glob}.y*ml") end end def region_parameter_glob - File.join(base_dir, 'parameters', "#{region}", "#{stack_name_glob}.y*ml") + File.join(parameters_dir, "#{region}", "#{stack_name_glob}.y*ml") end def default_parameter_glob - File.join(base_dir, 'parameters', "#{stack_name_glob}.y*ml") + File.join(parameters_dir, "#{stack_name_glob}.y*ml") end def stack_name_glob diff --git a/spec/stack_master/stack_definition_spec.rb b/spec/stack_master/stack_definition_spec.rb index b9deb16e..acd1d336 100644 --- a/spec/stack_master/stack_definition_spec.rb +++ b/spec/stack_master/stack_definition_spec.rb @@ -5,7 +5,8 @@ stack_name: stack_name, template: template, tags: tags, - base_dir: base_dir) + base_dir: base_dir, + parameter_files: parameter_files) end let(:region) { 'us-east-1' } @@ -13,6 +14,7 @@ let(:template) { 'template.json' } let(:tags) { {'environment' => 'production'} } let(:base_dir) { '/base_dir' } + let(:parameter_files) { nil } before do allow(Dir).to receive(:glob).with( @@ -35,7 +37,7 @@ end it 'has default and region specific parameter file locations' do - expect(stack_definition.parameter_files).to eq([ + expect(stack_definition.all_parameter_files).to eq([ "/base_dir/parameters/#{stack_name}.yaml", "/base_dir/parameters/#{stack_name}.yml", "/base_dir/parameters/#{region}/#{stack_name}.yaml", @@ -75,7 +77,7 @@ end it 'includes a parameter lookup dir for it' do - expect(stack_definition.parameter_files).to eq([ + expect(stack_definition.all_parameter_files).to eq([ "/base_dir/parameters/#{stack_name}.yaml", "/base_dir/parameters/#{stack_name}.yml", "/base_dir/parameters/#{region}/#{stack_name}.yaml", @@ -109,4 +111,12 @@ it 'defaults ejson_file_kms to true' do expect(stack_definition.ejson_file_kms).to eq true end + + context "with explicit parameter_files" do + let(:parameter_files) { ["my-stack.yml", "../my-stack.yml"] } + + it "resolves them relative to parameters_dir" do + expect(stack_definition.parameter_files).to eq ["/base_dir/parameters/my-stack.yml", "/base_dir/my-stack.yml"] + end + end end From d2bdefd337dd7bac9add117b21314e6e9d3f0eea Mon Sep 17 00:00:00 2001 From: Steve Hodgkiss Date: Wed, 10 Jun 2020 16:39:08 +0400 Subject: [PATCH 03/17] Fix an example due to change in behaviour with JSON.pretty_generate(sparkle_template) --- features/apply_with_s3.feature | 2 +- features/step_definitions/stack_steps.rb | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/features/apply_with_s3.feature b/features/apply_with_s3.feature index 5f957b25..f6683a0f 100644 --- a/features/apply_with_s3.feature +++ b/features/apply_with_s3.feature @@ -69,7 +69,7 @@ Feature: Apply command | Parameters diff: | | KeyName: my-key | And the output should match /2020-10-29 00:00:00 (\+|\-)[0-9]{4} myapp-vpc AWS::CloudFormation::Stack CREATE_COMPLETE/ - And an S3 file in bucket "my-bucket" with key "cfn_templates/my-app/myapp_vpc.json" exists with content: + And an S3 file in bucket "my-bucket" with key "cfn_templates/my-app/myapp_vpc.json" exists with JSON content: """ { "Description": "Test template", diff --git a/features/step_definitions/stack_steps.rb b/features/step_definitions/stack_steps.rb index c55a6849..0c11d1b0 100644 --- a/features/step_definitions/stack_steps.rb +++ b/features/step_definitions/stack_steps.rb @@ -73,3 +73,9 @@ def extract_hash_from_kv_string(string) file = StackMaster.s3_driver.find_file(bucket: bucket, object_key: key) expect(file).to eq body end + +When(/^an S3 file in bucket "([^"]*)" with key "([^"]*)" exists with JSON content:$/) do |bucket, key, body| + file = StackMaster.s3_driver.find_file(bucket: bucket, object_key: key) + parsed_file = JSON.parse(file) + expect(parsed_file).to eq JSON.parse(body) +end From 258aa440db09f33a34d335bde3abc9a575f4379b Mon Sep 17 00:00:00 2001 From: Steve Hodgkiss Date: Wed, 10 Jun 2020 16:54:08 +0400 Subject: [PATCH 04/17] Don't mix parameter_files and parameter glob files, choose one or the other --- features/apply_with_explicit_parameter_files.feature | 12 ++++++++++++ lib/stack_master/stack_definition.rb | 6 +++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/features/apply_with_explicit_parameter_files.feature b/features/apply_with_explicit_parameter_files.feature index c2506606..4a445445 100644 --- a/features/apply_with_explicit_parameter_files.feature +++ b/features/apply_with_explicit_parameter_files.feature @@ -13,6 +13,10 @@ Feature: Apply command with explicit parameter files parameter_files: - myapp-web-parameters.yml """ + And a file named "parameters/us-east-1/myapp-web.yml" with: + """ + Color: blue + """ And a file named "parameters/myapp-web-parameters.yml" with: """ KeyName: my-key @@ -28,6 +32,12 @@ Feature: Apply command with explicit parameter files type 'String' end + parameters.color do + description 'Color' + type 'String' + default 'red' + end + resources.instance do type 'AWS::EC2::Instance' properties do @@ -50,4 +60,6 @@ Feature: Apply command with explicit parameter files | Parameters diff: | | KeyName: my-key | | Proposed change set: | + And the output should not contain "Color: blue" + And the output should contain "Color: red" And the exit status should be 0 diff --git a/lib/stack_master/stack_definition.rb b/lib/stack_master/stack_definition.rb index ed53c97e..313f0464 100644 --- a/lib/stack_master/stack_definition.rb +++ b/lib/stack_master/stack_definition.rb @@ -93,7 +93,11 @@ def s3_template_file_name end def all_parameter_files - parameter_files_from_globs + parameter_files + if parameter_files.empty? + parameter_files_from_globs + else + parameter_files + end end def parameter_files_from_globs From fce1a1e157445852ebe9fa83e8677aab67a5d3d0 Mon Sep 17 00:00:00 2001 From: Steve Hodgkiss Date: Wed, 10 Jun 2020 16:54:58 +0400 Subject: [PATCH 05/17] Add a changelog entry --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a122740e..1bf6047f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,15 @@ The format is based on [Keep a Changelog], and this project adheres to ## [Unreleased] +### Added + +- `parameter_dir` is now configurable to match the existing `template_dir`. +- A `parameter_files` array allows configuring an array of *explicit* parameter + files relative to `parameter_dir`. If this option is specified, automatic + parameter files based on region and the stack name will be be used. +- A `parameters` hash key allows defining parameters directly on the stack + definition rather than requiring an external parameter file. + ### Fixed - JSON template bodies with whitespace on leading lines would incorrectly be From d2d7e3f3f0a41015c1953da04c75eb324adf5c6b Mon Sep 17 00:00:00 2001 From: Steve Hodgkiss Date: Wed, 10 Jun 2020 16:56:44 +0400 Subject: [PATCH 06/17] Undo unnecessary spec change --- spec/stack_master/template_compiler_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/stack_master/template_compiler_spec.rb b/spec/stack_master/template_compiler_spec.rb index 5b018c9b..403de7f7 100644 --- a/spec/stack_master/template_compiler_spec.rb +++ b/spec/stack_master/template_compiler_spec.rb @@ -13,7 +13,7 @@ def self.compile(template_dir, template, compile_time_parameters, compile_option context 'when a template compiler is explicitly specified' do it 'uses it' do expect(StackMaster::TemplateCompilers::SparkleFormation).to receive(:compile).with('/base_dir/templates', 'template', compile_time_parameters, anything) - StackMaster::TemplateCompiler.compile(config, :sparkle_formation, '/base_dir/templates', 'template', compile_time_parameters, {}) + StackMaster::TemplateCompiler.compile(config, :sparkle_formation, '/base_dir/templates', 'template', compile_time_parameters, compile_time_parameters) end end From bb770f6df4110f39c809988e3ba10736084623e8 Mon Sep 17 00:00:00 2001 From: Steve Hodgkiss Date: Wed, 10 Jun 2020 16:58:31 +0400 Subject: [PATCH 07/17] Remove unused compiler= method --- lib/stack_master/stack_definition.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/stack_master/stack_definition.rb b/lib/stack_master/stack_definition.rb index 313f0464..ea257034 100644 --- a/lib/stack_master/stack_definition.rb +++ b/lib/stack_master/stack_definition.rb @@ -63,10 +63,6 @@ def ==(other) @compiler_options == other.compiler_options end - def compiler=(compiler) - @compiler = compiler&.to_sym - end - def template_file_path return unless template File.expand_path(File.join(template_dir, template)) From ae4096283db9e39c311df93411bcee7c5bf8f947 Mon Sep 17 00:00:00 2001 From: Steve Hodgkiss Date: Thu, 11 Jun 2020 09:42:53 +0400 Subject: [PATCH 08/17] Update CHANGELOG.md Co-authored-by: Peter Vandoros --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bf6047f..12c6a195 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ The format is based on [Keep a Changelog], and this project adheres to - `parameter_dir` is now configurable to match the existing `template_dir`. - A `parameter_files` array allows configuring an array of *explicit* parameter files relative to `parameter_dir`. If this option is specified, automatic - parameter files based on region and the stack name will be be used. + parameter files based on region and the stack name will not be used. - A `parameters` hash key allows defining parameters directly on the stack definition rather than requiring an external parameter file. From 17c92ad2d7a293c0f81d203e186e47b60cf1922a Mon Sep 17 00:00:00 2001 From: Steve Hodgkiss Date: Thu, 11 Jun 2020 09:57:36 +0400 Subject: [PATCH 09/17] Update changelog --- CHANGELOG.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12c6a195..e530e045 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,12 +12,12 @@ The format is based on [Keep a Changelog], and this project adheres to ### Added -- `parameter_dir` is now configurable to match the existing `template_dir`. -- A `parameter_files` array allows configuring an array of *explicit* parameter - files relative to `parameter_dir`. If this option is specified, automatic - parameter files based on region and the stack name will not be used. -- A `parameters` hash key allows defining parameters directly on the stack - definition rather than requiring an external parameter file. +- `parameters_dir` is now configurable to match the existing `template_dir`. +- `parameter_files` configures an array of parameter files relative to + `parameters_dir` that will be used instead of automatic parameter file globs + based on region and stack name. +- `parameters` configures stack parameters directly on the stack definition + rather than requiring an external parameter file. ### Fixed From 00d541f6ac4495f266e5a04f294ffd464d63212b Mon Sep 17 00:00:00 2001 From: Steve Hodgkiss Date: Thu, 11 Jun 2020 10:00:14 +0400 Subject: [PATCH 10/17] Update readme --- README.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1649874f..dd90d731 100644 --- a/README.md +++ b/README.md @@ -157,7 +157,8 @@ template_compilers: ## Parameters -Parameters are loaded from multiple YAML files, merged from the following lookup paths from bottom to top: +By default, parameters are loaded from multiple YAML files, merged from the +following lookup paths from bottom to top: - parameters/[stack_name].yaml - parameters/[stack_name].yml @@ -172,6 +173,30 @@ A simple parameter file could look like this: key_name: myapp-us-east-1 ``` +Alternatively, a `parameter_files` array can be defined to explicitly list +parameter files that will be loaded. If `parameter_files` are defined, the +automatic search locations will not be used. + +```yaml +parameters_dir: parameters # the default +stacks: + us-east-1: + my-app: + parameter_files: + - my-app.yml # parameters/my-app.yml +``` + +Parameters can also be defined inline with stack definitions: + +```yaml +stacks: + us-east-1: + my-app: + parameters: + VpcId: + stack_output: my-vpc/VpcId +``` + ### Compile Time Parameters Compile time parameters can be used for [SparkleFormation](http://www.sparkleformation.io) templates. It conforms and From 1c45dc590b8cc018bc9aca51e6bca22bd95cf4f0 Mon Sep 17 00:00:00 2001 From: Steve Hodgkiss Date: Thu, 11 Jun 2020 10:38:19 +0400 Subject: [PATCH 11/17] Update spec --- spec/stack_master/stack_definition_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/stack_master/stack_definition_spec.rb b/spec/stack_master/stack_definition_spec.rb index acd1d336..f0af5f07 100644 --- a/spec/stack_master/stack_definition_spec.rb +++ b/spec/stack_master/stack_definition_spec.rb @@ -115,8 +115,8 @@ context "with explicit parameter_files" do let(:parameter_files) { ["my-stack.yml", "../my-stack.yml"] } - it "resolves them relative to parameters_dir" do - expect(stack_definition.parameter_files).to eq ["/base_dir/parameters/my-stack.yml", "/base_dir/my-stack.yml"] + it "ignores parameter globs and resolves them relative to parameters_dir" do + expect(stack_definition.all_parameter_files).to eq ["/base_dir/parameters/my-stack.yml", "/base_dir/my-stack.yml"] end end end From 31ab0c3269ac9d29062d58107ba1404f150f0368 Mon Sep 17 00:00:00 2001 From: Steve Hodgkiss Date: Fri, 12 Jun 2020 13:48:16 +0400 Subject: [PATCH 12/17] Update error message for missing params when using parameter_files --- lib/stack_master/parameter_validator.rb | 31 ++++++++++++++----- spec/stack_master/parameter_validator_spec.rb | 20 ++++++++++-- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/lib/stack_master/parameter_validator.rb b/lib/stack_master/parameter_validator.rb index 1c405d15..b7b36b06 100644 --- a/lib/stack_master/parameter_validator.rb +++ b/lib/stack_master/parameter_validator.rb @@ -9,15 +9,14 @@ def initialize(stack:, stack_definition:) def error_message return nil unless missing_parameters? - message = "Empty/blank parameters detected. Please provide values for these parameters:" + message = "Empty/blank parameters detected. Please provide values for these parameters:\n" missing_parameters.each do |parameter_name| - message << "\n - #{parameter_name}" + message << " - #{parameter_name}\n" end - message << "\nParameters will be read from files matching the following globs:" - base_dir = Pathname.new(@stack_definition.base_dir) - @stack_definition.parameter_file_globs.each do |glob| - parameter_file = Pathname.new(glob).relative_path_from(base_dir) - message << "\n - #{parameter_file}" + if @stack_definition.parameter_files.empty? + message << message_for_parameter_globs + else + message << message_for_parameter_files end message end @@ -28,6 +27,24 @@ def missing_parameters? private + def message_for_parameter_files + "Parameters are configured to be read from the following files:\n".tap do |message| + @stack_definition.parameter_files.each do |parameter_file| + message << " - #{parameter_file}\n" + end + end + end + + def message_for_parameter_globs + "Parameters will be read from files matching the following globs:\n".tap do |message| + base_dir = Pathname.new(@stack_definition.base_dir) + @stack_definition.parameter_file_globs.each do |glob| + parameter_file = Pathname.new(glob).relative_path_from(base_dir) + message << " - #{parameter_file}\n" + end + end + end + def missing_parameters @missing_parameters ||= @stack.parameters_with_defaults.select { |_key, value| value.nil? }.keys diff --git a/spec/stack_master/parameter_validator_spec.rb b/spec/stack_master/parameter_validator_spec.rb index 1faf6746..79d5cdd7 100644 --- a/spec/stack_master/parameter_validator_spec.rb +++ b/spec/stack_master/parameter_validator_spec.rb @@ -2,7 +2,8 @@ subject(:parameter_validator) { described_class.new(stack: stack, stack_definition: stack_definition) } let(:stack) { StackMaster::Stack.new(parameters: parameters, template_body: '{}', template_format: :json) } - let(:stack_definition) { StackMaster::StackDefinition.new(base_dir: '/base_dir', region: 'ap-southeast-2', stack_name: 'stack_name') } + let(:parameter_files) { nil } + let(:stack_definition) { StackMaster::StackDefinition.new(base_dir: '/base_dir', region: 'ap-southeast-2', stack_name: 'stack_name', parameter_files: parameter_files) } describe '#missing_parameters?' do subject { parameter_validator.missing_parameters? } @@ -27,7 +28,7 @@ let(:parameters) { {'Param1' => true, 'Param2' => nil, 'Param3' => 'string', 'Param4' => nil} } it 'returns a descriptive message' do - expect(error_message).to eq(<<~MESSAGE.chomp) + expect(error_message).to eq(<<~MESSAGE) Empty/blank parameters detected. Please provide values for these parameters: - Param2 - Param4 @@ -38,6 +39,21 @@ end end + context 'when the stack definition is using explicit parameter files' do + let(:parameters) { {'Param1' => true, 'Param2' => nil, 'Param3' => 'string', 'Param4' => nil} } + let(:parameter_files) { ["params.yml"] } + + it 'returns a descriptive message' do + expect(error_message).to eq(<<~MESSAGE) + Empty/blank parameters detected. Please provide values for these parameters: + - Param2 + - Param4 + Parameters are configured to be read from the following files: + - /base_dir/parameters/params.yml + MESSAGE + end + end + context 'when no parameers have a nil value' do let(:parameters) { {'Param' => '1'} } From 263730e6f0896fc51d17b7e28969341f6f62b7f4 Mon Sep 17 00:00:00 2001 From: Steve Hodgkiss Date: Sat, 13 Jun 2020 13:55:19 +0400 Subject: [PATCH 13/17] Use Then --- features/step_definitions/stack_steps.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/features/step_definitions/stack_steps.rb b/features/step_definitions/stack_steps.rb index 0c11d1b0..d40540bb 100644 --- a/features/step_definitions/stack_steps.rb +++ b/features/step_definitions/stack_steps.rb @@ -69,12 +69,12 @@ def extract_hash_from_kv_string(string) allow(StackMaster.cloud_formation_driver.class).to receive(:new).and_return(StackMaster.cloud_formation_driver) end -When(/^an S3 file in bucket "([^"]*)" with key "([^"]*)" exists with content:$/) do |bucket, key, body| +Then(/^an S3 file in bucket "([^"]*)" with key "([^"]*)" exists with content:$/) do |bucket, key, body| file = StackMaster.s3_driver.find_file(bucket: bucket, object_key: key) expect(file).to eq body end -When(/^an S3 file in bucket "([^"]*)" with key "([^"]*)" exists with JSON content:$/) do |bucket, key, body| +Then(/^an S3 file in bucket "([^"]*)" with key "([^"]*)" exists with JSON content:$/) do |bucket, key, body| file = StackMaster.s3_driver.find_file(bucket: bucket, object_key: key) parsed_file = JSON.parse(file) expect(parsed_file).to eq JSON.parse(body) From 44c4c6ea1d3800e7b8f0d98da23d27abcfbe77ed Mon Sep 17 00:00:00 2001 From: Steve Hodgkiss Date: Sat, 13 Jun 2020 13:56:22 +0400 Subject: [PATCH 14/17] Update spec - move param to file --- features/apply_with_explicit_parameter_files.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/apply_with_explicit_parameter_files.feature b/features/apply_with_explicit_parameter_files.feature index 4a445445..4679ea60 100644 --- a/features/apply_with_explicit_parameter_files.feature +++ b/features/apply_with_explicit_parameter_files.feature @@ -20,6 +20,7 @@ Feature: Apply command with explicit parameter files And a file named "parameters/myapp-web-parameters.yml" with: """ KeyName: my-key + Color: red """ And a directory named "templates" And a file named "templates/myapp.rb" with: @@ -35,7 +36,6 @@ Feature: Apply command with explicit parameter files parameters.color do description 'Color' type 'String' - default 'red' end resources.instance do From 029a0889cfe06698a7d65f6c0f5873332aec5716 Mon Sep 17 00:00:00 2001 From: Steve Hodgkiss Date: Sat, 13 Jun 2020 13:58:29 +0400 Subject: [PATCH 15/17] Simplify File.expand_path --- lib/stack_master/stack_definition.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/stack_master/stack_definition.rb b/lib/stack_master/stack_definition.rb index ea257034..939e0c4c 100644 --- a/lib/stack_master/stack_definition.rb +++ b/lib/stack_master/stack_definition.rb @@ -65,7 +65,7 @@ def ==(other) def template_file_path return unless template - File.expand_path(File.join(template_dir, template)) + File.expand_path(template, template_dir) end def files_dir @@ -114,7 +114,7 @@ def s3_configured? def parameter_files Array(@parameter_files).map do |file| - File.expand_path(File.join(parameters_dir, file)) + File.expand_path(file, parameters_dir) end end From 4d27c42132c5fba739cd47a53d2c1a462ed3daf1 Mon Sep 17 00:00:00 2001 From: Steve Hodgkiss Date: Mon, 15 Jun 2020 13:13:16 +0400 Subject: [PATCH 16/17] Bump version to 2.7.0 --- lib/stack_master/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/stack_master/version.rb b/lib/stack_master/version.rb index 9ae3d87c..b2e2bc44 100644 --- a/lib/stack_master/version.rb +++ b/lib/stack_master/version.rb @@ -1,3 +1,3 @@ module StackMaster - VERSION = "2.6.1" + VERSION = "2.7.0" end From bb4918b8debbfd38d2ff937ae1e592ac1049b825 Mon Sep 17 00:00:00 2001 From: Steve Hodgkiss Date: Mon, 15 Jun 2020 13:13:53 +0400 Subject: [PATCH 17/17] Update changelog for 2.7.0 release --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e530e045..35de9059 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ The format is based on [Keep a Changelog], and this project adheres to [Keep a Changelog]: https://keepachangelog.com/en/1.0.0/ [Semantic Versioning]: https://semver.org/spec/v2.0.0.html -## [Unreleased] +## [2.7.0] - 2020-06-15 ### Added