From 679dca14c75a336150c80b5c36e715baa2ff7a80 Mon Sep 17 00:00:00 2001 From: Rory Breuk Date: Thu, 6 Apr 2017 16:24:59 +0200 Subject: [PATCH 1/6] Added caching of lookups --- README.md | 6 ++ hiera-vault.gemspec | 2 +- lib/hiera/backend/vault_backend.rb | 115 +++++++++++++++++++---------- 3 files changed, 81 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 82476e1..7b87f6b 100644 --- a/README.md +++ b/README.md @@ -203,3 +203,9 @@ SSL can be configured with the following config variables: :ssl_ca_path: /path/to/ca/ :ssl_verify: false :ssl_ciphers: "MY:SSL:CIPHER:CONFIG" + +## Caching +Caching can be configured with the following config variables: + :vault: + :cache_timeout: 10 + :cache_clean_interval: 3600 diff --git a/hiera-vault.gemspec b/hiera-vault.gemspec index 435a834..cb94306 100644 --- a/hiera-vault.gemspec +++ b/hiera-vault.gemspec @@ -3,7 +3,7 @@ require 'rubygems/package_task' spec = Gem::Specification.new do |gem| gem.name = "hiera-vault" - gem.version = "0.2.1" + gem.version = "0.2.2" gem.license = "Apache-2.0" gem.summary = "Module for using vault as a hiera backend" gem.email = "jonathan.sokolowski@gmail.com" diff --git a/lib/hiera/backend/vault_backend.rb b/lib/hiera/backend/vault_backend.rb index 8cba891..c05c947 100644 --- a/lib/hiera/backend/vault_backend.rb +++ b/lib/hiera/backend/vault_backend.rb @@ -1,9 +1,7 @@ -# Vault backend for Hiera class Hiera module Backend class Vault_backend - - def initialize() + def initialize require 'json' require 'vault' @@ -28,7 +26,7 @@ def initialize() begin @vault = Vault::Client.new @vault.configure do |config| - config.address = @config[:addr] unless @config[:addr].nil? + config.address = @config[:address] unless @config[:address].nil? config.token = @config[:token] unless @config[:token].nil? config.ssl_pem_file = @config[:ssl_pem_file] unless @config[:ssl_pem_file].nil? config.ssl_verify = @config[:ssl_verify] unless @config[:ssl_verify].nil? @@ -43,77 +41,112 @@ def initialize() @vault = nil Hiera.warn("[hiera-vault] Skipping backend. Configuration error: #{e}") end + + @cache = {} + @config[':cache_timeout'] ||= 10 + @config[:cache_clean_interval] ||= 3600 end def lookup(key, scope, order_override, resolution_type) return nil if @vault.nil? - Hiera.debug("[hiera-vault] Looking up #{key} in vault backend") - answer = nil found = false + Hiera.debug("Lookup #{key} in Vault backend") + Hiera.debug(Backend.parse_string(@config[:log_id], scope)) + # Only generic mounts supported so far @config[:mounts][:generic].each do |mount| path = Backend.parse_string(mount, scope, { 'key' => key }) Backend.datasources(scope, order_override) do |source| - Hiera.debug("Looking in path #{path}/#{source}/") - new_answer = lookup_generic("#{path}/#{source}/#{key}", scope) - #Hiera.debug("[hiera-vault] Answer: #{new_answer}:#{new_answer.class}") - next if new_answer.nil? + data = lookup_generic_with_cache("#{path}/#{source}/#{key}") + + next if data.nil? + found = true + + Hiera.debug("Found #{key} in #{path}/#{source}") + + new_answer = Backend.parse_answer(data, scope) case resolution_type when :array - raise Exception, "Hiera type mismatch: expected Array and got #{new_answer.class}" unless new_answer.kind_of? Array or new_answer.kind_of? String + raise Exception, "Hiera type mismatch for key '#{key}': expected Array and got #{new_answer.class}" unless new_answer.kind_of? Array or new_answer.kind_of? String answer ||= [] answer << new_answer when :hash - raise Exception, "Hiera type mismatch: expected Hash and got #{new_answer.class}" unless new_answer.kind_of? Hash + raise Exception, "Hiera type mismatch for key '#{key}': expected Hash and got #{new_answer.class}" unless new_answer.kind_of? Hash answer ||= {} answer = Backend.merge_answer(new_answer,answer) else answer = new_answer - found = true break end end - break if found end - return answer end - def lookup_generic(key, scope) - begin - secret = @vault.logical.read(key) - rescue Vault::HTTPConnectionError - Hiera.debug("[hiera-vault] Could not connect to read secret: #{key}") - rescue Vault::HTTPError => e - Hiera.warn("[hiera-vault] Could not read secret #{key}: #{e.errors.join("\n").rstrip}") - end + private + + def lookup_generic_with_cache(key) + return lookup_generic(key) if @config[:cache_timeout] <= 0 + + now = Time.now.to_i + expired_at = now + @config[:cache_timeout] - return nil if secret.nil? - - Hiera.debug("[hiera-vault] Read secret: #{key}") - if @config[:default_field] and (@config[:default_field_behavior] == 'ignore' or (secret.data.has_key?(@config[:default_field].to_sym) and secret.data.length == 1)) - return nil if not secret.data.has_key?(@config[:default_field].to_sym) - # Return just our default_field - data = secret.data[@config[:default_field].to_sym] - if @config[:default_field_parse] == 'json' - begin - data = JSON.parse(data) - rescue JSON::ParserError => e - Hiera.debug("[hiera-vault] Could not parse string as json: #{e}") - end + periodically_clean_cache(now) unless @config[:cache_clean_interval] == 0 + + if !@cache[key] || @cache[key][:expired_at] < now + Hiera.debug("[hiera-vault] Lookup #{key} in vault") + @cache[key] = { + :expired_at => expired_at, + :result => lookup_generic(key) + } + else + Hiera.debug("[hiera-vault] #{key} found in cache") + end + return @cache[key][:result] + end + + def lookup_generic(key) + begin + secret = @vault.logical.read(key) + rescue Vault::HTTPConnectionError + Hiera.debug("[hiera-vault] Could not connect to vault server") + rescue Vault::HTTPError => e + Hiera.warn("[hiera-vault] Could not read secret: #{e.errors.join("\n").rstrip}") + end + + return nil if secret.nil? + + if @config[:default_field] && (@config[:default_field_behavior] == 'ignore' || + (secret.data.has_key?(@config[:default_field].to_sym) && secret.data.length == 1)) + return nil if not secret.data.has_key?(@config[:default_field].to_sym) + + data = secret.data[@config[:default_field].to_sym] + if @config[:default_field_parse] == 'json' + begin + data = JSON.parse(data) + rescue JSON::ParserError => e + Hiera.debug("[hiera-vault] Could not parse string as json: #{e}") end - else - # Turn secret's hash keys into strings - data = secret.data.inject({}) { |h, (k, v)| h[k.to_s] = v; h } end - #Hiera.debug("[hiera-vault] Data: #{data}:#{data.class}") + else + data = secret.data.inject({}) { |h, (k, v)| h[k.to_s] = v; h } + end + Hiera.warn("[hiera-vault] Data: #{data}:#{data.class}") - return Backend.parse_answer(data, scope) + return data end + def periodically_clean_cache(now) + return if now < @clean_cache_at.to_i + + @clean_cache_at = now + @config[:cache_clean_interval] + @cache.delete_if do |_, entry| + entry[:expired_at] < now + end + end end end end From a78e2b01093edbda501339b6f7567810fd10ae35 Mon Sep 17 00:00:00 2001 From: Rory Breuk Date: Thu, 8 Jun 2017 12:15:04 +0200 Subject: [PATCH 2/6] Updated gemspec --- hiera-vault.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hiera-vault.gemspec b/hiera-vault.gemspec index cb94306..a651ea0 100644 --- a/hiera-vault.gemspec +++ b/hiera-vault.gemspec @@ -3,7 +3,7 @@ require 'rubygems/package_task' spec = Gem::Specification.new do |gem| gem.name = "hiera-vault" - gem.version = "0.2.2" + gem.version = "0.3.1" gem.license = "Apache-2.0" gem.summary = "Module for using vault as a hiera backend" gem.email = "jonathan.sokolowski@gmail.com" From 0554df0ccb4b3fb860da39b41a16828e62d59d1c Mon Sep 17 00:00:00 2001 From: Rory Breuk Date: Thu, 8 Jun 2017 14:56:31 +0200 Subject: [PATCH 3/6] Reverted some changes to make merge easier --- hiera-vault.gemspec | 2 +- lib/hiera/backend/vault_backend.rb | 32 +++++++++++++++--------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/hiera-vault.gemspec b/hiera-vault.gemspec index a651ea0..cb94306 100644 --- a/hiera-vault.gemspec +++ b/hiera-vault.gemspec @@ -3,7 +3,7 @@ require 'rubygems/package_task' spec = Gem::Specification.new do |gem| gem.name = "hiera-vault" - gem.version = "0.3.1" + gem.version = "0.2.2" gem.license = "Apache-2.0" gem.summary = "Module for using vault as a hiera backend" gem.email = "jonathan.sokolowski@gmail.com" diff --git a/lib/hiera/backend/vault_backend.rb b/lib/hiera/backend/vault_backend.rb index c05c947..b235f50 100644 --- a/lib/hiera/backend/vault_backend.rb +++ b/lib/hiera/backend/vault_backend.rb @@ -26,7 +26,7 @@ def initialize begin @vault = Vault::Client.new @vault.configure do |config| - config.address = @config[:address] unless @config[:address].nil? + config.address = @config[:addr] unless @config[:addr].nil? config.token = @config[:token] unless @config[:token].nil? config.ssl_pem_file = @config[:ssl_pem_file] unless @config[:ssl_pem_file].nil? config.ssl_verify = @config[:ssl_verify] unless @config[:ssl_verify].nil? @@ -50,31 +50,29 @@ def initialize def lookup(key, scope, order_override, resolution_type) return nil if @vault.nil? + Hiera.debug("[hiera-vault] Looking up #{key} in vault backend") + answer = nil found = false - Hiera.debug("Lookup #{key} in Vault backend") - Hiera.debug(Backend.parse_string(@config[:log_id], scope)) - # Only generic mounts supported so far @config[:mounts][:generic].each do |mount| path = Backend.parse_string(mount, scope, { 'key' => key }) Backend.datasources(scope, order_override) do |source| + Hiera.debug("[hiera-vault] Looking in path #{path}/#{source}/") data = lookup_generic_with_cache("#{path}/#{source}/#{key}") - + #Hiera.debug("[hiera-vault] Answer: #{new_answer}:#{new_answer.class}") next if data.nil? found = true - - Hiera.debug("Found #{key} in #{path}/#{source}") - new_answer = Backend.parse_answer(data, scope) + case resolution_type when :array - raise Exception, "Hiera type mismatch for key '#{key}': expected Array and got #{new_answer.class}" unless new_answer.kind_of? Array or new_answer.kind_of? String + raise Exception, "Hiera type mismatch: expected Array and got #{new_answer.class}" unless new_answer.kind_of? Array or new_answer.kind_of? String answer ||= [] answer << new_answer when :hash - raise Exception, "Hiera type mismatch for key '#{key}': expected Hash and got #{new_answer.class}" unless new_answer.kind_of? Hash + raise Exception, "Hiera type mismatch: expected Hash and got #{new_answer.class}" unless new_answer.kind_of? Hash answer ||= {} answer = Backend.merge_answer(new_answer,answer) else @@ -83,6 +81,7 @@ def lookup(key, scope, order_override, resolution_type) end end end + return answer end @@ -112,17 +111,17 @@ def lookup_generic(key) begin secret = @vault.logical.read(key) rescue Vault::HTTPConnectionError - Hiera.debug("[hiera-vault] Could not connect to vault server") + Hiera.debug("[hiera-vault] Could not connect to read secret: #{key}") rescue Vault::HTTPError => e - Hiera.warn("[hiera-vault] Could not read secret: #{e.errors.join("\n").rstrip}") + Hiera.warn("[hiera-vault] Could not read secret #{key}: #{e.errors.join("\n").rstrip}") end return nil if secret.nil? - if @config[:default_field] && (@config[:default_field_behavior] == 'ignore' || - (secret.data.has_key?(@config[:default_field].to_sym) && secret.data.length == 1)) + Hiera.debug("[hiera-vault] Read secret: #{key}") + if @config[:default_field] and (@config[:default_field_behavior] == 'ignore' or (secret.data.has_key?(@config[:default_field].to_sym) and secret.data.length == 1)) return nil if not secret.data.has_key?(@config[:default_field].to_sym) - + # Return just our default_field data = secret.data[@config[:default_field].to_sym] if @config[:default_field_parse] == 'json' begin @@ -134,7 +133,7 @@ def lookup_generic(key) else data = secret.data.inject({}) { |h, (k, v)| h[k.to_s] = v; h } end - Hiera.warn("[hiera-vault] Data: #{data}:#{data.class}") + #Hiera.debug("[hiera-vault] Data: #{data}:#{data.class}") return data end @@ -147,6 +146,7 @@ def periodically_clean_cache(now) entry[:expired_at] < now end end + end end end From fcc76d73e701805cddab7abc65a355f7743b621e Mon Sep 17 00:00:00 2001 From: Rory Breuk Date: Thu, 8 Jun 2017 14:57:31 +0200 Subject: [PATCH 4/6] Reverted some changes to make merge easier --- lib/hiera/backend/vault_backend.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/hiera/backend/vault_backend.rb b/lib/hiera/backend/vault_backend.rb index b235f50..bdac732 100644 --- a/lib/hiera/backend/vault_backend.rb +++ b/lib/hiera/backend/vault_backend.rb @@ -1,3 +1,4 @@ +# Vault backend for Hiera class Hiera module Backend class Vault_backend From 34c6715c4ae33f0996198e60d8f91a942305e171 Mon Sep 17 00:00:00 2001 From: Rory Date: Mon, 9 Mar 2020 12:53:51 +0100 Subject: [PATCH 5/6] Created by OWASP Threat Dragon --- ThreatDragonModels/Hiera vault/Hiera vault.json | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 ThreatDragonModels/Hiera vault/Hiera vault.json diff --git a/ThreatDragonModels/Hiera vault/Hiera vault.json b/ThreatDragonModels/Hiera vault/Hiera vault.json new file mode 100644 index 0000000..1b09f4a --- /dev/null +++ b/ThreatDragonModels/Hiera vault/Hiera vault.json @@ -0,0 +1,10 @@ +{ + "summary": { + "title": "Hiera vault", + "owner": "Rory" + }, + "detail": { + "contributors": [], + "diagrams": [] + } +} \ No newline at end of file From e6976477c427c97bae192e1b0a584b21b34cbe97 Mon Sep 17 00:00:00 2001 From: Rory Date: Mon, 9 Mar 2020 12:54:27 +0100 Subject: [PATCH 6/6] Updated by OWASP Threat Dragon --- ThreatDragonModels/Hiera vault/Hiera vault.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ThreatDragonModels/Hiera vault/Hiera vault.json b/ThreatDragonModels/Hiera vault/Hiera vault.json index 1b09f4a..0fdd113 100644 --- a/ThreatDragonModels/Hiera vault/Hiera vault.json +++ b/ThreatDragonModels/Hiera vault/Hiera vault.json @@ -5,6 +5,12 @@ }, "detail": { "contributors": [], - "diagrams": [] + "diagrams": [ + { + "title": "Test", + "thumbnail": "./public/content/images/thumbnail.jpg", + "id": 0 + } + ] } } \ No newline at end of file