diff --git a/README.md b/README.md index 82476e1..7b87f6b 100644 --- a/README.md +++ b/README.md @@ -203,3 +203,9 @@ SSL can be configured with the following config variables: :ssl_ca_path: /path/to/ca/ :ssl_verify: false :ssl_ciphers: "MY:SSL:CIPHER:CONFIG" + +## Caching +Caching can be configured with the following config variables: + :vault: + :cache_timeout: 10 + :cache_clean_interval: 3600 diff --git a/ThreatDragonModels/Hiera vault/Hiera vault.json b/ThreatDragonModels/Hiera vault/Hiera vault.json new file mode 100644 index 0000000..0fdd113 --- /dev/null +++ b/ThreatDragonModels/Hiera vault/Hiera vault.json @@ -0,0 +1,16 @@ +{ + "summary": { + "title": "Hiera vault", + "owner": "Rory" + }, + "detail": { + "contributors": [], + "diagrams": [ + { + "title": "Test", + "thumbnail": "./public/content/images/thumbnail.jpg", + "id": 0 + } + ] + } +} \ No newline at end of file diff --git a/hiera-vault.gemspec b/hiera-vault.gemspec index 435a834..cb94306 100644 --- a/hiera-vault.gemspec +++ b/hiera-vault.gemspec @@ -3,7 +3,7 @@ require 'rubygems/package_task' spec = Gem::Specification.new do |gem| gem.name = "hiera-vault" - gem.version = "0.2.1" + gem.version = "0.2.2" gem.license = "Apache-2.0" gem.summary = "Module for using vault as a hiera backend" gem.email = "jonathan.sokolowski@gmail.com" diff --git a/lib/hiera/backend/vault_backend.rb b/lib/hiera/backend/vault_backend.rb index 8cba891..bdac732 100644 --- a/lib/hiera/backend/vault_backend.rb +++ b/lib/hiera/backend/vault_backend.rb @@ -2,8 +2,7 @@ class Hiera module Backend class Vault_backend - - def initialize() + def initialize require 'json' require 'vault' @@ -43,12 +42,16 @@ def initialize() @vault = nil Hiera.warn("[hiera-vault] Skipping backend. Configuration error: #{e}") end + + @cache = {} + @config[':cache_timeout'] ||= 10 + @config[:cache_clean_interval] ||= 3600 end def lookup(key, scope, order_override, resolution_type) return nil if @vault.nil? - Hiera.debug("[hiera-vault] Looking up #{key} in vault backend") + Hiera.debug("[hiera-vault] Looking up #{key} in vault backend") answer = nil found = false @@ -57,10 +60,13 @@ def lookup(key, scope, order_override, resolution_type) @config[:mounts][:generic].each do |mount| path = Backend.parse_string(mount, scope, { 'key' => key }) Backend.datasources(scope, order_override) do |source| - Hiera.debug("Looking in path #{path}/#{source}/") - new_answer = lookup_generic("#{path}/#{source}/#{key}", scope) + Hiera.debug("[hiera-vault] Looking in path #{path}/#{source}/") + data = lookup_generic_with_cache("#{path}/#{source}/#{key}") #Hiera.debug("[hiera-vault] Answer: #{new_answer}:#{new_answer.class}") - next if new_answer.nil? + next if data.nil? + found = true + new_answer = Backend.parse_answer(data, scope) + case resolution_type when :array raise Exception, "Hiera type mismatch: expected Array and got #{new_answer.class}" unless new_answer.kind_of? Array or new_answer.kind_of? String @@ -72,46 +78,74 @@ def lookup(key, scope, order_override, resolution_type) answer = Backend.merge_answer(new_answer,answer) else answer = new_answer - found = true break end end - break if found end return answer end - def lookup_generic(key, scope) - begin - secret = @vault.logical.read(key) - rescue Vault::HTTPConnectionError - Hiera.debug("[hiera-vault] Could not connect to read secret: #{key}") - rescue Vault::HTTPError => e - Hiera.warn("[hiera-vault] Could not read secret #{key}: #{e.errors.join("\n").rstrip}") - end + private + + def lookup_generic_with_cache(key) + return lookup_generic(key) if @config[:cache_timeout] <= 0 + + now = Time.now.to_i + expired_at = now + @config[:cache_timeout] + + periodically_clean_cache(now) unless @config[:cache_clean_interval] == 0 + + if !@cache[key] || @cache[key][:expired_at] < now + Hiera.debug("[hiera-vault] Lookup #{key} in vault") + @cache[key] = { + :expired_at => expired_at, + :result => lookup_generic(key) + } + else + Hiera.debug("[hiera-vault] #{key} found in cache") + end + return @cache[key][:result] + end + + def lookup_generic(key) + begin + secret = @vault.logical.read(key) + rescue Vault::HTTPConnectionError + Hiera.debug("[hiera-vault] Could not connect to read secret: #{key}") + rescue Vault::HTTPError => e + Hiera.warn("[hiera-vault] Could not read secret #{key}: #{e.errors.join("\n").rstrip}") + end - return nil if secret.nil? - - Hiera.debug("[hiera-vault] Read secret: #{key}") - if @config[:default_field] and (@config[:default_field_behavior] == 'ignore' or (secret.data.has_key?(@config[:default_field].to_sym) and secret.data.length == 1)) - return nil if not secret.data.has_key?(@config[:default_field].to_sym) - # Return just our default_field - data = secret.data[@config[:default_field].to_sym] - if @config[:default_field_parse] == 'json' - begin - data = JSON.parse(data) - rescue JSON::ParserError => e - Hiera.debug("[hiera-vault] Could not parse string as json: #{e}") - end + return nil if secret.nil? + + Hiera.debug("[hiera-vault] Read secret: #{key}") + if @config[:default_field] and (@config[:default_field_behavior] == 'ignore' or (secret.data.has_key?(@config[:default_field].to_sym) and secret.data.length == 1)) + return nil if not secret.data.has_key?(@config[:default_field].to_sym) + # Return just our default_field + data = secret.data[@config[:default_field].to_sym] + if @config[:default_field_parse] == 'json' + begin + data = JSON.parse(data) + rescue JSON::ParserError => e + Hiera.debug("[hiera-vault] Could not parse string as json: #{e}") end - else - # Turn secret's hash keys into strings - data = secret.data.inject({}) { |h, (k, v)| h[k.to_s] = v; h } end - #Hiera.debug("[hiera-vault] Data: #{data}:#{data.class}") + else + data = secret.data.inject({}) { |h, (k, v)| h[k.to_s] = v; h } + end + #Hiera.debug("[hiera-vault] Data: #{data}:#{data.class}") + + return data + end - return Backend.parse_answer(data, scope) + def periodically_clean_cache(now) + return if now < @clean_cache_at.to_i + + @clean_cache_at = now + @config[:cache_clean_interval] + @cache.delete_if do |_, entry| + entry[:expired_at] < now + end end end