snapshot/restore wip

playbooks/vault_restore.yml

@@ -0,0 +1,21 @@
+---
+- name: "[VAULT] Restore"
+  hosts: "{{ scope | default('hashistack_masters') }}"
+  become: false
+  gather_facts: false
+
+  vars:
+    ansible_ssh_user: "vault-snapshot"
+
+  tasks:
+    - name: "Vault"
+      include_role:
+        name: "vault"
+        tasks_from: "__restore.yml"
+        apply:
+          tags:
+            - vault
+      tags:
+        - vault
+
+

playbooks/vault_snapshot.yml

@@ -1,8 +1,8 @@
 ---
 - name: "[VAULT] Snapshot"
-  hosts: "hashistack_masters[0]"
+  hosts: "{{ scope | default('hashistack_masters') }}"
   become: false
-  gather_facts: true
+  gather_facts: false
 
   vars:
     ansible_ssh_user: "vault-snapshot"

playbooks/vault_unseal.yml

@@ -0,0 +1,18 @@
+---
+- name: "[VAULT] Snapshot"
+  hosts: "hashistack_masters"
+  become: true
+  gather_facts: false
+
+  tasks:
+    - name: "Vault"
+      include_role:
+        name: "vault"
+        tasks_from: "_inplace_init_unseal.yml"
+        apply:
+          tags:
+            - vault
+      tags:
+        - vault
+
+

requirements.yml

@@ -19,4 +19,4 @@ collections:
   - name: cloud.terraform
     version: "1.1.1"
   - name: rtnp.galaxie_clans
-    version: "1.16.0"
+    version: "1.16.2"

roles/common_vars/defaults/main.yml

@@ -51,3 +51,5 @@ tf_module_dest: "{{ hs_workspace_tf_modules_dir }}/{{ tf_module_name }}"
 glxclans_host_service_user_name: "caretaker"
 
 hs_install_vault_sidecar: true
+
+hs_workspace_local_backup_dir: "{{ hs_workspace_root }}/backups"

roles/common_vars/tasks/main.yml

@@ -1,4 +1,12 @@
 ---
+- name: "Gather useful facts"
+  setup:
+    gather_subset:
+      - "!hardware"
+      - "!virtual"
+      - "!ohai"
+      - "!facter"
+
 - name: Load os-specific vars
   include_vars: "{{ _current_os_vars }}"
   with_first_found:

roles/vault/files/snapshot/policies/snapshot.hcl

@@ -2,3 +2,10 @@ path "/sys/storage/raft/snapshot" {
   capabilities = ["read"]
 }
 
+path "/sys/leader" {
+  capabilities = ["read", "list"]
+}
+
+path "/sys/storage/raft/snapshot-force" {
+  capabilities = ["create", "update"]
+}

roles/vault/handlers/main.yml

@@ -1,11 +1,11 @@
 ---
-- name: Restart vault
+- name: "Restart vault"
   systemd:
     name: vault
     state: restarted
     daemon_reload: true
     enabled: true
 
-- name: Update ca trust
+- name: "Update ca trust"
   command: >-
     {{ __hs_vault_update_trust_command }}

roles/vault/tasks/__restore.yml

@@ -0,0 +1,51 @@
+---
+# Implementation of:
+# https://developer.hashicorp.com/vault/tutorials/standard-procedures/sop-restore
+- name: "Load collection common vars"
+  import_role:
+    name: "vault_vars"
+  tags:
+    - always
+
+- name: "Gather vault leader"
+  uri:
+    url: "{{ __hs_vault_api.leader }}"
+    return_content: true
+    headers:
+      X-Vault-Token: "{{ hs_vault_snapshot_token }}"
+  environment:
+    SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
+  register: _current_vault_leader
+
+- name: "Cook variables"
+  set_fact:
+    _hs_vault_is_leader: >-
+      {{ _current_vault_leader.json.leader_address == __hs_vault_conf_api_addr }}
+
+- name: "Synchronize duplicity datastore"
+  synchronize:
+    src: "{{ hs_workspace_local_backup_dir }}/vault/"
+    mode: push
+    dest: "{{ __hs_vault_snapshot_duplicity_dir }}/"
+    rsync_opts: '-e "ssh -F {{ hs_workspace_root }}/ssh.cfg"'
+  when: _hs_vault_is_leader
+
+- name: "Duplicity restore"
+  shell:
+    cmd: >-
+      duplicity restore
+      file://{{ __hs_vault_snapshot_duplicity_dir }}
+      {{ __hs_vault_snapshot_home_dir }}/snapshots
+    executable: /usr/bin/bash
+  environment:
+    PASSPHRASE: "{{ hs_vault_snapshot_passphrase }}"
+  when: _hs_vault_is_leader
+
+- name: "Vault restore"
+  shell:
+    cmd: >-
+      source {{ __hs_vault_snapshot_home_dir }}/.bash_profile &&
+      vault operator raft snapshot restore -force {{ __hs_vault_snapshot_dir }}/vault.snapshot
+    executable: /usr/bin/bash
+  when: _hs_vault_is_leader
+

roles/vault/tasks/__snapshot.yml

@@ -7,21 +7,53 @@
   tags:
     - always
 
+- name: "Gather vault leader"
+  uri:
+    url: "{{ __hs_vault_api.leader }}"
+    return_content: true
+    headers:
+      X-Vault-Token: "{{ hs_vault_snapshot_token }}"
+  environment:
+    SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
+  register: _current_vault_leader
+
+- name: "Cook variables"
+  set_fact:
+    _hs_vault_is_leader: >-
+      {{ _current_vault_leader.json.leader_address == __hs_vault_conf_api_addr }}
+
 - name: "Snapshot"
   shell:
     cmd: >-
       source {{ __hs_vault_snapshot_home_dir }}/.bash_profile &&
-      vault operator raft snapshot save
-      {{ __hs_vault_snapshot_home_dir }}/snapshots/vault.{{ ansible_date_time.iso8601_basic_short }}.snapshot
+      vault operator raft snapshot save {{ __hs_vault_snapshot_dir }}/vault.snapshot
     executable: /usr/bin/bash
-    chdir: "{{ __hs_vault_snapshot_home_dir }}/snapshots"
+  when: _hs_vault_is_leader
 
 - name: "Create archive"
   shell:
     cmd: >-
-      duplicity backup {{ __hs_vault_snapshot_home_dir }}/snapshots
-      file://{{ __hs_vault_snapshot_home_dir }}
+      duplicity backup
+      {{ __hs_vault_snapshot_home_dir }}/snapshots
+      file://{{ __hs_vault_snapshot_duplicity_dir }}
     executable: /usr/bin/bash
   environment:
     PASSPHRASE: "{{ hs_vault_snapshot_passphrase }}"
+  when: _hs_vault_is_leader
+
+- name: "[LOCAL] Prepare backup dir"
+  file:
+    path: "{{ hs_workspace_local_backup_dir }}/vault"
+    state: directory
+    mode: 0750
+  delegate_to: localhost
+  run_once: true
+  become: false
 
+- name: "Synchronize duplicity datastore"
+  synchronize:
+    src: "{{ __hs_vault_snapshot_duplicity_dir }}/"
+    mode: pull
+    dest: "{{ hs_workspace_local_backup_dir }}/vault/"
+    rsync_opts: '-e "ssh -F {{ hs_workspace_root }}/ssh.cfg"'
+  when: _hs_vault_is_leader

roles/vault/tasks/_inplace_init_unseal.yml

@@ -1,11 +1,17 @@
 ---
-- name: Explicitly load secret dir
+- name: "Load vault role vars"
+  import_role:
+    name: "vault_vars"
+  tags:
+    - always
+
+- name: "Explicitly load secret dir"
   include_vars:
     dir: "{{ hs_vault_local_secret_dir }}"
     ignore_unknown_extensions: true
   no_log: true
 
-- name: Init first node
+- name: "Init first node"
   include_tasks: "_inplace_init.yml"
   when: __hs_vault_is_first_master
 
@@ -17,7 +23,7 @@
   delay: 5
   tags: unseal
 
-- name: Wait for port 8200 to become open on the host, don't start checking for 10 seconds
+- name: "Wait for port 8200 to become open on the host (delay: 10 seconds)"
   ansible.builtin.wait_for:
     host: "{{ hs_vault_api_address }}"
     port: "{{ hs_vault_api_port }}"

roles/vault/tasks/common/_install.yml

@@ -1,20 +1,20 @@
 ---
-- name: Unarchive
+- name: "Unarchive"
   unarchive:
     src: "{{ __hs_vault_local_archive_path }}"
     dest: "/usr/bin"
     creates: "/usr/bin/vault"
 
-- name: Set permissions on vault binary
+- name: "Set permissions on vault binary"
   file:
     path: "/usr/bin/vault"
     owner: root
     group: vault
     mode: 0750
 
-- name: Render system service
+- name: "Render system service"
   template:
-    src: vault.service.j2
+    src: "vault.service.j2"
     dest: "/lib/systemd/system/vault.service"
     mode: 0644
   notify: Restart vault

roles/vault/tasks/main.yml

@@ -1,22 +1,10 @@
 ---
-- name: "Load collection common vars"
+- name: "Load vault role vars"
   import_role:
     name: "vault_vars"
   tags:
     - always
 
-- name: "Load role os vars"
-  include_vars: "{{ _current_os_vars }}"
-  with_first_found:
-    - skip: true
-      files: "{{ __hs_role_vars_precedence }}"
-  loop_control:
-    loop_var: _current_os_vars
-  when:
-    - ansible_distribution is defined
-    - ansible_distribution | length > 0
-  tags: always
-
 - name: "Execute os tasks"
   include_tasks: "{{ _current_os_tasks }}"
   with_first_found:
@@ -26,10 +14,10 @@
     loop_var: _current_os_tasks
   tags: always
 
-- name: Start master vault services
+- name: "Start master vault services"
   systemd:
-    state: started
     name: vault
+    state: started
     daemon_reload: true
     enabled: true
   when: __hs_vault_is_master

roles/vault_vars/vars/main.yml

@@ -14,6 +14,7 @@ __hs_vault_api:
   genroot: "{{ __hs_vault_conf_api_addr }}/v1/sys/generate-root/attempt"
   genroot_update: "{{ __hs_vault_conf_api_addr }}/v1/sys/generate-root/update"
   genroot_decode: "{{ __hs_vault_conf_api_addr }}/v1/sys/decode-token"
+  leader: "{{ __hs_vault_conf_api_addr }}/v1/sys/leader"
 
 
 __hs_vault_cluster_protocol: "https"
@@ -25,7 +26,11 @@ __hs_vault_ssl_cert_group: "ssl-cert"
 
 __hs_vault_conf_dir: "/etc/vault.d"
 __hs_vault_home_dir: "/opt/vault"
+
 __hs_vault_snapshot_home_dir: "/opt/vault-snapshot"
+__hs_vault_snapshot_dir: "{{ __hs_vault_snapshot_home_dir }}/snapshots"
+__hs_vault_snapshot_duplicity_dir: "{{ __hs_vault_snapshot_home_dir }}/duplicity"
+
 __hs_vault_data_dir: "{{ __hs_vault_home_dir }}/data"
 
 __hs_vault_conf_file: "{{ __hs_vault_conf_dir }}/vault.hcl"