Template terraform using a basic inventory

.github/workflows/stackhpc.yml

@@ -53,21 +53,13 @@ jobs:
         shell: bash
         env:
           arcus_CLOUDS_YAML: ${{ secrets.ARCUS_CLOUDS_YAML }}
-
-      - name: Provision ports, inventory and other infrastructure apart from nodes
-        run: |
-          . venv/bin/activate
-          . environments/${{ matrix.cloud }}/activate
-          cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
-          TF_VAR_create_nodes=false terraform apply -auto-approve
-
+
       - name: Setup environment-specific inventory/terraform inputs
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
           ansible-playbook ansible/adhoc/generate-passwords.yml
           echo vault_testuser_password: "$TESTUSER_PASSWORD" > $APPLIANCES_ENVIRONMENT_ROOT/inventory/group_vars/all/test_user.yml
-          ansible-playbook ansible/adhoc/template-cloud-init.yml
         env:
           TESTUSER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 

ansible/.gitignore

@@ -33,4 +33,6 @@ roles/*
 !roles/mysql/
 !roles/mysql/**
 !roles/systemd/
-!roles/systemd/**
+!roles/systemd/**
+!roles/terraform/
+!roles/terraform/**

ansible/adhoc/provision.yml

@@ -0,0 +1,16 @@
+- hosts: all
+  become: no
+  gather_facts: no
+  tasks:
+    - import_role:
+        name: terraform
+        tasks_from: template.yml
+      when: "tf_state != 'absent'"
+
+- hosts: localhost
+  become: no
+  gather_facts: no
+  tasks:
+    - import_role:
+        name: terraform
+        tasks_from: apply.yml

ansible/bootstrap.yml

@@ -14,17 +14,12 @@
       when: "'secrets_openhpc_' in (hostvars[inventory_hostname] | join)"
 
 - hosts: etc_hosts
-  gather_facts: false
+  gather_facts: yes
   tags: etc_hosts
   become: yes
   tasks:
-    - name: Template /etc/hosts
-      copy:
-        content: "{{ etc_hosts_template }}"
-        dest: /etc/hosts
-        owner: root
-        group: root
-        mode: u=rw,og=r
+    - import_role:
+        name: etc_hosts
 
 - hosts: cluster
   gather_facts: false

ansible/roles/basic_users/defaults/main.yml

@@ -1,4 +1,4 @@
-basic_users_manage_homedir: "{{ (ansible_hostname == (ansible_play_hosts | first)) }}"
+basic_users_manage_homedir: "{{ (inventory_hostname == (ansible_play_hosts | first)) }}"
 basic_users_userdefaults:
   state: present
   create_home: "{{ basic_users_manage_homedir }}"

ansible/roles/etc_hosts/README.md

@@ -1,5 +1,7 @@
 # etc_hosts
 
-This role provides documentation only.
+Hosts in the `etc_hosts` groups have `/etc/hosts` created. The generated file defines all hosts in this group using `ansible_host` as the IP address and `hostname` as the canonical hostname. This may need overriding for multi-homed hosts. See `environments/common/inventory/group_vars/all/cloud_init.yml` for configuration.
 
-Hosts in the `etc_hosts` groups get `/etc/hosts` created via `cloud-init`. The generated file defines all hosts in this group using `ansible_host` as the IP address and `inventory_hostname` as the canonical hostname. This may need overriding for multi-homed hosts. See `environments/common/inventory/group_vars/all/cloud_init.yml` for configuration.
+# Variables:
+
+- `etc_hosts_template`: Template file to use. Default uses in-role template.

ansible/roles/etc_hosts/defaults/main.yml

@@ -0,0 +1 @@
+etc_hosts_template: hosts.j2

ansible/roles/etc_hosts/tasks/main.yml

@@ -0,0 +1,8 @@
+- name: Template out /etc/hosts
+  template:
+    src: "{{ etc_hosts_template }}"
+    dest: /etc/hosts
+    owner: root
+    group: root
+    mode: 0644
+  become: yes

ansible/roles/etc_hosts/templates/hosts.j2

@@ -0,0 +1,6 @@
+127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
+::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
+
+{% for inventory_hostname in groups['etc_hosts'] | sort -%}
+{{ hostvars[inventory_hostname]['ansible_host'] }} {{ hostvars[inventory_hostname]['hostname'] }}
+{% endfor -%}

ansible/roles/terraform/defaults/main.yml

@@ -0,0 +1,9 @@
+tf_project_path: "{{ appliances_environment_root }}/terraform/"
+tf_template_path: # default uses normal ansible search path
+tf_single_templates:
+  - main.tf.j2
+  - network.tf.j2
+  - inventory.tf.j2
+tf_host_templates:
+  - node.tf.j2
+tf_state: present

ansible/roles/terraform/tasks/apply.yml

@@ -0,0 +1,34 @@
+- name: Create Terraform plan
+  community.general.terraform:
+    project_path: "{{ tf_project_path }}"
+    state: planned
+    plan_file: tf.plan
+    force_init: yes
+  register: _tf_plan
+
+- name: Show Terraform plan
+  debug:
+    msg: "{{ _tf_plan.stdout }}"
+
+- name: Confirm Terraform plan execution
+  pause:
+    prompt: "Do you want to execute this plan? (Only 'yes' executes)"
+  register: confirm_plan
+  when: "'No changes. Your infrastructure matches the configuration.' not in _tf_plan.stdout"
+  # TODO: add override for autodeployment, e.g. CI!
+
+- name: Execute Terraform plan
+  community.general.terraform:
+    project_path: "{{ tf_project_path }}"
+    state: "{{ tf_state }}"
+    plan_file: tf.plan
+    force_init: yes
+  when:
+    - not confirm_plan.skipped | default(false)
+    - confirm_plan.user_input | bool
+  register: _tf_apply
+  # TODO: add override for autodeployment, e.g. CI!
+
+- debug:
+    msg: "{{ _tf_apply.stdout }}"
+  when: "'stdout' in _tf_apply"

ansible/roles/terraform/tasks/template.yml

@@ -0,0 +1,14 @@
+- name: Template host-independent Terraform configuration files
+  template:
+    src: "{{ [tf_template_path, item ] | flatten(skip_nulls=True) | join('/') }}"
+    dest: "{{ tf_project_path }}/{{ item | splitext | first }}"
+  loop: "{{ tf_single_templates }}"
+  run_once: true
+  delegate_to: localhost
+
+- name: Template host-dependent Terraform configuration files
+  template:
+    src: "{{ [tf_template_path, item ] | flatten(skip_nulls=True) | join('/') }}"
+    dest: "{{ tf_project_path }}/{{inventory_hostname }}_{{ item | splitext | first }}"
+  loop: "{{ tf_host_templates }}"
+  delegate_to: localhost

ansible/roles/terraform/templates/inventory.tf.j2

@@ -0,0 +1,20 @@
+
+resource "local_file" "hosts" {
+    content  = <<EOT
+{% for host in groups['all'] | sort %}
+{{ host }} ansible_host=${openstack_networking_port_v2.{{ host }}.all_fixed_ips[0]}
+{% endfor %}
+EOT
+    filename = "../inventory/hosts"
+}
+
+resource "local_file" "partitions" {
+    content = <<EOT
+{% for part in openhpc_slurm_partitions %}
+{{ cluster_name }}_{{ part.name }}:
+    children:
+        {{ part.name }}
+{% endfor %}
+EOT
+    filename = "../inventory/partitions.yml"
+}

ansible/roles/terraform/templates/main.tf.j2

@@ -0,0 +1,8 @@
+terraform {
+  required_version = ">= 0.14"
+  required_providers {
+    openstack = {
+      source = "terraform-provider-openstack/openstack"
+    }
+  }
+}

ansible/roles/terraform/templates/network.tf.j2

@@ -0,0 +1,16 @@
+
+data "openstack_networking_network_v2" "cluster_net" {
+  name           = "{{ network_name }}"
+}
+
+data "openstack_networking_subnet_v2" "cluster_subnet" {
+
+  name            = "{{ subnet_name }}"
+}
+
+{% for secgroup in hostvars.values() | map(attribute='security_groups', default=[]) | flatten | unique %}
+data "openstack_networking_secgroup_v2" "{{ secgroup }}" {
+
+  name = "{{ secgroup }}"
+}
+{% endfor %}

ansible/roles/terraform/templates/network.yml.j2

@@ -0,0 +1,17 @@
+
+data "openstack_networking_network_v2" "cluster_net" {
+  name           = "{{ network_name }}"
+}
+
+data "openstack_networking_subnet_v2" "cluster_subnet" {
+
+  name            = "{{ subnet_name }}"
+}
+
+{% for inventory_hostname in groups['all'] %}
+data "openstack_networking_secgroup_v2" "{{ inventory_hostname }}" {
+  for_each = toset({{ hostvars[inventory_hostname].security_groups | default(["default"]) | to_json }} ) {# json forces double-quotes #}
+
+  name = each.key
+}
+{% endfor %}

ansible/roles/terraform/templates/node.tf.j2

@@ -0,0 +1,83 @@
+data "openstack_images_image_v2" "{{ inventory_hostname }}" {
+  name = "{{ hostvars[inventory_hostname].image_name }}"
+}
+
+{% for volume in hostvars[inventory_hostname].volumes | default([]) %}
+
+resource "openstack_blockstorage_volume_v3" "{{ inventory_hostname }}_{{ volume.label }}" {
+    name = "{{ cluster_name }}-{{ inventory_hostname }}-{{ volume.label }}"
+    description = "{{ volume.description | default('') }}"
+    size = {{ volume.size }}
+}
+
+{% endfor %}
+
+resource "openstack_networking_port_v2" "{{ inventory_hostname }}" {
+
+  name = "{{ cluster_name }}-{{ inventory_hostname }}"
+  network_id = data.openstack_networking_network_v2.cluster_net.id
+  admin_state_up = "true"
+
+  fixed_ip {
+    subnet_id = data.openstack_networking_subnet_v2.cluster_subnet.id
+  }
+
+  security_group_ids = [
+  {% for secgroup in hostvars[inventory_hostname].security_groups %}
+    data.openstack_networking_secgroup_v2.{{ secgroup }}.id,
+  {% endfor %}
+  ]
+
+  binding {
+    vnic_type = "{{ hostvars[inventory_hostname].vnic_type }}"
+    profile = "{{ hostvars[inventory_hostname].vnic_profile | to_json }}"
+  }
+}
+
+resource "openstack_compute_instance_v2" "{{ inventory_hostname }}" {
+
+  name = "{{ hostname }}"
+  image_name = data.openstack_images_image_v2.{{ inventory_hostname }}.name
+  flavor_name = "{{ hostvars[inventory_hostname].flavor }}"
+  key_pair = "{{ hostvars[inventory_hostname].key_pair }}"
+
+  {% if hostvars[inventory_hostname].volumes | default([]) | length %}
+  # root device:
+  block_device {
+      uuid = data.openstack_images_image_v2.{{ inventory_hostname }}.id
+      source_type  = "image"
+      destination_type = "local"
+      boot_index = 0
+      delete_on_termination = true
+  }
+  {% endif %}
+
+  {% for volume in hostvars[inventory_hostname].volumes | default([]) %}
+  block_device {
+      destination_type = "volume"
+      source_type  = "volume"
+      boot_index = -1
+      uuid = openstack_blockstorage_volume_v3.{{ inventory_hostname }}_{{ volume.label }}.id
+  }
+  {% endfor %}
+
+  network {
+    port = openstack_networking_port_v2.{{ inventory_hostname }}.id
+    access_network = true
+  }
+
+  metadata = {
+    environment_root = "{{ appliances_environment_root }}"
+  }
+
+  {% if user_data is defined %}
+  user_data = <<-EOT
+      {{ user_data | indent(6) }}
+  EOT
+  {% endif %}
+
+  lifecycle{
+    ignore_changes = [{{ tf_ignore_changes | join(', ') }}]
+  }
+
+}

environments/arcus/inventory/cluster.yml

@@ -0,0 +1,29 @@
+all:
+  vars:
+    # required:
+    cluster_name: templatetf # FIXME: needs defining for each arcus cluster!
+    image_name: openhpc-221118-1422.qcow2 # https://github.com/stackhpc/slurm_image_builder/pull/12
+    flavor: vm.alaska.cpu.general.small
+    key_pair: slurm-app-ci
+    network_name: WCDC-iLab-60
+    subnet_name: WCDC-iLab-60
+
+    # overrides:
+    vnic_type: direct
+    block_device_prefix: sd
+    home_volume_size: 10
+    state_volume_size: 10
+
+    openhpc_slurm_partitions:
+      - name: small
+      - name: extra
+
+# use child groups of compute with names in openhpc_slurm_partitions to define partitions
+compute:
+  children:
+    small:
+      hosts:
+        compute[0:1]:
+    extra:
+      hosts:
+        compute[2:3]: