Add role for deploying EXPO

nova/core/roles/expo/README.md

@@ -0,0 +1,23 @@
+# expo
+
+This role is used to deploy Exercise Portal (EXPO) to a VM.
+
+## Requirements
+
+Certificates, which are defined in the defaults.
+
+## Role Variables
+
+See [defaults/main.yml](https://github.com/novateams/nova.core/blob/main/nova/core/roles/expo/defaults/main.yml) for the full list of variables.
+
+## Dependencies
+
+- Depends on Docker and Docker Compose being installed on the host. Docker can be installed using the [nova.core.docker](https://github.com/novateams/nova.core/tree/main/nova/core/roles/docker) role.
+
+## Example
+
+```yaml
+- name: Including connection role
+  include_role:
+    name: nova.core.expo
+```

nova/core/roles/expo/defaults/main.yml

@@ -0,0 +1,103 @@
+---
+### general ###
+expo_git_repo:
+expo_fqdn: expo.localhost
+expo_configuration_folder: XS/2023/XS23TR
+expo_project_mode: production
+expo_readonly_mode: false
+expo_profiles: production ### "archive,docs,availability,consumer"
+expo_debug_mode: true
+expo_node_tls_check: 0
+
+### MKDOCS ###
+expo_mkdocs_fqdn: expo-mkdocs.localhost
+expo_mkdocs_build_content: all # guides or all
+
+### EXTERNAL ###
+expo_providentia_token:
+expo_gitlab_token:
+
+### EVIDENCE ###
+expo_evidence_public: ""
+expo_evidence_private: ""
+
+### BACKEND ###
+expo_backend_service_replicas: 6
+expo_session_secret: ""
+
+### BACKEND WORKER ###
+expo_backend_worker_service_replicas: 6
+
+### FRONTEND ###
+expo_frontend_service_replicas: 3
+
+### KEYCLOAK ###
+expo_keycloak_legacy: false
+expo_keycloak_fqdn: keycloak.localhost
+expo_keycloak_realm: EXPO
+expo_keycloak_client_secret: ""
+expo_keycloak_client_scope: resources
+expo_keycloak_resource_prefix: Re_EXPO_
+expo_keycloak_role_prefix: Ro_EXPO_
+expo_keycloak_client_id: expo
+expo_keycloak_local_admin_username: admin
+expo_keycloak_local_admin_password: admin-pass
+expo_keycloak_log_level: INFO
+
+### TRAEFIK ###
+expo_traefik_fqdn: expo-traefik.localhost
+expo_traefik_log_level: DEBUG
+expo_traefik_certificate_expo_fullchain_path: ./data/expo.localhost.crt
+expo_traefik_certificate_expo_key_path: ./data/expo.localhost.key
+expo_traefik_certificate_mkdocs_fullchain_path: ./data/mkdocs.localhost.crt
+expo_traefik_certificate_mkdocs_key_path: ./data/mkdocs.localhost.key
+expo_traefik_certificate_keycloak_fullchain_path: ./data/keycloak.localhost.crt
+expo_traefik_certificate_keycloak_key_path: ./data/keycloak.localhost.key
+expo_traefik_certificate_kafdrop_fullchain_path: ./data/kafdrop.localhost.crt
+expo_traefik_certificate_kafdrop_key_path: ./data/kafdrop.localhost.key
+expo_traefik_certificate_traefik_fullchain_path: ./data/traefik.localhost.crt
+expo_traefik_certificate_traefik_key_path: ./data/traefik.localhost.key
+
+### MONGO ###
+expo_mongo_pool_size: 500
+expo_mongo_query_profile: 1
+expo_mongo_query_slowms: 50
+expo_mongo_query_sample: 1.0
+expo_mongo_root_username: root
+expo_mongo_root_password: ""
+expo_mongo_dababase_name: expo
+expo_mongo_database_username: expo
+expo_mongo_database_password: ""
+
+### GUACAMOLE ###
+expo_guacamole_enabled: false
+expo_guacamole_api_username: guacadmin
+expo_guacamole_api_password: ""
+expo_guacamole_gt_for_gt_password: ""
+expo_guacamole_gt_for_bt_password: ""
+
+### DIRTY SOCKS ###
+expo_dirty_socks_enabled: false
+
+### HALL OF FAME ###
+expo_hall_of_fame_enabled: false
+
+### KAFKA ###
+expo_kafdrop_fqdn: expo-kafdrop.localhost
+expo_kafka_enabled: false
+expo_kafka_external: false
+expo_kafka_username: expo
+expo_kafka_password: ""
+
+### SENTRY ###
+expo_sentry_frontend_enabled: false
+expo_sentry_frontend_dsn:
+expo_sentry_frontend_tracing: 1
+expo_sentry_backend_enabled: false
+expo_sentry_backend_dsn:
+expo_sentry_backend_tracing: 1
+
+### NETWORK ###
+expo_network_name: expo-network
+expo_network_ipv4_subnet: 172.19.0.0/16
+expo_network_ipv6_subnet: fd71::/64

nova/core/roles/expo/tasks/dependencies.yml

@@ -0,0 +1,32 @@
+---
+- name: Ensure we have required packages ...
+  ansible.builtin.apt:
+    name:
+      - make
+      - curl
+      - git
+      - jq
+    state: latest
+
+- name: Performance tuning for host ...
+  ansible.builtin.template:
+    src: etc/sysctl.d/11-expo.conf
+    dest: /etc/sysctl.d/11-expo.conf
+  register: expo_host_config
+
+- name: Reload expo sysctl values if conf file changed, ignore errors as some paths change over time  ...
+  ansible.builtin.command: sysctl -p /etc/sysctl.d/11-expo.conf
+  ignore_errors: true
+  when: expo_host_config.changed
+
+- name: Disable all kernel mitigations for more raw cpu ...
+  ansible.builtin.lineinfile:
+    dest: /etc/default/grub
+    regexp: ^GRUB_CMDLINE_LINUX_DEFAULT.*
+    line: GRUB_CMDLINE_LINUX_DEFAULT="autoinstall quiet splash mitigations=off"
+    state: present
+  register: grub_config
+
+- name: Update-grub if grub config changed ...
+  ansible.builtin.command: update-grub
+  when: grub_config.changed

nova/core/roles/expo/tasks/deploy-expo.yml

@@ -0,0 +1,20 @@
+---
+- name: Sync EXPO code to target ...
+  ansible.builtin.include_tasks: sync-code.yml
+
+- name: Prepare app environment ...
+  community.general.make:
+    chdir: /srv/expo/
+    target: prepare
+
+- name: Build new app containers, this takes around 5 minutes  ...
+  community.general.make:
+    chdir: /srv/expo/
+    target: build
+
+- name: Run app with profiles {{ expo_profiles }} ...
+  community.general.make:
+    chdir: /srv/expo/
+    target: start-with-profiles-from-environment
+  environment:
+    COMPOSE_PROFILES: "{{ expo_profiles }}"

nova/core/roles/expo/tasks/main.yml

@@ -0,0 +1,13 @@
+---
+- name: Check if expo already present ...
+  ansible.builtin.stat:
+    path: /srv/expo
+  register: expo_folder_existence
+
+### install dependencies only if expo folder does not exist
+- name: Install dependencies ...
+  ansible.builtin.include_tasks: dependencies.yml
+  when: not expo_folder_existence.stat.exists
+
+- name: Deploy EXPO app ...
+  ansible.builtin.include_tasks: deploy-expo.yml

nova/core/roles/expo/tasks/sync-code.yml

@@ -0,0 +1,69 @@
+---
+- name: Stat for /tmp/expo-{{ fqdn }} ...
+  ansible.builtin.stat:
+    path: /tmp/expo-{{ fqdn }}
+  delegate_to: localhost
+  become: false
+  register: app_folder_existence
+
+- name: Localhost delete /tmp/expo-{{ fqdn }} ...
+  ansible.builtin.file:
+    state: absent
+    path: /tmp/expo-{{ fqdn }}
+  delegate_to: localhost
+  become: false
+  when: app_folder_existence.stat.exists
+
+- name: Clone git repository to localhost /tmp/expo-{{ fqdn }} ...
+  ansible.builtin.git:
+    repo: "{{ expo_git_repo }}"
+    force: true
+    accept_hostkey: true
+    clone: true
+    dest: /tmp/expo-{{ fqdn }}
+    recursive: true
+    depth: 1
+  delegate_to: localhost
+  become: false
+
+- name: Create folder /srv/expo ...
+  ansible.builtin.file:
+    state: directory
+    path: /srv/expo
+    owner: "{{ ansible_user }}"
+    group: "{{ ansible_user }}"
+
+- name: Sync files to target ...
+  ansible.posix.synchronize:
+    src: /tmp/expo-{{ fqdn }}/
+    dest: /srv/expo
+    recursive: true
+    delete: true
+    rsync_opts:
+      - --exclude=.git
+      - --exclude=.gitmodules
+      - --exclude=.gitlab-ci.yml
+      - --exclude=.idea
+      - --exclude=.vscode
+      - --exclude=*/override.*.env
+      - --exclude=personal-functions.makerc
+      - --exclude=data
+      - --exclude=logs
+      - --exclude=expo-mongo/sync/
+      - --exclude=/test*
+    use_ssh_args: true
+  become: false
+
+- name: Localhost delete /tmp/expo-{{ fqdn }} ...
+  ansible.builtin.file:
+    state: absent
+    path: /tmp/expo-{{ fqdn }}
+  delegate_to: localhost
+  become: false
+
+- name: Template the configuration files ...
+  ansible.builtin.template:
+    src: "{{ item }}"
+    dest: /srv/expo/env/{{ item | basename }}
+  with_fileglob:
+    - templates/srv/expo/env/*.env

nova/core/roles/expo/templates/etc/sysctl.d/11-expo.conf

@@ -0,0 +1,54 @@
+### expo related sysctl values
+fs.inotify.max_user_watches=524288
+vm.swappiness=1
+vm.overcommit_memory=1
+vm.max_map_count=9999999
+
+net.ipv4.netfilter.ip_conntrack_generic_timeout=120
+net.ipv4.netfilter.ip_conntrack_tcp_timeout_established=54000
+net.ipv4.netfilter.ip_conntrack_max=196608
+
+net.ipv6.netfilter.ip_conntrack_generic_timeout=120
+net.ipv6.netfilter.ip_conntrack_tcp_timeout_established=54000
+net.ipv6.netfilter.ip_conntrack_max=196608
+
+net.netfilter.nf_conntrack_generic_timeout=120
+net.netfilter.nf_conntrack_tcp_timeout_established=54000
+net.netfilter.nf_conntrack_sctp_timeout_established=54000
+
+net.netfilter.nf_conntrack_max=524288
+
+
+# Default Socket Receive Buffer
+net.core.rmem_default = 31457280
+
+# Maximum Socket Receive Buffer
+net.core.rmem_max = 33554432
+
+# Default Socket Send Buffer
+net.core.wmem_default = 31457280
+
+# Maximum Socket Send Buffer
+net.core.wmem_max = 33554432
+
+# Increase number of incoming connections
+net.core.somaxconn = 65535
+
+# Increase number of incoming connections backlog
+net.core.netdev_max_backlog = 65536
+
+# Increase the maximum amount of option memory buffers
+net.core.optmem_max = 25165824
+
+# Increase the maximum total buffer-space allocatable
+# This is measured in units of pages (4096 bytes)
+net.ipv4.tcp_mem = 786432 1048576 26777216
+net.ipv4.udp_mem = 65536 131072 262144
+
+# Increase the read-buffer space allocatable
+net.ipv4.tcp_rmem = 8192 87380 33554432
+net.ipv4.udp_rmem_min = 16384
+
+# Increase the write-buffer-space allocatable
+net.ipv4.tcp_wmem = 8192 65536 33554432
+net.ipv4.udp_wmem_min = 16384

nova/core/roles/expo/templates/srv/expo/env/override.expo-backend-availability.env

@@ -0,0 +1,3 @@
+##################################################################################################
+### BACKEND AVAILABILITY ENV - override values with env/override.expo-backend-availability.env ###
+##################################################################################################

nova/core/roles/expo/templates/srv/expo/env/override.expo-backend-consumer.env

@@ -0,0 +1,3 @@
+##########################################################################################
+### BACKEND CONSUMER ENV - override values with env/override.expo-backend-consumer.env ###
+##########################################################################################

nova/core/roles/expo/templates/srv/expo/env/override.expo-backend-worker.env

@@ -0,0 +1,5 @@
+######################################################################################
+### BACKEND WORKER ENV - override values with env/override.expo-backend-worker.env ###
+######################################################################################
+
+BACKEND_WORKER_SERVICE_REPLICAS={{ expo_backend_worker_service_replicas }}

nova/core/roles/expo/templates/srv/expo/env/override.expo-backend.env

@@ -0,0 +1,42 @@
+########################################################################
+### BACKEND ENV - override values with env/override.expo-backend.env ###
+########################################################################
+
+### BACKEND ###
+BACKEND_SERVICE_REPLICAS={{ expo_backend_service_replicas }}
+
+### SESSION ###
+SESSION_SECRET="{{ expo_session_secret }}"
+GRAPHQL_CURSOR_SECRET="{{ expo_session_secret }}"
+
+### SENTRY ###
+BACKEND_SENTRY_ENABLED={{ expo_sentry_backend_enabled | upper }}
+BACKEND_SENTRY_DSN="{{ expo_sentry_backend_dsn }}"
+BACKEND_SENTRY_TRACING={{ expo_sentry_backend_tracing }}
+
+### EXTERNAL ###
+EXTERNAL_PROVIDENTIA_TOKEN="{{ expo_providentia_token }}"
+EXTERNAL_GITLAB_TOKEN="{{ expo_gitlab_token }}"
+
+### EVIDENCE ###
+EVIDENCE_PUBLIC_KEY="{{ expo_evidence_public }}"
+EVIDENCE_PRIVATE_KEY="{{ expo_evidence_private }}"
+
+### DIRTY SOCKS ###
+DIRTY_SOCKS_ENABLED={{ expo_dirty_socks_enabled | upper }}
+
+### HALL OF FAME ###
+HALL_OF_FAME_ENABLED={{ expo_hall_of_fame_enabled | upper }}
+
+### GUACAMOLE ###
+GUACAMOLE_ENABLED={{ expo_guacamole_enabled | upper }}
+EXTERNAL_GUACAMOLE_API_USERNAME={{ expo_guacamole_api_username }}
+EXTERNAL_GUACAMOLE_API_PASSWORD={{ expo_guacamole_api_password }}
+EXTERNAL_GUACAMOLE_GT_PASSWORD_FOR_GT_VMS={{ expo_guacamole_gt_for_gt_password }}
+EXTERNAL_GUACAMOLE_GT_PASSWORD_FOR_BT_VMS={{ expo_guacamole_gt_for_bt_password }}
+
+### KAFKA ###
+KAFKA_ENABLED={{ expo_kafka_enabled | upper }}
+KAFKA_EXTERNAL={{ expo_kafka_external | upper }}
+KAFKA_USERNAME="{{ expo_kafka_username }}"
+KAFKA_PASSWORD="{{ expo_kafka_password }}"

nova/core/roles/expo/templates/srv/expo/env/override.expo-frontend.env

@@ -0,0 +1,11 @@
+########################################################################
+### FRONTEND ENV - override values with env/override.expo-frontend.env ###
+########################################################################
+
+### FRONTEND ###
+FRONTEND_SERVICE_REPLICAS={{ expo_frontend_service_replicas }}
+
+### SENTRY ###
+FRONTEND_SENTRY_ENABLED={{ expo_sentry_frontend_enabled | upper }}
+FRONTEND_SENTRY_DSN="{{ expo_sentry_frontend_dsn }}"
+FRONTEND_SENTRY_TRACING={{ expo_sentry_frontend_tracing }}

nova/core/roles/expo/templates/srv/expo/env/override.expo-kafdrop.env

@@ -0,0 +1,5 @@
+########################################################################
+### KAFDROP ENV - override values with env/override.expo-kafdrop.env ###
+########################################################################
+
+KAFDROP_FQDN={{ expo_kafdrop_fqdn }}