From 1eefbce8082cea0c79b7f08cf14a6e65ae1b68de Mon Sep 17 00:00:00 2001
From: Jean Mainguy <9969006+jhandguy@users.noreply.github.com>
Date: Fri, 7 Jul 2023 15:58:40 +0200
Subject: [PATCH] feat: add haproxy module for load balancing
---
.github/workflows/ci.yaml | 2 +-
.gitignore | 3 --
Makefile | 2 +-
README.md | 32 ++++++++--------
sql/Makefile | 3 +-
sql/helm/templates/ingress.yaml | 28 ++++++++++++++
sql/helm/templates/issuer.yaml | 8 ++++
sql/helm/values.yaml | 8 ++--
sql/k6/script.js | 8 ++--
.../{redis => haproxy}/.terraform.lock.hcl | 0
.../environments/{redis => haproxy}/README.md | 8 ++++
.../{redis => haproxy}/modules.tf | 23 ++++++++++-
.../{redis => haproxy}/outputs.tf | 20 ++++++++++
.../{redis => haproxy}/providers.tf | 0
.../{redis => haproxy}/randoms.tf | 9 +++++
.../{redis => haproxy}/requirements.tf | 0
terraform/modules/haproxy/README.md | 38 +++++++++++++++++++
terraform/modules/haproxy/helm_releases.tf | 34 +++++++++++++++++
terraform/modules/haproxy/outputs.tf | 9 +++++
terraform/modules/haproxy/requirements.tf | 8 ++++
terraform/modules/haproxy/variables.tf | 15 ++++++++
terraform/modules/sql/README.md | 1 +
terraform/modules/sql/helm_releases.tf | 5 +++
terraform/modules/sql/variables.tf | 5 +++
24 files changed, 239 insertions(+), 30 deletions(-)
create mode 100644 sql/helm/templates/ingress.yaml
create mode 100644 sql/helm/templates/issuer.yaml
rename terraform/environments/{redis => haproxy}/.terraform.lock.hcl (100%)
rename terraform/environments/{redis => haproxy}/README.md (79%)
rename terraform/environments/{redis => haproxy}/modules.tf (84%)
rename terraform/environments/{redis => haproxy}/outputs.tf (79%)
rename terraform/environments/{redis => haproxy}/providers.tf (100%)
rename terraform/environments/{redis => haproxy}/randoms.tf (71%)
rename terraform/environments/{redis => haproxy}/requirements.tf (100%)
create mode 100644 terraform/modules/haproxy/README.md
create mode 100644 terraform/modules/haproxy/helm_releases.tf
create mode 100644 terraform/modules/haproxy/outputs.tf
create mode 100644 terraform/modules/haproxy/requirements.tf
create mode 100644 terraform/modules/haproxy/variables.tf
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 4edd3b3..81bbe7a 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -215,5 +215,5 @@ jobs:
- name: Run all tests
env:
- ENVIRONMENT: redis
+ ENVIRONMENT: haproxy
run: make rust_ci
diff --git a/.gitignore b/.gitignore
index 353dc9e..6e27bca 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,3 @@
-.DS_STORE
-.idea
-
**/.terraform/*
**/terraform.tfvars
diff --git a/Makefile b/Makefile
index 2aa4849..afe49c7 100644
--- a/Makefile
+++ b/Makefile
@@ -87,7 +87,7 @@ update_terraform:
update:
make update_terraform ENVIRONMENT=consul
make update_terraform ENVIRONMENT=nginx
- make update_terraform ENVIRONMENT=redis
+ make update_terraform ENVIRONMENT=haproxy
make -j update_s3 update_dynamo update_gateway update_cli
update_%:
diff --git a/README.md b/README.md
index 43700a6..7a6d9bb 100644
--- a/README.md
+++ b/README.md
@@ -17,28 +17,28 @@ brew install protobuf protoc-gen-go protoc-gen-go-grpc kind terraform k6
| Consul | `make setup ENVIRONMENT=consul` |
| Nginx | `make setup ENVIRONMENT=nginx` |
| Nginx (ArgoRollouts) | `make setup ENVIRONMENT=nginx TF_VAR_argorollouts_enabled=true` |
-| Redis | `make setup ENVIRONMENT=redis` |
+| HAProxy | `make setup ENVIRONMENT=haproxy` |
## Run Tests
-| Environment | Command |
-|-------------------------|------------------------------------|
-| Consul | `make go_test ENVIRONMENT=consul` |
-| Nginx | `make go_test ENVIRONMENT=nginx` |
-| Redis | `make rust_test ENVIRONMENT=redis` |
+| Environment | Command |
+|-------------|--------------------------------------|
+| Consul | `make go_test ENVIRONMENT=consul` |
+| Nginx | `make go_test ENVIRONMENT=nginx` |
+| HAProxy | `make rust_test ENVIRONMENT=haproxy` |
## Run Load Tests
-| Environment | Command |
-|-------------------------|------------------------------------|
-| Consul | `make go_load ENVIRONMENT=consul` |
-| Nginx | `make go_load ENVIRONMENT=nginx` |
-| Redis | `make rust_load ENVIRONMENT=redis` |
+| Environment | Command |
+|-------------|--------------------------------------|
+| Consul | `make go_load ENVIRONMENT=consul` |
+| Nginx | `make go_load ENVIRONMENT=nginx` |
+| HAProxy | `make rust_load ENVIRONMENT=haproxy` |
## Destroy Infrastructure
-| Environment | Command |
-|-------------------------|------------------------------------|
-| Consul | `make teardown ENVIRONMENT=consul` |
-| Nginx | `make teardown ENVIRONMENT=nginx` |
-| Redis | `make teardown ENVIRONMENT=redis` |
+| Environment | Command |
+|-------------|-------------------------------------|
+| Consul | `make teardown ENVIRONMENT=consul` |
+| Nginx | `make teardown ENVIRONMENT=nginx` |
+| HAProxy | `make teardown ENVIRONMENT=haproxy` |
diff --git a/sql/Makefile b/sql/Makefile
index bf553a9..3879baa 100644
--- a/sql/Makefile
+++ b/sql/Makefile
@@ -18,7 +18,8 @@ test:
cargo test --features $(FEATURE) -- --include-ignored
load: export SQL_TOKEN=$(shell terraform -chdir=$(CHDIR) output -json sql_$(FEATURE)_token | jq -r .)
-load: export SQL_URL=$(shell terraform -chdir=$(CHDIR) output -json sql_$(FEATURE)_url | jq -r .)
+load: export SQL_HOST=$(shell terraform -chdir=$(CHDIR) output -json sql_$(FEATURE)_ingress_host | jq -r .)
+load: export SQL_URL=$(shell terraform -chdir=$(CHDIR) output -json sql_$(FEATURE)_ingress_url | jq -r .)
load:
k6 run k6/script.js
diff --git a/sql/helm/templates/ingress.yaml b/sql/helm/templates/ingress.yaml
new file mode 100644
index 0000000..2ecbd56
--- /dev/null
+++ b/sql/helm/templates/ingress.yaml
@@ -0,0 +1,28 @@
+{{- with .Values.ingress }}
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+ name: {{ $.Release.Name }}
+ namespace: {{ $.Release.Namespace }}
+ labels:
+ app: {{ $.Release.Name }}
+ annotations:
+ cert-manager.io/issuer: {{ $.Release.Name }}
+spec:
+ rules:
+ - host: {{ .host }}
+ http:
+ paths:
+ - path: /
+ pathType: Prefix
+ backend:
+ service:
+ name: {{ $.Release.Name }}
+ port:
+ name: http
+ tls:
+ - secretName: {{ $.Release.Name }}-tls
+ hosts:
+ - {{ .host }}
+{{- end }}
diff --git a/sql/helm/templates/issuer.yaml b/sql/helm/templates/issuer.yaml
new file mode 100644
index 0000000..0862d91
--- /dev/null
+++ b/sql/helm/templates/issuer.yaml
@@ -0,0 +1,8 @@
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+ name: {{ .Release.Name }}
+ namespace: {{ .Release.Namespace }}
+spec:
+ selfSigned: {}
diff --git a/sql/helm/values.yaml b/sql/helm/values.yaml
index 31a7988..6ab0d4c 100644
--- a/sql/helm/values.yaml
+++ b/sql/helm/values.yaml
@@ -1,11 +1,11 @@
replicas: 1
resources:
limits:
- cpu: 50m
- memory: 75Mi
+ cpu: 75m
+ memory: 100Mi
requests:
- cpu: 25m
- memory: 75Mi
+ cpu: 50m
+ memory: 100Mi
horizontalPodAutoscaler:
minReplicas: 1
maxReplicas: 2
diff --git a/sql/k6/script.js b/sql/k6/script.js
index 3afb763..c64962f 100644
--- a/sql/k6/script.js
+++ b/sql/k6/script.js
@@ -3,10 +3,11 @@ import {check, sleep} from 'k6';
import {randomItem, randomString, uuidv4} from 'https://jslib.k6.io/k6-utils/1.4.0/index.js';
export const options = {
+ setupTimeout: '2m',
stages: [
- {target: 20, duration: '20s'},
- {target: 20, duration: '20s'},
- {target: 0, duration: '20s'},
+ {target: 20, duration: '40s'},
+ {target: 20, duration: '40s'},
+ {target: 0, duration: '40s'},
],
thresholds: {
'checks': ['rate>0.9'],
@@ -20,6 +21,7 @@ const url = `http://${__ENV.SQL_URL}`;
const params = {
headers: {
'Authorization': `Bearer ${__ENV.SQL_TOKEN}`,
+ 'Host': __ENV.SQL_HOST,
'Content-Type': 'application/json',
'X-Redis-Enabled': __ENV.REDIS_ENABLED,
},
diff --git a/terraform/environments/redis/.terraform.lock.hcl b/terraform/environments/haproxy/.terraform.lock.hcl
similarity index 100%
rename from terraform/environments/redis/.terraform.lock.hcl
rename to terraform/environments/haproxy/.terraform.lock.hcl
diff --git a/terraform/environments/redis/README.md b/terraform/environments/haproxy/README.md
similarity index 79%
rename from terraform/environments/redis/README.md
rename to terraform/environments/haproxy/README.md
index 3049fef..ccfb341 100644
--- a/terraform/environments/redis/README.md
+++ b/terraform/environments/haproxy/README.md
@@ -15,6 +15,8 @@
| Name | Source | Version |
|------|--------|---------|
+| [certmanager](#module\_certmanager) | ../../modules/certmanager | n/a |
+| [haproxy](#module\_haproxy) | ../../modules/haproxy | n/a |
| [kind](#module\_kind) | ../../modules/kind | n/a |
| [loki](#module\_loki) | ../../modules/loki | n/a |
| [metrics](#module\_metrics) | ../../modules/metrics | n/a |
@@ -36,6 +38,8 @@
| [random_pet.mysql_user](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) | resource |
| [random_pet.postgres_database](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) | resource |
| [random_pet.postgres_user](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) | resource |
+| [random_pet.sql_mysql_host](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) | resource |
+| [random_pet.sql_postgres_host](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) | resource |
## Inputs
@@ -57,8 +61,12 @@ No inputs.
| [prometheus\_url](#output\_prometheus\_url) | Prometheus URL |
| [redis\_password](#output\_redis\_password) | Redis password |
| [redis\_url](#output\_redis\_url) | Redis URL |
+| [sql\_mysql\_ingress\_host](#output\_sql\_mysql\_ingress\_host) | SQL MySQL Ingress host |
+| [sql\_mysql\_ingress\_url](#output\_sql\_mysql\_ingress\_url) | SQL MySQL Ingress URL |
| [sql\_mysql\_token](#output\_sql\_mysql\_token) | SQL MySQL token |
| [sql\_mysql\_url](#output\_sql\_mysql\_url) | SQL MySQL URL |
+| [sql\_postgres\_ingress\_host](#output\_sql\_postgres\_ingress\_host) | SQL Postgres Ingress host |
+| [sql\_postgres\_ingress\_url](#output\_sql\_postgres\_ingress\_url) | SQL Postgres Ingress URL |
| [sql\_postgres\_token](#output\_sql\_postgres\_token) | SQL Postgres token |
| [sql\_postgres\_url](#output\_sql\_postgres\_url) | SQL Postgres URL |
diff --git a/terraform/environments/redis/modules.tf b/terraform/environments/haproxy/modules.tf
similarity index 84%
rename from terraform/environments/redis/modules.tf
rename to terraform/environments/haproxy/modules.tf
index 9fe2145..cfccf73 100644
--- a/terraform/environments/redis/modules.tf
+++ b/terraform/environments/haproxy/modules.tf
@@ -1,7 +1,7 @@
module "kind" {
source = "../../modules/kind"
- cluster_name = "redis"
+ cluster_name = "haproxy"
node_ports = [
"postgresql",
"mysql",
@@ -13,6 +13,9 @@ module "kind" {
"prometheus",
"alertmanager",
"grafana",
+ "haproxy_http",
+ "haproxy_https",
+ "haproxy_stat",
]
}
@@ -49,9 +52,11 @@ module "sql_postgres" {
source = "../../modules/sql"
feature = "postgres"
+ ingress_host = random_pet.sql_postgres_host.id
node_ip = module.kind.node_ip
node_ports = [module.kind.node_ports["sql_postgres_http"], module.kind.node_ports["sql_postgres_metrics"]]
prometheus_enabled = true
+ replicas = 2
secrets = {
"database_url" = module.postgresql.cluster_url
"database_user" = module.postgresql.user_name
@@ -69,9 +74,11 @@ module "sql_mysql" {
source = "../../modules/sql"
feature = "mysql"
+ ingress_host = random_pet.sql_mysql_host.id
node_ip = module.kind.node_ip
node_ports = [module.kind.node_ports["sql_mysql_http"], module.kind.node_ports["sql_mysql_metrics"]]
prometheus_enabled = true
+ replicas = 2
secrets = {
"database_url" = module.mysql.cluster_url
"database_user" = module.mysql.user_name
@@ -115,3 +122,17 @@ module "metrics" {
depends_on = [module.kind]
source = "../../modules/metrics"
}
+
+module "haproxy" {
+ depends_on = [module.prometheus]
+ source = "../../modules/haproxy"
+
+ node_ip = module.kind.node_ip
+ node_ports = [module.kind.node_ports["haproxy_http"], module.kind.node_ports["haproxy_https"], module.kind.node_ports["haproxy_stat"]]
+ prometheus_enabled = true
+}
+
+module "certmanager" {
+ depends_on = [module.kind]
+ source = "../../modules/certmanager"
+}
diff --git a/terraform/environments/redis/outputs.tf b/terraform/environments/haproxy/outputs.tf
similarity index 79%
rename from terraform/environments/redis/outputs.tf
rename to terraform/environments/haproxy/outputs.tf
index 05237c0..439e539 100644
--- a/terraform/environments/redis/outputs.tf
+++ b/terraform/environments/haproxy/outputs.tf
@@ -52,6 +52,16 @@ output "sql_postgres_token" {
sensitive = true
}
+output "sql_postgres_ingress_url" {
+ value = module.haproxy.url
+ description = "SQL Postgres Ingress URL"
+}
+
+output "sql_postgres_ingress_host" {
+ value = random_pet.sql_postgres_host.id
+ description = "SQL Postgres Ingress host"
+}
+
output "sql_mysql_url" {
value = module.sql_mysql.url
description = "SQL MySQL URL"
@@ -63,6 +73,16 @@ output "sql_mysql_token" {
sensitive = true
}
+output "sql_mysql_ingress_url" {
+ value = module.haproxy.url
+ description = "SQL MySQL Ingress URL"
+}
+
+output "sql_mysql_ingress_host" {
+ value = random_pet.sql_mysql_host.id
+ description = "SQL MySQL Ingress host"
+}
+
output "prometheus_url" {
value = module.prometheus.prometheus_url
description = "Prometheus URL"
diff --git a/terraform/environments/redis/providers.tf b/terraform/environments/haproxy/providers.tf
similarity index 100%
rename from terraform/environments/redis/providers.tf
rename to terraform/environments/haproxy/providers.tf
diff --git a/terraform/environments/redis/randoms.tf b/terraform/environments/haproxy/randoms.tf
similarity index 71%
rename from terraform/environments/redis/randoms.tf
rename to terraform/environments/haproxy/randoms.tf
index 02b44c5..a173eab 100644
--- a/terraform/environments/redis/randoms.tf
+++ b/terraform/environments/haproxy/randoms.tf
@@ -23,3 +23,12 @@ resource "random_password" "sql_mysql_token" {
length = 32
special = false
}
+
+resource "random_pet" "sql_postgres_host" {
+ length = 3
+ separator = "."
+}
+resource "random_pet" "sql_mysql_host" {
+ length = 3
+ separator = "."
+}
diff --git a/terraform/environments/redis/requirements.tf b/terraform/environments/haproxy/requirements.tf
similarity index 100%
rename from terraform/environments/redis/requirements.tf
rename to terraform/environments/haproxy/requirements.tf
diff --git a/terraform/modules/haproxy/README.md b/terraform/modules/haproxy/README.md
new file mode 100644
index 0000000..a49ab5b
--- /dev/null
+++ b/terraform/modules/haproxy/README.md
@@ -0,0 +1,38 @@
+
+## Requirements
+
+| Name | Version |
+|------|---------|
+| [terraform](#requirement\_terraform) | ~> 1 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| [helm](#provider\_helm) | n/a |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [helm_release.haproxy](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| [node\_ip](#input\_node\_ip) | Node ip | `string` | n/a | yes |
+| [node\_ports](#input\_node\_ports) | Node ports | `tuple([number, number, number])` | n/a | yes |
+| [prometheus\_enabled](#input\_prometheus\_enabled) | Enable Prometheus | `bool` | `false` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| [cluster\_url](#output\_cluster\_url) | Cluster URL |
+| [url](#output\_url) | URL |
+
diff --git a/terraform/modules/haproxy/helm_releases.tf b/terraform/modules/haproxy/helm_releases.tf
new file mode 100644
index 0000000..4522529
--- /dev/null
+++ b/terraform/modules/haproxy/helm_releases.tf
@@ -0,0 +1,34 @@
+resource "helm_release" "haproxy" {
+ name = "haproxy"
+ namespace = "haproxy"
+ repository = "https://haproxytech.github.io/helm-charts"
+ chart = "kubernetes-ingress"
+ create_namespace = true
+ wait = true
+ version = "1.32.0"
+
+ values = [
+ <<-EOF
+ controller:
+ replicaCount: 1
+ ingressClassResource:
+ default: true
+ config:
+ ssl-redirect: "false"
+%{if var.prometheus_enabled}
+ serviceMonitor:
+ enabled: true
+ extraLabels:
+ release: prometheus
+%{endif}
+ service:
+ type: NodePort
+ nodePorts:
+ http: ${var.node_ports.0}
+ https: ${var.node_ports.1}
+ stat: ${var.node_ports.2}
+ defaultBackend:
+ enabled: true
+ EOF
+ ]
+}
diff --git a/terraform/modules/haproxy/outputs.tf b/terraform/modules/haproxy/outputs.tf
new file mode 100644
index 0000000..74bd9ff
--- /dev/null
+++ b/terraform/modules/haproxy/outputs.tf
@@ -0,0 +1,9 @@
+output "url" {
+ value = "${var.node_ip}:${var.node_ports.0}"
+ description = "URL"
+}
+
+output "cluster_url" {
+ value = "${helm_release.haproxy.name}-kubernetes-ingress.${helm_release.haproxy.namespace}.svc.cluster.local:80"
+ description = "Cluster URL"
+}
diff --git a/terraform/modules/haproxy/requirements.tf b/terraform/modules/haproxy/requirements.tf
new file mode 100644
index 0000000..6620326
--- /dev/null
+++ b/terraform/modules/haproxy/requirements.tf
@@ -0,0 +1,8 @@
+terraform {
+ required_version = "~> 1"
+ required_providers {
+ helm = {
+ source = "hashicorp/helm"
+ }
+ }
+}
diff --git a/terraform/modules/haproxy/variables.tf b/terraform/modules/haproxy/variables.tf
new file mode 100644
index 0000000..9a95e08
--- /dev/null
+++ b/terraform/modules/haproxy/variables.tf
@@ -0,0 +1,15 @@
+variable "node_ip" {
+ type = string
+ description = "Node ip"
+}
+
+variable "node_ports" {
+ type = tuple([number, number, number])
+ description = "Node ports"
+}
+
+variable "prometheus_enabled" {
+ type = bool
+ default = false
+ description = "Enable Prometheus"
+}
diff --git a/terraform/modules/sql/README.md b/terraform/modules/sql/README.md
index 7138581..2e919d0 100644
--- a/terraform/modules/sql/README.md
+++ b/terraform/modules/sql/README.md
@@ -26,6 +26,7 @@ No modules.
| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
| [feature](#input\_feature) | Feature | `string` | n/a | yes |
+| [ingress\_host](#input\_ingress\_host) | Ingress host | `string` | n/a | yes |
| [node\_ip](#input\_node\_ip) | Node ip | `string` | n/a | yes |
| [node\_ports](#input\_node\_ports) | Node ports | `tuple([number, number])` | n/a | yes |
| [prometheus\_enabled](#input\_prometheus\_enabled) | Enable Prometheus | `bool` | `false` | no |
diff --git a/terraform/modules/sql/helm_releases.tf b/terraform/modules/sql/helm_releases.tf
index c0e5b0a..8d74a20 100644
--- a/terraform/modules/sql/helm_releases.tf
+++ b/terraform/modules/sql/helm_releases.tf
@@ -12,9 +12,14 @@ resource "helm_release" "sql" {
nodePorts:
http: ${var.node_ports.0}
metrics: ${var.node_ports.1}
+ ingress:
+ host: ${var.ingress_host}
prometheus:
enabled: ${var.prometheus_enabled}
groupName: ${var.feature == "mysql" ? "MySQL" : title(var.feature)}
+ horizontalPodAutoscaler:
+ minReplicas: ${var.replicas}
+ maxReplicas: ${var.replicas * 2}
EOF
]
diff --git a/terraform/modules/sql/variables.tf b/terraform/modules/sql/variables.tf
index e928f57..6a4643c 100644
--- a/terraform/modules/sql/variables.tf
+++ b/terraform/modules/sql/variables.tf
@@ -30,6 +30,11 @@ variable "replicas" {
description = "Replicas"
}
+variable "ingress_host" {
+ type = string
+ description = "Ingress host"
+}
+
variable "prometheus_enabled" {
type = bool
default = false