Skip to content

PoC: Use of ingress controller for Nomad cluster #1248

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

PoC: Use of ingress controller for Nomad cluster #1248

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6b9f175
Traefik ingress running on api nodes
sitole Sep 25, 2025
b44e506
ingress routing, removed port mapping for other services
sitole Sep 25, 2025
c12ffdf
routing for api, edge api, docker proxy, sandboxes
sitole Sep 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 24 additions & 15 deletions iac/provider-gcp/nomad-cluster/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,15 +92,15 @@ module "network" {

gcp_project_id = var.gcp_project_id

api_port = var.api_port
docker_reverse_proxy_port = var.docker_reverse_proxy_port
network_name = var.network_name
domain_name = var.domain_name
additional_domains = var.additional_domains
// api_port = var.api_port
// docker_reverse_proxy_port = var.docker_reverse_proxy_port
network_name = var.network_name
domain_name = var.domain_name
additional_domains = var.additional_domains

client_instance_group = google_compute_region_instance_group_manager.client_pool.instance_group
client_proxy_port = var.edge_proxy_port
client_proxy_health_port = var.edge_api_port
client_instance_group = google_compute_region_instance_group_manager.client_pool.instance_group
//client_proxy_port = var.edge_proxy_port
//client_proxy_health_port = var.edge_api_port

api_instance_group = google_compute_instance_group_manager.api_pool.instance_group
build_instance_group = google_compute_instance_group_manager.build_pool.instance_group
Expand All @@ -115,14 +115,23 @@ module "network" {
labels = var.labels
prefix = var.prefix

additional_api_path_rules = [
for service in var.additional_api_services : {
paths = service.paths
service_id = service.service_id
}
]
//additional_api_path_rules = [
// for service in var.additional_api_services : {
// paths = service.paths
// service_id = service.service_id
// }
//]

additional_ports = [for service in var.additional_api_services : service.api_node_group_port]
//additional_ports = [for service in var.additional_api_services : service.api_node_group_port]


ingress = {
port_name = "ingress"
port = 8800
health_port_name = "ingress-health"
health_port = 8900
health_path = "/ping"
}
}

module "filestore" {
Expand Down
108 changes: 46 additions & 62 deletions iac/provider-gcp/nomad-cluster/network/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,16 @@ locals {

parts = split(".", var.domain_name)
is_subdomain = length(local.parts) > 2

// Take everything except last 2 parts
subdomain = local.is_subdomain ? join(".", slice(local.parts, 0, length(local.parts) - 2)) : ""

// Take last 2 parts (1 dot)
root_domain = local.is_subdomain ? join(".", slice(local.parts, length(local.parts) - 2, length(local.parts))) : var.domain_name

backends = {

/*
session = {
protocol = "HTTP"
port = var.client_proxy_port.port
Expand Down Expand Up @@ -66,6 +70,32 @@ locals {
}
groups = [{ group = var.build_instance_group }]
}
*/










ingress = {
protocol = "HTTP"
port = var.ingress.port
port_name = var.ingress.port_name
timeout_sec = 86400
connection_draining_timeout_sec = 1
http_health_check = {
request_path = var.ingress.health_path
port = var.ingress.health_port
timeout_sec = 3
check_interval_sec = 3
}
groups = [{ group = var.api_instance_group }]
}

nomad = {
protocol = "HTTP"
port = 80
Expand All @@ -78,6 +108,7 @@ locals {
}
groups = [{ group = var.server_instance_group }]
}

consul = {
protocol = "HTTP"
port = 80
Expand Down Expand Up @@ -251,17 +282,7 @@ resource "google_certificate_manager_certificate_map_entry" "subdomains_map_entr
# Load balancers
resource "google_compute_url_map" "orch_map" {
name = "${var.prefix}orch-map"
default_service = google_compute_backend_service.default["nomad"].self_link

host_rule {
hosts = concat(["api.${var.domain_name}"], [for d in var.additional_domains : "api.${d}"])
path_matcher = "api-paths"
}

host_rule {
hosts = concat(["docker.${var.domain_name}"], [for d in var.additional_domains : "docker.${d}"])
path_matcher = "docker-reverse-proxy-paths"
}
default_service = google_compute_backend_service.default["ingress"].self_link

host_rule {
hosts = concat(["nomad.${var.domain_name}"], [for d in var.additional_domains : "nomad.${d}"])
Expand All @@ -273,33 +294,10 @@ resource "google_compute_url_map" "orch_map" {
path_matcher = "consul-paths"
}

host_rule {
hosts = concat(["*.${var.domain_name}"], [for d in var.additional_domains : "*.${d}"])
path_matcher = "session-paths"
}

path_matcher {
name = "api-paths"
default_service = google_compute_backend_service.default["api"].self_link

dynamic "path_rule" {
for_each = var.additional_api_path_rules
content {
paths = path_rule.value.paths
service = path_rule.value.service_id
}
}
}

path_matcher {
name = "docker-reverse-proxy-paths"
default_service = google_compute_backend_service.default["docker-reverse-proxy"].self_link
}

path_matcher {
name = "session-paths"
default_service = google_compute_backend_service.default["session"].self_link
}
//host_rule {
// hosts = concat(["*.${var.domain_name}"], [for d in var.additional_domains : "*.${d}"])
// path_matcher = "ingress-paths"
//}

path_matcher {
name = "nomad-paths"
Expand All @@ -322,6 +320,11 @@ resource "google_compute_url_map" "orch_map" {
name = "consul-paths"
default_service = google_compute_backend_service.default["consul"].self_link
}

//path_matcher {
// name = "ingress-paths"
// default_service = google_compute_backend_service.default["ingress"].self_link
//}
}

### IPv4 block ###
Expand Down Expand Up @@ -526,32 +529,10 @@ resource "google_compute_firewall" "default-hc" {
}
}

dynamic "allow" {
for_each = toset(var.additional_ports)

content {
protocol = "tcp"
ports = [allow.value]
}
}
}

resource "google_compute_firewall" "client_proxy_firewall_ingress" {
name = "${var.prefix}${var.cluster_tag_name}-client-proxy-firewall-ingress"
network = var.network_name

allow {
protocol = "tcp"
ports = ["3002"]
ports = [var.ingress.port]
}

priority = 999

direction = "INGRESS"
target_tags = [var.cluster_tag_name]
# Load balancer health check IP ranges
# https://cloud.google.com/load-balancing/docs/health-check-concepts
source_ranges = ["130.211.0.0/22", "35.191.0.0/16"]
}

resource "google_compute_firewall" "logs_collector_firewall_ingress" {
Expand Down Expand Up @@ -629,6 +610,9 @@ resource "google_compute_firewall" "orch_firewall_egress" {
target_tags = [var.cluster_tag_name]
}

/*
todo


# Security policy
resource "google_compute_security_policy_rule" "api-throttling-api-key" {
Expand Down Expand Up @@ -760,7 +744,7 @@ resource "google_compute_security_policy_rule" "disable-consul" {
}
}
}

*/
resource "google_compute_security_policy" "disable-bots-log-collector" {
name = "disable-bots-log-collector"

Expand Down
65 changes: 39 additions & 26 deletions iac/provider-gcp/nomad-cluster/network/variables.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,34 +32,14 @@ variable "cloudflare_api_token_secret_name" {
type = string
}

variable "api_port" {
variable "ingress" {
type = object({
name = string
port = number
health_path = string
})
}
port_name = string
port = number

variable "docker_reverse_proxy_port" {
type = object({
name = string
port = number
health_path = string
})
}

variable "client_proxy_health_port" {
type = object({
name = string
port = number
path = string
})
}

variable "client_proxy_port" {
type = object({
name = string
port = number
health_port_name = string
health_port = number
health_path = string
})
}

Expand Down Expand Up @@ -103,6 +83,7 @@ variable "labels" {
type = map(string)
}

/*
variable "additional_api_path_rules" {
description = "Additional path rules to add to the load balancer routing."
type = list(object({
Expand All @@ -115,3 +96,35 @@ variable "additional_ports" {
description = "Additional ports to expose on the load balancer."
type = list(number)
}

variable "api_port" {
type = object({
name = string
port = number
health_path = string
})
}

variable "docker_reverse_proxy_port" {
type = object({
name = string
port = number
health_path = string
})
}

variable "client_proxy_health_port" {
type = object({
name = string
port = number
path = string
})
}

variable "client_proxy_port" {
type = object({
name = string
port = number
})
}
*/
21 changes: 4 additions & 17 deletions iac/provider-gcp/nomad-cluster/nodepool-api.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,26 +52,13 @@ resource "google_compute_instance_group_manager" "api_pool" {
}

named_port {
name = var.edge_api_port.name
port = var.edge_api_port.port
name = "ingress"
port = 8800
}

named_port {
name = var.edge_proxy_port.name
port = var.edge_proxy_port.port
}

named_port {
name = var.api_port.name
port = var.api_port.port
}

dynamic "named_port" {
for_each = local.api_additional_ports
content {
name = "${var.prefix}${named_port.value.name}"
port = named_port.value.port
}
name = "ingress-health"
port = 8900
}

auto_healing_policies {
Expand Down
5 changes: 0 additions & 5 deletions iac/provider-gcp/nomad-cluster/nodepool-build.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,6 @@ resource "google_compute_instance_group_manager" "build_pool" {
instance_template = google_compute_instance_template.build.id
}

named_port {
name = var.docker_reverse_proxy_port.name
port = var.docker_reverse_proxy_port.port
}

auto_healing_policies {
health_check = google_compute_health_check.build_nomad_check.id
initial_delay_sec = 600
Expand Down
5 changes: 0 additions & 5 deletions iac/provider-gcp/nomad-cluster/nodepool-clickhouse.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,6 @@ resource "google_compute_instance_group_manager" "clickhouse_pool" {
instance_template = google_compute_instance_template.clickhouse.id
}

named_port {
name = var.clickhouse_health_port.name
port = var.clickhouse_health_port.port
}

auto_healing_policies {
health_check = google_compute_health_check.clickhouse_nomad_check.id
initial_delay_sec = 600
Expand Down
Loading
Loading