-
Notifications
You must be signed in to change notification settings - Fork 1
/
eks.tf
196 lines (181 loc) · 9.99 KB
/
eks.tf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
// Copyright 2022 Isovalent, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Used to list AMIs according to the specified filter.
data "aws_ami" "workers" {
for_each = var.self_managed_node_groups
most_recent = true
owners = var.ami_owners
filter {
name = "name"
values = [
each.value.ami_name_filter
]
}
}
// Used to make sure the VPC has been created and introduce proper dependencies between 'data' blocks.
data "aws_vpc" "vpc" {
id = var.vpc_id
}
// Used to wait for the subnets that should be used for the EKS control-plane to exist in two different AZs.
// Unfortunately there doesn't seem to be a better way to do this in Terraform.
resource "null_resource" "wait_for_control_plane_subnets" {
provisioner "local-exec" {
command = "${path.module}/scripts/wait-for-control-plane-subnets.sh ${data.aws_vpc.vpc.id} ${var.region}"
}
}
// Used to list all subnets usable for the EKS control-plane.
data "aws_subnets" "eks_control_plane" {
depends_on = [
null_resource.wait_for_control_plane_subnets,
]
filter {
name = "vpc-id"
values = [
data.aws_vpc.vpc.id
]
}
filter {
name = "tag:eks-control-plane"
values = [
"true"
]
}
}
// Used to list all private subnets in the VPC.
data "aws_subnets" "private" {
depends_on = [
null_resource.wait_for_control_plane_subnets,
]
filter {
name = "vpc-id"
values = [
data.aws_vpc.vpc.id
]
}
filter {
name = "tag:type"
values = [
"private"
]
}
}
// Used to list all public subnets in the VPC.
data "aws_subnets" "public" {
filter {
name = "vpc-id"
values = [
data.aws_vpc.vpc.id
]
}
filter {
name = "tag:type"
values = [
"public"
]
}
}
// EKS cluster.
module "main" {
source = "terraform-aws-modules/eks/aws"
version = "~> 20.0"
enable_cluster_creator_admin_permissions = true // Give access to person/bot running terraform access to the cluster
cluster_endpoint_public_access = true // Enable public access to the Kubernetes API server.
authentication_mode = "API_AND_CONFIG_MAP" // Authentication mode for EKS. Will move to API only in v21 of the upstream module
cluster_name = var.name // The name of the EKS cluster.
cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr // The CIDR block to assign Kubernetes service IP addresses from.
cluster_version = var.kubernetes_version // The version of EKS to use.
control_plane_subnet_ids = length(var.control_plane_subnet_ids) > 0 ? var.control_plane_subnet_ids : data.aws_subnets.eks_control_plane.ids // The set of all subnets in which the EKS control-plane can be placed.
enable_irsa = true // Enable IAM roles for service accounts. These are used extensively.
subnet_ids = var.include_public_subnets ? setunion(data.aws_subnets.private.ids, data.aws_subnets.public.ids) : data.aws_subnets.private.ids // The set of all subnets in which worker nodes can be placed.
tags = var.tags // The tags placed on the EKS cluster.
vpc_id = data.aws_vpc.vpc.id // The ID of the VPC in which to create the cluster.
self_managed_node_groups = { // The set of self-managed node groups.
for key, g in var.self_managed_node_groups :
key => {
ami_type = g.ami_type // The AMI family to use for worker nodes, "AL2_x86_64" etc. https://docs.aws.amazon.com/eks/latest/APIReference/API_CreateNodegroup.html#API_CreateNodegroup_RequestBody
ami_id = data.aws_ami.workers[key].image_id // The ID of the AMI to use for worker nodes.
create_security_group = false // Don't create a dedicated security group. A common one is used instead.
desired_size = g.min_nodes // Set the desired size of the worker group to the minimum.
key_name = g.key_name != "" ? g.key_name : aws_key_pair.ssh_access.key_name // The name of the SSH key to use for the nodes.
bootstrap_extra_args = g.ami_type == "BOTTLEROCKET_x86_64" ? g.kubelet_extra_args : "--kubelet-extra-args '${g.kubelet_extra_args}'" // The set of extra arguments to the bootstrap script. Used to pass extra flags to the kubelet, and namely to set labels and taints. For bottlerocket this needs to be a TOML(https://bottlerocket.dev/en/os/1.19.x/api/settings/kubernetes/) since it doesn't use kubelet to pass the args.
iam_role_additional_policies = { // The set of additional policies to add to the worker group IAM role.
for index, arn in var.worker_node_additional_policies :
arn => arn
}
max_size = g.max_nodes // The maximum size of the worker group.
min_size = g.min_nodes // The minimum size of the worker group.
name = "${var.name}-${g.name}" // Prefix the worker group name with the name of the EKS cluster.
instance_type = g.instance_type // The instance type to use for worker nodes.
pre_bootstrap_user_data = g.pre_bootstrap_user_data // The pre-bootstrap user data to use for worker nodes.
post_bootstrap_user_data = g.post_bootstrap_user_data // The pre-bootstrap user data to use for worker nodes.
iam_role_additional_policies = g.iam_role_additional_policies
iam_role_use_name_prefix = g.iam_role_use_name_prefix
subnet_ids = length(g.subnet_ids) > 0 ? g.subnet_ids : data.aws_subnets.private.ids // Only place nodes in private subnets. This may change in the future.
tags = merge(g.extra_tags, { // The set of tags placed on each worker node.
"k8s.io/cluster-autoscaler/enabled" = "true", // Required by the cluster autoscaler.
"k8s.io/cluster-autoscaler/${var.name}" = "owned", // Required by the cluster autoscaler.
})
block_device_mappings = {
(g.root_volume_id) = {
device_name = g.root_volume_id
ebs = {
volume_size = g.root_volume_size // The size of the root volume of each worker node.
volume_type = g.root_volume_type // The type of the root volume of each worker node.
encrypted = true // Encrypt the volume.
delete_on_termination = true // Delete the volume on instance termination.
}
}
}
metadata_options = {
http_endpoint = "enabled"
http_protocol_ipv6 = "disabled"
http_put_response_hop_limit = 2
instance_metadata_tags = "disabled"
http_tokens = var.allow_imdsv1 ? "optional" : "required"
}
}
}
}
// Make sure the kubeconfig file always exists.
// This is necessary because running 'terraform init' may replace the directory containing it when run.
resource "null_resource" "kubeconfig" {
depends_on = [
module.main.cluster_arn // Do not run before the EKS cluster is up.
]
triggers = {
always_run = timestamp()
}
provisioner "local-exec" {
command = "aws eks update-kubeconfig --name ${var.name} --region ${var.region}"
environment = {
CLUSTER_NAME = var.name,
KUBECONFIG = local.path_to_kubeconfig_file,
REGION = var.region,
}
}
}
resource "null_resource" "disable_aws_vpc_cni_plugin" {
count = var.disable_aws_vpc_cni_plugin ? 1 : 0
depends_on = [
null_resource.kubeconfig,
]
provisioner "local-exec" {
command = <<EOF
kubectl -n kube-system patch ds aws-node -p '{"spec":{"template":{"spec":{"nodeSelector":{"io.cilium/aws-node-enabled":"true"}}}}}'
EOF
environment = {
KUBECONFIG = local.path_to_kubeconfig_file,
}
}
}