Skip to content

Commit ae60d52

Browse files
discordianfishdiscordianfish
committed
Initial commit
0 parents  commit ae60d52

File tree

9 files changed

+1689
-0
lines changed

9 files changed

+1689
-0
lines changed

Diff for: ‎.dockerignore

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
generated/

Diff for: ‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
generated/

Diff for: ‎Dockerfile

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
FROM debian:sid
2+
3+
ENV KUBE_VERSION v1.8.4
4+
ENV KUBEADM_URL https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/amd64/kubeadm
5+
6+
RUN apt-get -qy update && apt-get -qy install curl make awscli golang-cfssl \
7+
&& useradd -m user \
8+
&& curl -Lfo /usr/bin/kubeadm "$KUBEADM_URL" \
9+
&& chmod a+x /usr/bin/kubeadm
10+
11+
COPY . /usr/src
12+
WORKDIR /usr/src
13+
ENTRYPOINT [ "make" ]

Diff for: ‎Makefile

+116
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
NAME ?= int
2+
DOMAIN_ROOT ?= example.com
3+
DOMAIN_NAME := $(NAME).$(DOMAIN_ROOT)
4+
CONTROLLER_SUBDOMAIN := api
5+
CONTROLLER_FQDN := $(CONTROLLER_SUBDOMAIN).$(DOMAIN_NAME)
6+
CONTROLLER_POOL_SIZE := 3
7+
8+
REGION ?= us-east-1
9+
ASSET_BUCKET ?= example-asset-bucket
10+
11+
TOP := $(shell pwd)
12+
TLS_CA_CSR ?= $(TOP)/cfssl/csr/ca-csr.json
13+
14+
BUILD ?= generated/$(NAME)
15+
BUILD_TLS := $(BUILD)/tls
16+
BUILD_KUBEADM := $(BUILD)/kubeadm
17+
18+
PUBLIC_SUBNET_CIDR_PREFIX ?= 172.20.15
19+
PRIVATE_SUBNET_CIDR_PREFIX ?= 172.20.16
20+
21+
PARENT_ZONEID ?= ZABCD
22+
23+
VPCID ?= vpc-1234
24+
IGW ?= igw-1234
25+
26+
CLUSTER_STATE ?= existing
27+
28+
define kv_pair
29+
{ "ParameterKey": "$(1)", "ParameterValue": "$(2)" }
30+
endef
31+
32+
define cfn_params
33+
[
34+
$(call kv_pair,DomainName,$(DOMAIN_NAME)),
35+
$(call kv_pair,ControllerSubdomain,$(CONTROLLER_SUBDOMAIN)),
36+
$(call kv_pair,assetBucket,$(ASSET_BUCKET)),
37+
$(call kv_pair,PrivateSubnetCidrPrefix,$(PRIVATE_SUBNET_CIDR_PREFIX)),
38+
$(call kv_pair,PublicSubnetCidrPrefix,$(PUBLIC_SUBNET_CIDR_PREFIX)),
39+
$(call kv_pair,VPCID,$(VPCID)),
40+
$(call kv_pair,InternetGateway,$(IGW)),
41+
$(call kv_pair,ParentZoneID,$(PARENT_ZONEID)),
42+
$(call kv_pair,ClusterState,$(CLUSTER_STATE))
43+
]
44+
endef
45+
export cfn_params
46+
47+
OBJS := $(BUILD_TLS) $(BUILD_TLS)/ca.pem $(BUILD_TLS)/server-key.pem \
48+
$(BUILD_TLS)/peer-key.pem $(BUILD_KUBEADM)/ca.crt \
49+
$(BUILD_KUBEADM)/front-proxy-ca.crt $(BUILD_KUBEADM)/sa.pub \
50+
$(BUILD_KUBEADM)/admin.conf
51+
52+
all: $(OBJS)
53+
upload: all
54+
aws s3 cp --recursive $(BUILD_TLS) s3://$(ASSET_BUCKET)/$(DOMAIN_NAME)/etcd
55+
aws s3 cp --recursive $(BUILD_KUBEADM) s3://$(ASSET_BUCKET)/$(DOMAIN_NAME)/kubeadm
56+
57+
require-op:
58+
ifndef OP
59+
$(error OP required)
60+
endif
61+
62+
params:
63+
echo $$cfn_params
64+
65+
create-cluster:
66+
OP=create-stack CLUSTER_STATE=new make cloudformation
67+
68+
update-cluster:
69+
OP=update-stack make cloudformation
70+
71+
cloudformation: require-op upload
72+
aws --region $(REGION) cloudformation $(OP) \
73+
--stack-name $(NAME) \
74+
--capabilities CAPABILITY_IAM \
75+
--parameters "$$cfn_params" \
76+
--template-body "$$(cat kubernetes.yaml)" $(OPTS)
77+
78+
$(BUILD):
79+
mkdir -p $@
80+
81+
$(BUILD_TLS):
82+
mkdir -p $@
83+
84+
$(BUILD_TLS)/ca.pem:
85+
cd $(BUILD_TLS); cfssl gencert -initca $(TLS_CA_CSR) | cfssljson -bare ca -
86+
87+
$(BUILD_TLS)/server-key.pem:
88+
cd $(BUILD_TLS); \
89+
echo '{"CN":"$(CONTROLLER_FQDN)","hosts":["$(DOMAIN_NAME)","*.$(DOMAIN_NAME)","*.ec2.internal","localhost", "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster.local" ],"key":{"algo":"rsa","size":2048}}' \
90+
| cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=$(TOP)/cfssl/ca-config.json -profile=server -hostname="$(HOSTNAME_API)" - \
91+
| cfssljson -bare server
92+
93+
$(BUILD_TLS)/peer-key.pem:
94+
cd $(BUILD_TLS); \
95+
echo '{"CN":"$(CONTROLLER_FQDN)","hosts":["$(DOMAIN_NAME)","*.$(DOMAIN_NAME)","*.ec2.internal","localhost"],"key":{"algo":"rsa","size":2048}}' \
96+
| cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=$(TOP)/cfssl/ca-config.json -profile=peer -hostname="$(HOSTNAME_API)" - \
97+
| cfssljson -bare peer
98+
99+
$(BUILD_KUBEADM)/ca.crt:
100+
kubeadm alpha phase certs ca --cert-dir $(TOP)/$(dir $@)
101+
102+
$(BUILD_KUBEADM)/front-proxy-ca.crt:
103+
kubeadm alpha phase certs front-proxy-ca --cert-dir $(TOP)/$(dir $@)
104+
105+
$(BUILD_KUBEADM)/sa.pub:
106+
kubeadm alpha phase certs sa --cert-dir $(TOP)/$(dir $@)
107+
108+
$(BUILD_KUBEADM)/admin.conf: $(BUILD_KUBEADM)/ca.crt
109+
kubeadm alpha phase kubeconfig admin \
110+
--apiserver-advertise-address $(CONTROLLER_FQDN) \
111+
--cert-dir $(TOP)/$(dir $@)
112+
# FIXME: Somehow --apiserver-advertise-address isn't working so we need to
113+
# patch file manually.
114+
sed 's|\(server: https://\)[^:]*\(.*\)|\1$(CONTROLLER_FQDN)\2|' \
115+
/etc/kubernetes/admin.conf \
116+
| install -m600 /dev/stdin $(dir $@)/admin.conf

Diff for: ‎README.md

+81
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# kubecfn
2+
*Cloudformation based installer for secure multi-node kubeadm cluster.*
3+
4+
# Operations
5+
You can either edit the Makefile or use environment variable to override
6+
specific settings.
7+
8+
- After deploying the cluster, you need to install kube2iam. Set your AWS
9+
account ID in [manifests/kube2iam.yaml](manifests/kube2iam.yaml) and apply the
10+
manifest.
11+
12+
## Known issues / README FIRST / FIXME
13+
- Run all kubeadm generation steps in the Docker container (if in doubt, run
14+
all). This is required since `kubeadm alpha phase kubeconfig` has hardcoded
15+
file locations.
16+
17+
- When deleting the stack, it will fail to delete the hosted zone because we
18+
created DNS records from the lambda. In this case delete the records manually
19+
and retry deletion.
20+
21+
- Same is true for cloud-provider-integration managed resources like ELBs. These
22+
should be deleted in kubernetes first. If that's not possible, the resources
23+
need to be deleted manually so cloudformation deletion can finish.
24+
25+
- On rolling upgrades etcd-members are suppose to remove themself from the
26+
cluster. This isn't working reliably yet. If this happens, the new replacement
27+
node can't join the cluster. This will block the rollout. In this case make
28+
sure the old node is actually terminated and remove it from the cluster with
29+
`/etc/etcdctl-wrapper member remove`.
30+
31+
- Rolling upgrades for workers is disabled right now. When updating, kill the
32+
old instances manually.
33+
34+
- Sometimes kubeadm fails, probably when it comes up before etcd reached quorum
35+
and fails. Just restart it. FIXME
36+
37+
- Sometimes ignition fails to get assets from s3 and reboots as a slow form or
38+
'retry': https://github.com/coreos/bugs/issues/2280
39+
40+
## Create cluster
41+
- `docker build -t cfn-make .`
42+
- `docker run -e AWS_ACCESS_KEY_ID=xx -e AWS_SECRET_ACCESS_KEY=yy cfn-make \
43+
create-cluster`
44+
- Wait for the cluster to come up and update it again. This will flip the
45+
*initial-cluster-state* flag in etcd, making sure that further updates can be
46+
rolled back and forward reliably:
47+
- `docker run -e AWS_ACCESS_KEY_ID=xx -e AWS_SECRET_ACCESS_KEY=yy cfn-make \
48+
update-cluster`
49+
- Install networking plugin:
50+
- `kubectl apply -f manifests/kube-flanne.yaml`
51+
52+
## "Dry run"
53+
Cloudformation supports [Change
54+
Sets](http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-changesets-create.html)
55+
which can be used to get the changes CloudFormation will do without actually
56+
updating the stack.
57+
58+
Create ChangeSet:
59+
```
60+
docker run -e AWS_ACCESS_KEY_ID=.. -e AWS_SECRET_ACCESS_KEY=.. -v $PWD:/usr/src/ \
61+
cfn-make cloudformation OP=create-change-set OPTS=--change-set-name=test2
62+
```
63+
64+
To view the change set run:
65+
```
66+
aws --region us-east-1 cloudformation describe-change-set \
67+
--stack-name int2 --change-set-name test2
68+
```
69+
70+
## Create custom cluster
71+
To create a second cluster, you need to override the name of the cloudformation
72+
stack. This can be done with the NAME environment variable.
73+
Since the stack uses a existing VPC but brings it's own subnets, the network
74+
ranges need to be adjusted too:
75+
76+
```
77+
docker run -e AWS_ACCESS_KEY_ID=.. -e AWS_SECRET_ACCESS_KEY=.. -v $PWD:/usr/src/ \
78+
cfn-make create-cluster NAME=int3 \
79+
PUBLIC_SUBNET_CIDR_PREFIX=172.20.15 \
80+
PRIVATE_SUBNET_CIDR_PREFIX=172.20.16
81+
```

Diff for: ‎cfssl/ca-config.json

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
{
2+
"signing": {
3+
"default": {
4+
"expiry": "43800h"
5+
},
6+
"profiles": {
7+
"server": {
8+
"expiry": "43800h",
9+
"usages": [
10+
"signing",
11+
"key encipherment",
12+
"server auth"
13+
]
14+
},
15+
"client": {
16+
"expiry": "43800h",
17+
"usages": [
18+
"signing",
19+
"key encipherment",
20+
"client auth"
21+
]
22+
},
23+
"peer": {
24+
"expiry": "43800h",
25+
"usages": [
26+
"signing",
27+
"key encipherment",
28+
"server auth",
29+
"client auth"
30+
]
31+
}
32+
}
33+
}
34+
}

Diff for: ‎cfssl/csr/ca-csr.json

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"CN": "kubernetes CA",
3+
"key": {
4+
"algo": "rsa",
5+
"size": 2048
6+
},
7+
"names": [
8+
{
9+
"C": "US",
10+
"L": "NYC",
11+
"O": "Foo",
12+
"ST": "Bar",
13+
"OU": "Baz",
14+
"OU": "Infrastructure"
15+
}
16+
]
17+
}

0 commit comments

Comments
 (0)