Skip to content

kubernetes_api_server_link_update #308

kubernetes_api_server_link_update

kubernetes_api_server_link_update #308

Workflow file for this run

name: e2e
on:
push:
branches:
- 'master'
tags:
- 'v*'
pull_request:
branches: [ master ]
workflow_dispatch:
env:
GO_TOOLCHAIN: "golang"
# TODO: match BASEIMAGE with Makefile default (nonroot variant)
BASEIMAGE: "gcr.io/distroless/static-debian11"
KIND_CLUSTER_NAME: "kind"
jobs:
build:
name: build
runs-on: ubuntu-20.04
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
id: go
- name: Build
run: |
mkdir _output
make docker-build/proxy-agent-amd64 REGISTRY=gcr.io/k8s-staging-kas-network-proxy TAG=local BASEIMAGE=${{ env.BASEIMAGE }}
docker tag gcr.io/k8s-staging-kas-network-proxy/proxy-agent-amd64:local gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master
docker save gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master > _output/konnectivity-agent.tar
make docker-build/proxy-server-amd64 REGISTRY=gcr.io/k8s-staging-kas-network-proxy TAG=local BASEIMAGE=${{ env.BASEIMAGE }}
docker tag gcr.io/k8s-staging-kas-network-proxy/proxy-server-amd64:local gcr.io/k8s-staging-kas-network-proxy/proxy-server:master
docker save gcr.io/k8s-staging-kas-network-proxy/proxy-server:master > _output/konnectivity-server.tar
- uses: actions/upload-artifact@v4
with:
name: konnectivity-server
path: _output/konnectivity-server.tar
- uses: actions/upload-artifact@v4
with:
name: konnectivity-agent
path: _output/konnectivity-agent.tar
kind-e2e:
name: kind-e2e
runs-on: ubuntu-20.04
timeout-minutes: 100
needs:
- build
env:
REGISTRY: gcr.io/k8s-staging-kas-network-proxy
KIND_IMAGE: kindest/node:${{ matrix.k8s }}
TAG: master
CONNECTION_MODE: ${{ matrix.connection-mode }}
strategy:
fail-fast: false
matrix:
k8s: [ v1.28.7, v1.29.2, v1.30.4 ]
connection-mode: [ grpc, http-connect ]
steps:
- name: Install kind
run: |
curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.23.0/kind-linux-amd64
chmod +x ./kind
sudo mv ./kind /usr/local/bin/kind
- name: Check out code
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
id: go
- name: Download prebuilt konnectivity-server image
uses: actions/download-artifact@v4
with:
name: konnectivity-server
- name: Download prebuilt konnectivity-agent image
uses: actions/download-artifact@v4
with:
name: konnectivity-agent
- name: Load prebuilt konnectivity images
run: |
docker load --input konnectivity-server.tar
docker load --input konnectivity-agent.tar
- name: Fix konnectivity docker image tags
run: |
docker tag gcr.io/k8s-staging-kas-network-proxy/proxy-server:master gcr.io/k8s-staging-kas-network-proxy/proxy-server-amd64:master
docker tag gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master gcr.io/k8s-staging-kas-network-proxy/proxy-agent-amd64:master
- name: Run e2e tests
run: make test-e2e-ci
e2e:
name: e2e
runs-on: ubuntu-20.04
timeout-minutes: 100
needs:
- build
strategy:
fail-fast: false
matrix:
ipFamily: ["ipv4", "ipv6", "dual"]
k8s: [ v1.28.7, v1.29.2, v1.30.4 ]
env:
JOB_NAME: "kindnetd-e2e-${{ matrix.ipFamily }}"
IP_FAMILY: ${{ matrix.ipFamily }}
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Enable ipv4 and ipv6 forwarding
run: |
sudo sysctl -w net.ipv6.conf.all.forwarding=1
sudo sysctl -w net.ipv4.ip_forward=1
- name: Set up environment (download dependencies)
run: |
TMP_DIR=$(mktemp -d)
# Test binaries
curl -L https://dl.k8s.io/${{ matrix.k8s }}/kubernetes-test-linux-amd64.tar.gz -o ${TMP_DIR}/kubernetes-test-linux-amd64.tar.gz
tar xvzf ${TMP_DIR}/kubernetes-test-linux-amd64.tar.gz \
--directory ${TMP_DIR} \
--strip-components=3 kubernetes/test/bin/ginkgo kubernetes/test/bin/e2e.test
# kubectl
curl -L https://dl.k8s.io/${{ matrix.k8s }}/bin/linux/amd64/kubectl -o ${TMP_DIR}/kubectl
# kind
curl -Lo ${TMP_DIR}/kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64
# Install
sudo cp ${TMP_DIR}/ginkgo /usr/local/bin/ginkgo
sudo cp ${TMP_DIR}/e2e.test /usr/local/bin/e2e.test
sudo cp ${TMP_DIR}/kubectl /usr/local/bin/kubectl
sudo cp ${TMP_DIR}/kind /usr/local/bin/kind
sudo chmod +x /usr/local/bin/*
- name: Create multi node cluster
run: |
# output_dir
mkdir -p _artifacts
# create cluster
cat <<EOF | /usr/local/bin/kind create cluster \
--name ${{ env.KIND_CLUSTER_NAME}} \
--image kindest/node:${{ matrix.k8s }} \
-v7 --wait 1m --retain --config=-
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
networking:
ipFamily: ${IP_FAMILY}
nodes:
- role: control-plane
image: ${{ matrix.k8s }}
kubeadmConfigPatchesJSON6902:
- kind: ClusterConfiguration
patch: |
- op: add
path: /apiServer/certSANs/-
value: konnectivity-server.kube-system.svc.cluster.local
kubeadmConfigPatches:
- |
kind: ClusterConfiguration
apiServer:
extraArgs:
"egress-selector-config-file": "/etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml"
extraVolumes:
- name: egress-selector-config-file
hostPath: "/etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml"
mountPath: "/etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml"
readOnly: true
- name: konnectivity-server
hostPath: "/etc/kubernetes/konnectivity-server"
mountPath: "/etc/kubernetes/konnectivity-server"
readOnly: true
extraMounts:
- hostPath: ./examples/kind/egress_selector_configuration.yaml
containerPath: /etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml
- role: worker
image: ${{ matrix.k8s }}
- role: worker
image: ${{ matrix.k8s }}
EOF
# dump the kubeconfig for later
/usr/local/bin/kind get kubeconfig --name ${{ env.KIND_CLUSTER_NAME}} > _artifacts/kubeconfig.conf
- uses: actions/download-artifact@v4
with:
name: konnectivity-server
- uses: actions/download-artifact@v4
with:
name: konnectivity-agent
- name: Install konnectivity
run: |
# preload konnectivity images
docker load --input konnectivity-server.tar
docker load --input konnectivity-agent.tar
/usr/local/bin/kind load docker-image gcr.io/k8s-staging-kas-network-proxy/proxy-server:master --name ${{ env.KIND_CLUSTER_NAME}}
/usr/local/bin/kind load docker-image gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master --name ${{ env.KIND_CLUSTER_NAME}}
kubectl apply -f examples/kind/konnectivity-server.yaml
kubectl apply -f examples/kind/konnectivity-agent-ds.yaml
- name: Get Cluster status
run: |
# wait network is ready
sleep 5
/usr/local/bin/kubectl get nodes -o wide
/usr/local/bin/kubectl get pods -A
/usr/local/bin/kubectl wait --timeout=1m --for=condition=ready pods --namespace=kube-system -l k8s-app=kube-dns
# smoke test
/usr/local/bin/kubectl run test --image httpd:2
/usr/local/bin/kubectl wait --timeout=1m --for=condition=ready pods test
/usr/local/bin/kubectl get pods -A -owide
/usr/local/bin/kubectl wait --timeout=1m --for=condition=ready pods --namespace=kube-system -l k8s-app=konnectivity-agent
/usr/local/bin/kubectl logs test
- name: Workaround CoreDNS for IPv6 airgapped
if: ${{ matrix.ipFamily == 'ipv6' }}
run: |
# Patch CoreDNS to work in Github CI
# 1. Github CI doesn´t offer IPv6 connectivity, so CoreDNS should be configured
# to work in an offline environment:
# https://github.com/coredns/coredns/issues/2494#issuecomment-457215452
# 2. Github CI adds following domains to resolv.conf search field:
# .net.
# CoreDNS should handle those domains and answer with NXDOMAIN instead of SERVFAIL
# otherwise pods stops trying to resolve the domain.
# Get the current config
original_coredns=$(/usr/local/bin/kubectl get -oyaml -n=kube-system configmap/coredns)
echo "Original CoreDNS config:"
echo "${original_coredns}"
# Patch it
fixed_coredns=$(
printf '%s' "${original_coredns}" | sed \
-e 's/^.*kubernetes cluster\.local/& net/' \
-e '/^.*upstream$/d' \
-e '/^.*fallthrough.*$/d' \
-e '/^.*forward . \/etc\/resolv.conf$/d' \
-e '/^.*loop$/d' \
)
echo "Patched CoreDNS config:"
echo "${fixed_coredns}"
printf '%s' "${fixed_coredns}" | /usr/local/bin/kubectl apply -f -
- name: Run tests
run: |
export KUBERNETES_CONFORMANCE_TEST='y'
export E2E_REPORT_DIR=${PWD}/_artifacts
# Run tests
/usr/local/bin/ginkgo --nodes=25 \
--focus="\[Conformance\]" \
--skip="Feature|Federation|machinery|PerformanceDNS|DualStack|Disruptive|Serial|Slow|KubeProxy|LoadBalancer|GCE|Netpol|NetworkPolicy|NodeConformance" \
/usr/local/bin/e2e.test \
-- \
--kubeconfig=${PWD}/_artifacts/kubeconfig.conf \
--provider=local \
--dump-logs-on-failure=false \
--report-dir=${E2E_REPORT_DIR} \
--disable-log-dump=true
- name: Upload Junit Reports
if: always()
uses: actions/upload-artifact@v4
with:
name: kind-junit-${{ env.JOB_NAME }}-${{ matrix.k8s }}-${{ matrix.ipFamily }}-${{ github.run_id }}
path: './_artifacts/*.xml'
- name: Export logs
if: always()
run: |
/usr/local/bin/kind export logs --name ${KIND_CLUSTER_NAME} --loglevel=debug ./_artifacts/logs
- name: Upload logs
if: always()
uses: actions/upload-artifact@v4
with:
name: kind-logs-${{ env.JOB_NAME }}-${{ matrix.k8s }}-${{ matrix.ipFamily }}-${{ github.run_id }}
path: ./_artifacts/logs