Skip to content

Commit

Permalink
fix: ignore metallb admission (#400)
Browse files Browse the repository at this point in the history
* fix: ignore metallb admission

Set the metallb admission controller to ignore. Our IPAddressPools and
L2Advertisements are static and work. The admission controller has a
habit of not coming online and blocking the deploy. Ignoring its
failures avoids this.

Add a kustomize utility and refactor the metallb deploy to use it.

* chore: fix kustomization fields

Co-authored-by: Patryk Małek <[email protected]>
  • Loading branch information
rainest and pmalek committed Oct 3, 2022
1 parent ece09e2 commit dbe01b1
Show file tree
Hide file tree
Showing 3 changed files with 114 additions and 5 deletions.
57 changes: 54 additions & 3 deletions pkg/clusters/addons/metallb/metallb.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,14 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/dynamic"
"sigs.k8s.io/kustomize/api/types"
kustomize "sigs.k8s.io/kustomize/api/types"
"sigs.k8s.io/kustomize/kyaml/resid"

"github.com/kong/kubernetes-testing-framework/pkg/clusters"
"github.com/kong/kubernetes-testing-framework/pkg/clusters/types/kind"
"github.com/kong/kubernetes-testing-framework/pkg/utils/docker"
"github.com/kong/kubernetes-testing-framework/pkg/utils/kubernetes/kubectl"
"github.com/kong/kubernetes-testing-framework/pkg/utils/networking"
)

Expand Down Expand Up @@ -140,7 +144,7 @@ func (a *addon) DumpDiagnostics(ctx context.Context, cluster clusters.Cluster) (
var (
defaultStartIP = net.ParseIP("0.0.0.100")
defaultEndIP = net.ParseIP("0.0.0.250")
metalManifest = "https://raw.githubusercontent.com/metallb/metallb/v0.13.5/config/manifests/metallb-native.yaml"
metalManifest = "https://github.com/metallb/metallb/config/native?ref=v0.13.5"
secretKeyLen = 128
)

Expand Down Expand Up @@ -308,23 +312,64 @@ func getIPRangeForMetallb(network net.IPNet) (startIP, endIP net.IP) {
// TODO: needs to be replaced with non-kubectl, just used this originally for speed.
//
// See: https://github.com/Kong/kubernetes-testing-framework/issues/25

const admissionPatch = `
- op: replace
# ipaddresspoolvalidationwebhook.metallb.io
path: /webhooks/5/failurePolicy
value: "Ignore"
- op: replace
# l2advertisementvalidationwebhook.metallb.io
path: /webhooks/6/failurePolicy
value: "Ignore"
`

func getManifest() (io.Reader, error) {
return kubectl.GetKustomizedManifest(kustomize.Kustomization{
Resources: []string{metalManifest},
Patches: []kustomize.Patch{
{
Patch: admissionPatch,
Target: &types.Selector{
ResId: resid.ResId{
Gvk: resid.Gvk{
Group: "admissionregistration.k8s.io",
Version: "v1",
Kind: "ValidatingWebhookConfiguration",
},
Name: "metallb-webhook-configuration",
Namespace: "metallb-system",
},
},
},
},
})
}

func metallbDeployHack(cluster clusters.Cluster) error {
// generate a temporary kubeconfig since we're going to be using kubectl
kubeconfig, err := clusters.TempKubeconfig(cluster)
if err != nil {
return err
}

defer os.Remove(kubeconfig.Name())

deployArgs := []string{
"--kubeconfig", kubeconfig.Name(),
"apply", "-f", metalManifest,
"apply", "-f", "-",
}

manifest, err := getManifest()
if err != nil {
return fmt.Errorf("could not deploy metallb: %w", err)
}

stderr := new(bytes.Buffer)
cmd := exec.Command("kubectl", deployArgs...)
cmd.Stdout = io.Discard
cmd.Stderr = stderr
cmd.Stdin = manifest

if err := cmd.Run(); err != nil {
return fmt.Errorf("%s: %w", stderr.String(), err)
Expand All @@ -336,13 +381,19 @@ func metallbDeployHack(cluster clusters.Cluster) error {
func metallbDeleteHack(kubeconfig *os.File) error {
deployArgs := []string{
"--kubeconfig", kubeconfig.Name(),
"delete", "-f", metalManifest,
"delete", "-f", "-",
}

manifest, err := getManifest()
if err != nil {
return fmt.Errorf("could not delete metallb: %w", err)
}

stderr := new(bytes.Buffer)
cmd := exec.Command("kubectl", deployArgs...)
cmd.Stdout = io.Discard
cmd.Stderr = stderr
cmd.Stdin = manifest

if err := cmd.Run(); err != nil {
return fmt.Errorf("%s: %w", stderr.String(), err)
Expand Down
4 changes: 2 additions & 2 deletions pkg/clusters/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ func CleanupGeneratedResources(ctx context.Context, cluster Cluster, creatorID s
func KustomizeDeployForCluster(ctx context.Context, cluster Cluster, kustomizeURL string) error {
// generate the kustomize YAML
stdout, stderr := new(bytes.Buffer), new(bytes.Buffer)
cmd := exec.CommandContext(ctx, "kubectl", "kustomize", kustomizeURL)
cmd := exec.CommandContext(ctx, "kubectl", "-v9", "kustomize", kustomizeURL)
cmd.Stdout = stdout
cmd.Stderr = stderr
if err := cmd.Run(); err != nil {
Expand All @@ -225,7 +225,7 @@ func KustomizeDeployForCluster(ctx context.Context, cluster Cluster, kustomizeUR
func KustomizeDeleteForCluster(ctx context.Context, cluster Cluster, kustomizeURL string) error {
// generate the kustomize YAML
stdout, stderr := new(bytes.Buffer), new(bytes.Buffer)
cmd := exec.CommandContext(ctx, "kubectl", "kustomize", kustomizeURL)
cmd := exec.CommandContext(ctx, "kubectl", "-v9", "kustomize", kustomizeURL)
cmd.Stdout = stdout
cmd.Stderr = stderr
if err := cmd.Run(); err != nil {
Expand Down
58 changes: 58 additions & 0 deletions pkg/utils/kubernetes/kubectl/kustomize.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
package kubectl

import (
"bytes"
"fmt"
"io"
"os"
"path/filepath"

"github.com/ghodss/yaml"
"sigs.k8s.io/kustomize/api/krusty"
"sigs.k8s.io/kustomize/api/types"
"sigs.k8s.io/kustomize/kyaml/filesys"
)

// GetKustomizedManifest takes a kustomization and any number of manifest readers. It adds the manifests to the
// kustomization's resources and returns a reader with the rendered kustomization.
func GetKustomizedManifest(kustomization types.Kustomization, manifests ...io.Reader) (io.Reader, error) {
workDir, err := os.MkdirTemp("", "ktf.")
if err != nil {
return nil, err
}
defer os.RemoveAll(workDir)
for i, manifest := range manifests {
orig, err := io.ReadAll(manifest)
if err != nil {
return nil, err
}
err = os.WriteFile(filepath.Join(workDir, fmt.Sprintf("resource_%d.yaml", i)), orig, 0o600) //nolint:gomnd
if err != nil {
return nil, err
}
kustomization.Resources = append(kustomization.Resources, fmt.Sprintf("resource_%d.yaml", i))
}
marshalled, err := yaml.Marshal(kustomization)
if err != nil {
return nil, err
}
err = os.WriteFile(filepath.Join(workDir, "kustomization.yaml"), marshalled, 0o600) //nolint:gomnd
if err != nil {
return nil, err
}
kustomized, err := runKustomize(workDir)
if err != nil {
return nil, err
}
return bytes.NewReader(kustomized), nil
}

// runKustomize runs kustomize on a path and returns the YAML output.
func runKustomize(path string) ([]byte, error) {
k := krusty.MakeKustomizer(krusty.MakeDefaultOptions())
m, err := k.Run(filesys.MakeFsOnDisk(), path)
if err != nil {
return []byte{}, err
}
return m.AsYaml()
}

0 comments on commit dbe01b1

Please sign in to comment.