diff --git a/e2e/helper/helper.go b/e2e/helper/helper.go new file mode 100644 index 0000000..ecd753d --- /dev/null +++ b/e2e/helper/helper.go @@ -0,0 +1,113 @@ +// helper is a package that offers e2e test helpers. It's a badly named package. +// If it grows we should refactor it into something a little more manageable +// but it's fine for now. +package helper + +import ( + "bytes" + "context" + "fmt" + "strconv" + "strings" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + controllerruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/e2e-framework/klient/k8s/resources" + "sigs.k8s.io/e2e-framework/klient/wait" + "sigs.k8s.io/e2e-framework/klient/wait/conditions" + "sigs.k8s.io/e2e-framework/pkg/envconf" +) + +const debugDeploymentName = "debugy" + +// EnsureDebugContainer ensures that the helper debug container is installed right namespace. This allows us to make requests from inside the cluster +// regardless of the external network configuration. +func EnsureDebugContainer(t *testing.T, ctx context.Context, cfg *envconf.Config, namespace string) { + t.Helper() + + client, err := cfg.NewClient() + if err != nil { + t.Fatal(err) + } + + // Deploy a debug container so that we can test that the app is available later + deployment := newDebugDeployment(namespace, debugDeploymentName, 1, debugDeploymentName) + if err = client.Resources().Create(ctx, deployment); controllerruntimeclient.IgnoreAlreadyExists(err) != nil { + t.Fatal(err) + } + + err = wait.For( + conditions.New(client.Resources()). + DeploymentConditionMatch(deployment, appsv1.DeploymentAvailable, corev1.ConditionTrue), + wait.WithTimeout(time.Minute*5)) + if err != nil { + t.Fatal(err) + } +} + +// PostToSpinApp is a crude function for using the debug pod to post to a spin app +// within the cluster. It allows customization of the route, but all requests +// are currently `POST`. +func PostToSpinApp(t *testing.T, ctx context.Context, cfg *envconf.Config, namespace, spinAppName, route, body string) (string, int, error) { + t.Helper() + + client, err := cfg.NewClient() + if err != nil { + t.Fatal(err) + } + + // Find the debug pod + pods := &corev1.PodList{} + err = client.Resources(namespace).List(ctx, pods, resources.WithLabelSelector("app=pod-exec")) + if err != nil || pods.Items == nil || len(pods.Items) == 0 { + return "", -1, fmt.Errorf("failed to get debug pods: %w", err) + } + + debugPod := pods.Items[0] + + podName := debugPod.Name + + command := []string{"curl", "-s", "-m", "5", "-w", "\n%{http_code}\n", "http://" + spinAppName + "." + namespace + route, "--data", body, "-o", "-"} + + var stdout, stderr bytes.Buffer + if err := client.Resources().ExecInPod(ctx, namespace, podName, debugDeploymentName, command, &stdout, &stderr); err != nil { + t.Logf("Curl Spin App failed, err: %v.\nstdout:\n%s\n\nstderr:\n%s\n", err, stdout.String(), stderr.String()) + return "", -1, err + } + + parts := strings.SplitN(stdout.String(), "\n", 2) + if len(parts) != 2 { + t.Fatalf("Curl Spin App failed, unexpected response format: %s", &stdout) + } + + strStatus := strings.Trim(parts[1], "\n") + statusCode, err := strconv.Atoi(strStatus) + if err != nil { + t.Logf("error parsing status code: %v", err) + return parts[0], statusCode, err + } + t.Logf("Curl Spin App response: %s, status code: %d, err: %v", parts[0], statusCode, err) + return parts[0], statusCode, nil + +} + +func newDebugDeployment(namespace string, name string, replicas int32, containerName string) *appsv1.Deployment { + labels := map[string]string{"app": "pod-exec"} + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: containerName, Image: "nginx"}}}, + }, + }, + } +} diff --git a/e2e/redis_test.go b/e2e/redis_test.go new file mode 100644 index 0000000..498dffb --- /dev/null +++ b/e2e/redis_test.go @@ -0,0 +1,342 @@ +package e2e + +import ( + "context" + "io" + "strings" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + controllerruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/e2e-framework/klient" + "sigs.k8s.io/e2e-framework/klient/k8s" + "sigs.k8s.io/e2e-framework/klient/wait" + "sigs.k8s.io/e2e-framework/klient/wait/conditions" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/features" + "sigs.k8s.io/e2e-framework/support/utils" + + spinapps_v1alpha1 "github.com/spinkube/spin-operator/api/v1alpha1" + "github.com/spinkube/spin-operator/e2e/helper" + "github.com/spinkube/spin-operator/internal/generics" +) + +// TestShimRedis checks that the shim has basic runtime config support +// by being able to integrate with redis as the backend for a kv store +func TestShimRedis(t *testing.T) { + var client klient.Client + + // TODO: Use an image from a sample app in this repository + appImage := "ghcr.io/calebschoepp/spin-checklist:v0.1.0" + testSpinAppName := "test-shim-redis" + + defaultTest := features.New("default and most minimal setup"). + Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + + client = cfg.Client() + + if err := spinapps_v1alpha1.AddToScheme(client.Resources(testNamespace).GetScheme()); err != nil { + t.Fatalf("failed to register the spinapps_v1alpha1 types with Kubernetes scheme: %s", err) + } + + return ctx + }). + Assess("redis is setup", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + if err := client.Resources().Create(ctx, redisDeployment()); err != nil { + t.Fatalf("Failed to create redis deployment: %s", err) + } + + if err := client.Resources().Create(ctx, redisService()); err != nil { + t.Fatalf("Failed to create redis service: %s", err) + } + + return ctx + }). + Assess("spin app custom resource is created", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + testSpinApp := newSpinAppUsingRedis(testSpinAppName, appImage, "containerd-shim-spin") + + if err := client.Resources().Create(ctx, testSpinApp); err != nil { + t.Fatalf("Failed to create spinapp: %s", err) + } + // wait for spinapp to be created + if err := wait.For( + conditions.New(client.Resources()).ResourceMatch(testSpinApp, func(object k8s.Object) bool { + return true + }), + wait.WithTimeout(3*time.Minute), + wait.WithInterval(30*time.Second), + ); err != nil { + t.Fatal(err) + } + return ctx + }). + Assess("spin app deployment and service are available", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + // wait for deployment to be ready + if err := wait.For( + conditions.New(client.Resources()).DeploymentAvailable(testSpinAppName, testNamespace), + wait.WithTimeout(3*time.Minute), + wait.WithInterval(30*time.Second), + ); err != nil { + t.Fatal(err) + } + + svc := &v1.ServiceList{ + Items: []v1.Service{ + {ObjectMeta: metav1.ObjectMeta{Name: testSpinAppName, Namespace: testNamespace}}, + }, + } + + if err := wait.For( + conditions.New(client.Resources()).ResourcesFound(svc), + wait.WithTimeout(3*time.Minute), + wait.WithInterval(30*time.Second), + ); err != nil { + t.Fatal(err) + } + return ctx + }). + Assess("spin app is using redis", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + helper.EnsureDebugContainer(t, ctx, cfg, testNamespace) + + _, status, err := helper.PostToSpinApp(t, ctx, cfg, testNamespace, testSpinAppName, "/api", "{\"key\": \"foo\", \"value\": \"bar\"}") + + if err != nil { + t.Fatal(err) + } + if status != 200 { + t.Fatalf("expected 200 but got %d", status) + } + + p := utils.RunCommand("kubectl exec -n default deployment/redis -- bash -c \"echo KEYS * | redis-cli\"") + if p.Err() != nil { + t.Log(p.Out()) + t.Fatal(p.Err()) + } + buf := new(strings.Builder) + _, err = io.Copy(buf, p.Out()) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(buf.String(), "foo") { + t.Fatalf("expected 'foo' but got '%s'", buf.String()) + } + + return ctx + }). + Teardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + if err := client.Resources().Delete(ctx, redisDeployment()); err != nil { + t.Fatalf("Failed to delete redis deployment: %s", err) + } + if err := client.Resources().Delete(ctx, redisService()); err != nil { + t.Fatalf("Failed to delete redis service: %s", err) + } + return ctx + }). + Feature() + testEnv.Test(t, defaultTest) +} + +// TestSpintainerRedis checks that Spintainer has basic runtime config support +// by being able to integrate with redis as the backend for a kv store +// +//nolint:gocyclo +func TestSpintainerRedis(t *testing.T) { + var client klient.Client + + // TODO: Use an image from a sample app in this repository + appImage := "ghcr.io/calebschoepp/spin-checklist:v0.1.0" + testSpinAppName := "test-spintainer-redis" + + defaultTest := features.New("default and most minimal setup"). + Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + + client = cfg.Client() + + if err := spinapps_v1alpha1.AddToScheme(client.Resources(testNamespace).GetScheme()); err != nil { + t.Fatalf("failed to register the spinapps_v1alpha1 types with Kubernetes scheme: %s", err) + } + + return ctx + }). + Assess("redis is setup", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + if err := client.Resources().Create(ctx, redisDeployment()); err != nil { + t.Fatalf("Failed to create redis deployment: %s", err) + } + + if err := client.Resources().Create(ctx, redisService()); err != nil { + t.Fatalf("Failed to create redis service: %s", err) + } + + return ctx + }). + Assess("spin app custom resource is created", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + testSpinApp := newSpinAppUsingRedis(testSpinAppName, appImage, "spintainer") + + if err := client.Resources().Create(ctx, newSpintainerExecutor(testNamespace)); controllerruntimeclient.IgnoreAlreadyExists(err) != nil { + t.Fatalf("Failed to create spinappexecutor: %s", err) + } + + if err := client.Resources().Create(ctx, testSpinApp); err != nil { + t.Fatalf("Failed to create spinapp: %s", err) + } + // wait for spinapp to be created + if err := wait.For( + conditions.New(client.Resources()).ResourceMatch(testSpinApp, func(object k8s.Object) bool { + return true + }), + wait.WithTimeout(3*time.Minute), + wait.WithInterval(30*time.Second), + ); err != nil { + t.Fatal(err) + } + return ctx + }). + Assess("spin app deployment and service are available", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + // wait for deployment to be ready + if err := wait.For( + conditions.New(client.Resources()).DeploymentAvailable(testSpinAppName, testNamespace), + wait.WithTimeout(3*time.Minute), + wait.WithInterval(30*time.Second), + ); err != nil { + t.Fatal(err) + } + + svc := &v1.ServiceList{ + Items: []v1.Service{ + {ObjectMeta: metav1.ObjectMeta{Name: testSpinAppName, Namespace: testNamespace}}, + }, + } + + if err := wait.For( + conditions.New(client.Resources()).ResourcesFound(svc), + wait.WithTimeout(3*time.Minute), + wait.WithInterval(30*time.Second), + ); err != nil { + t.Fatal(err) + } + return ctx + }). + Assess("spin app is using redis", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + helper.EnsureDebugContainer(t, ctx, cfg, testNamespace) + + _, status, err := helper.PostToSpinApp(t, ctx, cfg, testNamespace, testSpinAppName, "/api", "{\"key\": \"foo\", \"value\": \"bar\"}") + + if err != nil { + t.Fatal(err) + } + if status != 200 { + t.Fatalf("expected 200 but got %d", status) + } + + p := utils.RunCommand("kubectl exec -n default deployment/redis -- bash -c \"echo KEYS * | redis-cli\"") + if p.Err() != nil { + t.Log(p.Out()) + t.Fatal(p.Err()) + } + buf := new(strings.Builder) + _, err = io.Copy(buf, p.Out()) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(buf.String(), "foo") { + t.Fatalf("expected 'foo' but got '%s'", buf.String()) + } + + return ctx + }). + Teardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + if err := client.Resources().Delete(ctx, redisDeployment()); err != nil { + t.Fatalf("Failed to delete redis deployment: %s", err) + } + if err := client.Resources().Delete(ctx, redisService()); err != nil { + t.Fatalf("Failed to delete redis service: %s", err) + } + return ctx + }). + Feature() + testEnv.Test(t, defaultTest) +} + +func redisDeployment() *appsv1.Deployment { + return &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "redis", + Namespace: "default", + }, + Spec: appsv1.DeploymentSpec{ + Replicas: generics.Ptr(int32(1)), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "redis"}, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "redis"}, + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{}, + InitContainers: []v1.Container{}, + Containers: []v1.Container{{ + Name: "redis", + Image: "redis:latest", + Ports: []v1.ContainerPort{{ + ContainerPort: 6379, + }}, + }}, + }, + }, + }, + } +} + +func redisService() *v1.Service { + return &v1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "redis", + Namespace: "default", + }, + Spec: v1.ServiceSpec{ + Selector: map[string]string{"app": "redis"}, + Ports: []v1.ServicePort{{ + Protocol: v1.ProtocolTCP, + Port: 6379, + TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: 6379}, + }}, + }, + } +} + +func newSpinAppUsingRedis(name, image, executor string) *spinapps_v1alpha1.SpinApp { + return &spinapps_v1alpha1.SpinApp{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: testNamespace, + }, + Spec: spinapps_v1alpha1.SpinAppSpec{ + Replicas: 1, + Image: image, + Executor: executor, + RuntimeConfig: spinapps_v1alpha1.RuntimeConfig{ + KeyValueStores: []spinapps_v1alpha1.KeyValueStoreConfig{{ + Name: "default", + Type: "redis", + Options: []spinapps_v1alpha1.RuntimeConfigOption{{ + Name: "url", + Value: "redis://redis.default.svc.cluster.local:6379", + }}, + }}, + }, + }, + } +} diff --git a/e2e/spintainer_test.go b/e2e/spintainer_test.go index 720c1a3..2d60b1b 100644 --- a/e2e/spintainer_test.go +++ b/e2e/spintainer_test.go @@ -7,6 +7,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + controllerruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/e2e-framework/klient" "sigs.k8s.io/e2e-framework/klient/k8s" "sigs.k8s.io/e2e-framework/klient/wait" @@ -40,7 +41,7 @@ func TestSpintainer(t *testing.T) { Assess("spin app custom resource is created", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { testSpinApp := newSpinAppCR(testSpinAppName, helloWorldImage, "spintainer") - if err := client.Resources().Create(ctx, newSpintainerExecutor(testNamespace)); err != nil { + if err := client.Resources().Create(ctx, newSpintainerExecutor(testNamespace)); controllerruntimeclient.IgnoreAlreadyExists(err) != nil { t.Fatalf("Failed to create spinappexecutor: %s", err) }