Skip to content

Commit

Permalink
Test runtimextension integration
Browse files Browse the repository at this point in the history
Signed-off-by: Danil-Grigorev <[email protected]>
  • Loading branch information
Danil-Grigorev committed Nov 12, 2024
1 parent 5b7d26f commit 7560e66
Show file tree
Hide file tree
Showing 6 changed files with 74 additions and 7 deletions.
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -618,6 +618,7 @@ generate-e2e-templates-main: $(KUSTOMIZE)
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-ignition --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-ignition.yaml
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/clusterclass-quick-start-kcp-only --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/clusterclass-quick-start-kcp-only.yaml
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-cross-ns-topology --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-cross-ns-topology.yaml
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-cross-ns-upgrades-runtimesdk --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-cross-ns-upgrades-runtimesdk.yaml
$(KUSTOMIZE) build $(INMEMORY_TEMPLATES)/main/cluster-template --load-restrictor LoadRestrictionsNone > $(INMEMORY_TEMPLATES)/main/cluster-template.yaml

.PHONY: generate-metrics-config
Expand Down
33 changes: 26 additions & 7 deletions test/e2e/cluster_upgrade_runtimesdk.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,9 @@ type ClusterUpgradeWithRuntimeSDKSpecInput struct {
ExtensionServiceNamespace string
// ExtensionServiceName is the name of the service to configure in the test-namespace scoped ExtensionConfig.
ExtensionServiceName string

// ClassNamespace is an optional class namespace reference, configuring cross-namespace cluster class reference
ClassNamespace bool
}

// ClusterUpgradeWithRuntimeSDKSpec implements a spec that upgrades a cluster and runs the Kubernetes conformance suite.
Expand All @@ -109,9 +112,9 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl
)

var (
input ClusterUpgradeWithRuntimeSDKSpecInput
namespace *corev1.Namespace
cancelWatches context.CancelFunc
input ClusterUpgradeWithRuntimeSDKSpecInput
namespace, infraNamespace *corev1.Namespace
cancelWatches, cancelInfraWatches context.CancelFunc

controlPlaneMachineCount int64
workerMachineCount int64
Expand Down Expand Up @@ -148,6 +151,7 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl

// Set up a Namespace where to host objects for this spec and create a watcher for the Namespace events.
namespace, cancelWatches = framework.SetupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, input.PostNamespaceCreated)
infraNamespace, cancelInfraWatches = framework.SetupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, input.PostNamespaceCreated)
clusterName = fmt.Sprintf("%s-%s", specName, util.RandomString(6))
clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult)
})
Expand All @@ -162,7 +166,7 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl
By("Deploy Test Extension ExtensionConfig")

Expect(input.BootstrapClusterProxy.GetClient().Create(ctx,
extensionConfig(specName, namespace.Name, input.ExtensionServiceNamespace, input.ExtensionServiceName))).
extensionConfig(specName, input.ExtensionServiceNamespace, input.ExtensionServiceName, namespace.Name, infraNamespace.Name))).
To(Succeed(), "Failed to create the extension config")

By("Creating a workload cluster; creation waits for BeforeClusterCreateHook to gate the operation")
Expand All @@ -177,6 +181,11 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl
infrastructureProvider = *input.InfrastructureProvider
}

variables := map[string]string{}
if input.ClassNamespace {
variables["CLUSTER_CLASS_NAMESPACE"] = infraNamespace.Name
}

clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
ClusterProxy: input.BootstrapClusterProxy,
ConfigCluster: clusterctl.ConfigClusterInput{
Expand All @@ -190,6 +199,7 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl
KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeFrom),
ControlPlaneMachineCount: ptr.To[int64](controlPlaneMachineCount),
WorkerMachineCount: ptr.To[int64](workerMachineCount),
ClusterctlVariables: variables,
},
PreWaitForCluster: func() {
beforeClusterCreateTestHandler(ctx,
Expand Down Expand Up @@ -304,7 +314,7 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl
if !input.SkipCleanup {
// Delete the extensionConfig first to ensure the BeforeDeleteCluster hook doesn't block deletion.
Eventually(func() error {
return input.BootstrapClusterProxy.GetClient().Delete(ctx, extensionConfig(specName, namespace.Name, input.ExtensionServiceNamespace, input.ExtensionServiceName))
return input.BootstrapClusterProxy.GetClient().Delete(ctx, extensionConfig(specName, input.ExtensionServiceNamespace, input.ExtensionServiceName))
}, 10*time.Second, 1*time.Second).Should(Succeed(), "delete extensionConfig failed")

Byf("Deleting cluster %s", klog.KObj(clusterResources.Cluster))
Expand All @@ -322,8 +332,17 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl
Deleter: input.BootstrapClusterProxy.GetClient(),
Name: namespace.Name,
})

if input.ClassNamespace {
Byf("Deleting namespace used for optionally hosting the %q infrastructure spec", specName)
framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{
Deleter: input.BootstrapClusterProxy.GetClient(),
Name: infraNamespace.Name,
})
}
}
cancelWatches()
cancelInfraWatches()
})
}

Expand Down Expand Up @@ -429,7 +448,7 @@ func machineSetPreflightChecksTestHandler(ctx context.Context, c client.Client,
// We make sure this cluster-wide object does not conflict with others by using a random generated
// name and a NamespaceSelector selecting on the namespace of the current test.
// Thus, this object is "namespaced" to the current test even though it's a cluster-wide object.
func extensionConfig(name, namespace, extensionServiceNamespace, extensionServiceName string) *runtimev1.ExtensionConfig {
func extensionConfig(name, extensionServiceNamespace, extensionServiceName string, namespaces ...string) *runtimev1.ExtensionConfig {
return &runtimev1.ExtensionConfig{
ObjectMeta: metav1.ObjectMeta{
// Note: We have to use a constant name here as we have to be able to reference it in the ClusterClass
Expand All @@ -454,7 +473,7 @@ func extensionConfig(name, namespace, extensionServiceNamespace, extensionServic
{
Key: "kubernetes.io/metadata.name",
Operator: metav1.LabelSelectorOpIn,
Values: []string{namespace},
Values: namespaces,
},
},
},
Expand Down
33 changes: 33 additions & 0 deletions test/e2e/cluster_upgrade_runtimesdk_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,3 +60,36 @@ var _ = Describe("When upgrading a workload cluster using ClusterClass with Runt
}
})
})

var _ = Describe("When upgrading a workload cluster using ClusterClass in a different NS with RuntimeSDK [PR-Blocking] [ClusterClass]", func() {
ClusterUpgradeWithRuntimeSDKSpec(ctx, func() ClusterUpgradeWithRuntimeSDKSpecInput {
version, err := semver.ParseTolerant(e2eConfig.GetVariable(KubernetesVersionUpgradeFrom))
Expect(err).ToNot(HaveOccurred(), "Invalid argument, KUBERNETES_VERSION_UPGRADE_FROM is not a valid version")
if version.LT(semver.MustParse("1.24.0")) {
Fail("This test only supports upgrades from Kubernetes >= v1.24.0")
}

return ClusterUpgradeWithRuntimeSDKSpecInput{
E2EConfig: e2eConfig,
ClusterctlConfigPath: clusterctlConfigPath,
BootstrapClusterProxy: bootstrapClusterProxy,
ArtifactFolder: artifactFolder,
SkipCleanup: skipCleanup,
InfrastructureProvider: ptr.To("docker"),
PostUpgrade: func(proxy framework.ClusterProxy, namespace, clusterName string) {
// This check ensures that the resourceVersions are stable, i.e. it verifies there are no
// continuous reconciles when everything should be stable.
framework.ValidateResourceVersionStable(ctx, proxy, namespace, clusterctlcluster.FilterClusterObjectsWithNameFilter(clusterName))
},
// "upgrades" is the same as the "topology" flavor but with an additional MachinePool.
Flavor: ptr.To("cross-ns-upgrades-runtimesdk"),
ClassNamespace: true,
// The runtime extension gets deployed to the test-extension-system namespace and is exposed
// by the test-extension-webhook-service.
// The below values are used when creating the cluster-wide ExtensionConfig to refer
// the actual service.
ExtensionServiceNamespace: "test-extension-system",
ExtensionServiceName: "test-extension-webhook-service",
}
})
})
1 change: 1 addition & 0 deletions test/e2e/config/docker.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -348,6 +348,7 @@ providers:
- sourcePath: "../data/infrastructure-docker/main/cluster-template-topology.yaml"
- sourcePath: "../data/infrastructure-docker/main/cluster-template-ignition.yaml"
- sourcePath: "../data/infrastructure-docker/main/cluster-template-cross-ns-topology.yaml"
- sourcePath: "../data/infrastructure-docker/main/cluster-template-cross-ns-upgrades-runtimesdk.yaml"
- sourcePath: "../data/infrastructure-docker/main/clusterclass-quick-start.yaml"
- sourcePath: "../data/infrastructure-docker/main/clusterclass-quick-start-kcp-only.yaml"
- sourcePath: "../data/infrastructure-docker/main/clusterclass-quick-start-runtimesdk.yaml"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
---
apiVersion: cluster.x-k8s.io/v1beta1
kind: Cluster
metadata:
name: '${CLUSTER_NAME}'
spec:
topology:
classNamespace: '${CLUSTER_CLASS_NAMESPACE}'
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
resources:
- ../cluster-template-upgrades-runtimesdk

patches:
- path: cluster.yaml

0 comments on commit 7560e66

Please sign in to comment.