diff --git a/test/e2e/data/e2e_conf.yaml b/test/e2e/data/e2e_conf.yaml index bc7f6da450..766f525d78 100644 --- a/test/e2e/data/e2e_conf.yaml +++ b/test/e2e/data/e2e_conf.yaml @@ -226,6 +226,8 @@ providers: - sourcePath: "./infrastructure-aws/generated/cluster-template-upgrades.yaml" - sourcePath: "./infrastructure-aws/generated/cluster-template-external-infrastructure.yaml" - sourcePath: "./infrastructure-aws/generated/cluster-template-external-securitygroups.yaml" + - sourcePath: "./infrastructure-aws/generated/cluster-template-peered-remote.yaml" + - sourcePath: "./infrastructure-aws/generated/cluster-template-internal-elb.yaml" - sourcePath: "./infrastructure-aws/kustomize_sources/topology/clusterclass-quick-start.yaml" - sourcePath: "./shared/v1beta1_provider/metadata.yaml" replacements: diff --git a/test/e2e/data/infrastructure-aws/kustomize_sources/internal-elb/kustomization.yaml b/test/e2e/data/infrastructure-aws/kustomize_sources/internal-elb/kustomization.yaml new file mode 100644 index 0000000000..dc72a029d5 --- /dev/null +++ b/test/e2e/data/infrastructure-aws/kustomize_sources/internal-elb/kustomization.yaml @@ -0,0 +1,5 @@ +resources: + - ../remote-management-cluster +patchesStrategicMerge: + - patches/internal-elb.yaml + diff --git a/test/e2e/data/infrastructure-aws/kustomize_sources/internal-elb/patches/internal-elb.yaml b/test/e2e/data/infrastructure-aws/kustomize_sources/internal-elb/patches/internal-elb.yaml new file mode 100644 index 0000000000..3edf80a347 --- /dev/null +++ b/test/e2e/data/infrastructure-aws/kustomize_sources/internal-elb/patches/internal-elb.yaml @@ -0,0 +1,14 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AWSCluster +metadata: + name: "${CLUSTER_NAME}" +spec: + controlPlaneLoadBalancer: + scheme: internal + network: + vpc: + id: "${WL_VPC_ID}" + subnets: + - id: "${WL_PUBLIC_SUBNET_ID}" + - id: "${WL_PRIVATE_SUBNET_ID}" + diff --git a/test/e2e/data/infrastructure-aws/kustomize_sources/peered-remote/kustomization.yaml b/test/e2e/data/infrastructure-aws/kustomize_sources/peered-remote/kustomization.yaml new file mode 100644 index 0000000000..2a80f9b142 --- /dev/null +++ b/test/e2e/data/infrastructure-aws/kustomize_sources/peered-remote/kustomization.yaml @@ -0,0 +1,5 @@ +resources: + - ../remote-management-cluster +patchesStrategicMerge: + - patches/management.yaml + diff --git a/test/e2e/data/infrastructure-aws/kustomize_sources/peered-remote/patches/management.yaml b/test/e2e/data/infrastructure-aws/kustomize_sources/peered-remote/patches/management.yaml new file mode 100644 index 0000000000..4443bfa259 --- /dev/null +++ b/test/e2e/data/infrastructure-aws/kustomize_sources/peered-remote/patches/management.yaml @@ -0,0 +1,12 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AWSCluster +metadata: + name: "${CLUSTER_NAME}" +spec: + network: + vpc: + id: "${MGMT_VPC_ID}" + subnets: + - id: "${MGMT_PUBLIC_SUBNET_ID}" + - id: "${MGMT_PRIVATE_SUBNET_ID}" + diff --git a/test/e2e/shared/aws.go b/test/e2e/shared/aws.go index acf97213b7..12f8e871dd 100644 --- a/test/e2e/shared/aws.go +++ b/test/e2e/shared/aws.go @@ -47,6 +47,7 @@ import ( cfn_iam "github.com/awslabs/goformation/v4/cloudformation/iam" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "k8s.io/utils/pointer" "sigs.k8s.io/yaml" cfn_bootstrap "sigs.k8s.io/cluster-api-provider-aws/cmd/clusterawsadm/cloudformation/bootstrap" @@ -57,6 +58,207 @@ import ( "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/wait" ) +type AWSInfrastructureSpec struct { + ClusterName, VpcCidr, PublicSubnetCidr, PrivateSubnetCidr, AvailabilityZone string + ExternalSecurityGroups bool +} + +type AWSInfrastructureState struct { + PrivateSubnetID *string + PrivateSubnetState *string + PublicSubnetID *string + PublicSubnetState *string + VpcState *string + NatGatewayState *string + PublicRouteTableID *string + PrivateRouteTableID *string +} + +type AWSInfrastructure struct { + Spec AWSInfrastructureSpec + Context *E2EContext + VPC *ec2.Vpc + Subnets []*ec2.Subnet + RouteTables []*ec2.RouteTable + InternetGateway *ec2.InternetGateway + ElasticIP *ec2.Address + NatGateway *ec2.NatGateway + State AWSInfrastructureState +} + +func (i *AWSInfrastructure) New(ais AWSInfrastructureSpec, e2eCtx *E2EContext) AWSInfrastructure { + i.Spec = ais + i.Context = e2eCtx + return *i +} + +func (i *AWSInfrastructure) CreateVPC() AWSInfrastructure { + cv, err := CreateVPC(i.Context, i.Spec.ClusterName+"-vpc", i.Spec.VpcCidr) + if err != nil { + return *i + } + + i.VPC = cv + i.State.VpcState = cv.State + return *i +} + +func (i *AWSInfrastructure) RefreshVPCState() AWSInfrastructure { + vpc, err := GetVPC(i.Context, *i.VPC.VpcId) + if err != nil { + return *i + } + if vpc != nil { + i.VPC = vpc + i.State.VpcState = vpc.State + } + return *i +} + +func (i *AWSInfrastructure) CreatePublicSubnet() AWSInfrastructure { + subnet, err := CreateSubnet(i.Context, i.Spec.ClusterName, i.Spec.PublicSubnetCidr, i.Spec.AvailabilityZone, *i.VPC.VpcId, "public") + if err != nil { + i.State.PublicSubnetState = pointer.String("failed") + return *i + } + i.State.PublicSubnetID = subnet.SubnetId + i.State.PublicSubnetState = subnet.State + i.Subnets = append(i.Subnets, subnet) + return *i +} + +func (i *AWSInfrastructure) CreatePrivateSubnet() AWSInfrastructure { + subnet, err := CreateSubnet(i.Context, i.Spec.ClusterName, i.Spec.PrivateSubnetCidr, i.Spec.AvailabilityZone, *i.VPC.VpcId, "private") + if err != nil { + i.State.PrivateSubnetState = pointer.String("failed") + return *i + } + i.State.PrivateSubnetID = subnet.SubnetId + i.State.PrivateSubnetState = subnet.State + i.Subnets = append(i.Subnets, subnet) + return *i +} + +func (i *AWSInfrastructure) CreateInternetGateway() AWSInfrastructure { + igwC, err := CreateInternetGateway(i.Context, i.Spec.ClusterName+"-igw") + if err != nil { + return *i + } + _, aerr := AttachInternetGateway(i.Context, *igwC.InternetGatewayId, *i.VPC.VpcId) + if aerr != nil { + i.InternetGateway = igwC + return *i + } + i.InternetGateway = igwC + return *i +} + +func (i *AWSInfrastructure) AllocateAddress() AWSInfrastructure { + aa, err := AllocateAddress(i.Context, i.Spec.ClusterName+"-eip") + if err != nil { + return *i + } + + if addr, _ := GetAddress(i.Context, *aa.AllocationId); addr != nil { + i.ElasticIP = addr + } + return *i +} + +func (i *AWSInfrastructure) CreateNatGateway(ct string) AWSInfrastructure { + s, serr := GetSubnetByName(i.Context, i.Spec.ClusterName+"-subnet-"+ct) + if serr != nil { + return *i + } + ngwC, ngwce := CreateNatGateway(i.Context, i.Spec.ClusterName+"-nat", ct, *i.ElasticIP.AllocationId, *s.SubnetId) + if ngwce != nil { + return *i + } + if WaitForNatGatewayState(i.Context, *ngwC.NatGatewayId, 180, "available") { + ngw, _ := GetNatGateway(i.Context, *ngwC.NatGatewayId) + i.NatGateway = ngw + i.State.NatGatewayState = ngw.State + return *i + } + i.NatGateway = ngwC + return *i +} + +func (i *AWSInfrastructure) CreateRouteTable(subnetType string) AWSInfrastructure { + rt, err := CreateRouteTable(i.Context, i.Spec.ClusterName+"-rt-"+subnetType, *i.VPC.VpcId) + if err != nil { + return *i + } + switch subnetType { + case "public": + if a, _ := AssociateRouteTable(i.Context, *rt.RouteTableId, *i.State.PublicSubnetID); a != nil { + i.State.PublicRouteTableID = rt.RouteTableId + } + case "private": + if a, _ := AssociateRouteTable(i.Context, *rt.RouteTableId, *i.State.PrivateSubnetID); a != nil { + i.State.PrivateRouteTableID = rt.RouteTableId + } + } + return *i +} + +func (i *AWSInfrastructure) GetRouteTable(rtID string) AWSInfrastructure { + rt, err := GetRouteTable(i.Context, rtID) + if err != nil { + return *i + } + if rt != nil { + i.RouteTables = append(i.RouteTables, rt) + } + return *i +} + +func (i *AWSInfrastructure) CreateInfrastructure() AWSInfrastructure { + i.CreateVPC() + Expect(i.VPC).NotTo(BeNil()) + if i.VPC != nil { + i.CreatePublicSubnet() + i.CreatePrivateSubnet() + for t := 0; t < 30; t++ { + if *i.RefreshVPCState().State.VpcState == "available" { + break + } + time.Sleep(1 * time.Second) + } + } + i.CreateInternetGateway() + i.AllocateAddress() + i.CreateNatGateway("public") + WaitForNatGatewayState(i.Context, *i.NatGateway.NatGatewayId, 180, "available") + i.CreateRouteTable("public") + i.CreateRouteTable("private") + Expect(CreateRoute(i.Context, *i.State.PublicRouteTableID, "0.0.0.0/0", nil, i.InternetGateway.InternetGatewayId, nil)).To(BeTrue()) + Expect(CreateRoute(i.Context, *i.State.PrivateRouteTableID, "0.0.0.0/0", i.NatGateway.NatGatewayId, nil, nil)).To(BeTrue()) + i.GetRouteTable(*i.State.PublicRouteTableID) + i.GetRouteTable(*i.State.PrivateRouteTableID) + return *i +} + +func (i *AWSInfrastructure) DeleteInfrastructure() AWSInfrastructure { + for _, rt := range i.RouteTables { + for _, a := range rt.Associations { + DisassociateRouteTable(i.Context, *a.RouteTableAssociationId) + } + if !DeleteRouteTable(i.Context, *rt.RouteTableId) { + fmt.Printf("%+v", rt) + } + } + DeleteNatGateway(i.Context, *i.NatGateway.NatGatewayId) + WaitForNatGatewayState(i.Context, *i.NatGateway.NatGatewayId, 180, "deleted") + ReleaseAddress(i.Context, *i.ElasticIP.AllocationId) + Eventually(DetachInternetGateway(i.Context, *i.InternetGateway.InternetGatewayId, *i.VPC.VpcId), 60*time.Second).Should(BeTrue()) + DeleteInternetGateway(i.Context, *i.InternetGateway.InternetGatewayId) + DeleteSubnet(i.Context, *i.State.PrivateSubnetID) + DeleteSubnet(i.Context, *i.State.PublicSubnetID) + DeleteVPC(i.Context, *i.VPC.VpcId) + return *i +} + func NewAWSSession() client.ConfigProvider { By("Getting an AWS IAM session - from environment") region, err := credentials.ResolveRegion("") @@ -757,6 +959,9 @@ func GetVPC(e2eCtx *E2EContext, vpcID string) (*ec2.Vpc, error) { if err != nil { return nil, err } + if result.Vpcs == nil { + return nil, nil + } return result.Vpcs[0], nil } @@ -848,6 +1053,33 @@ func GetSubnet(e2eCtx *E2EContext, subnetID string) (*ec2.Subnet, error) { if err != nil { return nil, err } + if result.Subnets == nil { + return nil, nil + } + return result.Subnets[0], nil +} + +func GetSubnetByName(e2eCtx *E2EContext, name string) (*ec2.Subnet, error) { + ec2Svc := ec2.New(e2eCtx.AWSSession) + + filter := &ec2.Filter{ + Name: aws.String("tag:Name"), + Values: aws.StringSlice([]string{name}), + } + + input := &ec2.DescribeSubnetsInput{ + Filters: []*ec2.Filter{ + filter, + }, + } + + result, err := ec2Svc.DescribeSubnets(input) + if err != nil { + return nil, err + } + if result.Subnets == nil { + return nil, nil + } return result.Subnets[0], nil } @@ -912,6 +1144,30 @@ func DeleteSubnet(e2eCtx *E2EContext, subnetID string) bool { return true } +func GetAddress(e2eCtx *E2EContext, allocationID string) (*ec2.Address, error) { + ec2Svc := ec2.New(e2eCtx.AWSSession) + + filter := &ec2.Filter{ + Name: aws.String("allocation-id"), + Values: aws.StringSlice([]string{allocationID}), + } + + input := &ec2.DescribeAddressesInput{ + Filters: []*ec2.Filter{ + filter, + }, + } + + result, err := ec2Svc.DescribeAddresses(input) + if err != nil { + return nil, err + } + if result.Addresses == nil { + return nil, nil + } + return result.Addresses[0], nil +} + func AllocateAddress(e2eCtx *E2EContext, eipName string) (*ec2.AllocateAddressOutput, error) { ec2Svc := ec2.New(e2eCtx.AWSSession) @@ -950,11 +1206,11 @@ func DisassociateAddress(e2eCtx *E2EContext, assocID string) bool { return true } -func ReleaseAddress(e2eCtx *E2EContext, allocID string) bool { +func ReleaseAddress(e2eCtx *E2EContext, allocationID string) bool { ec2Svc := ec2.New(e2eCtx.AWSSession) input := &ec2.ReleaseAddressInput{ - AllocationId: aws.String(allocID), + AllocationId: aws.String(allocationID), } if _, err := ec2Svc.ReleaseAddress(input); err != nil { @@ -963,7 +1219,7 @@ func ReleaseAddress(e2eCtx *E2EContext, allocID string) bool { return true } -func CreateNatGateway(e2eCtx *E2EContext, gatewayName string, connectType string, allocID string, subnetID string) (*ec2.NatGateway, error) { +func CreateNatGateway(e2eCtx *E2EContext, gatewayName string, connectType string, allocationID string, subnetID string) (*ec2.NatGateway, error) { ec2Svc := ec2.New(e2eCtx.AWSSession) input := &ec2.CreateNatGatewayInput{ @@ -985,8 +1241,8 @@ func CreateNatGateway(e2eCtx *E2EContext, gatewayName string, connectType string input.ConnectivityType = aws.String(connectType) } - if allocID != "" { - input.AllocationId = aws.String(allocID) + if allocationID != "" { + input.AllocationId = aws.String(allocationID) } result, err := ec2Svc.CreateNatGateway(input) @@ -1014,6 +1270,9 @@ func GetNatGateway(e2eCtx *E2EContext, gatewayID string) (*ec2.NatGateway, error if err != nil { return nil, err } + if result.NatGateways == nil { + return nil, nil + } return result.NatGateways[0], nil } @@ -1086,6 +1345,9 @@ func GetInternetGateway(e2eCtx *E2EContext, gatewayID string) (*ec2.InternetGate if err != nil { return nil, err } + if result.InternetGateways == nil { + return nil, nil + } return result.InternetGateways[0], nil } @@ -1156,6 +1418,30 @@ func CreatePeering(e2eCtx *E2EContext, peerName string, vpcID string, peerVpcID return result.VpcPeeringConnection, nil } +func GetPeering(e2eCtx *E2EContext, peeringID string) (*ec2.VpcPeeringConnection, error) { + ec2Svc := ec2.New(e2eCtx.AWSSession) + + filter := &ec2.Filter{ + Name: aws.String("vpc-peering-connection-id"), + Values: aws.StringSlice([]string{peeringID}), + } + + input := &ec2.DescribeVpcPeeringConnectionsInput{ + Filters: []*ec2.Filter{ + filter, + }, + } + + result, err := ec2Svc.DescribeVpcPeeringConnections(input) + if err != nil { + return nil, err + } + if result.VpcPeeringConnections == nil { + return nil, nil + } + return result.VpcPeeringConnections[0], nil +} + func DeletePeering(e2eCtx *E2EContext, peeringID string) bool { ec2Svc := ec2.New(e2eCtx.AWSSession) @@ -1268,6 +1554,9 @@ func GetRouteTable(e2eCtx *E2EContext, rtID string) (*ec2.RouteTable, error) { if err != nil { return nil, err } + if result.RouteTables == nil { + return nil, nil + } return result.RouteTables[0], nil } @@ -1308,8 +1597,7 @@ func CreateRoute(e2eCtx *E2EContext, rtID string, destinationCidr string, natID if err != nil { return false, err } - created := *result.Return - return created, nil + return *result.Return, nil } func DeleteRoute(e2eCtx *E2EContext, rtID string, destinationCidr string) bool { @@ -1399,6 +1687,9 @@ func GetSecurityGroup(e2eCtx *E2EContext, sgID string) (*ec2.SecurityGroup, erro if err != nil { return nil, err } + if result.SecurityGroups == nil { + return nil, nil + } return result.SecurityGroups[0], nil } @@ -1454,6 +1745,9 @@ func GetSecurityGroupRule(e2eCtx *E2EContext, sgrID string) (*ec2.SecurityGroupR if err != nil { return nil, err } + if result.SecurityGroupRules == nil { + return nil, nil + } return result.SecurityGroupRules[0], nil } diff --git a/test/e2e/suites/unmanaged/unmanaged_functional_test.go b/test/e2e/suites/unmanaged/unmanaged_functional_test.go index 47ea7429b7..19ec62c805 100644 --- a/test/e2e/suites/unmanaged/unmanaged_functional_test.go +++ b/test/e2e/suites/unmanaged/unmanaged_functional_test.go @@ -32,8 +32,10 @@ import ( "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/config" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1" "sigs.k8s.io/cluster-api-provider-aws/exp/instancestate" @@ -666,4 +668,222 @@ var _ = ginkgo.Context("[unmanaged] [functional]", func() { Expect(shared.DeleteVPC(e2eCtx, vpcID)).To(BeTrue()) }) }) + + ginkgo.Describe("Peerings and internal ELB", func() { + ginkgo.It("should create external clusters in peered VPC and with an internal ELB", func() { + specName := "functional-test-peered-internal-elb" + requiredResources = &shared.TestResource{EC2Normal: 2 * e2eCtx.Settings.InstanceVCPU, IGW: 2, NGW: 2, VPC: 2, ClassicLB: 2, EIP: 5} + requiredResources.WriteRequestedResources(e2eCtx, specName) + Expect(shared.AcquireResources(requiredResources, config.GinkgoConfig.ParallelNode, flock.New(shared.ResourceQuotaFilePath))).To(Succeed()) + defer shared.ReleaseResources(requiredResources, config.GinkgoConfig.ParallelNode, flock.New(shared.ResourceQuotaFilePath)) + namespace := shared.SetupSpecNamespace(ctx, specName, e2eCtx) + defer shared.DumpSpecResourcesAndCleanup(ctx, "", namespace, e2eCtx) + + ginkgo.By("Creating the management cluster infrastructure") + mgmtClusterName := fmt.Sprintf("cluster-%s", util.RandomString(6)) + mgmtClusterInfra := new(shared.AWSInfrastructure) + mgmtClusterInfra.New(shared.AWSInfrastructureSpec{ + ClusterName: mgmtClusterName, + VpcCidr: "10.0.0.0/23", + PublicSubnetCidr: "10.0.0.0/24", + PrivateSubnetCidr: "10.0.1.0/24", + }, e2eCtx) + mgmtClusterInfra.CreateInfrastructure() + Expect(mgmtClusterInfra.VPC).NotTo(BeNil()) + Expect(*mgmtClusterInfra.State.VpcState).To(Equal("available")) + Expect(len(mgmtClusterInfra.Subnets)).To(Equal(2)) + Expect(mgmtClusterInfra.InternetGateway).NotTo(BeNil()) + Expect(mgmtClusterInfra.ElasticIP).NotTo(BeNil()) + Expect(mgmtClusterInfra.NatGateway).NotTo(BeNil()) + Expect(len(mgmtClusterInfra.RouteTables)).To(Equal(2)) + + ginkgo.By("Creating the workload cluster infrastructure") + wlClusterName := fmt.Sprintf("cluster-%s", util.RandomString(6)) + wlClusterInfra := new(shared.AWSInfrastructure) + wlClusterInfra.New(shared.AWSInfrastructureSpec{ + ClusterName: wlClusterName, + VpcCidr: "10.0.2.0/23", + PublicSubnetCidr: "10.0.2.0/24", + PrivateSubnetCidr: "10.0.3.0/24", + }, e2eCtx) + wlClusterInfra.CreateInfrastructure() + Expect(wlClusterInfra.VPC).NotTo(BeNil()) + Expect(*wlClusterInfra.State.VpcState).To(Equal("available")) + Expect(len(wlClusterInfra.Subnets)).To(Equal(2)) + Expect(wlClusterInfra.InternetGateway).NotTo(BeNil()) + Expect(wlClusterInfra.ElasticIP).NotTo(BeNil()) + Expect(wlClusterInfra.NatGateway).NotTo(BeNil()) + Expect(len(wlClusterInfra.RouteTables)).To(Equal(2)) + + shared.SetEnvVar("MGMT_VPC_ID", *mgmtClusterInfra.VPC.VpcId, false) + shared.SetEnvVar("WL_VPC_ID", *wlClusterInfra.VPC.VpcId, false) + shared.SetEnvVar("MGMT_PUBLIC_SUBNET_ID", *mgmtClusterInfra.State.PublicSubnetID, false) + shared.SetEnvVar("MGMT_PRIVATE_SUBNET_ID", *mgmtClusterInfra.State.PrivateSubnetID, false) + shared.SetEnvVar("WL_PUBLIC_SUBNET_ID", *wlClusterInfra.State.PublicSubnetID, false) + shared.SetEnvVar("WL_PRIVATE_SUBNET_ID", *wlClusterInfra.State.PrivateSubnetID, false) + + ginkgo.By("Creating VPC peerings") + cPeering, _ := shared.CreatePeering(e2eCtx, mgmtClusterName+"-"+wlClusterName, *mgmtClusterInfra.VPC.VpcId, *wlClusterInfra.VPC.VpcId) + Expect(cPeering).NotTo(BeNil()) + Eventually(func() bool { + aPeering, err := shared.AcceptPeering(e2eCtx, *cPeering.VpcPeeringConnectionId) + if err != nil { + return false + } + return aPeering != nil + }, 60*time.Second).Should(BeTrue()) + + ginkgo.By("Creating routes for peerings") + shared.CreateRoute(e2eCtx, *mgmtClusterInfra.State.PublicRouteTableID, "10.0.2.0/23", nil, nil, cPeering.VpcPeeringConnectionId) + shared.CreateRoute(e2eCtx, *mgmtClusterInfra.State.PrivateRouteTableID, "10.0.2.0/23", nil, nil, cPeering.VpcPeeringConnectionId) + shared.CreateRoute(e2eCtx, *wlClusterInfra.State.PublicRouteTableID, "10.0.0.0/23", nil, nil, cPeering.VpcPeeringConnectionId) + shared.CreateRoute(e2eCtx, *wlClusterInfra.State.PrivateRouteTableID, "10.0.0.0/23", nil, nil, cPeering.VpcPeeringConnectionId) + + ginkgo.By("Creating a management cluster in a peered VPC") + mgmtConfigCluster := defaultConfigCluster(mgmtClusterName, namespace.Name) + mgmtConfigCluster.WorkerMachineCount = pointer.Int64Ptr(1) + mgmtConfigCluster.Flavor = "peered-remote" + mgmtCluster, mgmtMD, _ := createCluster(ctx, mgmtConfigCluster, result) + + mgmtWM := framework.GetMachinesByMachineDeployments(ctx, framework.GetMachinesByMachineDeploymentsInput{ + Lister: e2eCtx.Environment.BootstrapClusterProxy.GetClient(), + ClusterName: mgmtClusterName, + Namespace: namespace.Name, + MachineDeployment: *mgmtMD[0], + }) + mgmtCPM := framework.GetControlPlaneMachinesByCluster(ctx, framework.GetControlPlaneMachinesByClusterInput{ + Lister: e2eCtx.Environment.BootstrapClusterProxy.GetClient(), + ClusterName: mgmtClusterName, + Namespace: namespace.Name, + }) + Expect(len(mgmtWM)).To(Equal(1)) + Expect(len(mgmtCPM)).To(Equal(1)) + + mgmtClusterProxy := e2eCtx.Environment.BootstrapClusterProxy.GetWorkloadCluster(ctx, mgmtCluster.Namespace, mgmtCluster.Name) + + shared.Byf("Creating a namespace for hosting the %s test spec", specName) + mgmtNamespace := framework.CreateNamespace(ctx, framework.CreateNamespaceInput{ + Creator: mgmtClusterProxy.GetClient(), + Name: namespace.Name, + }) + + ginkgo.By("Initializing the management cluster") + clusterctl.InitManagementClusterAndWatchControllerLogs(ctx, clusterctl.InitManagementClusterAndWatchControllerLogsInput{ + ClusterProxy: mgmtClusterProxy, + ClusterctlConfigPath: e2eCtx.Environment.ClusterctlConfigPath, + InfrastructureProviders: e2eCtx.E2EConfig.InfrastructureProviders(), + LogFolder: filepath.Join(e2eCtx.Settings.ArtifactFolder, "clusters", mgmtCluster.Name), + }, e2eCtx.E2EConfig.GetIntervals(specName, "wait-controllers")...) + + ginkgo.By("Ensure API servers are stable before doing the move") + Consistently(func() error { + kubeSystem := &corev1.Namespace{} + return e2eCtx.Environment.BootstrapClusterProxy.GetClient().Get(ctx, client.ObjectKey{Name: "kube-system"}, kubeSystem) + }, "5s", "100ms").Should(BeNil(), "Failed to assert bootstrap API server stability") + Consistently(func() error { + kubeSystem := &corev1.Namespace{} + return mgmtClusterProxy.GetClient().Get(ctx, client.ObjectKey{Name: "kube-system"}, kubeSystem) + }, "5s", "100ms").Should(BeNil(), "Failed to assert management API server stability") + + ginkgo.By("Moving the management cluster to be self hosted") + clusterctl.Move(ctx, clusterctl.MoveInput{ + LogFolder: filepath.Join(e2eCtx.Settings.ArtifactFolder, "clusters", "bootstrap"), + ClusterctlConfigPath: e2eCtx.Environment.ClusterctlConfigPath, + FromKubeconfigPath: e2eCtx.Environment.BootstrapClusterProxy.GetKubeconfigPath(), + ToKubeconfigPath: mgmtClusterProxy.GetKubeconfigPath(), + Namespace: namespace.Name, + }) + + mgmtCluster = framework.DiscoveryAndWaitForCluster(ctx, framework.DiscoveryAndWaitForClusterInput{ + Getter: mgmtClusterProxy.GetClient(), + Namespace: mgmtNamespace.Name, + Name: mgmtCluster.Name, + }, e2eCtx.E2EConfig.GetIntervals(specName, "wait-cluster")...) + + mgmtControlPlane := framework.GetKubeadmControlPlaneByCluster(ctx, framework.GetKubeadmControlPlaneByClusterInput{ + Lister: mgmtClusterProxy.GetClient(), + ClusterName: mgmtCluster.Name, + Namespace: mgmtCluster.Namespace, + }) + Expect(mgmtControlPlane).ToNot(BeNil()) + + ginkgo.By("Creating a namespace to host the internal-elb spec") + wlNamespace := framework.CreateNamespace(ctx, framework.CreateNamespaceInput{ + Creator: mgmtClusterProxy.GetClient(), + Name: wlClusterName, + }) + + ginkgo.By("Creating workload cluster with internal ELB") + wlConfigCluster := defaultConfigCluster(wlClusterName, wlNamespace.Name) + wlConfigCluster.WorkerMachineCount = pointer.Int64Ptr(1) + wlConfigCluster.Flavor = "internal-elb" + wlResult := &clusterctl.ApplyClusterTemplateAndWaitResult{} + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: mgmtClusterProxy, + ConfigCluster: wlConfigCluster, + WaitForClusterIntervals: e2eCtx.E2EConfig.GetIntervals("", "wait-cluster"), + WaitForControlPlaneIntervals: e2eCtx.E2EConfig.GetIntervals("", "wait-control-plane"), + WaitForMachineDeployments: e2eCtx.E2EConfig.GetIntervals("", "wait-worker-nodes"), + }, wlResult) + + wlWM := framework.GetMachinesByMachineDeployments(ctx, framework.GetMachinesByMachineDeploymentsInput{ + Lister: mgmtClusterProxy.GetClient(), + ClusterName: mgmtClusterName, + Namespace: wlNamespace.Name, + MachineDeployment: *wlResult.MachineDeployments[0], + }) + wlCPM := framework.GetControlPlaneMachinesByCluster(ctx, framework.GetControlPlaneMachinesByClusterInput{ + Lister: mgmtClusterProxy.GetClient(), + ClusterName: wlClusterName, + Namespace: wlNamespace.Name, + }) + Expect(len(wlWM)).To(Equal(1)) + Expect(len(wlCPM)).To(Equal(1)) + + ginkgo.By("Deleting the workload cluster") + framework.DeleteCluster(ctx, framework.DeleteClusterInput{ + Deleter: mgmtClusterProxy.GetClient(), + Cluster: wlResult.Cluster, + }) + + framework.WaitForClusterDeleted(ctx, framework.WaitForClusterDeletedInput{ + Getter: mgmtClusterProxy.GetClient(), + Cluster: wlResult.Cluster, + }, e2eCtx.E2EConfig.GetIntervals("", "wait-delete-cluster")...) + + ginkgo.By("Moving the management cluster back to bootstrap") + clusterctl.Move(ctx, clusterctl.MoveInput{ + LogFolder: filepath.Join(e2eCtx.Settings.ArtifactFolder, "clusters", mgmtCluster.Name), + ClusterctlConfigPath: e2eCtx.Environment.ClusterctlConfigPath, + FromKubeconfigPath: mgmtClusterProxy.GetKubeconfigPath(), + ToKubeconfigPath: e2eCtx.Environment.BootstrapClusterProxy.GetKubeconfigPath(), + Namespace: namespace.Name, + }) + + mgmtCluster = framework.DiscoveryAndWaitForCluster(ctx, framework.DiscoveryAndWaitForClusterInput{ + Getter: e2eCtx.Environment.BootstrapClusterProxy.GetClient(), + Namespace: mgmtNamespace.Name, + Name: mgmtCluster.Name, + }, e2eCtx.E2EConfig.GetIntervals(specName, "wait-cluster")...) + + mgmtControlPlane = framework.GetKubeadmControlPlaneByCluster(ctx, framework.GetKubeadmControlPlaneByClusterInput{ + Lister: e2eCtx.Environment.BootstrapClusterProxy.GetClient(), + ClusterName: mgmtCluster.Name, + Namespace: mgmtCluster.Namespace, + }) + Expect(mgmtControlPlane).ToNot(BeNil()) + + ginkgo.By("Deleting the management cluster") + deleteCluster(ctx, mgmtCluster) + + ginkgo.By("Deleting peering connection") + Expect(shared.DeletePeering(e2eCtx, *cPeering.VpcPeeringConnectionId)).To(BeTrue()) + + ginkgo.By("Deleting the workload cluster infrastructure") + wlClusterInfra.DeleteInfrastructure() + + ginkgo.By("Deleting the management cluster infrastructure") + mgmtClusterInfra.DeleteInfrastructure() + }) + }) })