From 5b08cef72c4a8ad1daa7a7aad168863752d0e738 Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Thu, 2 Jan 2025 18:15:07 +0000 Subject: [PATCH 1/2] improve log dumper get more networking information usefult to troubleshoot network issues. --- pkg/dump/dumper.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pkg/dump/dumper.go b/pkg/dump/dumper.go index 138642acffdfe..3c3aee7d6e2aa 100644 --- a/pkg/dump/dumper.go +++ b/pkg/dump/dumper.go @@ -345,7 +345,19 @@ func (n *logDumperNode) dump(ctx context.Context) []error { if err := n.shellToFile(ctx, "sudo iptables -t filter --list-rules", filepath.Join(n.dir, "iptables-filter.log")); err != nil { errors = append(errors, err) } - if err := n.shellToFile(ctx, "ip route", filepath.Join(n.dir, "ip-routes.log")); err != nil { + if err := n.shellToFile(ctx, "sudo nft list ruleset", filepath.Join(n.dir, "nftables-ruleset.log")); err != nil { + errors = append(errors, err) + } + if err := n.shellToFile(ctx, "ip route show table all", filepath.Join(n.dir, "ip-routes.log")); err != nil { + errors = append(errors, err) + } + if err := n.shellToFile(ctx, "ip rule list", filepath.Join(n.dir, "ip-rules.log")); err != nil { + errors = append(errors, err) + } + if err := n.shellToFile(ctx, "ip -s link", filepath.Join(n.dir, "ip-link.log")); err != nil { + errors = append(errors, err) + } + if err := n.shellToFile(ctx, "ss -s", filepath.Join(n.dir, "netstat.log")); err != nil { errors = append(errors, err) } From f2c239dd81e9b724bb34b245c8a8a8258fa91259 Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Sun, 29 Dec 2024 18:09:57 +0000 Subject: [PATCH 2/2] add kindnet network plugin add kindnet as an experimental network addon containerd adds the requirement to use the loopback cni plugin, kindnet provides that capability and containerd does not require it since containerd/containerd/pull/10238 Change-Id: I1397a90186885b02e98b5ffa444fe629c1046757 --- cmd/kops/create_cluster.go | 3 +- cmd/kops/integration_test.go | 10 + docs/cli/kops_create_cluster.md | 2 +- docs/networking.md | 2 + docs/networking/kindnet.md | 30 + k8s/crds/kops.k8s.io_clusters.yaml | 33 + nodeup/pkg/model/networking/kindnet.go | 38 + pkg/apis/kops/networking.go | 27 + pkg/apis/kops/v1alpha2/networking.go | 22 +- .../kops/v1alpha2/zz_generated.conversion.go | 112 + .../kops/v1alpha2/zz_generated.deepcopy.go | 87 + pkg/apis/kops/v1alpha3/networking.go | 19 + .../kops/v1alpha3/zz_generated.conversion.go | 112 + .../kops/v1alpha3/zz_generated.deepcopy.go | 87 + pkg/apis/kops/validation/validation.go | 26 + pkg/apis/kops/validation/validation_test.go | 67 + pkg/apis/kops/zz_generated.deepcopy.go | 87 + pkg/apis/nodeup/config.go | 4 + pkg/model/components/kindnet.go | 68 + pkg/model/iam/iam_builder.go | 18 + ...net.example.com-ASGLifecycle_event_pattern | 1 + ....com-InstanceScheduledChange_event_pattern | 1 + ...mple.com-InstanceStateChange_event_pattern | 1 + ...example.com-SpotInterruption_event_pattern | 1 + ...bastions.privatekindnet.example.com_policy | 10 + ..._masters.privatekindnet.example.com_policy | 10 + ...le_nodes.privatekindnet.example.com_policy | 10 + ...bastions.privatekindnet.example.com_policy | 10 + ..._masters.privatekindnet.example.com_policy | 277 ++ ...cy_nodes.privatekindnet.example.com_policy | 31 + ...4a6ed9aa889b9e2c39cd663eb9c7157_public_key | 1 + ...sters.privatekindnet.example.com_user_data | 138 + ...nodes.privatekindnet.example.com_user_data | 161 ++ ...s_s3_object_cluster-completed.spec_content | 229 ++ ...s3_object_etcd-cluster-spec-events_content | 4 + ...s_s3_object_etcd-cluster-spec-main_content | 4 + .../aws_s3_object_kops-version.txt_content | 1 + ...cdmanager-events-master-us-test-1a_content | 138 + ...etcdmanager-main-master-us-test-1a_content | 138 + ...-static-kube-apiserver-healthcheck_content | 33 + ...ect_nodeupconfig-master-us-test-1a_content | 333 +++ .../aws_s3_object_nodeupconfig-nodes_content | 66 + ...-controller.addons.k8s.io-k8s-1.18_content | 237 ++ ...-csi-driver.addons.k8s.io-k8s-1.17_content | 1122 ++++++++ ...ndnet.example.com-addons-bootstrap_content | 121 + ...ons-coredns.addons.k8s.io-k8s-1.12_content | 383 +++ ...-controller.addons.k8s.io-k8s-1.12_content | 138 + ...-controller.addons.k8s.io-k8s-1.16_content | 227 ++ ...let-api.rbac.addons.k8s.io-k8s-1.9_content | 17 + ...m-addons-limit-range.addons.k8s.io_content | 15 + ...addons-networking.kindnet-k8s-1.32_content | 163 ++ ...e-termination-handler.aws-k8s-1.11_content | 285 ++ ...-storage-aws.addons.k8s.io-v1.15.0_content | 118 + ...ueue_privatekindnet-example-com-nth_policy | 16 + .../update_cluster/privatekindnet/id_rsa.pub | 1 + .../privatekindnet/in-v1alpha2.yaml | 98 + .../privatekindnet/kubernetes.tf | 1467 +++++++++++ .../networking.kindnet/k8s-1.32.yaml.template | 2299 +++++++++++++++++ .../bootstrapchannelbuilder.go | 17 + upup/pkg/fi/cloudup/new_cluster.go | 2 + upup/pkg/fi/cloudup/new_cluster_test.go | 12 + upup/pkg/fi/cloudup/populate_cluster_spec.go | 1 + upup/pkg/fi/nodeup/command.go | 17 +- 63 files changed, 9201 insertions(+), 7 deletions(-) create mode 100644 docs/networking/kindnet.md create mode 100644 nodeup/pkg/model/networking/kindnet.go create mode 100644 pkg/model/components/kindnet.go create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_cloudwatch_event_rule_privatekindnet.example.com-ASGLifecycle_event_pattern create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_cloudwatch_event_rule_privatekindnet.example.com-InstanceScheduledChange_event_pattern create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_cloudwatch_event_rule_privatekindnet.example.com-InstanceStateChange_event_pattern create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_cloudwatch_event_rule_privatekindnet.example.com-SpotInterruption_event_pattern create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_iam_role_bastions.privatekindnet.example.com_policy create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_iam_role_masters.privatekindnet.example.com_policy create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_iam_role_nodes.privatekindnet.example.com_policy create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_iam_role_policy_bastions.privatekindnet.example.com_policy create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_iam_role_policy_masters.privatekindnet.example.com_policy create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_iam_role_policy_nodes.privatekindnet.example.com_policy create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_key_pair_kubernetes.privatekindnet.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_launch_template_master-us-test-1a.masters.privatekindnet.example.com_user_data create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_launch_template_nodes.privatekindnet.example.com_user_data create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_s3_object_cluster-completed.spec_content create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_s3_object_etcd-cluster-spec-events_content create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_s3_object_etcd-cluster-spec-main_content create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_s3_object_kops-version.txt_content create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_s3_object_manifests-etcdmanager-events-master-us-test-1a_content create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_s3_object_manifests-etcdmanager-main-master-us-test-1a_content create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_s3_object_manifests-static-kube-apiserver-healthcheck_content create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_s3_object_nodeupconfig-master-us-test-1a_content create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_s3_object_nodeupconfig-nodes_content create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-bootstrap_content create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-limit-range.addons.k8s.io_content create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-networking.kindnet-k8s-1.32_content create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-node-termination-handler.aws-k8s-1.11_content create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content create mode 100644 tests/integration/update_cluster/privatekindnet/data/aws_sqs_queue_privatekindnet-example-com-nth_policy create mode 100644 tests/integration/update_cluster/privatekindnet/id_rsa.pub create mode 100644 tests/integration/update_cluster/privatekindnet/in-v1alpha2.yaml create mode 100644 tests/integration/update_cluster/privatekindnet/kubernetes.tf create mode 100644 upup/models/cloudup/resources/addons/networking.kindnet/k8s-1.32.yaml.template diff --git a/cmd/kops/create_cluster.go b/cmd/kops/create_cluster.go index 054e5a60e50fa..cbd2ca4498345 100644 --- a/cmd/kops/create_cluster.go +++ b/cmd/kops/create_cluster.go @@ -314,7 +314,7 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command { cmd.Flags().StringVar(&options.EtcdStorageType, "etcd-storage-type", options.EtcdStorageType, "The default storage type for etcd members") cmd.RegisterFlagCompletionFunc("etcd-storage-type", completeStorageType) - cmd.Flags().StringVar(&options.Networking, "networking", options.Networking, "Networking mode. kubenet, external, flannel-vxlan (or flannel), flannel-udp, calico, canal, kube-router, amazonvpc, cilium, cilium-etcd, cni.") + cmd.Flags().StringVar(&options.Networking, "networking", options.Networking, "Networking mode. kubenet, external, flannel-vxlan (or flannel), flannel-udp, calico, canal, kube-router, amazonvpc, cilium, cilium-etcd, kindnet, cni.") cmd.RegisterFlagCompletionFunc("networking", completeNetworking(options)) cmd.Flags().StringVar(&options.DNSZone, "dns-zone", options.DNSZone, "DNS hosted zone (defaults to longest matching zone)") @@ -983,6 +983,7 @@ func completeNetworking(options *CreateClusterOptions) func(cmd *cobra.Command, "cilium", "cilium-eni", "cilium-etcd", + "kindnet", } if !options.IPv6 { diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index 7b07a9f7a57d9..61e1a1083a81c 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -218,6 +218,7 @@ const ( canalAddon = "networking.projectcalico.org.canal-k8s-1.25" ciliumAddon = "networking.cilium.io-k8s-1.16" flannelAddon = "networking.flannel-k8s-1.25" + kindnetAddon = "networking.kindnet-k8s-1.32" certManagerAddon = "certmanager.io-k8s-1.16" clusterAutoscalerAddon = "cluster-autoscaler.addons.k8s.io-k8s-1.15" @@ -618,6 +619,15 @@ func TestPrivateFlannel(t *testing.T) { runTestTerraformAWS(t) } +// TestPrivateKindnet runs the test on a configuration with private topology, flannel networking +func TestPrivateKindnet(t *testing.T) { + newIntegrationTest("privatekindnet.example.com", "privatekindnet"). + withPrivate(). + withDefaultAddons30(). + withAddons(kindnetAddon). + runTestTerraformAWS(t) +} + // TestPrivateCalico runs the test on a configuration with private topology, calico networking func TestPrivateCalico(t *testing.T) { newIntegrationTest("privatecalico.example.com", "privatecalico"). diff --git a/docs/cli/kops_create_cluster.md b/docs/cli/kops_create_cluster.md index 8bde12799ae2f..13bae47971217 100644 --- a/docs/cli/kops_create_cluster.md +++ b/docs/cli/kops_create_cluster.md @@ -100,7 +100,7 @@ kops create cluster [CLUSTER] [flags] --kubernetes-version string Version of Kubernetes to run (defaults to version in channel) --network-cidr strings Network CIDR(s) to use --network-id string Shared Network or VPC to use - --networking string Networking mode. kubenet, external, flannel-vxlan (or flannel), flannel-udp, calico, canal, kube-router, amazonvpc, cilium, cilium-etcd, cni. (default "cilium") + --networking string Networking mode. kubenet, external, flannel-vxlan (or flannel), flannel-udp, calico, canal, kube-router, amazonvpc, cilium, cilium-etcd, kindnet, cni. (default "cilium") --node-count int32 Total number of worker nodes. Defaults to one node per zone --node-image string Machine image for worker nodes. Takes precedence over --image --node-security-groups strings Additional pre-created security groups to add to worker nodes. diff --git a/docs/networking.md b/docs/networking.md index 34a3f2e9dc411..4a9707e83d0c1 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -23,6 +23,7 @@ As of kOps 1.26 the default network provider is Cilium. Prior to that the defaul | Cilium ENI | 1.18 | 1.26 | - | - | | Flannel udp | 1.5.2 | - | 1.27 | Kubernetes 1.28 | | Flannel vxlan | 1.8.0 | - | 1.27 | Kubernetes 1.28 | +| Kindnet | 1.32 | - | - | - | | Kopeio | 1.5 | - | - | - | | Kube-router | 1.6.2 | - | 1.27 | Kubernetes 1.28 | | Kubenet | 1.5 | 1.5 | - | - | @@ -69,6 +70,7 @@ Several CNI providers are currently built into kOps: * [Canal](networking/canal.md) * [Cilium](networking/cilium.md) * [Flannel](networking/flannel.md) +* [Kindnet](networking/kindnet.md) * [Kube-router](networking/kube-router.md) kOps makes it easy for cluster operators to choose one of these options. The manifests for the providers diff --git a/docs/networking/kindnet.md b/docs/networking/kindnet.md new file mode 100644 index 0000000000000..f6a2fafcc51b9 --- /dev/null +++ b/docs/networking/kindnet.md @@ -0,0 +1,30 @@ +# Kindnet + +***Kindnet support is experimental, and may be removed at any time*** + +## Introduction + +* [kindnet](http://kindnet.es) + +Kindnet focuses on providing essential networking functionality without unnecessary complexity. + +## Installing + +To install [kindnet](https://github.com/aojea/kindnet) - use `--networking kindnet`. + +```sh +export ZONES=mylistofzone +kops create cluster \ + --zones $ZONES \ + --networking kindnet \ + --yes \ + --name myclustername.mydns.io +``` + +## Getting help + +For problems with kindnet please post an issue to Github: + +- [Kindnet Issues](https://github.com/aojea/kindnet/issues) + +You can learn more about the different configurations options in https://kindnet.es/ diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index 782ee35692309..118c8d16135d5 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -5773,6 +5773,39 @@ spec: description: GCPNetworkingSpec is the specification of GCP's native networking mode, using IP aliases. type: object + kindnet: + description: KindnetNetworkingSpec configures Kindnet settings. + properties: + adminNetworkPolicies: + type: boolean + baselineAdminNetworkPolicies: + type: boolean + dnsCaching: + type: boolean + fastPathThreshold: + format: int32 + type: integer + logLevel: + format: int32 + type: integer + masquerade: + description: KindnetMasqueradeSpec configures Kindnet masquerading + settings. + properties: + enabled: + type: boolean + nonMasqueradeCIDRs: + items: + type: string + type: array + type: object + nat64: + type: boolean + networkPolicies: + type: boolean + version: + type: string + type: object kopeio: description: KopeioNetworkingSpec declares that we want Kopeio networking diff --git a/nodeup/pkg/model/networking/kindnet.go b/nodeup/pkg/model/networking/kindnet.go new file mode 100644 index 0000000000000..fa85fee6fde06 --- /dev/null +++ b/nodeup/pkg/model/networking/kindnet.go @@ -0,0 +1,38 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package networking + +import ( + "k8s.io/kops/nodeup/pkg/model" + "k8s.io/kops/upup/pkg/fi" +) + +// KindnetBuilder for Kindnet +type KindnetBuilder struct { + *model.NodeupModelContext +} + +var _ fi.NodeupModelBuilder = &KindnetBuilder{} + +// Build is responsible for performing setup for Kindnet. +func (b *KindnetBuilder) Build(c *fi.NodeupModelBuilderContext) error { + if b.NodeupConfig.Networking.Kindnet == nil { + return nil + } + + return nil +} diff --git a/pkg/apis/kops/networking.go b/pkg/apis/kops/networking.go index 3edb9b2c20eac..0406abdcef216 100644 --- a/pkg/apis/kops/networking.go +++ b/pkg/apis/kops/networking.go @@ -78,6 +78,7 @@ type NetworkingSpec struct { Cilium *CiliumNetworkingSpec `json:"cilium,omitempty"` LyftVPC *LyftVPCNetworkingSpec `json:"lyftvpc,omitempty"` GCP *GCPNetworkingSpec `json:"gcp,omitempty"` + Kindnet *KindnetNetworkingSpec `json:"kindnet,omitempty"` } // UsesKubenet returns true if our networking is derived from kubenet @@ -553,3 +554,29 @@ type LyftVPCNetworkingSpec struct { // GCPNetworkingSpec is the specification of GCP's native networking mode, using IP aliases. type GCPNetworkingSpec struct{} + +// KindnetNetworkingSpec configures Kindnet settings. +type KindnetNetworkingSpec struct { + // Version is the version of the kindnet agent. + // Default: v1.8.0 + Version string `json:"version,omitempty"` + // Enable network policies + NetworkPolicies *bool `json:"networkPolicies,omitempty"` + AdminNetworkPolicies *bool `json:"adminNetworkPolicies,omitempty"` + BaselineAdminNetworkPolicies *bool `json:"baselineAdminNetworkPolicies,omitempty"` + // enable dns caching + DNSCaching *bool `json:"dnsCaching,omitempty"` + // enable nat64 on ipv6 clusters + NAT64 *bool `json:"nat64,omitempty"` + // number of packets in a connection to offload it to the fast path + FastPathThreshold *int32 `json:"fastPathThreshold,omitempty"` + // node agent masquerading rules + Masquerade *KindnetMasqueradeSpec `json:"masquerade,omitempty"` + // log level + LogLevel *int32 `json:"logLevel,omitempty"` +} + +type KindnetMasqueradeSpec struct { + Enabled *bool `json:"enabled,omitempty"` + NonMasqueradeCIDRs []string `json:"nonMasqueradeCIDRs,omitempty"` +} diff --git a/pkg/apis/kops/v1alpha2/networking.go b/pkg/apis/kops/v1alpha2/networking.go index 09af2790af39a..f17de85b2bc7d 100644 --- a/pkg/apis/kops/v1alpha2/networking.go +++ b/pkg/apis/kops/v1alpha2/networking.go @@ -49,12 +49,13 @@ type NetworkingSpec struct { Cilium *CiliumNetworkingSpec `json:"cilium,omitempty"` LyftVPC *LyftVPCNetworkingSpec `json:"lyftvpc,omitempty"` GCP *GCPNetworkingSpec `json:"gce,omitempty"` + Kindnet *KindnetNetworkingSpec `json:"kindnet,omitempty"` } func (s *NetworkingSpec) IsEmpty() bool { return s.Classic == nil && s.Kubenet == nil && s.External == nil && s.CNI == nil && s.Kopeio == nil && s.Weave == nil && s.Flannel == nil && s.Calico == nil && s.Canal == nil && s.KubeRouter == nil && - s.Romana == nil && s.AmazonVPC == nil && s.Cilium == nil && s.LyftVPC == nil && s.GCP == nil + s.Romana == nil && s.AmazonVPC == nil && s.Cilium == nil && s.LyftVPC == nil && s.GCP == nil && s.Kindnet == nil } // ClassicNetworkingSpec is the specification of classic networking mode, integrated into kubernetes. @@ -674,3 +675,22 @@ type LyftVPCNetworkingSpec struct { // GCPNetworkingSpec is the specification of GCP's native networking mode, using IP aliases. type GCPNetworkingSpec struct{} + +// KindnetNetworkingSpec configures Kindnet settings. +type KindnetNetworkingSpec struct { + Version string `json:"version,omitempty"` + NetworkPolicies *bool `json:"networkPolicies,omitempty"` + AdminNetworkPolicies *bool `json:"adminNetworkPolicies,omitempty"` + BaselineAdminNetworkPolicies *bool `json:"baselineAdminNetworkPolicies,omitempty"` + DNSCaching *bool `json:"dnsCaching,omitempty"` + NAT64 *bool `json:"nat64,omitempty"` + FastPathThreshold *int32 `json:"fastPathThreshold,omitempty"` + Masquerade *KindnetMasqueradeSpec `json:"masquerade,omitempty"` + LogLevel *int32 `json:"logLevel,omitempty"` +} + +// KindnetMasqueradeSpec configures Kindnet masquerading settings. +type KindnetMasqueradeSpec struct { + Enabled *bool `json:"enabled,omitempty"` + NonMasqueradeCIDRs []string `json:"nonMasqueradeCIDRs,omitempty"` +} diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 898c243e04dc0..6913427497081 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -624,6 +624,26 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*KindnetMasqueradeSpec)(nil), (*kops.KindnetMasqueradeSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha2_KindnetMasqueradeSpec_To_kops_KindnetMasqueradeSpec(a.(*KindnetMasqueradeSpec), b.(*kops.KindnetMasqueradeSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*kops.KindnetMasqueradeSpec)(nil), (*KindnetMasqueradeSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kops_KindnetMasqueradeSpec_To_v1alpha2_KindnetMasqueradeSpec(a.(*kops.KindnetMasqueradeSpec), b.(*KindnetMasqueradeSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*KindnetNetworkingSpec)(nil), (*kops.KindnetNetworkingSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha2_KindnetNetworkingSpec_To_kops_KindnetNetworkingSpec(a.(*KindnetNetworkingSpec), b.(*kops.KindnetNetworkingSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*kops.KindnetNetworkingSpec)(nil), (*KindnetNetworkingSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kops_KindnetNetworkingSpec_To_v1alpha2_KindnetNetworkingSpec(a.(*kops.KindnetNetworkingSpec), b.(*KindnetNetworkingSpec), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*KopeioAuthenticationSpec)(nil), (*kops.KopeioAuthenticationSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_KopeioAuthenticationSpec_To_kops_KopeioAuthenticationSpec(a.(*KopeioAuthenticationSpec), b.(*kops.KopeioAuthenticationSpec), scope) }); err != nil { @@ -4831,6 +4851,80 @@ func Convert_kops_KeysetSpec_To_v1alpha2_KeysetSpec(in *kops.KeysetSpec, out *Ke return autoConvert_kops_KeysetSpec_To_v1alpha2_KeysetSpec(in, out, s) } +func autoConvert_v1alpha2_KindnetMasqueradeSpec_To_kops_KindnetMasqueradeSpec(in *KindnetMasqueradeSpec, out *kops.KindnetMasqueradeSpec, s conversion.Scope) error { + out.Enabled = in.Enabled + out.NonMasqueradeCIDRs = in.NonMasqueradeCIDRs + return nil +} + +// Convert_v1alpha2_KindnetMasqueradeSpec_To_kops_KindnetMasqueradeSpec is an autogenerated conversion function. +func Convert_v1alpha2_KindnetMasqueradeSpec_To_kops_KindnetMasqueradeSpec(in *KindnetMasqueradeSpec, out *kops.KindnetMasqueradeSpec, s conversion.Scope) error { + return autoConvert_v1alpha2_KindnetMasqueradeSpec_To_kops_KindnetMasqueradeSpec(in, out, s) +} + +func autoConvert_kops_KindnetMasqueradeSpec_To_v1alpha2_KindnetMasqueradeSpec(in *kops.KindnetMasqueradeSpec, out *KindnetMasqueradeSpec, s conversion.Scope) error { + out.Enabled = in.Enabled + out.NonMasqueradeCIDRs = in.NonMasqueradeCIDRs + return nil +} + +// Convert_kops_KindnetMasqueradeSpec_To_v1alpha2_KindnetMasqueradeSpec is an autogenerated conversion function. +func Convert_kops_KindnetMasqueradeSpec_To_v1alpha2_KindnetMasqueradeSpec(in *kops.KindnetMasqueradeSpec, out *KindnetMasqueradeSpec, s conversion.Scope) error { + return autoConvert_kops_KindnetMasqueradeSpec_To_v1alpha2_KindnetMasqueradeSpec(in, out, s) +} + +func autoConvert_v1alpha2_KindnetNetworkingSpec_To_kops_KindnetNetworkingSpec(in *KindnetNetworkingSpec, out *kops.KindnetNetworkingSpec, s conversion.Scope) error { + out.Version = in.Version + out.NetworkPolicies = in.NetworkPolicies + out.AdminNetworkPolicies = in.AdminNetworkPolicies + out.BaselineAdminNetworkPolicies = in.BaselineAdminNetworkPolicies + out.DNSCaching = in.DNSCaching + out.NAT64 = in.NAT64 + out.FastPathThreshold = in.FastPathThreshold + if in.Masquerade != nil { + in, out := &in.Masquerade, &out.Masquerade + *out = new(kops.KindnetMasqueradeSpec) + if err := Convert_v1alpha2_KindnetMasqueradeSpec_To_kops_KindnetMasqueradeSpec(*in, *out, s); err != nil { + return err + } + } else { + out.Masquerade = nil + } + out.LogLevel = in.LogLevel + return nil +} + +// Convert_v1alpha2_KindnetNetworkingSpec_To_kops_KindnetNetworkingSpec is an autogenerated conversion function. +func Convert_v1alpha2_KindnetNetworkingSpec_To_kops_KindnetNetworkingSpec(in *KindnetNetworkingSpec, out *kops.KindnetNetworkingSpec, s conversion.Scope) error { + return autoConvert_v1alpha2_KindnetNetworkingSpec_To_kops_KindnetNetworkingSpec(in, out, s) +} + +func autoConvert_kops_KindnetNetworkingSpec_To_v1alpha2_KindnetNetworkingSpec(in *kops.KindnetNetworkingSpec, out *KindnetNetworkingSpec, s conversion.Scope) error { + out.Version = in.Version + out.NetworkPolicies = in.NetworkPolicies + out.AdminNetworkPolicies = in.AdminNetworkPolicies + out.BaselineAdminNetworkPolicies = in.BaselineAdminNetworkPolicies + out.DNSCaching = in.DNSCaching + out.NAT64 = in.NAT64 + out.FastPathThreshold = in.FastPathThreshold + if in.Masquerade != nil { + in, out := &in.Masquerade, &out.Masquerade + *out = new(KindnetMasqueradeSpec) + if err := Convert_kops_KindnetMasqueradeSpec_To_v1alpha2_KindnetMasqueradeSpec(*in, *out, s); err != nil { + return err + } + } else { + out.Masquerade = nil + } + out.LogLevel = in.LogLevel + return nil +} + +// Convert_kops_KindnetNetworkingSpec_To_v1alpha2_KindnetNetworkingSpec is an autogenerated conversion function. +func Convert_kops_KindnetNetworkingSpec_To_v1alpha2_KindnetNetworkingSpec(in *kops.KindnetNetworkingSpec, out *KindnetNetworkingSpec, s conversion.Scope) error { + return autoConvert_kops_KindnetNetworkingSpec_To_v1alpha2_KindnetNetworkingSpec(in, out, s) +} + func autoConvert_v1alpha2_KopeioAuthenticationSpec_To_kops_KopeioAuthenticationSpec(in *KopeioAuthenticationSpec, out *kops.KopeioAuthenticationSpec, s conversion.Scope) error { return nil } @@ -6224,6 +6318,15 @@ func autoConvert_v1alpha2_NetworkingSpec_To_kops_NetworkingSpec(in *NetworkingSp } else { out.GCP = nil } + if in.Kindnet != nil { + in, out := &in.Kindnet, &out.Kindnet + *out = new(kops.KindnetNetworkingSpec) + if err := Convert_v1alpha2_KindnetNetworkingSpec_To_kops_KindnetNetworkingSpec(*in, *out, s); err != nil { + return err + } + } else { + out.Kindnet = nil + } return nil } @@ -6405,6 +6508,15 @@ func autoConvert_kops_NetworkingSpec_To_v1alpha2_NetworkingSpec(in *kops.Network } else { out.GCP = nil } + if in.Kindnet != nil { + in, out := &in.Kindnet, &out.Kindnet + *out = new(KindnetNetworkingSpec) + if err := Convert_kops_KindnetNetworkingSpec_To_v1alpha2_KindnetNetworkingSpec(*in, *out, s); err != nil { + return err + } + } else { + out.Kindnet = nil + } return nil } diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index 99f85e64b9a70..db9773eec5318 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -3011,6 +3011,88 @@ func (in *KeysetSpec) DeepCopy() *KeysetSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KindnetMasqueradeSpec) DeepCopyInto(out *KindnetMasqueradeSpec) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + if in.NonMasqueradeCIDRs != nil { + in, out := &in.NonMasqueradeCIDRs, &out.NonMasqueradeCIDRs + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KindnetMasqueradeSpec. +func (in *KindnetMasqueradeSpec) DeepCopy() *KindnetMasqueradeSpec { + if in == nil { + return nil + } + out := new(KindnetMasqueradeSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KindnetNetworkingSpec) DeepCopyInto(out *KindnetNetworkingSpec) { + *out = *in + if in.NetworkPolicies != nil { + in, out := &in.NetworkPolicies, &out.NetworkPolicies + *out = new(bool) + **out = **in + } + if in.AdminNetworkPolicies != nil { + in, out := &in.AdminNetworkPolicies, &out.AdminNetworkPolicies + *out = new(bool) + **out = **in + } + if in.BaselineAdminNetworkPolicies != nil { + in, out := &in.BaselineAdminNetworkPolicies, &out.BaselineAdminNetworkPolicies + *out = new(bool) + **out = **in + } + if in.DNSCaching != nil { + in, out := &in.DNSCaching, &out.DNSCaching + *out = new(bool) + **out = **in + } + if in.NAT64 != nil { + in, out := &in.NAT64, &out.NAT64 + *out = new(bool) + **out = **in + } + if in.FastPathThreshold != nil { + in, out := &in.FastPathThreshold, &out.FastPathThreshold + *out = new(int32) + **out = **in + } + if in.Masquerade != nil { + in, out := &in.Masquerade, &out.Masquerade + *out = new(KindnetMasqueradeSpec) + (*in).DeepCopyInto(*out) + } + if in.LogLevel != nil { + in, out := &in.LogLevel, &out.LogLevel + *out = new(int32) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KindnetNetworkingSpec. +func (in *KindnetNetworkingSpec) DeepCopy() *KindnetNetworkingSpec { + if in == nil { + return nil + } + out := new(KindnetNetworkingSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KopeioAuthenticationSpec) DeepCopyInto(out *KopeioAuthenticationSpec) { *out = *in @@ -4783,6 +4865,11 @@ func (in *NetworkingSpec) DeepCopyInto(out *NetworkingSpec) { *out = new(GCPNetworkingSpec) **out = **in } + if in.Kindnet != nil { + in, out := &in.Kindnet, &out.Kindnet + *out = new(KindnetNetworkingSpec) + (*in).DeepCopyInto(*out) + } return } diff --git a/pkg/apis/kops/v1alpha3/networking.go b/pkg/apis/kops/v1alpha3/networking.go index 2ccb675bf445f..7942cc8748bfe 100644 --- a/pkg/apis/kops/v1alpha3/networking.go +++ b/pkg/apis/kops/v1alpha3/networking.go @@ -82,6 +82,7 @@ type NetworkingSpec struct { Cilium *CiliumNetworkingSpec `json:"cilium,omitempty"` LyftVPC *kops.LyftVPCNetworkingSpec `json:"-"` GCP *GCPNetworkingSpec `json:"gcp,omitempty"` + Kindnet *KindnetNetworkingSpec `json:"kindnet,omitempty"` } // KubenetNetworkingSpec is the specification for kubenet networking, largely integrated but intended to replace classic @@ -510,3 +511,21 @@ type HubbleSpec struct { // GCPNetworkingSpec is the specification of GCP's native networking mode, using IP aliases. type GCPNetworkingSpec struct{} + +// KindnetNetworkingSpec configures Kindnet settings. +type KindnetNetworkingSpec struct { + Version string `json:"version,omitempty"` + NetworkPolicies *bool `json:"networkPolicies,omitempty"` + AdminNetworkPolicies *bool `json:"adminNetworkPolicies,omitempty"` + BaselineAdminNetworkPolicies *bool `json:"baselineAdminNetworkPolicies,omitempty"` + DNSCaching *bool `json:"dnsCaching,omitempty"` + NAT64 *bool `json:"nat64,omitempty"` + FastPathThreshold *int32 `json:"fastPathThreshold,omitempty"` + Masquerade *KindnetMasqueradeSpec `json:"masquerade,omitempty"` + LogLevel *int32 `json:"logLevel,omitempty"` +} + +type KindnetMasqueradeSpec struct { + Enabled *bool `json:"enabled,omitempty"` + NonMasqueradeCIDRs []string `json:"nonMasqueradeCIDRs,omitempty"` +} diff --git a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go index b979e1e79ae7b..5706825335e42 100644 --- a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go @@ -744,6 +744,26 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*KindnetMasqueradeSpec)(nil), (*kops.KindnetMasqueradeSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha3_KindnetMasqueradeSpec_To_kops_KindnetMasqueradeSpec(a.(*KindnetMasqueradeSpec), b.(*kops.KindnetMasqueradeSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*kops.KindnetMasqueradeSpec)(nil), (*KindnetMasqueradeSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kops_KindnetMasqueradeSpec_To_v1alpha3_KindnetMasqueradeSpec(a.(*kops.KindnetMasqueradeSpec), b.(*KindnetMasqueradeSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*KindnetNetworkingSpec)(nil), (*kops.KindnetNetworkingSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha3_KindnetNetworkingSpec_To_kops_KindnetNetworkingSpec(a.(*KindnetNetworkingSpec), b.(*kops.KindnetNetworkingSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*kops.KindnetNetworkingSpec)(nil), (*KindnetNetworkingSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kops_KindnetNetworkingSpec_To_v1alpha3_KindnetNetworkingSpec(a.(*kops.KindnetNetworkingSpec), b.(*KindnetNetworkingSpec), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*KopeioAuthenticationSpec)(nil), (*kops.KopeioAuthenticationSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha3_KopeioAuthenticationSpec_To_kops_KopeioAuthenticationSpec(a.(*KopeioAuthenticationSpec), b.(*kops.KopeioAuthenticationSpec), scope) }); err != nil { @@ -5227,6 +5247,80 @@ func Convert_kops_KeysetSpec_To_v1alpha3_KeysetSpec(in *kops.KeysetSpec, out *Ke return autoConvert_kops_KeysetSpec_To_v1alpha3_KeysetSpec(in, out, s) } +func autoConvert_v1alpha3_KindnetMasqueradeSpec_To_kops_KindnetMasqueradeSpec(in *KindnetMasqueradeSpec, out *kops.KindnetMasqueradeSpec, s conversion.Scope) error { + out.Enabled = in.Enabled + out.NonMasqueradeCIDRs = in.NonMasqueradeCIDRs + return nil +} + +// Convert_v1alpha3_KindnetMasqueradeSpec_To_kops_KindnetMasqueradeSpec is an autogenerated conversion function. +func Convert_v1alpha3_KindnetMasqueradeSpec_To_kops_KindnetMasqueradeSpec(in *KindnetMasqueradeSpec, out *kops.KindnetMasqueradeSpec, s conversion.Scope) error { + return autoConvert_v1alpha3_KindnetMasqueradeSpec_To_kops_KindnetMasqueradeSpec(in, out, s) +} + +func autoConvert_kops_KindnetMasqueradeSpec_To_v1alpha3_KindnetMasqueradeSpec(in *kops.KindnetMasqueradeSpec, out *KindnetMasqueradeSpec, s conversion.Scope) error { + out.Enabled = in.Enabled + out.NonMasqueradeCIDRs = in.NonMasqueradeCIDRs + return nil +} + +// Convert_kops_KindnetMasqueradeSpec_To_v1alpha3_KindnetMasqueradeSpec is an autogenerated conversion function. +func Convert_kops_KindnetMasqueradeSpec_To_v1alpha3_KindnetMasqueradeSpec(in *kops.KindnetMasqueradeSpec, out *KindnetMasqueradeSpec, s conversion.Scope) error { + return autoConvert_kops_KindnetMasqueradeSpec_To_v1alpha3_KindnetMasqueradeSpec(in, out, s) +} + +func autoConvert_v1alpha3_KindnetNetworkingSpec_To_kops_KindnetNetworkingSpec(in *KindnetNetworkingSpec, out *kops.KindnetNetworkingSpec, s conversion.Scope) error { + out.Version = in.Version + out.NetworkPolicies = in.NetworkPolicies + out.AdminNetworkPolicies = in.AdminNetworkPolicies + out.BaselineAdminNetworkPolicies = in.BaselineAdminNetworkPolicies + out.DNSCaching = in.DNSCaching + out.NAT64 = in.NAT64 + out.FastPathThreshold = in.FastPathThreshold + if in.Masquerade != nil { + in, out := &in.Masquerade, &out.Masquerade + *out = new(kops.KindnetMasqueradeSpec) + if err := Convert_v1alpha3_KindnetMasqueradeSpec_To_kops_KindnetMasqueradeSpec(*in, *out, s); err != nil { + return err + } + } else { + out.Masquerade = nil + } + out.LogLevel = in.LogLevel + return nil +} + +// Convert_v1alpha3_KindnetNetworkingSpec_To_kops_KindnetNetworkingSpec is an autogenerated conversion function. +func Convert_v1alpha3_KindnetNetworkingSpec_To_kops_KindnetNetworkingSpec(in *KindnetNetworkingSpec, out *kops.KindnetNetworkingSpec, s conversion.Scope) error { + return autoConvert_v1alpha3_KindnetNetworkingSpec_To_kops_KindnetNetworkingSpec(in, out, s) +} + +func autoConvert_kops_KindnetNetworkingSpec_To_v1alpha3_KindnetNetworkingSpec(in *kops.KindnetNetworkingSpec, out *KindnetNetworkingSpec, s conversion.Scope) error { + out.Version = in.Version + out.NetworkPolicies = in.NetworkPolicies + out.AdminNetworkPolicies = in.AdminNetworkPolicies + out.BaselineAdminNetworkPolicies = in.BaselineAdminNetworkPolicies + out.DNSCaching = in.DNSCaching + out.NAT64 = in.NAT64 + out.FastPathThreshold = in.FastPathThreshold + if in.Masquerade != nil { + in, out := &in.Masquerade, &out.Masquerade + *out = new(KindnetMasqueradeSpec) + if err := Convert_kops_KindnetMasqueradeSpec_To_v1alpha3_KindnetMasqueradeSpec(*in, *out, s); err != nil { + return err + } + } else { + out.Masquerade = nil + } + out.LogLevel = in.LogLevel + return nil +} + +// Convert_kops_KindnetNetworkingSpec_To_v1alpha3_KindnetNetworkingSpec is an autogenerated conversion function. +func Convert_kops_KindnetNetworkingSpec_To_v1alpha3_KindnetNetworkingSpec(in *kops.KindnetNetworkingSpec, out *KindnetNetworkingSpec, s conversion.Scope) error { + return autoConvert_kops_KindnetNetworkingSpec_To_v1alpha3_KindnetNetworkingSpec(in, out, s) +} + func autoConvert_v1alpha3_KopeioAuthenticationSpec_To_kops_KopeioAuthenticationSpec(in *KopeioAuthenticationSpec, out *kops.KopeioAuthenticationSpec, s conversion.Scope) error { return nil } @@ -6574,6 +6668,15 @@ func autoConvert_v1alpha3_NetworkingSpec_To_kops_NetworkingSpec(in *NetworkingSp } else { out.GCP = nil } + if in.Kindnet != nil { + in, out := &in.Kindnet, &out.Kindnet + *out = new(kops.KindnetNetworkingSpec) + if err := Convert_v1alpha3_KindnetNetworkingSpec_To_kops_KindnetNetworkingSpec(*in, *out, s); err != nil { + return err + } + } else { + out.Kindnet = nil + } return nil } @@ -6731,6 +6834,15 @@ func autoConvert_kops_NetworkingSpec_To_v1alpha3_NetworkingSpec(in *kops.Network } else { out.GCP = nil } + if in.Kindnet != nil { + in, out := &in.Kindnet, &out.Kindnet + *out = new(KindnetNetworkingSpec) + if err := Convert_kops_KindnetNetworkingSpec_To_v1alpha3_KindnetNetworkingSpec(*in, *out, s); err != nil { + return err + } + } else { + out.Kindnet = nil + } return nil } diff --git a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go index 0a84da4d2adec..dcd3ad4533ccb 100644 --- a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go @@ -2985,6 +2985,88 @@ func (in *KeysetSpec) DeepCopy() *KeysetSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KindnetMasqueradeSpec) DeepCopyInto(out *KindnetMasqueradeSpec) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + if in.NonMasqueradeCIDRs != nil { + in, out := &in.NonMasqueradeCIDRs, &out.NonMasqueradeCIDRs + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KindnetMasqueradeSpec. +func (in *KindnetMasqueradeSpec) DeepCopy() *KindnetMasqueradeSpec { + if in == nil { + return nil + } + out := new(KindnetMasqueradeSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KindnetNetworkingSpec) DeepCopyInto(out *KindnetNetworkingSpec) { + *out = *in + if in.NetworkPolicies != nil { + in, out := &in.NetworkPolicies, &out.NetworkPolicies + *out = new(bool) + **out = **in + } + if in.AdminNetworkPolicies != nil { + in, out := &in.AdminNetworkPolicies, &out.AdminNetworkPolicies + *out = new(bool) + **out = **in + } + if in.BaselineAdminNetworkPolicies != nil { + in, out := &in.BaselineAdminNetworkPolicies, &out.BaselineAdminNetworkPolicies + *out = new(bool) + **out = **in + } + if in.DNSCaching != nil { + in, out := &in.DNSCaching, &out.DNSCaching + *out = new(bool) + **out = **in + } + if in.NAT64 != nil { + in, out := &in.NAT64, &out.NAT64 + *out = new(bool) + **out = **in + } + if in.FastPathThreshold != nil { + in, out := &in.FastPathThreshold, &out.FastPathThreshold + *out = new(int32) + **out = **in + } + if in.Masquerade != nil { + in, out := &in.Masquerade, &out.Masquerade + *out = new(KindnetMasqueradeSpec) + (*in).DeepCopyInto(*out) + } + if in.LogLevel != nil { + in, out := &in.LogLevel, &out.LogLevel + *out = new(int32) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KindnetNetworkingSpec. +func (in *KindnetNetworkingSpec) DeepCopy() *KindnetNetworkingSpec { + if in == nil { + return nil + } + out := new(KindnetNetworkingSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KopeioAuthenticationSpec) DeepCopyInto(out *KopeioAuthenticationSpec) { *out = *in @@ -4734,6 +4816,11 @@ func (in *NetworkingSpec) DeepCopyInto(out *NetworkingSpec) { *out = new(GCPNetworkingSpec) **out = **in } + if in.Kindnet != nil { + in, out := &in.Kindnet, &out.Kindnet + *out = new(KindnetNetworkingSpec) + (*in).DeepCopyInto(*out) + } return } diff --git a/pkg/apis/kops/validation/validation.go b/pkg/apis/kops/validation/validation.go index 2695b1e2445a6..11b38a0a38e86 100644 --- a/pkg/apis/kops/validation/validation.go +++ b/pkg/apis/kops/validation/validation.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "net" + "net/netip" "net/url" "path/filepath" "regexp" @@ -1220,6 +1221,14 @@ func validateNetworking(cluster *kops.Cluster, v *kops.NetworkingSpec, fldPath * allErrs = append(allErrs, validateNetworkingGCP(cluster, v.GCP, fldPath.Child("gcp"))...) } + if v.Kindnet != nil { + if optionTaken { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("kindnet"), "only one networking option permitted")) + } + + allErrs = append(allErrs, validateNetworkingKindnet(cluster, v.Kindnet, fldPath.Child("kindnet"))...) + } + return allErrs } @@ -1397,6 +1406,23 @@ func validateNetworkingGCP(cluster *kops.Cluster, v *kops.GCPNetworkingSpec, fld return allErrs } +func validateNetworkingKindnet(cluster *kops.Cluster, v *kops.KindnetNetworkingSpec, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + if v.Masquerade != nil && v.Masquerade.Enabled != nil && *v.Masquerade.Enabled { + for _, cidr := range v.Masquerade.NonMasqueradeCIDRs { + if cidr == "" { + continue + } + _, err := netip.ParsePrefix(cidr) + if err != nil { + allErrs = append(allErrs, field.Invalid(fldPath, cidr, err.Error())) + } + } + } + return allErrs +} + func validateAdditionalPolicy(role string, policy string, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} diff --git a/pkg/apis/kops/validation/validation_test.go b/pkg/apis/kops/validation/validation_test.go index da2136c5c9db4..1c6b072f2b0ba 100644 --- a/pkg/apis/kops/validation/validation_test.go +++ b/pkg/apis/kops/validation/validation_test.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/upup/pkg/fi" + "k8s.io/utils/ptr" ) func Test_Validate_DNS(t *testing.T) { @@ -458,6 +459,72 @@ func Test_Validate_Networking_Flannel(t *testing.T) { } } +func Test_Validate_Networking_Kindnet(t *testing.T) { + grid := []struct { + Input kops.KindnetNetworkingSpec + ExpectedErrors []string + }{ + { + Input: kops.KindnetNetworkingSpec{ + Masquerade: &kops.KindnetMasqueradeSpec{ + Enabled: ptr.To(true), + }, + }, + }, + { + Input: kops.KindnetNetworkingSpec{ + Masquerade: &kops.KindnetMasqueradeSpec{ + Enabled: ptr.To(true), + NonMasqueradeCIDRs: []string{"10.0.0.0/24", "2001:db8::/64"}, + }, + }, + }, + { + Input: kops.KindnetNetworkingSpec{ + Masquerade: &kops.KindnetMasqueradeSpec{ + Enabled: ptr.To(true), + NonMasqueradeCIDRs: []string{"a.b.c.d/24", "2001:db8::/64"}, + }, + }, + ExpectedErrors: []string{"Invalid value::networking.kindnet"}, + }, + { + Input: kops.KindnetNetworkingSpec{ + Masquerade: &kops.KindnetMasqueradeSpec{ + Enabled: ptr.To(false), + NonMasqueradeCIDRs: []string{"a.b.c.d/24", "2001:db8::/64"}, + }, + }, + ExpectedErrors: []string{}, + }, + } + + for _, g := range grid { + cluster := &kops.Cluster{ + Spec: kops.ClusterSpec{ + KubernetesVersion: "1.27.0", + Networking: kops.NetworkingSpec{ + NetworkCIDR: "10.0.0.0/8", + NonMasqueradeCIDR: "100.64.0.0/10", + PodCIDR: "100.96.0.0/11", + ServiceClusterIPRange: "100.64.0.0/13", + Subnets: []kops.ClusterSubnetSpec{ + { + Name: "sg-test", + CIDR: "10.11.0.0/16", + Type: "Public", + }, + }, + Kindnet: &g.Input, + }, + }, + } + + errs := validateNetworking(cluster, &cluster.Spec.Networking, field.NewPath("networking"), true, &cloudProviderConstraints{}) + testErrors(t, g.Input, errs, g.ExpectedErrors) + } +} + func Test_Validate_Networking_OverlappingCIDR(t *testing.T) { grid := []struct { Name string diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index 87ef7670a9792..02a6774229e11 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -3072,6 +3072,88 @@ func (in *KeysetSpec) DeepCopy() *KeysetSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KindnetMasqueradeSpec) DeepCopyInto(out *KindnetMasqueradeSpec) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + if in.NonMasqueradeCIDRs != nil { + in, out := &in.NonMasqueradeCIDRs, &out.NonMasqueradeCIDRs + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KindnetMasqueradeSpec. +func (in *KindnetMasqueradeSpec) DeepCopy() *KindnetMasqueradeSpec { + if in == nil { + return nil + } + out := new(KindnetMasqueradeSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KindnetNetworkingSpec) DeepCopyInto(out *KindnetNetworkingSpec) { + *out = *in + if in.NetworkPolicies != nil { + in, out := &in.NetworkPolicies, &out.NetworkPolicies + *out = new(bool) + **out = **in + } + if in.AdminNetworkPolicies != nil { + in, out := &in.AdminNetworkPolicies, &out.AdminNetworkPolicies + *out = new(bool) + **out = **in + } + if in.BaselineAdminNetworkPolicies != nil { + in, out := &in.BaselineAdminNetworkPolicies, &out.BaselineAdminNetworkPolicies + *out = new(bool) + **out = **in + } + if in.DNSCaching != nil { + in, out := &in.DNSCaching, &out.DNSCaching + *out = new(bool) + **out = **in + } + if in.NAT64 != nil { + in, out := &in.NAT64, &out.NAT64 + *out = new(bool) + **out = **in + } + if in.FastPathThreshold != nil { + in, out := &in.FastPathThreshold, &out.FastPathThreshold + *out = new(int32) + **out = **in + } + if in.Masquerade != nil { + in, out := &in.Masquerade, &out.Masquerade + *out = new(KindnetMasqueradeSpec) + (*in).DeepCopyInto(*out) + } + if in.LogLevel != nil { + in, out := &in.LogLevel, &out.LogLevel + *out = new(int32) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KindnetNetworkingSpec. +func (in *KindnetNetworkingSpec) DeepCopy() *KindnetNetworkingSpec { + if in == nil { + return nil + } + out := new(KindnetNetworkingSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KopeioAuthenticationSpec) DeepCopyInto(out *KopeioAuthenticationSpec) { *out = *in @@ -4876,6 +4958,11 @@ func (in *NetworkingSpec) DeepCopyInto(out *NetworkingSpec) { *out = new(GCPNetworkingSpec) **out = **in } + if in.Kindnet != nil { + in, out := &in.Kindnet, &out.Kindnet + *out = new(KindnetNetworkingSpec) + (*in).DeepCopyInto(*out) + } return } diff --git a/pkg/apis/nodeup/config.go b/pkg/apis/nodeup/config.go index a00430d7235c9..5cd16c596324f 100644 --- a/pkg/apis/nodeup/config.go +++ b/pkg/apis/nodeup/config.go @@ -334,6 +334,10 @@ func NewConfig(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) (*Confi config.Networking.KubeRouter = &kops.KuberouterNetworkingSpec{} } + if cluster.Spec.Networking.Kindnet != nil { + config.Networking.Kindnet = &kops.KindnetNetworkingSpec{} + } + if instanceGroup.Spec.Kubelet != nil { config.KubeletConfig = *instanceGroup.Spec.Kubelet } diff --git a/pkg/model/components/kindnet.go b/pkg/model/components/kindnet.go new file mode 100644 index 0000000000000..556c3bfe6a9f1 --- /dev/null +++ b/pkg/model/components/kindnet.go @@ -0,0 +1,68 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package components + +import ( + "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/loader" +) + +// KindnetOptionsBuilder adds options for kindnet to the model +type KindnetOptionsBuilder struct { + Context *OptionsContext +} + +var _ loader.ClusterOptionsBuilder = &KindnetOptionsBuilder{} + +func (b *KindnetOptionsBuilder) BuildOptions(o *kops.Cluster) error { + clusterSpec := &o.Spec + c := clusterSpec.Networking.Kindnet + if c == nil { + return nil + } + + if c.Version == "" { + c.Version = "v1.8.0" + } + + // Kindnet should masquerade well known ranges if kops is not doing it + if c.Masquerade == nil { + c.Masquerade = &kops.KindnetMasqueradeSpec{ + Enabled: fi.PtrTo(true), + } + if clusterSpec.Networking.NetworkCIDR != "" { + c.Masquerade.NonMasqueradeCIDRs = append(c.Masquerade.NonMasqueradeCIDRs, clusterSpec.Networking.NetworkCIDR) + } + if clusterSpec.Networking.PodCIDR != "" { + c.Masquerade.NonMasqueradeCIDRs = append(c.Masquerade.NonMasqueradeCIDRs, clusterSpec.Networking.PodCIDR) + } + if clusterSpec.Networking.ServiceClusterIPRange != "" { + c.Masquerade.NonMasqueradeCIDRs = append(c.Masquerade.NonMasqueradeCIDRs, clusterSpec.Networking.ServiceClusterIPRange) + } + } + + if c.FastPathThreshold == nil { + c.FastPathThreshold = fi.PtrTo(int32(0)) + } + + if c.LogLevel == nil { + c.LogLevel = fi.PtrTo(int32(2)) + } + + return nil +} diff --git a/pkg/model/iam/iam_builder.go b/pkg/model/iam/iam_builder.go index 82bdd63e281d0..816f2f2282a67 100644 --- a/pkg/model/iam/iam_builder.go +++ b/pkg/model/iam/iam_builder.go @@ -374,6 +374,10 @@ func (r *NodeRoleAPIServer) BuildAWSPolicy(b *PolicyBuilder) (*Policy, error) { addCalicoSrcDstCheckPermissions(p) } + if b.Cluster.Spec.Networking.Kindnet != nil { + addKindnetSrcDstCheckPermissions(p) + } + return p, nil } @@ -438,6 +442,10 @@ func (r *NodeRoleMaster) BuildAWSPolicy(b *PolicyBuilder) (*Policy, error) { addCalicoSrcDstCheckPermissions(p) } + if b.Cluster.Spec.Networking.Kindnet != nil { + addKindnetSrcDstCheckPermissions(p) + } + return p, nil } @@ -469,6 +477,10 @@ func (r *NodeRoleNode) BuildAWSPolicy(b *PolicyBuilder) (*Policy, error) { addKubeRouterSrcDstCheckPermissions(p) } + if b.Cluster.Spec.Networking.Kindnet != nil { + addKindnetSrcDstCheckPermissions(p) + } + return p, nil } @@ -777,6 +789,12 @@ func addKubeRouterSrcDstCheckPermissions(p *Policy) { ) } +func addKindnetSrcDstCheckPermissions(p *Policy) { + p.unconditionalAction.Insert( + "ec2:ModifyInstanceAttribute", + ) +} + func (b *PolicyBuilder) addNodeupPermissions(p *Policy, enableHookSupport bool) { addCertIAMPolicies(p) addKMSGenerateRandomPolicies(p) diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_cloudwatch_event_rule_privatekindnet.example.com-ASGLifecycle_event_pattern b/tests/integration/update_cluster/privatekindnet/data/aws_cloudwatch_event_rule_privatekindnet.example.com-ASGLifecycle_event_pattern new file mode 100644 index 0000000000000..c8db9dbe9c41b --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_cloudwatch_event_rule_privatekindnet.example.com-ASGLifecycle_event_pattern @@ -0,0 +1 @@ +{"source":["aws.autoscaling"],"detail-type":["EC2 Instance-terminate Lifecycle Action"]} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_cloudwatch_event_rule_privatekindnet.example.com-InstanceScheduledChange_event_pattern b/tests/integration/update_cluster/privatekindnet/data/aws_cloudwatch_event_rule_privatekindnet.example.com-InstanceScheduledChange_event_pattern new file mode 100644 index 0000000000000..fb4ea7defdc52 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_cloudwatch_event_rule_privatekindnet.example.com-InstanceScheduledChange_event_pattern @@ -0,0 +1 @@ +{"source": ["aws.health"],"detail-type": ["AWS Health Event"],"detail": {"service": ["EC2"],"eventTypeCategory": ["scheduledChange"]}} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_cloudwatch_event_rule_privatekindnet.example.com-InstanceStateChange_event_pattern b/tests/integration/update_cluster/privatekindnet/data/aws_cloudwatch_event_rule_privatekindnet.example.com-InstanceStateChange_event_pattern new file mode 100644 index 0000000000000..8c2916419dc7d --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_cloudwatch_event_rule_privatekindnet.example.com-InstanceStateChange_event_pattern @@ -0,0 +1 @@ +{"source": ["aws.ec2"],"detail-type": ["EC2 Instance State-change Notification"]} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_cloudwatch_event_rule_privatekindnet.example.com-SpotInterruption_event_pattern b/tests/integration/update_cluster/privatekindnet/data/aws_cloudwatch_event_rule_privatekindnet.example.com-SpotInterruption_event_pattern new file mode 100644 index 0000000000000..2d0e83b4164b2 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_cloudwatch_event_rule_privatekindnet.example.com-SpotInterruption_event_pattern @@ -0,0 +1 @@ +{"source": ["aws.ec2"],"detail-type": ["EC2 Spot Instance Interruption Warning"]} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_bastions.privatekindnet.example.com_policy b/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_bastions.privatekindnet.example.com_policy new file mode 100644 index 0000000000000..66d5de1d5ae1e --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_bastions.privatekindnet.example.com_policy @@ -0,0 +1,10 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { "Service": "ec2.amazonaws.com"}, + "Action": "sts:AssumeRole" + } + ] +} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_masters.privatekindnet.example.com_policy b/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_masters.privatekindnet.example.com_policy new file mode 100644 index 0000000000000..66d5de1d5ae1e --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_masters.privatekindnet.example.com_policy @@ -0,0 +1,10 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { "Service": "ec2.amazonaws.com"}, + "Action": "sts:AssumeRole" + } + ] +} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_nodes.privatekindnet.example.com_policy b/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_nodes.privatekindnet.example.com_policy new file mode 100644 index 0000000000000..66d5de1d5ae1e --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_nodes.privatekindnet.example.com_policy @@ -0,0 +1,10 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { "Service": "ec2.amazonaws.com"}, + "Action": "sts:AssumeRole" + } + ] +} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_policy_bastions.privatekindnet.example.com_policy b/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_policy_bastions.privatekindnet.example.com_policy new file mode 100644 index 0000000000000..54912e12a5880 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_policy_bastions.privatekindnet.example.com_policy @@ -0,0 +1,10 @@ +{ + "Statement": [ + { + "Action": "ec2:DescribeRegions", + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_policy_masters.privatekindnet.example.com_policy b/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_policy_masters.privatekindnet.example.com_policy new file mode 100644 index 0000000000000..398c8269c17a5 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_policy_masters.privatekindnet.example.com_policy @@ -0,0 +1,277 @@ +{ + "Statement": [ + { + "Action": "ec2:AttachVolume", + "Condition": { + "StringEquals": { + "aws:ResourceTag/KubernetesCluster": "privatekindnet.example.com", + "aws:ResourceTag/k8s.io/role/master": "1" + } + }, + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "s3:Get*" + ], + "Effect": "Allow", + "Resource": "arn:aws-test:s3:::placeholder-read-bucket/clusters.example.com/privatekindnet.example.com/*" + }, + { + "Action": [ + "s3:DeleteObject", + "s3:DeleteObjectVersion", + "s3:GetObject", + "s3:PutObject" + ], + "Effect": "Allow", + "Resource": "arn:aws-test:s3:::placeholder-write-bucket/clusters.example.com/privatekindnet.example.com/backups/etcd/main/*" + }, + { + "Action": [ + "s3:DeleteObject", + "s3:DeleteObjectVersion", + "s3:GetObject", + "s3:PutObject" + ], + "Effect": "Allow", + "Resource": "arn:aws-test:s3:::placeholder-write-bucket/clusters.example.com/privatekindnet.example.com/backups/etcd/events/*" + }, + { + "Action": [ + "s3:GetBucketLocation", + "s3:GetEncryptionConfiguration", + "s3:ListBucket", + "s3:ListBucketVersions" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:s3:::placeholder-read-bucket" + ] + }, + { + "Action": [ + "s3:GetBucketLocation", + "s3:GetEncryptionConfiguration", + "s3:ListBucket", + "s3:ListBucketVersions" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:s3:::placeholder-write-bucket" + ] + }, + { + "Action": [ + "route53:ChangeResourceRecordSets", + "route53:GetHostedZone", + "route53:ListResourceRecordSets" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:route53:::hostedzone/Z1AFAKE1ZON3YO" + ] + }, + { + "Action": [ + "route53:GetChange" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:route53:::change/*" + ] + }, + { + "Action": [ + "route53:ListHostedZones", + "route53:ListTagsForResource" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": "ec2:CreateTags", + "Condition": { + "StringEquals": { + "aws:RequestTag/KubernetesCluster": "privatekindnet.example.com", + "ec2:CreateAction": [ + "CreateVolume", + "CreateSnapshot" + ] + } + }, + "Effect": "Allow", + "Resource": [ + "arn:aws-test:ec2:*:*:snapshot/*", + "arn:aws-test:ec2:*:*:volume/*" + ] + }, + { + "Action": [ + "ec2:CreateTags", + "ec2:DeleteTags" + ], + "Condition": { + "Null": { + "aws:RequestTag/KubernetesCluster": "true" + }, + "StringEquals": { + "aws:ResourceTag/KubernetesCluster": "privatekindnet.example.com" + } + }, + "Effect": "Allow", + "Resource": [ + "arn:aws-test:ec2:*:*:snapshot/*", + "arn:aws-test:ec2:*:*:volume/*" + ] + }, + { + "Action": "ec2:CreateTags", + "Condition": { + "StringEquals": { + "aws:RequestTag/KubernetesCluster": "privatekindnet.example.com", + "ec2:CreateAction": [ + "CreateSecurityGroup" + ] + } + }, + "Effect": "Allow", + "Resource": [ + "arn:aws-test:ec2:*:*:security-group/*" + ] + }, + { + "Action": [ + "ec2:CreateTags", + "ec2:DeleteTags" + ], + "Condition": { + "Null": { + "aws:RequestTag/KubernetesCluster": "true" + }, + "StringEquals": { + "aws:ResourceTag/KubernetesCluster": "privatekindnet.example.com" + } + }, + "Effect": "Allow", + "Resource": [ + "arn:aws-test:ec2:*:*:security-group/*" + ] + }, + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeScalingActivities", + "autoscaling:DescribeTags", + "ec2:DescribeAccountAttributes", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeImages", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstances", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeRegions", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeTags", + "ec2:DescribeVolumes", + "ec2:DescribeVolumesModifications", + "ec2:DescribeVpcs", + "ec2:GetInstanceTypesFromInstanceRequirements", + "ec2:ModifyInstanceAttribute", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DescribeLoadBalancerPolicies", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetHealth", + "iam:CreateServiceLinkedRole", + "iam:GetServerCertificate", + "iam:ListServerCertificates", + "kms:CreateGrant", + "kms:Decrypt", + "kms:DescribeKey", + "kms:Encrypt", + "kms:GenerateDataKey*", + "kms:GenerateRandom", + "kms:ReEncrypt*", + "sqs:DeleteMessage", + "sqs:ReceiveMessage" + ], + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "autoscaling:CompleteLifecycleAction", + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + "ec2:AttachVolume", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:DeleteSecurityGroup", + "ec2:DeleteVolume", + "ec2:DetachVolume", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyVolume", + "ec2:RevokeSecurityGroupIngress", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", + "elasticloadbalancing:AttachLoadBalancerToSubnets", + "elasticloadbalancing:ConfigureHealthCheck", + "elasticloadbalancing:CreateLoadBalancerListeners", + "elasticloadbalancing:CreateLoadBalancerPolicy", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:DeleteLoadBalancerListeners", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:DeregisterTargets", + "elasticloadbalancing:DetachLoadBalancerFromSubnets", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer", + "elasticloadbalancing:SetLoadBalancerPoliciesOfListener" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/KubernetesCluster": "privatekindnet.example.com" + } + }, + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "ec2:CreateSecurityGroup", + "ec2:CreateSnapshot", + "ec2:CreateVolume", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateTargetGroup" + ], + "Condition": { + "StringEquals": { + "aws:RequestTag/KubernetesCluster": "privatekindnet.example.com" + } + }, + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": "ec2:CreateSecurityGroup", + "Effect": "Allow", + "Resource": "arn:aws-test:ec2:*:*:vpc/*" + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_policy_nodes.privatekindnet.example.com_policy b/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_policy_nodes.privatekindnet.example.com_policy new file mode 100644 index 0000000000000..c733a127d787e --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_policy_nodes.privatekindnet.example.com_policy @@ -0,0 +1,31 @@ +{ + "Statement": [ + { + "Action": [ + "s3:GetBucketLocation", + "s3:GetEncryptionConfiguration", + "s3:ListBucket", + "s3:ListBucketVersions" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:s3:::placeholder-read-bucket" + ] + }, + { + "Action": [ + "autoscaling:DescribeAutoScalingInstances", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstances", + "ec2:DescribeRegions", + "ec2:ModifyInstanceAttribute", + "iam:GetServerCertificate", + "iam:ListServerCertificates", + "kms:GenerateRandom" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_key_pair_kubernetes.privatekindnet.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key b/tests/integration/update_cluster/privatekindnet/data/aws_key_pair_kubernetes.privatekindnet.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key new file mode 100644 index 0000000000000..81cb0127830e7 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_key_pair_kubernetes.privatekindnet.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ== diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_launch_template_master-us-test-1a.masters.privatekindnet.example.com_user_data b/tests/integration/update_cluster/privatekindnet/data/aws_launch_template_master-us-test-1a.masters.privatekindnet.example.com_user_data new file mode 100644 index 0000000000000..5cc3129fad62b --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_launch_template_master-us-test-1a.masters.privatekindnet.example.com_user_data @@ -0,0 +1,138 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 +NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 +NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 +NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + +export AWS_REGION=us-test-1 + + + + +sysctl -w net.core.rmem_max=16777216 || true +sysctl -w net.core.wmem_max=16777216 || true +sysctl -w net.ipv4.tcp_rmem='4096 87380 16777216' || true +sysctl -w net.ipv4.tcp_wmem='4096 87380 16777216' || true + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, urls +download-or-bust() { + echo "== Downloading $1 with hash $2 from $3 ==" + local -r file="$1" + local -r hash="$2" + local -a urls + mapfile -t urls < <(split-commas "$3") + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return 0 + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --compressed -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --compression=auto -O ${file} --connect-timeout=20 --tries=6 --wait=10" + "curl -f -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10" + "wget -O ${file} --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "== Downloading ${url} using ${cmd} ==" + if ! (${cmd} "${url}"); then + echo "== Failed to download ${url} using ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Failed to validate hash for ${url} ==" + rm -f "${file}" + else + echo "== Downloaded ${url} with hash ${hash} ==" + return 0 + fi + done + done + + echo "== All downloads failed; sleeping before retrying ==" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum "${file}" | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== File ${file} is corrupted; hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo "$1" | tr "," "\n" +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "== Running nodeup ==" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "== Failed to initialize the machine ID; ensure machine-id configured ==" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +CloudProvider: aws +ClusterName: privatekindnet.example.com +ConfigBase: memfs://clusters.example.com/privatekindnet.example.com +InstanceGroupName: master-us-test-1a +InstanceGroupRole: ControlPlane +NodeupConfigHash: U/D+9qWrP/UX/em0nez7EfMbcInS8VTwNKfm2DsFr38= + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_launch_template_nodes.privatekindnet.example.com_user_data b/tests/integration/update_cluster/privatekindnet/data/aws_launch_template_nodes.privatekindnet.example.com_user_data new file mode 100644 index 0000000000000..cda4ccde75052 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_launch_template_nodes.privatekindnet.example.com_user_data @@ -0,0 +1,161 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 +NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 +NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 +NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + +export AWS_REGION=us-test-1 + + + + +sysctl -w net.core.rmem_max=16777216 || true +sysctl -w net.core.wmem_max=16777216 || true +sysctl -w net.ipv4.tcp_rmem='4096 87380 16777216' || true +sysctl -w net.ipv4.tcp_wmem='4096 87380 16777216' || true + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, urls +download-or-bust() { + echo "== Downloading $1 with hash $2 from $3 ==" + local -r file="$1" + local -r hash="$2" + local -a urls + mapfile -t urls < <(split-commas "$3") + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return 0 + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --compressed -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --compression=auto -O ${file} --connect-timeout=20 --tries=6 --wait=10" + "curl -f -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10" + "wget -O ${file} --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "== Downloading ${url} using ${cmd} ==" + if ! (${cmd} "${url}"); then + echo "== Failed to download ${url} using ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Failed to validate hash for ${url} ==" + rm -f "${file}" + else + echo "== Downloaded ${url} with hash ${hash} ==" + return 0 + fi + done + done + + echo "== All downloads failed; sleeping before retrying ==" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum "${file}" | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== File ${file} is corrupted; hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo "$1" | tr "," "\n" +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "== Running nodeup ==" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "== Failed to initialize the machine ID; ensure machine-id configured ==" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +CloudProvider: aws +ClusterName: privatekindnet.example.com +ConfigServer: + CACertificates: | + -----BEGIN CERTIFICATE----- + MIIBbjCCARigAwIBAgIMFpANqBD8NSD82AUSMA0GCSqGSIb3DQEBCwUAMBgxFjAU + BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwODAwWhcNMzEwNzA3MDcw + ODAwWjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD + SwAwSAJBANFI3zr0Tk8krsW8vwjfMpzJOlWQ8616vG3YPa2qAgI7V4oKwfV0yIg1 + jt+H6f4P/wkPAPTPTfRp9Iy8oHEEFw0CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG + MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNG3zVjTcLlJwDsJ4/K9DV7KohUA + MA0GCSqGSIb3DQEBCwUAA0EAB8d03fY2w7WKpfO29qI295pu2C4ca9AiVGOpgSc8 + tmQsq6rcxt3T+rb589PVtz0mw/cKTxOk6gH2CCC+yHfy2w== + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBbjCCARigAwIBAgIMFpANvmSa0OAlYmXKMA0GCSqGSIb3DQEBCwUAMBgxFjAU + BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwOTM2WhcNMzEwNzA3MDcw + OTM2WjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD + SwAwSAJBAMF6F4aZdpe0RUpyykaBpWwZCnwbffhYGOw+fs6RdLuUq7QCNmJm/Eq7 + WWOziMYDiI9SbclpD+6QiJ0N3EqppVUCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG + MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFLImp6ARjPDAH6nhI+scWVt3Q9bn + MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA + 9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw== + -----END CERTIFICATE----- + servers: + - https://kops-controller.internal.privatekindnet.example.com:3988/ +InstanceGroupName: nodes +InstanceGroupRole: Node +NodeupConfigHash: O0n+dX3fOwnIfrtsVAdOGEyfJph+eqTZmsTZXpP00sE= + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_cluster-completed.spec_content new file mode 100644 index 0000000000000..c684ce9f29ce2 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_cluster-completed.spec_content @@ -0,0 +1,229 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2016-12-12T04:13:14Z" + name: privatekindnet.example.com +spec: + api: + loadBalancer: + class: Classic + type: Public + authorization: + alwaysAllow: {} + channel: stable + cloudConfig: + awsEBSCSIDriver: + version: v1.33.0 + manageStorageClasses: true + cloudControllerManager: + allocateNodeCIDRs: true + clusterCIDR: 100.96.0.0/11 + clusterName: privatekindnet.example.com + configureCloudRoutes: false + image: registry.k8s.io/provider-aws/cloud-controller-manager:v1.30.3 + leaderElection: + leaderElect: true + cloudProvider: aws + clusterDNSDomain: cluster.local + configBase: memfs://clusters.example.com/privatekindnet.example.com + containerd: + logLevel: info + runc: + version: 1.2.4 + version: 1.7.24 + dnsZone: Z1AFAKE1ZON3YO + etcdClusters: + - backups: + backupStore: memfs://clusters.example.com/privatekindnet.example.com/backups/etcd/main + etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + manager: + backupRetentionDays: 90 + name: main + version: 3.5.17 + - backups: + backupStore: memfs://clusters.example.com/privatekindnet.example.com/backups/etcd/events + etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + manager: + backupRetentionDays: 90 + name: events + version: 3.5.17 + externalDns: + provider: dns-controller + iam: + legacy: false + keyStore: memfs://clusters.example.com/privatekindnet.example.com/pki + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: external + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + featureGates: + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-apiserver:v1.30.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.privatekindnet.example.com + serviceAccountJWKSURI: https://api.internal.privatekindnet.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: external + clusterCIDR: 100.96.0.0/11 + clusterName: privatekindnet.example.com + configureCloudRoutes: false + featureGates: + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-controller-manager:v1.30.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeDNS: + cacheMaxConcurrent: 150 + cacheMaxSize: 1000 + cpuRequest: 100m + domain: cluster.local + memoryLimit: 170Mi + memoryRequest: 70Mi + nodeLocalDNS: + cpuRequest: 25m + enabled: false + image: registry.k8s.io/dns/k8s-dns-node-cache:1.23.0 + memoryRequest: 5Mi + provider: CoreDNS + serverIP: 100.64.0.10 + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + image: registry.k8s.io/kube-proxy:v1.30.0 + logLevel: 2 + kubeScheduler: + featureGates: + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-scheduler:v1.30.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: external + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + InTreePluginAWSUnregister: "true" + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + podInfraContainerImage: registry.k8s.io/pause:3.9 + podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true + registerSchedulable: true + shutdownGracePeriod: 30s + shutdownGracePeriodCriticalPods: 10s + kubernetesApiAccess: + - 0.0.0.0/0 + kubernetesVersion: 1.30.0 + masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: external + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + InTreePluginAWSUnregister: "true" + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + podInfraContainerImage: registry.k8s.io/pause:3.9 + podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true + registerSchedulable: true + shutdownGracePeriod: 30s + shutdownGracePeriodCriticalPods: 10s + masterPublicName: api.privatekindnet.example.com + networkCIDR: 172.20.0.0/16 + networking: + kindnet: + fastPathThreshold: 0 + logLevel: 2 + masquerade: + enabled: true + nonMasqueradeCIDRs: + - 172.20.0.0/16 + - 100.96.0.0/11 + - 100.64.0.0/13 + version: v1.8.0 + nodeTerminationHandler: + cpuRequest: 50m + deleteSQSMsgIfNodeNotFound: false + enableRebalanceDraining: false + enableRebalanceMonitoring: false + enableScheduledEventDraining: true + enableSpotInterruptionDraining: true + enabled: true + excludeFromLoadBalancers: true + managedASGTag: aws-node-termination-handler/managed + memoryRequest: 64Mi + podTerminationGracePeriod: -1 + prometheusEnable: false + taintNode: false + version: v1.22.0 + nonMasqueradeCIDR: 100.64.0.0/10 + podCIDR: 100.96.0.0/11 + secretStore: memfs://clusters.example.com/privatekindnet.example.com/secrets + serviceClusterIPRange: 100.64.0.0/13 + sshAccess: + - 0.0.0.0/0 + subnets: + - cidr: 172.20.32.0/19 + name: us-test-1a + type: Private + zone: us-test-1a + - cidr: 172.20.4.0/22 + name: utility-us-test-1a + type: Utility + zone: us-test-1a + topology: + dns: + type: Public diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_etcd-cluster-spec-events_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_etcd-cluster-spec-events_content new file mode 100644 index 0000000000000..432cd0d5554c5 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_etcd-cluster-spec-events_content @@ -0,0 +1,4 @@ +{ + "memberCount": 1, + "etcdVersion": "3.5.17" +} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_etcd-cluster-spec-main_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_etcd-cluster-spec-main_content new file mode 100644 index 0000000000000..432cd0d5554c5 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_etcd-cluster-spec-main_content @@ -0,0 +1,4 @@ +{ + "memberCount": 1, + "etcdVersion": "3.5.17" +} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_kops-version.txt_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_kops-version.txt_content new file mode 100644 index 0000000000000..b7340298dcdd5 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_kops-version.txt_content @@ -0,0 +1 @@ +1.21.0-alpha.1 diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_manifests-etcdmanager-events-master-us-test-1a_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_manifests-etcdmanager-events-master-us-test-1a_content new file mode 100644 index 0000000000000..658872a810d43 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_manifests-etcdmanager-events-master-us-test-1a_content @@ -0,0 +1,138 @@ +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + labels: + k8s-app: etcd-manager-events + name: etcd-manager-events + namespace: kube-system +spec: + containers: + - command: + - /bin/sh + - -c + - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /etcd-manager + --backup-store=memfs://clusters.example.com/privatekindnet.example.com/backups/etcd/events + --client-urls=https://__name__:4002 --cluster-name=etcd-events --containerized=true + --dns-suffix=.internal.privatekindnet.example.com --grpc-port=3997 --peer-urls=https://__name__:2381 + --quarantine-client-urls=https://__name__:3995 --v=6 --volume-name-tag=k8s.io/etcd/events + --volume-provider=aws --volume-tag=k8s.io/etcd/events --volume-tag=k8s.io/role/control-plane=1 + --volume-tag=kubernetes.io/cluster/privatekindnet.example.com=owned > /tmp/pipe + 2>&1 + env: + - name: ETCD_MANAGER_DAILY_BACKUPS_RETENTION + value: 90d + image: registry.k8s.io/etcd-manager/etcd-manager-slim:v3.0.20241012 + name: etcd-manager + resources: + requests: + cpu: 200m + memory: 100Mi + securityContext: + privileged: true + volumeMounts: + - mountPath: /rootfs + name: rootfs + - mountPath: /run + name: run + - mountPath: /etc/kubernetes/pki/etcd-manager + name: pki + - mountPath: /opt + name: opt + - mountPath: /var/log/etcd.log + name: varlogetcd + hostNetwork: true + hostPID: true + initContainers: + - args: + - --target-dir=/opt/kops-utils/ + - --src=/ko-app/kops-utils-cp + command: + - /ko-app/kops-utils-cp + image: registry.k8s.io/kops/kops-utils-cp:1.31.0-beta.1 + name: kops-utils-cp + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + - args: + - --target-dir=/opt/etcd-v3.4.13 + - --src=/usr/local/bin/etcd + - --src=/usr/local/bin/etcdctl + command: + - /opt/kops-utils/kops-utils-cp + image: registry.k8s.io/etcd:3.4.13-0 + name: init-etcd-3-4-13 + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + - args: + - --target-dir=/opt/etcd-v3.5.17 + - --src=/usr/local/bin/etcd + - --src=/usr/local/bin/etcdctl + command: + - /opt/kops-utils/kops-utils-cp + image: registry.k8s.io/etcd:3.5.17-0 + name: init-etcd-3-5-17 + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + - args: + - --symlink + - --target-dir=/opt/etcd-v3.4.3 + - --src=/opt/etcd-v3.4.13/etcd + - --src=/opt/etcd-v3.4.13/etcdctl + command: + - /opt/kops-utils/kops-utils-cp + image: registry.k8s.io/kops/kops-utils-cp:1.31.0-beta.1 + name: init-etcd-symlinks-3-4-13 + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + - args: + - --symlink + - --target-dir=/opt/etcd-v3.5.0 + - --target-dir=/opt/etcd-v3.5.1 + - --target-dir=/opt/etcd-v3.5.13 + - --target-dir=/opt/etcd-v3.5.3 + - --target-dir=/opt/etcd-v3.5.4 + - --target-dir=/opt/etcd-v3.5.6 + - --target-dir=/opt/etcd-v3.5.7 + - --target-dir=/opt/etcd-v3.5.9 + - --src=/opt/etcd-v3.5.17/etcd + - --src=/opt/etcd-v3.5.17/etcdctl + command: + - /opt/kops-utils/kops-utils-cp + image: registry.k8s.io/kops/kops-utils-cp:1.31.0-beta.1 + name: init-etcd-symlinks-3-5-17 + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + priorityClassName: system-cluster-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + volumes: + - hostPath: + path: / + type: Directory + name: rootfs + - hostPath: + path: /run + type: DirectoryOrCreate + name: run + - hostPath: + path: /etc/kubernetes/pki/etcd-manager-events + type: DirectoryOrCreate + name: pki + - emptyDir: {} + name: opt + - hostPath: + path: /var/log/etcd-events.log + type: FileOrCreate + name: varlogetcd +status: {} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_manifests-etcdmanager-main-master-us-test-1a_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_manifests-etcdmanager-main-master-us-test-1a_content new file mode 100644 index 0000000000000..e685e41191f0d --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_manifests-etcdmanager-main-master-us-test-1a_content @@ -0,0 +1,138 @@ +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + labels: + k8s-app: etcd-manager-main + name: etcd-manager-main + namespace: kube-system +spec: + containers: + - command: + - /bin/sh + - -c + - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /etcd-manager + --backup-store=memfs://clusters.example.com/privatekindnet.example.com/backups/etcd/main + --client-urls=https://__name__:4001 --cluster-name=etcd --containerized=true + --dns-suffix=.internal.privatekindnet.example.com --grpc-port=3996 --peer-urls=https://__name__:2380 + --quarantine-client-urls=https://__name__:3994 --v=6 --volume-name-tag=k8s.io/etcd/main + --volume-provider=aws --volume-tag=k8s.io/etcd/main --volume-tag=k8s.io/role/control-plane=1 + --volume-tag=kubernetes.io/cluster/privatekindnet.example.com=owned > /tmp/pipe + 2>&1 + env: + - name: ETCD_MANAGER_DAILY_BACKUPS_RETENTION + value: 90d + image: registry.k8s.io/etcd-manager/etcd-manager-slim:v3.0.20241012 + name: etcd-manager + resources: + requests: + cpu: 200m + memory: 100Mi + securityContext: + privileged: true + volumeMounts: + - mountPath: /rootfs + name: rootfs + - mountPath: /run + name: run + - mountPath: /etc/kubernetes/pki/etcd-manager + name: pki + - mountPath: /opt + name: opt + - mountPath: /var/log/etcd.log + name: varlogetcd + hostNetwork: true + hostPID: true + initContainers: + - args: + - --target-dir=/opt/kops-utils/ + - --src=/ko-app/kops-utils-cp + command: + - /ko-app/kops-utils-cp + image: registry.k8s.io/kops/kops-utils-cp:1.31.0-beta.1 + name: kops-utils-cp + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + - args: + - --target-dir=/opt/etcd-v3.4.13 + - --src=/usr/local/bin/etcd + - --src=/usr/local/bin/etcdctl + command: + - /opt/kops-utils/kops-utils-cp + image: registry.k8s.io/etcd:3.4.13-0 + name: init-etcd-3-4-13 + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + - args: + - --target-dir=/opt/etcd-v3.5.17 + - --src=/usr/local/bin/etcd + - --src=/usr/local/bin/etcdctl + command: + - /opt/kops-utils/kops-utils-cp + image: registry.k8s.io/etcd:3.5.17-0 + name: init-etcd-3-5-17 + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + - args: + - --symlink + - --target-dir=/opt/etcd-v3.4.3 + - --src=/opt/etcd-v3.4.13/etcd + - --src=/opt/etcd-v3.4.13/etcdctl + command: + - /opt/kops-utils/kops-utils-cp + image: registry.k8s.io/kops/kops-utils-cp:1.31.0-beta.1 + name: init-etcd-symlinks-3-4-13 + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + - args: + - --symlink + - --target-dir=/opt/etcd-v3.5.0 + - --target-dir=/opt/etcd-v3.5.1 + - --target-dir=/opt/etcd-v3.5.13 + - --target-dir=/opt/etcd-v3.5.3 + - --target-dir=/opt/etcd-v3.5.4 + - --target-dir=/opt/etcd-v3.5.6 + - --target-dir=/opt/etcd-v3.5.7 + - --target-dir=/opt/etcd-v3.5.9 + - --src=/opt/etcd-v3.5.17/etcd + - --src=/opt/etcd-v3.5.17/etcdctl + command: + - /opt/kops-utils/kops-utils-cp + image: registry.k8s.io/kops/kops-utils-cp:1.31.0-beta.1 + name: init-etcd-symlinks-3-5-17 + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + priorityClassName: system-cluster-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + volumes: + - hostPath: + path: / + type: Directory + name: rootfs + - hostPath: + path: /run + type: DirectoryOrCreate + name: run + - hostPath: + path: /etc/kubernetes/pki/etcd-manager-main + type: DirectoryOrCreate + name: pki + - emptyDir: {} + name: opt + - hostPath: + path: /var/log/etcd.log + type: FileOrCreate + name: varlogetcd +status: {} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_manifests-static-kube-apiserver-healthcheck_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_manifests-static-kube-apiserver-healthcheck_content new file mode 100644 index 0000000000000..21a36c9bb0d56 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_manifests-static-kube-apiserver-healthcheck_content @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null +spec: + containers: + - args: + - --ca-cert=/secrets/ca.crt + - --client-cert=/secrets/client.crt + - --client-key=/secrets/client.key + image: registry.k8s.io/kops/kube-apiserver-healthcheck:1.31.0-beta.1 + livenessProbe: + httpGet: + host: 127.0.0.1 + path: /.kube-apiserver-healthcheck/healthz + port: 3990 + initialDelaySeconds: 5 + timeoutSeconds: 5 + name: healthcheck + resources: {} + securityContext: + runAsNonRoot: true + runAsUser: 10012 + volumeMounts: + - mountPath: /secrets + name: healthcheck-secrets + readOnly: true + volumes: + - hostPath: + path: /etc/kubernetes/kube-apiserver-healthcheck/secrets + type: Directory + name: healthcheck-secrets +status: {} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_nodeupconfig-master-us-test-1a_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_nodeupconfig-master-us-test-1a_content new file mode 100644 index 0000000000000..6ea9515af205b --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_nodeupconfig-master-us-test-1a_content @@ -0,0 +1,333 @@ +APIServerConfig: + API: + publicName: api.privatekindnet.example.com + ClusterDNSDomain: cluster.local + KubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: external + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + featureGates: + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-apiserver:v1.30.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.privatekindnet.example.com + serviceAccountJWKSURI: https://api.internal.privatekindnet.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + ServiceAccountPublicKeys: | + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm + XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ== + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF + Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ== + -----END RSA PUBLIC KEY----- +Assets: + amd64: + - 32a32ec3d7e7f8b2648c9dd503ce9ef63b4af1d1677f5b5aed7846fb02d66f18@https://dl.k8s.io/release/v1.30.0/bin/linux/amd64/kubelet,https://cdn.dl.k8s.io/release/v1.30.0/bin/linux/amd64/kubelet + - 7c3807c0f5c1b30110a2ff1e55da1d112a6d0096201f1beb81b269f582b5d1c5@https://dl.k8s.io/release/v1.30.0/bin/linux/amd64/kubectl,https://cdn.dl.k8s.io/release/v1.30.0/bin/linux/amd64/kubectl + - 5035d7814c95cd3cedbc5efb447ef25a4942ef05caab2159746d55ce1698c74a@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/amd64/ecr-credential-provider-linux-amd64 + - 1511f6c003ace805eafeb1132727791326283cff88a923d76329e1892bba7a10@https://storage.googleapis.com/k8s-artifacts-cni/release/v1.4.1/cni-plugins-linux-amd64-v1.4.1.tgz,https://github.com/containernetworking/plugins/releases/download/v1.4.1/cni-plugins-linux-amd64-v1.4.1.tgz + - 1a94f15139f37633f39e24f08a4071f4533b285df3cbee6478972d26147bcaef@https://github.com/containerd/containerd/releases/download/v1.7.24/containerd-1.7.24-linux-amd64.tar.gz + - e83565aa78ec8f52a4d2b4eb6c4ca262b74c5f6770c1f43670c3029c20175502@https://github.com/opencontainers/runc/releases/download/v1.2.4/runc.amd64 + - 71aee9d987b7fad0ff2ade50b038ad7e2356324edc02c54045960a3521b3e6a7@https://github.com/containerd/nerdctl/releases/download/v1.7.4/nerdctl-1.7.4-linux-amd64.tar.gz + - d16a1ffb3938f5a19d5c8f45d363bd091ef89c0bc4d44ad16b933eede32fdcbb@https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.29.0/crictl-v1.29.0-linux-amd64.tar.gz + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 + arm64: + - fa887647422d34f3c7cc5b30fefcf97084d2c3277eff237c5808685ba8e4b15a@https://dl.k8s.io/release/v1.30.0/bin/linux/arm64/kubelet,https://cdn.dl.k8s.io/release/v1.30.0/bin/linux/arm64/kubelet + - 669af0cf520757298ea60a8b6eb6b719ba443a9c7d35f36d3fb2fd7513e8c7d2@https://dl.k8s.io/release/v1.30.0/bin/linux/arm64/kubectl,https://cdn.dl.k8s.io/release/v1.30.0/bin/linux/arm64/kubectl + - b3d567bda9e2996fc1fbd9d13506bd16763d3865b5c7b0b3c4b48c6088c04481@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/arm64/ecr-credential-provider-linux-arm64 + - 72644e13557cda8a5b39baf97fc5e93d23fdf7baba7700000e7e9efd8bdf9234@https://storage.googleapis.com/k8s-artifacts-cni/release/v1.4.1/cni-plugins-linux-arm64-v1.4.1.tgz,https://github.com/containernetworking/plugins/releases/download/v1.4.1/cni-plugins-linux-arm64-v1.4.1.tgz + - 420406d2b34ebb422ab3755fbeede59bf3bfcfccf5cfa584b558c93769d99064@https://github.com/containerd/containerd/releases/download/v1.7.24/containerd-1.7.24-linux-arm64.tar.gz + - 285f6c4c3de1d78d9f536a0299ae931219527b2ebd9ad89df5a1072896b7e82a@https://github.com/opencontainers/runc/releases/download/v1.2.4/runc.arm64 + - d8df47708ca57b9cd7f498055126ba7dcfc811d9ba43aae1830c93a09e70e22d@https://github.com/containerd/nerdctl/releases/download/v1.7.4/nerdctl-1.7.4-linux-arm64.tar.gz + - 0b615cfa00c331fb9c4524f3d4058a61cc487b33a3436d1269e7832cf283f925@https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.29.0/crictl-v1.29.0-linux-arm64.tar.gz + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 +CAs: + apiserver-aggregator-ca: | + -----BEGIN CERTIFICATE----- + MIIBgjCCASygAwIBAgIMFo3gINaZLHjisEcbMA0GCSqGSIb3DQEBCwUAMCIxIDAe + BgNVBAMTF2FwaXNlcnZlci1hZ2dyZWdhdG9yLWNhMB4XDTIxMDYzMDA0NTExMloX + DTMxMDYzMDA0NTExMlowIjEgMB4GA1UEAxMXYXBpc2VydmVyLWFnZ3JlZ2F0b3It + Y2EwXDANBgkqhkiG9w0BAQEFAANLADBIAkEAyyE71AOU3go5XFegLQ6fidI0LhhM + x7CzpTzh2xWKcHUfbNI7itgJvC/+GlyG5W+DF5V7ba0IJiQLsFve0oLdewIDAQAB + o0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU + ALfqF5ZmfqvqORuJIFilZYKF3d0wDQYJKoZIhvcNAQELBQADQQAHAomFKsF4jvYX + WM/UzQXDj9nSAFTf8dBPCXyZZNotsOH7+P6W4mMiuVs8bAuGiXGUdbsQ2lpiT/Rk + CzMeMdr4 + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBgjCCASygAwIBAgIMFo3gM0nxQpiX/agfMA0GCSqGSIb3DQEBCwUAMCIxIDAe + BgNVBAMTF2FwaXNlcnZlci1hZ2dyZWdhdG9yLWNhMB4XDTIxMDYzMDA0NTIzMVoX + DTMxMDYzMDA0NTIzMVowIjEgMB4GA1UEAxMXYXBpc2VydmVyLWFnZ3JlZ2F0b3It + Y2EwXDANBgkqhkiG9w0BAQEFAANLADBIAkEAyyE71AOU3go5XFegLQ6fidI0LhhM + x7CzpTzh2xWKcHUfbNI7itgJvC/+GlyG5W+DF5V7ba0IJiQLsFve0oLdewIDAQAB + o0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU + ALfqF5ZmfqvqORuJIFilZYKF3d0wDQYJKoZIhvcNAQELBQADQQCXsoezoxXu2CEN + QdlXZOfmBT6cqxIX/RMHXhpHwRiqPsTO8IO2bVA8CSzxNwMuSv/ZtrMHoh8+PcVW + HLtkTXH8 + -----END CERTIFICATE----- + etcd-clients-ca: | + -----BEGIN CERTIFICATE----- + MIIBcjCCARygAwIBAgIMFo1ogHnr26DL9YkqMA0GCSqGSIb3DQEBCwUAMBoxGDAW + BgNVBAMTD2V0Y2QtY2xpZW50cy1jYTAeFw0yMTA2MjgxNjE5MDFaFw0zMTA2Mjgx + NjE5MDFaMBoxGDAWBgNVBAMTD2V0Y2QtY2xpZW50cy1jYTBcMA0GCSqGSIb3DQEB + AQUAA0sAMEgCQQDYlt4Xx03Cp8QooPrloaVWznx9aQDSpl1UsrDyoBPNEElOLWep + uPaQBHiDLL8LwzGi7G9r+ib13tKrwprnlPv7AgMBAAGjQjBAMA4GA1UdDwEB/wQE + AwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQjlt4Ue54AbJPWlDpRM51s + x+PeBDANBgkqhkiG9w0BAQsFAANBAAZAdf8ROEVkr3Rf7I+s+CQOil2toadlKWOY + qCeJ2XaEROfp9aUTEIU1MGM3g57MPyAPPU7mURskuOQz6B1UFaY= + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBcjCCARygAwIBAgIMFo1olfBnC/CsT+dqMA0GCSqGSIb3DQEBCwUAMBoxGDAW + BgNVBAMTD2V0Y2QtY2xpZW50cy1jYTAeFw0yMTA2MjgxNjIwMzNaFw0zMTA2Mjgx + NjIwMzNaMBoxGDAWBgNVBAMTD2V0Y2QtY2xpZW50cy1jYTBcMA0GCSqGSIb3DQEB + AQUAA0sAMEgCQQDYlt4Xx03Cp8QooPrloaVWznx9aQDSpl1UsrDyoBPNEElOLWep + uPaQBHiDLL8LwzGi7G9r+ib13tKrwprnlPv7AgMBAAGjQjBAMA4GA1UdDwEB/wQE + AwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQjlt4Ue54AbJPWlDpRM51s + x+PeBDANBgkqhkiG9w0BAQsFAANBAF1xUz77PlUVUnd9duF8F7plou0TONC9R6/E + YQ8C6vM1b+9NSDGjCW8YmwEU2fBgskb/BBX2lwVZ32/RUEju4Co= + -----END CERTIFICATE----- + etcd-manager-ca-events: | + -----BEGIN CERTIFICATE----- + MIIBgDCCASqgAwIBAgIMFo+bKjm04vB4rNtaMA0GCSqGSIb3DQEBCwUAMCExHzAd + BgNVBAMTFmV0Y2QtbWFuYWdlci1jYS1ldmVudHMwHhcNMjEwNzA1MjAwOTU2WhcN + MzEwNzA1MjAwOTU2WjAhMR8wHQYDVQQDExZldGNkLW1hbmFnZXItY2EtZXZlbnRz + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKiC8tndMlEFZ7qzeKxeKqFVjaYpsh/H + g7RxWo15+1kgH3suO0lxp9+RxSVv97hnsfbySTPZVhy2cIQj7eZtZt8CAwEAAaNC + MEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFBg6 + CEZkQNnRkARBwFce03AEWa+sMA0GCSqGSIb3DQEBCwUAA0EAJMnBThok/uUe8q8O + sS5q19KUuE8YCTUzMDj36EBKf6NX4NoakCa1h6kfQVtlMtEIMWQZCjbm8xGK5ffs + GS/VUw== + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBgDCCASqgAwIBAgIMFo+bQ+EgIiBmGghjMA0GCSqGSIb3DQEBCwUAMCExHzAd + BgNVBAMTFmV0Y2QtbWFuYWdlci1jYS1ldmVudHMwHhcNMjEwNzA1MjAxMTQ2WhcN + MzEwNzA1MjAxMTQ2WjAhMR8wHQYDVQQDExZldGNkLW1hbmFnZXItY2EtZXZlbnRz + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKFhHVVxxDGv8d1jBvtdSxz7KIVoBOjL + DMxsmTsINiQkTQaFlb+XPlnY1ar4+RhE519AFUkqfhypk4Zxqf1YFXUCAwEAAaNC + MEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNuW + LLH5c8kDubDbr6BHgedW0iJ9MA0GCSqGSIb3DQEBCwUAA0EAiKUoBoaGu7XzboFE + hjfKlX0TujqWuW3qMxDEJwj4dVzlSLrAoB/G01MJ+xxYKh456n48aG6N827UPXhV + cPfVNg== + -----END CERTIFICATE----- + etcd-manager-ca-main: | + -----BEGIN CERTIFICATE----- + MIIBfDCCASagAwIBAgIMFo+bKjm1c3jfv6hIMA0GCSqGSIb3DQEBCwUAMB8xHTAb + BgNVBAMTFGV0Y2QtbWFuYWdlci1jYS1tYWluMB4XDTIxMDcwNTIwMDk1NloXDTMx + MDcwNTIwMDk1NlowHzEdMBsGA1UEAxMUZXRjZC1tYW5hZ2VyLWNhLW1haW4wXDAN + BgkqhkiG9w0BAQEFAANLADBIAkEAxbkDbGYmCSShpRG3r+lzTOFujyuruRfjOhYm + ZRX4w1Utd5y63dUc98sjc9GGUYMHd+0k1ql/a48tGhnK6N6jJwIDAQABo0IwQDAO + BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUWZLkbBFx + GAgPU4i62c52unSo7RswDQYJKoZIhvcNAQELBQADQQAj6Pgd0va/8FtkyMlnohLu + Gf4v8RJO6zk3Y6jJ4+cwWziipFM1ielMzSOZfFcCZgH3m5Io40is4hPSqyq2TOA6 + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBfDCCASagAwIBAgIMFo+bQ+Eg8Si30gr4MA0GCSqGSIb3DQEBCwUAMB8xHTAb + BgNVBAMTFGV0Y2QtbWFuYWdlci1jYS1tYWluMB4XDTIxMDcwNTIwMTE0NloXDTMx + MDcwNTIwMTE0NlowHzEdMBsGA1UEAxMUZXRjZC1tYW5hZ2VyLWNhLW1haW4wXDAN + BgkqhkiG9w0BAQEFAANLADBIAkEAw33jzcd/iosN04b0WXbDt7B0c3sJ3aafcGLP + vG3xRB9N5bYr9+qZAq3mzAFkxscn4j1ce5b1/GKTDEAClmZgdQIDAQABo0IwQDAO + BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUE/h+3gDP + DvKwHRyiYlXM8voZ1wowDQYJKoZIhvcNAQELBQADQQBXuimeEoAOu5HN4hG7NqL9 + t40K3ZRhRZv3JQWnRVJCBDjg1rD0GQJR/n+DoWvbeijI5C9pNjr2pWSIYR1eYCvd + -----END CERTIFICATE----- + etcd-peers-ca-events: | + -----BEGIN CERTIFICATE----- + MIIBfDCCASagAwIBAgIMFo+bKjmxTPh3/lYJMA0GCSqGSIb3DQEBCwUAMB8xHTAb + BgNVBAMTFGV0Y2QtcGVlcnMtY2EtZXZlbnRzMB4XDTIxMDcwNTIwMDk1NloXDTMx + MDcwNTIwMDk1NlowHzEdMBsGA1UEAxMUZXRjZC1wZWVycy1jYS1ldmVudHMwXDAN + BgkqhkiG9w0BAQEFAANLADBIAkEAv5g4HF2xmrYyouJfY9jXx1M3gPLD/pupvxPY + xyjJw5pNCy5M5XGS3iTqRD5RDE0fWudVHFZKLIe8WPc06NApXwIDAQABo0IwQDAO + BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUf6xiDI+O + Yph1ziCGr2hZaQYt+fUwDQYJKoZIhvcNAQELBQADQQBBxj5hqEQstonTb8lnqeGB + DEYtUeAk4eR/HzvUMjF52LVGuvN3XVt+JTrFeKNvb6/RDUbBNRj3azalcUkpPh6V + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBfDCCASagAwIBAgIMFo+bQ+Eq69jgzpKwMA0GCSqGSIb3DQEBCwUAMB8xHTAb + BgNVBAMTFGV0Y2QtcGVlcnMtY2EtZXZlbnRzMB4XDTIxMDcwNTIwMTE0NloXDTMx + MDcwNTIwMTE0NlowHzEdMBsGA1UEAxMUZXRjZC1wZWVycy1jYS1ldmVudHMwXDAN + BgkqhkiG9w0BAQEFAANLADBIAkEAo5Nj2CjX1qp3mEPw1H5nHAFWLoGNSLSlRFJW + 03NxaNPMFzL5PrCoyOXrX8/MWczuZYw0Crf8EPOOQWi2+W0XLwIDAQABo0IwQDAO + BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUxauhhKQh + cvdZND78rHe0RQVTTiswDQYJKoZIhvcNAQELBQADQQB+cq4jIS9q0zXslaRa+ViI + J+dviA3sMygbmSJO0s4DxYmoazKJblux5q0ASSvS9iL1l9ShuZ1dWyp2tpZawHyb + -----END CERTIFICATE----- + etcd-peers-ca-main: | + -----BEGIN CERTIFICATE----- + MIIBeDCCASKgAwIBAgIMFo+bKjmuLDDLcDHsMA0GCSqGSIb3DQEBCwUAMB0xGzAZ + BgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjAeFw0yMTA3MDUyMDA5NTZaFw0zMTA3 + MDUyMDA5NTZaMB0xGzAZBgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjBcMA0GCSqG + SIb3DQEBAQUAA0sAMEgCQQCyRaXWpwgN6INQqws9p/BvPElJv2Rno9dVTFhlQqDA + aUJXe7MBmiO4NJcW76EozeBh5ztR3/4NE1FM2x8TisS3AgMBAAGjQjBAMA4GA1Ud + DwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQtE1d49uSvpURf + OQ25Vlu6liY20DANBgkqhkiG9w0BAQsFAANBAAgLVaetJZcfOA3OIMMvQbz2Ydrt + uWF9BKkIad8jrcIrm3IkOtR8bKGmDIIaRKuG/ZUOL6NMe2fky3AAfKwleL4= + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBeDCCASKgAwIBAgIMFo+bQ+EuVthBfuZvMA0GCSqGSIb3DQEBCwUAMB0xGzAZ + BgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjAeFw0yMTA3MDUyMDExNDZaFw0zMTA3 + MDUyMDExNDZaMB0xGzAZBgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjBcMA0GCSqG + SIb3DQEBAQUAA0sAMEgCQQCxNbycDZNx5V1ZOiXxZSvaFpHRwKeHDfcuMUitdoPt + naVMlMTGDWAMuCVmFHFAWohIYynemEegmZkZ15S7AErfAgMBAAGjQjBAMA4GA1Ud + DwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTAjQ8T4HclPIsC + qipEfUIcLP6jqTANBgkqhkiG9w0BAQsFAANBAJdZ17TN3HlWrH7HQgfR12UBwz8K + G9DurDznVaBVUYaHY8Sg5AvAXeb+yIF2JMmRR+bK+/G1QYY2D3/P31Ic2Oo= + -----END CERTIFICATE----- + kubernetes-ca: | + -----BEGIN CERTIFICATE----- + MIIBbjCCARigAwIBAgIMFpANqBD8NSD82AUSMA0GCSqGSIb3DQEBCwUAMBgxFjAU + BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwODAwWhcNMzEwNzA3MDcw + ODAwWjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD + SwAwSAJBANFI3zr0Tk8krsW8vwjfMpzJOlWQ8616vG3YPa2qAgI7V4oKwfV0yIg1 + jt+H6f4P/wkPAPTPTfRp9Iy8oHEEFw0CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG + MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNG3zVjTcLlJwDsJ4/K9DV7KohUA + MA0GCSqGSIb3DQEBCwUAA0EAB8d03fY2w7WKpfO29qI295pu2C4ca9AiVGOpgSc8 + tmQsq6rcxt3T+rb589PVtz0mw/cKTxOk6gH2CCC+yHfy2w== + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBbjCCARigAwIBAgIMFpANvmSa0OAlYmXKMA0GCSqGSIb3DQEBCwUAMBgxFjAU + BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwOTM2WhcNMzEwNzA3MDcw + OTM2WjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD + SwAwSAJBAMF6F4aZdpe0RUpyykaBpWwZCnwbffhYGOw+fs6RdLuUq7QCNmJm/Eq7 + WWOziMYDiI9SbclpD+6QiJ0N3EqppVUCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG + MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFLImp6ARjPDAH6nhI+scWVt3Q9bn + MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA + 9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw== + -----END CERTIFICATE----- +ClusterName: privatekindnet.example.com +ControlPlaneConfig: + KubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: external + clusterCIDR: 100.96.0.0/11 + clusterName: privatekindnet.example.com + configureCloudRoutes: false + featureGates: + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-controller-manager:v1.30.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + KubeScheduler: + featureGates: + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-scheduler:v1.30.0 + leaderElection: + leaderElect: true + logLevel: 2 +DNSZone: Z1AFAKE1ZON3YO +EtcdClusterNames: +- main +- events +FileAssets: +- content: | + apiVersion: kubescheduler.config.k8s.io/v1 + clientConnection: + kubeconfig: /var/lib/kube-scheduler/kubeconfig + kind: KubeSchedulerConfiguration + path: /var/lib/kube-scheduler/config.yaml +Hooks: +- null +- null +InstallCNIAssets: true +KeypairIDs: + apiserver-aggregator-ca: "6980187172486667078076483355" + etcd-clients-ca: "6979622252718071085282986282" + etcd-manager-ca-events: "6982279354000777253151890266" + etcd-manager-ca-main: "6982279354000936168671127624" + etcd-peers-ca-events: "6982279353999767935825892873" + etcd-peers-ca-main: "6982279353998887468930183660" + kubernetes-ca: "6982820025135291416230495506" + service-account: "2" +KubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + image: registry.k8s.io/kube-proxy:v1.30.0 + logLevel: 2 +KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: external + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + InTreePluginAWSUnregister: "true" + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + node-role.kubernetes.io/control-plane: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + podInfraContainerImage: registry.k8s.io/pause:3.9 + podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true + registerSchedulable: true + shutdownGracePeriod: 30s + shutdownGracePeriodCriticalPods: 10s + taints: + - node-role.kubernetes.io/control-plane=:NoSchedule +KubernetesVersion: 1.30.0 +Networking: + kindnet: {} + nonMasqueradeCIDR: 100.64.0.0/10 + serviceClusterIPRange: 100.64.0.0/13 +UpdatePolicy: automatic +channels: +- memfs://clusters.example.com/privatekindnet.example.com/addons/bootstrap-channel.yaml +configStore: + keypairs: memfs://clusters.example.com/privatekindnet.example.com/pki + secrets: memfs://clusters.example.com/privatekindnet.example.com/secrets +containerdConfig: + logLevel: info + runc: + version: 1.2.4 + version: 1.7.24 +etcdManifests: +- memfs://clusters.example.com/privatekindnet.example.com/manifests/etcd/main-master-us-test-1a.yaml +- memfs://clusters.example.com/privatekindnet.example.com/manifests/etcd/events-master-us-test-1a.yaml +staticManifests: +- key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml +usesLegacyGossip: false +usesNoneDNS: false diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_nodeupconfig-nodes_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_nodeupconfig-nodes_content new file mode 100644 index 0000000000000..a3cae26db25b9 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_nodeupconfig-nodes_content @@ -0,0 +1,66 @@ +Assets: + amd64: + - 32a32ec3d7e7f8b2648c9dd503ce9ef63b4af1d1677f5b5aed7846fb02d66f18@https://dl.k8s.io/release/v1.30.0/bin/linux/amd64/kubelet,https://cdn.dl.k8s.io/release/v1.30.0/bin/linux/amd64/kubelet + - 7c3807c0f5c1b30110a2ff1e55da1d112a6d0096201f1beb81b269f582b5d1c5@https://dl.k8s.io/release/v1.30.0/bin/linux/amd64/kubectl,https://cdn.dl.k8s.io/release/v1.30.0/bin/linux/amd64/kubectl + - 5035d7814c95cd3cedbc5efb447ef25a4942ef05caab2159746d55ce1698c74a@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/amd64/ecr-credential-provider-linux-amd64 + - 1511f6c003ace805eafeb1132727791326283cff88a923d76329e1892bba7a10@https://storage.googleapis.com/k8s-artifacts-cni/release/v1.4.1/cni-plugins-linux-amd64-v1.4.1.tgz,https://github.com/containernetworking/plugins/releases/download/v1.4.1/cni-plugins-linux-amd64-v1.4.1.tgz + - 1a94f15139f37633f39e24f08a4071f4533b285df3cbee6478972d26147bcaef@https://github.com/containerd/containerd/releases/download/v1.7.24/containerd-1.7.24-linux-amd64.tar.gz + - e83565aa78ec8f52a4d2b4eb6c4ca262b74c5f6770c1f43670c3029c20175502@https://github.com/opencontainers/runc/releases/download/v1.2.4/runc.amd64 + - 71aee9d987b7fad0ff2ade50b038ad7e2356324edc02c54045960a3521b3e6a7@https://github.com/containerd/nerdctl/releases/download/v1.7.4/nerdctl-1.7.4-linux-amd64.tar.gz + - d16a1ffb3938f5a19d5c8f45d363bd091ef89c0bc4d44ad16b933eede32fdcbb@https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.29.0/crictl-v1.29.0-linux-amd64.tar.gz + arm64: + - fa887647422d34f3c7cc5b30fefcf97084d2c3277eff237c5808685ba8e4b15a@https://dl.k8s.io/release/v1.30.0/bin/linux/arm64/kubelet,https://cdn.dl.k8s.io/release/v1.30.0/bin/linux/arm64/kubelet + - 669af0cf520757298ea60a8b6eb6b719ba443a9c7d35f36d3fb2fd7513e8c7d2@https://dl.k8s.io/release/v1.30.0/bin/linux/arm64/kubectl,https://cdn.dl.k8s.io/release/v1.30.0/bin/linux/arm64/kubectl + - b3d567bda9e2996fc1fbd9d13506bd16763d3865b5c7b0b3c4b48c6088c04481@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/arm64/ecr-credential-provider-linux-arm64 + - 72644e13557cda8a5b39baf97fc5e93d23fdf7baba7700000e7e9efd8bdf9234@https://storage.googleapis.com/k8s-artifacts-cni/release/v1.4.1/cni-plugins-linux-arm64-v1.4.1.tgz,https://github.com/containernetworking/plugins/releases/download/v1.4.1/cni-plugins-linux-arm64-v1.4.1.tgz + - 420406d2b34ebb422ab3755fbeede59bf3bfcfccf5cfa584b558c93769d99064@https://github.com/containerd/containerd/releases/download/v1.7.24/containerd-1.7.24-linux-arm64.tar.gz + - 285f6c4c3de1d78d9f536a0299ae931219527b2ebd9ad89df5a1072896b7e82a@https://github.com/opencontainers/runc/releases/download/v1.2.4/runc.arm64 + - d8df47708ca57b9cd7f498055126ba7dcfc811d9ba43aae1830c93a09e70e22d@https://github.com/containerd/nerdctl/releases/download/v1.7.4/nerdctl-1.7.4-linux-arm64.tar.gz + - 0b615cfa00c331fb9c4524f3d4058a61cc487b33a3436d1269e7832cf283f925@https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.29.0/crictl-v1.29.0-linux-arm64.tar.gz +CAs: {} +ClusterName: privatekindnet.example.com +Hooks: +- null +- null +InstallCNIAssets: true +KeypairIDs: + kubernetes-ca: "6982820025135291416230495506" +KubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + image: registry.k8s.io/kube-proxy:v1.30.0 + logLevel: 2 +KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: external + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + InTreePluginAWSUnregister: "true" + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + nodeLabels: + node-role.kubernetes.io/node: "" + podInfraContainerImage: registry.k8s.io/pause:3.9 + podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true + registerSchedulable: true + shutdownGracePeriod: 30s + shutdownGracePeriodCriticalPods: 10s +KubernetesVersion: 1.30.0 +Networking: + kindnet: {} + nonMasqueradeCIDR: 100.64.0.0/10 + serviceClusterIPRange: 100.64.0.0/13 +UpdatePolicy: automatic +containerdConfig: + logLevel: info + runc: + version: 1.2.4 + version: 1.7.24 +usesLegacyGossip: false +usesNoneDNS: false diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content new file mode 100644 index 0000000000000..a58ca59706518 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content @@ -0,0 +1,237 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: aws-cloud-controller.addons.k8s.io + k8s-app: aws-cloud-controller-manager + name: aws-cloud-controller-manager + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + template: + metadata: + creationTimestamp: null + labels: + k8s-app: aws-cloud-controller-manager + kops.k8s.io/managed-by: kops + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + containers: + - args: + - --allocate-node-cidrs=true + - --cluster-cidr=100.96.0.0/11 + - --cluster-name=privatekindnet.example.com + - --configure-cloud-routes=false + - --leader-elect=true + - --v=2 + - --cloud-provider=aws + - --use-service-account-credentials=true + - --cloud-config=/etc/kubernetes/cloud.config + env: + - name: KUBERNETES_SERVICE_HOST + value: 127.0.0.1 + image: registry.k8s.io/provider-aws/cloud-controller-manager:v1.30.3 + imagePullPolicy: IfNotPresent + name: aws-cloud-controller-manager + resources: + requests: + cpu: 200m + volumeMounts: + - mountPath: /etc/kubernetes/cloud.config + name: cloudconfig + readOnly: true + hostNetwork: true + nodeSelector: null + priorityClassName: system-cluster-critical + serviceAccountName: aws-cloud-controller-manager + tolerations: + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + - effect: NoSchedule + key: node.kubernetes.io/not-ready + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + - effect: NoSchedule + key: node-role.kubernetes.io/master + volumes: + - hostPath: + path: /etc/kubernetes/cloud.config + type: "" + name: cloudconfig + updateStrategy: + type: RollingUpdate + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: aws-cloud-controller.addons.k8s.io + name: aws-cloud-controller-manager + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: aws-cloud-controller.addons.k8s.io + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- apiGroup: "" + kind: ServiceAccount + name: aws-cloud-controller-manager + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: aws-cloud-controller.addons.k8s.io + name: system:cloud-controller-manager +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - '*' +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +- apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch +- apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update +- apiGroups: + - "" + resources: + - secrets + verbs: + - list + - watch +- apiGroups: + - "" + resourceNames: + - node-controller + - service-controller + - route-controller + resources: + - serviceaccounts/token + verbs: + - create + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: aws-cloud-controller.addons.k8s.io + name: system:cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager +subjects: +- apiGroup: "" + kind: ServiceAccount + name: aws-cloud-controller-manager + namespace: kube-system diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content new file mode 100644 index 0000000000000..78ac12d3d37f3 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content @@ -0,0 +1,1122 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-controller + namespace: kube-system +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + +--- + +apiVersion: v1 +automountServiceAccountToken: true +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +apiVersion: v1 +automountServiceAccountToken: true +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-node-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-external-attacher-role +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-node-role +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - patch +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-external-provisioner-role +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - patch + - delete +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list +- apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - volumeattributesclasses + verbs: + - get + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-external-resizer-role +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - volumeattributesclasses + verbs: + - get + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-external-snapshotter-role +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + - patch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-attacher-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role +subjects: +- kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-node-getter-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role +subjects: +- kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-provisioner-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role +subjects: +- kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-resizer-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role +subjects: +- kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-snapshotter-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role +subjects: +- kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-leases-role + namespace: kube-system +rules: +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-leases-rolebinding + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ebs-csi-leases-role +subjects: +- kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app: ebs-csi-controller + app.kubernetes.io/managed-by: kops + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-controller + namespace: kube-system +spec: + ports: + - name: metrics + port: 3301 + targetPort: 3301 + selector: + app: ebs-csi-controller + type: ClusterIP + +--- + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-node + namespace: kube-system +spec: + revisionHistoryLimit: 10 + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + creationTimestamp: null + labels: + app: ebs-csi-node + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + kops.k8s.io/managed-by: kops + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: topology.kubernetes.io/zone + operator: Exists + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + - key: node.kubernetes.io/instance-type + operator: NotIn + values: + - a1.medium + - a1.large + - a1.xlarge + - a1.2xlarge + - a1.4xlarge + containers: + - args: + - node + - --endpoint=$(CSI_ENDPOINT) + - --logging-format=text + - --v=2 + env: + - name: AWS_REGION + value: us-test-1 + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver:v1.33.0 + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - /bin/aws-ebs-csi-driver + - pre-stop-hook + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + privileged: true + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=2 + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + image: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar:v2.11.0-eks-1-30-10 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 30 + periodSeconds: 90 + timeoutSeconds: 15 + name: node-driver-registrar + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - mountPath: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + name: probe-dir + - args: + - --csi-address=/csi/csi.sock + image: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe:v2.13.0-eks-1-30-10 + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + hostNetwork: false + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + serviceAccountName: ebs-csi-node-sa + terminationGracePeriodSeconds: 30 + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + - emptyDir: {} + name: probe-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-controller + namespace: kube-system +spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + creationTimestamp: null + labels: + app: ebs-csi-controller + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + kops.k8s.io/managed-by: kops + spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + weight: 1 + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - key: kubernetes.io/os + operator: In + values: + - linux + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - key: kubernetes.io/os + operator: In + values: + - linux + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - ebs-csi-controller + topologyKey: kubernetes.io/hostname + weight: 100 + containers: + - args: + - controller + - --endpoint=$(CSI_ENDPOINT) + - --k8s-tag-cluster-id=privatekindnet.example.com + - --extra-tags=KubernetesCluster=privatekindnet.example.com + - --http-endpoint=0.0.0.0:3301 + - --batching=true + - --logging-format=text + - --v=5 + env: + - name: AWS_REGION + value: us-test-1 + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + - name: AWS_EC2_ENDPOINT + valueFrom: + configMapKeyRef: + key: endpoint + name: aws-meta + optional: true + image: public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver:v1.33.0 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9811 + name: healthz + protocol: TCP + - containerPort: 3301 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --timeout=60s + - --csi-address=$(ADDRESS) + - --v=5 + - --feature-gates=Topology=true + - --extra-create-metadata + - --leader-election=true + - --default-fstype=ext4 + - --kube-api-qps=20 + - --kube-api-burst=100 + - --worker-threads=100 + - --retry-interval-max=30m + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner:v5.0.1-eks-1-30-10 + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --timeout=60s + - --csi-address=$(ADDRESS) + - --v=5 + - --leader-election=true + - --kube-api-qps=20 + - --kube-api-burst=100 + - --worker-threads=100 + - --retry-interval-max=5m + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: public.ecr.aws/eks-distro/kubernetes-csi/external-attacher:v4.6.1-eks-1-30-10 + imagePullPolicy: IfNotPresent + name: csi-attacher + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --timeout=60s + - --csi-address=$(ADDRESS) + - --v=5 + - --leader-election=true + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: public.ecr.aws/ebs-csi-driver/volume-modifier-for-k8s:v0.3.0 + imagePullPolicy: IfNotPresent + name: volumemodifier + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --timeout=60s + - --csi-address=$(ADDRESS) + - --v=5 + - --handle-volume-inuse-error=false + - --leader-election=true + - --kube-api-qps=20 + - --kube-api-burst=100 + - --workers=100 + - --retry-interval-max=30m + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: public.ecr.aws/eks-distro/kubernetes-csi/external-resizer:v1.11.1-eks-1-30-10 + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=/csi/csi.sock + - --http-endpoint=0.0.0.0:9811 + image: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe:v2.13.0-eks-1-30-10 + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + serviceAccountName: ebs-csi-controller-sa + tolerations: + - operator: Exists + topologySpreadConstraints: + - labelSelector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + - labelSelector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + volumes: + - emptyDir: {} + name: socket-dir + +--- + +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs.csi.aws.com +spec: + attachRequired: true + podInfoOnMount: false diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-bootstrap_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-bootstrap_content new file mode 100644 index 0000000000000..47149341ba4d3 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-bootstrap_content @@ -0,0 +1,121 @@ +kind: Addons +metadata: + creationTimestamp: null + name: bootstrap +spec: + addons: + - id: k8s-1.16 + manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml + manifestHash: eb16662bddba029d68576c2ef6cfc8975355a4e201a636992d0440b55466ea24 + name: kops-controller.addons.k8s.io + needsRollingUpdate: control-plane + selector: + k8s-addon: kops-controller.addons.k8s.io + version: 9.99.0 + - id: k8s-1.12 + manifest: coredns.addons.k8s.io/k8s-1.12.yaml + manifestHash: 591e3b40d00949575616698ce1c9230db8cb00bdab4f8a0d5ef14080a1d7a93c + name: coredns.addons.k8s.io + selector: + k8s-addon: coredns.addons.k8s.io + version: 9.99.0 + - id: k8s-1.9 + manifest: kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml + manifestHash: 01c120e887bd98d82ef57983ad58a0b22bc85efb48108092a24c4b82e4c9ea81 + name: kubelet-api.rbac.addons.k8s.io + selector: + k8s-addon: kubelet-api.rbac.addons.k8s.io + version: 9.99.0 + - manifest: limit-range.addons.k8s.io/v1.5.0.yaml + manifestHash: 2d55c3bc5e354e84a3730a65b42f39aba630a59dc8d32b30859fcce3d3178bc2 + name: limit-range.addons.k8s.io + selector: + k8s-addon: limit-range.addons.k8s.io + version: 9.99.0 + - id: k8s-1.12 + manifest: dns-controller.addons.k8s.io/k8s-1.12.yaml + manifestHash: 7a1b4740d79bc35f7fe0018fdd192743ef961feae4c065714dd6b59767125deb + name: dns-controller.addons.k8s.io + selector: + k8s-addon: dns-controller.addons.k8s.io + version: 9.99.0 + - id: k8s-1.11 + manifest: node-termination-handler.aws/k8s-1.11.yaml + manifestHash: e368ee43ac848123dc417eb01da535dd834120490be6456dcf9f1c58817f1cd3 + name: node-termination-handler.aws + prune: + kinds: + - kind: ConfigMap + labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops + - kind: Service + labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops + - kind: ServiceAccount + labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops + namespaces: + - kube-system + - group: admissionregistration.k8s.io + kind: MutatingWebhookConfiguration + labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops + - group: admissionregistration.k8s.io + kind: ValidatingWebhookConfiguration + labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops + - group: apps + kind: DaemonSet + labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops + - group: apps + kind: Deployment + labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops + namespaces: + - kube-system + - group: apps + kind: StatefulSet + labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops + - group: policy + kind: PodDisruptionBudget + labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops + namespaces: + - kube-system + - group: rbac.authorization.k8s.io + kind: ClusterRole + labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops + - group: rbac.authorization.k8s.io + kind: ClusterRoleBinding + labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops + - group: rbac.authorization.k8s.io + kind: Role + labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops + - group: rbac.authorization.k8s.io + kind: RoleBinding + labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops + selector: + k8s-addon: node-termination-handler.aws + version: 9.99.0 + - id: v1.15.0 + manifest: storage-aws.addons.k8s.io/v1.15.0.yaml + manifestHash: 4e2cda50cd5048133aad1b5e28becb60f4629d3f9e09c514a2757c27998b4200 + name: storage-aws.addons.k8s.io + selector: + k8s-addon: storage-aws.addons.k8s.io + version: 9.99.0 + - id: k8s-1.32 + manifest: networking.kindnet/k8s-1.32.yaml + manifestHash: fa9023e8e738acb584c86cfec376cab6ace06f314e896cdcc62a9a214b20eb11 + name: networking.kindnet + needsRollingUpdate: all + selector: + role.kubernetes.io/networking: "1" + version: 9.99.0 + - id: k8s-1.18 + manifest: aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml + manifestHash: cdab0df9fba966d97e240ae0ba6304d15c3c53906b8ff79365a934e2b660245d + name: aws-cloud-controller.addons.k8s.io + selector: + k8s-addon: aws-cloud-controller.addons.k8s.io + version: 9.99.0 + - id: k8s-1.17 + manifest: aws-ebs-csi-driver.addons.k8s.io/k8s-1.17.yaml + manifestHash: 0c2d807c601dcfc092c270e3c338a9245dd09bceee62b2e5697f7b45dd627e7b + name: aws-ebs-csi-driver.addons.k8s.io + selector: + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + version: 9.99.0 diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content new file mode 100644 index 0000000000000..31d86455d155c --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content @@ -0,0 +1,383 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + kubernetes.io/cluster-service: "true" + name: coredns + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns +subjects: +- kind: ServiceAccount + name: coredns + namespace: kube-system + +--- + +apiVersion: v1 +data: + Corefile: |- + .:53 { + errors + health { + lameduck 5s + } + ready + kubernetes cluster.local. in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + } + prometheus :9153 + forward . /etc/resolv.conf { + max_concurrent 1000 + } + cache 30 + loop + reload + loadbalance + } +kind: ConfigMap +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + addonmanager.kubernetes.io/mode: EnsureExists + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + name: coredns + namespace: kube-system + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: CoreDNS + name: coredns + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kube-dns + strategy: + rollingUpdate: + maxSurge: 10% + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + creationTimestamp: null + labels: + k8s-app: kube-dns + kops.k8s.io/managed-by: kops + spec: + containers: + - args: + - -conf + - /etc/coredns/Corefile + image: registry.k8s.io/coredns/coredns:v1.11.3 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + successThreshold: 1 + timeoutSeconds: 5 + name: coredns + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9153 + name: metrics + protocol: TCP + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + resources: + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - all + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /etc/coredns + name: config-volume + readOnly: true + dnsPolicy: Default + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: coredns + tolerations: + - key: CriticalAddonsOnly + operator: Exists + topologySpreadConstraints: + - labelSelector: + matchLabels: + k8s-app: kube-dns + maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + - labelSelector: + matchLabels: + k8s-app: kube-dns + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + volumes: + - configMap: + name: coredns + name: config-volume + +--- + +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "9153" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: CoreDNS + name: kube-dns + namespace: kube-system + resourceVersion: "0" +spec: + clusterIP: 100.64.0.10 + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP + - name: metrics + port: 9153 + protocol: TCP + selector: + k8s-app: kube-dns + +--- + +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + name: kube-dns + namespace: kube-system +spec: + maxUnavailable: 50% + selector: + matchLabels: + k8s-app: kube-dns + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + name: coredns-autoscaler + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + name: coredns-autoscaler +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - replicationcontrollers/scale + verbs: + - get + - update +- apiGroups: + - extensions + - apps + resources: + - deployments/scale + - replicasets/scale + verbs: + - get + - update +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - create + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + name: coredns-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: coredns-autoscaler +subjects: +- kind: ServiceAccount + name: coredns-autoscaler + namespace: kube-system + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + k8s-app: coredns-autoscaler + kubernetes.io/cluster-service: "true" + name: coredns-autoscaler + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: coredns-autoscaler + template: + metadata: + creationTimestamp: null + labels: + k8s-app: coredns-autoscaler + kops.k8s.io/managed-by: kops + spec: + containers: + - command: + - /cluster-proportional-autoscaler + - --namespace=kube-system + - --configmap=coredns-autoscaler + - --target=Deployment/coredns + - --default-params={"linear":{"coresPerReplica":256,"nodesPerReplica":16,"preventSinglePointFailure":true}} + - --logtostderr=true + - --v=2 + image: registry.k8s.io/cpa/cluster-proportional-autoscaler:v1.8.9 + name: autoscaler + resources: + requests: + cpu: 20m + memory: 10Mi + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: coredns-autoscaler + tolerations: + - key: CriticalAddonsOnly + operator: Exists diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content new file mode 100644 index 0000000000000..9c176638a5d93 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content @@ -0,0 +1,138 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: dns-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: dns-controller.addons.k8s.io + k8s-app: dns-controller + version: v1.31.0-beta.1 + name: dns-controller + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: dns-controller + strategy: + type: Recreate + template: + metadata: + creationTimestamp: null + labels: + k8s-addon: dns-controller.addons.k8s.io + k8s-app: dns-controller + kops.k8s.io/managed-by: kops + version: v1.31.0-beta.1 + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + containers: + - args: + - --watch-ingress=false + - --dns=aws-route53 + - --zone=*/Z1AFAKE1ZON3YO + - --internal-ipv4 + - --zone=*/* + - -v=2 + command: null + env: + - name: KUBERNETES_SERVICE_HOST + value: 127.0.0.1 + image: registry.k8s.io/kops/dns-controller:1.31.0-beta.1 + name: dns-controller + resources: + requests: + cpu: 50m + memory: 50Mi + securityContext: + runAsNonRoot: true + dnsPolicy: Default + hostNetwork: true + nodeSelector: null + priorityClassName: system-cluster-critical + serviceAccount: dns-controller + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + operator: Exists + - key: node.kubernetes.io/not-ready + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + - key: node-role.kubernetes.io/master + operator: Exists + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: dns-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: dns-controller.addons.k8s.io + name: dns-controller + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: dns-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: dns-controller.addons.k8s.io + name: kops:dns-controller +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - ingress + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: dns-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: dns-controller.addons.k8s.io + name: kops:dns-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kops:dns-controller +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:serviceaccount:kube-system:dns-controller diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content new file mode 100644 index 0000000000000..8605a556876f7 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content @@ -0,0 +1,227 @@ +apiVersion: v1 +data: + config.yaml: | + {"clusterName":"privatekindnet.example.com","cloud":"aws","configBase":"memfs://clusters.example.com/privatekindnet.example.com","secretStore":"memfs://clusters.example.com/privatekindnet.example.com/secrets","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["nodes.privatekindnet.example.com"],"Region":"us-test-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server","kube-proxy"]}} +kind: ConfigMap +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system + +--- + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + k8s-app: kops-controller + version: v1.31.0-beta.1 + name: kops-controller + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kops-controller + template: + metadata: + annotations: + dns.alpha.kubernetes.io/internal: kops-controller.internal.privatekindnet.example.com + creationTimestamp: null + labels: + k8s-addon: kops-controller.addons.k8s.io + k8s-app: kops-controller + kops.k8s.io/managed-by: kops + version: v1.31.0-beta.1 + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - key: kops.k8s.io/kops-controller-pki + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - key: kops.k8s.io/kops-controller-pki + operator: Exists + containers: + - args: + - --v=2 + - --conf=/etc/kubernetes/kops-controller/config/config.yaml + command: null + env: + - name: KUBERNETES_SERVICE_HOST + value: 127.0.0.1 + - name: KOPS_RUN_TOO_NEW_VERSION + value: "1" + image: registry.k8s.io/kops/kops-controller:1.31.0-beta.1 + name: kops-controller + resources: + requests: + cpu: 50m + memory: 50Mi + securityContext: + runAsNonRoot: true + runAsUser: 10011 + volumeMounts: + - mountPath: /etc/kubernetes/kops-controller/config/ + name: kops-controller-config + - mountPath: /etc/kubernetes/kops-controller/pki/ + name: kops-controller-pki + dnsPolicy: Default + hostNetwork: true + nodeSelector: null + priorityClassName: system-cluster-critical + serviceAccount: kops-controller + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + operator: Exists + - key: node.kubernetes.io/not-ready + operator: Exists + - key: node-role.kubernetes.io/master + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + volumes: + - configMap: + name: kops-controller + name: kops-controller-config + - hostPath: + path: /etc/kubernetes/kops-controller/ + type: Directory + name: kops-controller-pki + updateStrategy: + type: OnDelete + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - patch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kops-controller +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:serviceaccount:kube-system:kops-controller + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - create +- apiGroups: + - "" + - coordination.k8s.io + resourceNames: + - kops-controller-leader + resources: + - configmaps + - leases + verbs: + - get + - list + - watch + - patch + - update + - delete +- apiGroups: + - "" + - coordination.k8s.io + resources: + - configmaps + - leases + verbs: + - create + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kops-controller +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:serviceaccount:kube-system:kops-controller diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content new file mode 100644 index 0000000000000..36761e1c56255 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kubelet-api.rbac.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kubelet-api.rbac.addons.k8s.io + name: kops:system:kubelet-api-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kubelet-api-admin +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: kubelet-api diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-limit-range.addons.k8s.io_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-limit-range.addons.k8s.io_content new file mode 100644 index 0000000000000..4dcdce48b9ab9 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-limit-range.addons.k8s.io_content @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: LimitRange +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: limit-range.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: limit-range.addons.k8s.io + name: limits + namespace: default +spec: + limits: + - defaultRequest: + cpu: 100m + type: Container diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-networking.kindnet-k8s-1.32_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-networking.kindnet-k8s-1.32_content new file mode 100644 index 0000000000000..a302a14fd4688 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-networking.kindnet-k8s-1.32_content @@ -0,0 +1,163 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: networking.kindnet + app.kubernetes.io/managed-by: kops + role.kubernetes.io/networking: "1" + name: kindnet +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch + - patch +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get +- apiGroups: + - "" + resources: + - pods + - namespaces + verbs: + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: networking.kindnet + app.kubernetes.io/managed-by: kops + role.kubernetes.io/networking: "1" + name: kindnet +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kindnet +subjects: +- kind: ServiceAccount + name: kindnet + namespace: kube-system + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: networking.kindnet + app.kubernetes.io/managed-by: kops + role.kubernetes.io/networking: "1" + name: kindnet + namespace: kube-system + +--- + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: networking.kindnet + app: kindnet + app.kubernetes.io/managed-by: kops + k8s-app: kindnet + role.kubernetes.io/networking: "1" + tier: node + name: kindnet + namespace: kube-system +spec: + selector: + matchLabels: + app: kindnet + template: + metadata: + creationTimestamp: null + labels: + app: kindnet + k8s-app: kindnet + kops.k8s.io/managed-by: kops + tier: node + spec: + containers: + - args: + - /bin/kindnetd + - --hostname-override=$(NODE_NAME) + - --v=2 + - --fastpath-threshold=0 + - --masquerading=true + - --no-masquerade-cidr=172.20.0.0/16,100.96.0.0/11,100.64.0.0/13 + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: ghcr.io/aojea/kindnetd:v1.8.0 + name: kindnet-cni + resources: + requests: + cpu: 100m + memory: 50Mi + securityContext: + privileged: true + volumeMounts: + - mountPath: /etc/cni/net.d + name: cni-cfg + - mountPath: /var/lib/cni-kindnet + name: var-lib-kindnet + hostNetwork: true + initContainers: + - command: + - sh + - -c + - cat /opt/cni/bin/cni-kindnet > /cni/cni-kindnet ; chmod +x /cni/cni-kindnet + image: ghcr.io/aojea/kindnetd:v1.8.0 + name: install-cni-bin + volumeMounts: + - mountPath: /cni + name: cni-bin + serviceAccountName: kindnet + tolerations: + - effect: NoSchedule + operator: Exists + volumes: + - hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + name: cni-bin + - hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + name: cni-cfg + - hostPath: + path: /var/lib/cni-kindnet + type: DirectoryOrCreate + name: var-lib-kindnet diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-node-termination-handler.aws-k8s-1.11_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-node-termination-handler.aws-k8s-1.11_content new file mode 100644 index 0000000000000..12dee2372a19a --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-node-termination-handler.aws-k8s-1.11_content @@ -0,0 +1,285 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: node-termination-handler.aws + app.kubernetes.io/instance: aws-node-termination-handler + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-node-termination-handler + app.kubernetes.io/part-of: aws-node-termination-handler + app.kubernetes.io/version: v1.22.0 + k8s-addon: node-termination-handler.aws + k8s-app: aws-node-termination-handler + name: aws-node-termination-handler + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: node-termination-handler.aws + app.kubernetes.io/instance: aws-node-termination-handler + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-node-termination-handler + app.kubernetes.io/part-of: aws-node-termination-handler + app.kubernetes.io/version: v1.22.0 + k8s-addon: node-termination-handler.aws + name: aws-node-termination-handler +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - patch + - update +- apiGroups: + - "" + resources: + - pods + verbs: + - list + - get +- apiGroups: + - "" + resources: + - pods/eviction + verbs: + - create +- apiGroups: + - extensions + resources: + - daemonsets + verbs: + - get +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - get +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: node-termination-handler.aws + app.kubernetes.io/instance: aws-node-termination-handler + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-node-termination-handler + app.kubernetes.io/part-of: aws-node-termination-handler + app.kubernetes.io/version: v1.22.0 + k8s-addon: node-termination-handler.aws + name: aws-node-termination-handler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: aws-node-termination-handler +subjects: +- kind: ServiceAccount + name: aws-node-termination-handler + namespace: kube-system + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: node-termination-handler.aws + app.kubernetes.io/component: deployment + app.kubernetes.io/instance: aws-node-termination-handler + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-node-termination-handler + app.kubernetes.io/part-of: aws-node-termination-handler + app.kubernetes.io/version: v1.22.0 + k8s-addon: node-termination-handler.aws + k8s-app: aws-node-termination-handler + name: aws-node-termination-handler + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/instance: aws-node-termination-handler + app.kubernetes.io/name: aws-node-termination-handler + kubernetes.io/os: linux + template: + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: deployment + app.kubernetes.io/instance: aws-node-termination-handler + app.kubernetes.io/name: aws-node-termination-handler + k8s-app: aws-node-termination-handler + kops.k8s.io/managed-by: kops + kops.k8s.io/nth-mode: sqs + kubernetes.io/os: linux + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + containers: + - env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ENABLE_PROBES_SERVER + value: "true" + - name: PROBES_SERVER_PORT + value: "8080" + - name: PROBES_SERVER_ENDPOINT + value: /healthz + - name: LOG_LEVEL + value: info + - name: JSON_LOGGING + value: "true" + - name: LOG_FORMAT_VERSION + value: "2" + - name: ENABLE_PROMETHEUS_SERVER + value: "false" + - name: PROMETHEUS_SERVER_PORT + value: "9092" + - name: CHECK_TAG_BEFORE_DRAINING + value: "true" + - name: MANAGED_TAG + value: aws-node-termination-handler/managed + - name: USE_PROVIDER_ID + value: "true" + - name: DRY_RUN + value: "false" + - name: CORDON_ONLY + value: "false" + - name: TAINT_NODE + value: "false" + - name: EXCLUDE_FROM_LOAD_BALANCERS + value: "true" + - name: DELETE_LOCAL_DATA + value: "true" + - name: IGNORE_DAEMON_SETS + value: "true" + - name: POD_TERMINATION_GRACE_PERIOD + value: "-1" + - name: NODE_TERMINATION_GRACE_PERIOD + value: "120" + - name: EMIT_KUBERNETES_EVENTS + value: "true" + - name: COMPLETE_LIFECYCLE_ACTION_DELAY_SECONDS + value: "-1" + - name: ENABLE_SQS_TERMINATION_DRAINING + value: "true" + - name: QUEUE_URL + value: https://sqs.us-test-1.amazonaws.com/123456789012/privatekindnet-example-com-nth + - name: DELETE_SQS_MSG_IF_NODE_NOT_FOUND + value: "false" + - name: WORKERS + value: "10" + image: public.ecr.aws/aws-ec2/aws-node-termination-handler:v1.22.0 + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 + name: aws-node-termination-handler + ports: + - containerPort: 8080 + name: liveness-probe + protocol: TCP + - containerPort: 9092 + name: metrics + protocol: TCP + resources: + requests: + cpu: 50m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + hostNetwork: true + nodeSelector: null + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + serviceAccountName: aws-node-termination-handler + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - key: node-role.kubernetes.io/master + operator: Exists + topologySpreadConstraints: + - labelSelector: + matchLabels: + app.kubernetes.io/instance: aws-node-termination-handler + app.kubernetes.io/name: aws-node-termination-handler + kops.k8s.io/nth-mode: sqs + maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + - labelSelector: + matchLabels: + app.kubernetes.io/instance: aws-node-termination-handler + app.kubernetes.io/name: aws-node-termination-handler + kops.k8s.io/nth-mode: sqs + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + +--- + +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: node-termination-handler.aws + app.kubernetes.io/instance: aws-node-termination-handler + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-node-termination-handler + k8s-addon: node-termination-handler.aws + name: aws-node-termination-handler + namespace: kube-system +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/instance: aws-node-termination-handler + app.kubernetes.io/name: aws-node-termination-handler + kops.k8s.io/nth-mode: sqs diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content new file mode 100644 index 0000000000000..bea3e88be3ce7 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content @@ -0,0 +1,118 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: storage-aws.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: storage-aws.addons.k8s.io + name: default +parameters: + type: gp2 +provisioner: kubernetes.io/aws-ebs + +--- + +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + annotations: + storageclass.kubernetes.io/is-default-class: "false" + creationTimestamp: null + labels: + addon.kops.k8s.io/name: storage-aws.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: storage-aws.addons.k8s.io + name: gp2 +parameters: + type: gp2 +provisioner: kubernetes.io/aws-ebs + +--- + +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + annotations: + storageclass.kubernetes.io/is-default-class: "false" + creationTimestamp: null + labels: + addon.kops.k8s.io/name: storage-aws.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: storage-aws.addons.k8s.io + name: kops-ssd-1-17 +parameters: + encrypted: "true" + type: gp2 +provisioner: kubernetes.io/aws-ebs +volumeBindingMode: WaitForFirstConsumer + +--- + +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + annotations: + storageclass.kubernetes.io/is-default-class: "true" + creationTimestamp: null + labels: + addon.kops.k8s.io/name: storage-aws.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: storage-aws.addons.k8s.io + name: kops-csi-1-21 +parameters: + encrypted: "true" + type: gp3 +provisioner: ebs.csi.aws.com +volumeBindingMode: WaitForFirstConsumer + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: storage-aws.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: storage-aws.addons.k8s.io + name: system:aws-cloud-provider +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - patch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: storage-aws.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: storage-aws.addons.k8s.io + name: system:aws-cloud-provider +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:aws-cloud-provider +subjects: +- kind: ServiceAccount + name: aws-cloud-provider + namespace: kube-system diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_sqs_queue_privatekindnet-example-com-nth_policy b/tests/integration/update_cluster/privatekindnet/data/aws_sqs_queue_privatekindnet-example-com-nth_policy new file mode 100644 index 0000000000000..5acbcace3b129 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_sqs_queue_privatekindnet-example-com-nth_policy @@ -0,0 +1,16 @@ +{ + "Statement": [ + { + "Action": "sqs:SendMessage", + "Effect": "Allow", + "Principal": { + "Service": [ + "events.amazonaws.com", + "sqs.amazonaws.com" + ] + }, + "Resource": "arn:aws-test:sqs:us-test-1:123456789012:privatekindnet-example-com-nth" + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/privatekindnet/id_rsa.pub b/tests/integration/update_cluster/privatekindnet/id_rsa.pub new file mode 100644 index 0000000000000..81cb0127830e7 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ== diff --git a/tests/integration/update_cluster/privatekindnet/in-v1alpha2.yaml b/tests/integration/update_cluster/privatekindnet/in-v1alpha2.yaml new file mode 100644 index 0000000000000..a51a43fbfc88d --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/in-v1alpha2.yaml @@ -0,0 +1,98 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2016-12-12T04:13:14Z" + name: privatekindnet.example.com +spec: + kubernetesApiAccess: + - 0.0.0.0/0 + channel: stable + cloudProvider: aws + configBase: memfs://clusters.example.com/privatekindnet.example.com + etcdClusters: + - etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + name: main + - etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + name: events + iam: {} + kubelet: + anonymousAuth: false + kubernetesVersion: v1.30.0 + masterPublicName: api.privatekindnet.example.com + networkCIDR: 172.20.0.0/16 + networking: + kindnet: {} + nonMasqueradeCIDR: 100.64.0.0/10 + sshAccess: + - 0.0.0.0/0 + subnets: + - cidr: 172.20.32.0/19 + name: us-test-1a + type: Private + zone: us-test-1a + - cidr: 172.20.4.0/22 + name: utility-us-test-1a + type: Utility + zone: us-test-1a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2016-12-12T04:13:15Z" + name: master-us-test-1a + labels: + kops.k8s.io/cluster: privatekindnet.example.com +spec: + associatePublicIp: true + image: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220404 + machineType: m3.medium + maxSize: 1 + minSize: 1 + role: Master + subnets: + - us-test-1a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2016-12-12T04:13:15Z" + name: nodes + labels: + kops.k8s.io/cluster: privatekindnet.example.com +spec: + associatePublicIp: true + image: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220404 + machineType: t2.medium + maxSize: 2 + minSize: 2 + role: Node + subnets: + - us-test-1a + + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2016-12-14T15:32:41Z" + name: bastion + labels: + kops.k8s.io/cluster: privatekindnet.example.com +spec: + associatePublicIp: true + image: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220404 + machineType: t2.micro + maxSize: 1 + minSize: 1 + role: Bastion + subnets: + - utility-us-test-1a diff --git a/tests/integration/update_cluster/privatekindnet/kubernetes.tf b/tests/integration/update_cluster/privatekindnet/kubernetes.tf new file mode 100644 index 0000000000000..8d4f87b558d5b --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/kubernetes.tf @@ -0,0 +1,1467 @@ +locals { + bastion_autoscaling_group_ids = [aws_autoscaling_group.bastion-privatekindnet-example-com.id] + bastion_security_group_ids = [aws_security_group.bastion-privatekindnet-example-com.id] + bastions_role_arn = aws_iam_role.bastions-privatekindnet-example-com.arn + bastions_role_name = aws_iam_role.bastions-privatekindnet-example-com.name + cluster_name = "privatekindnet.example.com" + master_autoscaling_group_ids = [aws_autoscaling_group.master-us-test-1a-masters-privatekindnet-example-com.id] + master_security_group_ids = [aws_security_group.masters-privatekindnet-example-com.id] + masters_role_arn = aws_iam_role.masters-privatekindnet-example-com.arn + masters_role_name = aws_iam_role.masters-privatekindnet-example-com.name + node_autoscaling_group_ids = [aws_autoscaling_group.nodes-privatekindnet-example-com.id] + node_security_group_ids = [aws_security_group.nodes-privatekindnet-example-com.id] + node_subnet_ids = [aws_subnet.us-test-1a-privatekindnet-example-com.id] + nodes_role_arn = aws_iam_role.nodes-privatekindnet-example-com.arn + nodes_role_name = aws_iam_role.nodes-privatekindnet-example-com.name + region = "us-test-1" + route_table_private-us-test-1a_id = aws_route_table.private-us-test-1a-privatekindnet-example-com.id + route_table_public_id = aws_route_table.privatekindnet-example-com.id + subnet_us-test-1a_id = aws_subnet.us-test-1a-privatekindnet-example-com.id + subnet_utility-us-test-1a_id = aws_subnet.utility-us-test-1a-privatekindnet-example-com.id + vpc_cidr_block = aws_vpc.privatekindnet-example-com.cidr_block + vpc_id = aws_vpc.privatekindnet-example-com.id + vpc_ipv6_cidr_block = aws_vpc.privatekindnet-example-com.ipv6_cidr_block + vpc_ipv6_cidr_length = local.vpc_ipv6_cidr_block == "" ? null : tonumber(regex(".*/(\\d+)", local.vpc_ipv6_cidr_block)[0]) +} + +output "bastion_autoscaling_group_ids" { + value = [aws_autoscaling_group.bastion-privatekindnet-example-com.id] +} + +output "bastion_security_group_ids" { + value = [aws_security_group.bastion-privatekindnet-example-com.id] +} + +output "bastions_role_arn" { + value = aws_iam_role.bastions-privatekindnet-example-com.arn +} + +output "bastions_role_name" { + value = aws_iam_role.bastions-privatekindnet-example-com.name +} + +output "cluster_name" { + value = "privatekindnet.example.com" +} + +output "master_autoscaling_group_ids" { + value = [aws_autoscaling_group.master-us-test-1a-masters-privatekindnet-example-com.id] +} + +output "master_security_group_ids" { + value = [aws_security_group.masters-privatekindnet-example-com.id] +} + +output "masters_role_arn" { + value = aws_iam_role.masters-privatekindnet-example-com.arn +} + +output "masters_role_name" { + value = aws_iam_role.masters-privatekindnet-example-com.name +} + +output "node_autoscaling_group_ids" { + value = [aws_autoscaling_group.nodes-privatekindnet-example-com.id] +} + +output "node_security_group_ids" { + value = [aws_security_group.nodes-privatekindnet-example-com.id] +} + +output "node_subnet_ids" { + value = [aws_subnet.us-test-1a-privatekindnet-example-com.id] +} + +output "nodes_role_arn" { + value = aws_iam_role.nodes-privatekindnet-example-com.arn +} + +output "nodes_role_name" { + value = aws_iam_role.nodes-privatekindnet-example-com.name +} + +output "region" { + value = "us-test-1" +} + +output "route_table_private-us-test-1a_id" { + value = aws_route_table.private-us-test-1a-privatekindnet-example-com.id +} + +output "route_table_public_id" { + value = aws_route_table.privatekindnet-example-com.id +} + +output "subnet_us-test-1a_id" { + value = aws_subnet.us-test-1a-privatekindnet-example-com.id +} + +output "subnet_utility-us-test-1a_id" { + value = aws_subnet.utility-us-test-1a-privatekindnet-example-com.id +} + +output "vpc_cidr_block" { + value = aws_vpc.privatekindnet-example-com.cidr_block +} + +output "vpc_id" { + value = aws_vpc.privatekindnet-example-com.id +} + +output "vpc_ipv6_cidr_block" { + value = aws_vpc.privatekindnet-example-com.ipv6_cidr_block +} + +output "vpc_ipv6_cidr_length" { + value = local.vpc_ipv6_cidr_block == "" ? null : tonumber(regex(".*/(\\d+)", local.vpc_ipv6_cidr_block)[0]) +} + +provider "aws" { + region = "us-test-1" +} + +provider "aws" { + alias = "files" + region = "us-test-1" +} + +resource "aws_autoscaling_group" "bastion-privatekindnet-example-com" { + enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] + launch_template { + id = aws_launch_template.bastion-privatekindnet-example-com.id + version = aws_launch_template.bastion-privatekindnet-example-com.latest_version + } + max_instance_lifetime = 0 + max_size = 1 + metrics_granularity = "1Minute" + min_size = 1 + name = "bastion.privatekindnet.example.com" + protect_from_scale_in = false + tag { + key = "KubernetesCluster" + propagate_at_launch = true + value = "privatekindnet.example.com" + } + tag { + key = "Name" + propagate_at_launch = true + value = "bastion.privatekindnet.example.com" + } + tag { + key = "aws-node-termination-handler/managed" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/role/bastion" + propagate_at_launch = true + value = "1" + } + tag { + key = "kops.k8s.io/instancegroup" + propagate_at_launch = true + value = "bastion" + } + tag { + key = "kubernetes.io/cluster/privatekindnet.example.com" + propagate_at_launch = true + value = "owned" + } + target_group_arns = [aws_lb_target_group.bastion-privatekindnet-ex-70mv5p.id] + vpc_zone_identifier = [aws_subnet.utility-us-test-1a-privatekindnet-example-com.id] +} + +resource "aws_autoscaling_group" "master-us-test-1a-masters-privatekindnet-example-com" { + enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] + launch_template { + id = aws_launch_template.master-us-test-1a-masters-privatekindnet-example-com.id + version = aws_launch_template.master-us-test-1a-masters-privatekindnet-example-com.latest_version + } + load_balancers = [aws_elb.api-privatekindnet-example-com.id] + max_instance_lifetime = 0 + max_size = 1 + metrics_granularity = "1Minute" + min_size = 1 + name = "master-us-test-1a.masters.privatekindnet.example.com" + protect_from_scale_in = false + tag { + key = "KubernetesCluster" + propagate_at_launch = true + value = "privatekindnet.example.com" + } + tag { + key = "Name" + propagate_at_launch = true + value = "master-us-test-1a.masters.privatekindnet.example.com" + } + tag { + key = "aws-node-termination-handler/managed" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/role/control-plane" + propagate_at_launch = true + value = "1" + } + tag { + key = "k8s.io/role/master" + propagate_at_launch = true + value = "1" + } + tag { + key = "kops.k8s.io/instancegroup" + propagate_at_launch = true + value = "master-us-test-1a" + } + tag { + key = "kubernetes.io/cluster/privatekindnet.example.com" + propagate_at_launch = true + value = "owned" + } + vpc_zone_identifier = [aws_subnet.us-test-1a-privatekindnet-example-com.id] +} + +resource "aws_autoscaling_group" "nodes-privatekindnet-example-com" { + enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] + launch_template { + id = aws_launch_template.nodes-privatekindnet-example-com.id + version = aws_launch_template.nodes-privatekindnet-example-com.latest_version + } + max_instance_lifetime = 0 + max_size = 2 + metrics_granularity = "1Minute" + min_size = 2 + name = "nodes.privatekindnet.example.com" + protect_from_scale_in = false + tag { + key = "KubernetesCluster" + propagate_at_launch = true + value = "privatekindnet.example.com" + } + tag { + key = "Name" + propagate_at_launch = true + value = "nodes.privatekindnet.example.com" + } + tag { + key = "aws-node-termination-handler/managed" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/role/node" + propagate_at_launch = true + value = "1" + } + tag { + key = "kops.k8s.io/instancegroup" + propagate_at_launch = true + value = "nodes" + } + tag { + key = "kubernetes.io/cluster/privatekindnet.example.com" + propagate_at_launch = true + value = "owned" + } + vpc_zone_identifier = [aws_subnet.us-test-1a-privatekindnet-example-com.id] +} + +resource "aws_autoscaling_lifecycle_hook" "bastion-NTHLifecycleHook" { + autoscaling_group_name = aws_autoscaling_group.bastion-privatekindnet-example-com.id + default_result = "CONTINUE" + heartbeat_timeout = 300 + lifecycle_transition = "autoscaling:EC2_INSTANCE_TERMINATING" + name = "bastion-NTHLifecycleHook" +} + +resource "aws_autoscaling_lifecycle_hook" "master-us-test-1a-NTHLifecycleHook" { + autoscaling_group_name = aws_autoscaling_group.master-us-test-1a-masters-privatekindnet-example-com.id + default_result = "CONTINUE" + heartbeat_timeout = 300 + lifecycle_transition = "autoscaling:EC2_INSTANCE_TERMINATING" + name = "master-us-test-1a-NTHLifecycleHook" +} + +resource "aws_autoscaling_lifecycle_hook" "nodes-NTHLifecycleHook" { + autoscaling_group_name = aws_autoscaling_group.nodes-privatekindnet-example-com.id + default_result = "CONTINUE" + heartbeat_timeout = 300 + lifecycle_transition = "autoscaling:EC2_INSTANCE_TERMINATING" + name = "nodes-NTHLifecycleHook" +} + +resource "aws_cloudwatch_event_rule" "privatekindnet-example-com-ASGLifecycle" { + event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_privatekindnet.example.com-ASGLifecycle_event_pattern") + name = "privatekindnet.example.com-ASGLifecycle" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "privatekindnet.example.com-ASGLifecycle" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_cloudwatch_event_rule" "privatekindnet-example-com-InstanceScheduledChange" { + event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_privatekindnet.example.com-InstanceScheduledChange_event_pattern") + name = "privatekindnet.example.com-InstanceScheduledChange" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "privatekindnet.example.com-InstanceScheduledChange" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_cloudwatch_event_rule" "privatekindnet-example-com-InstanceStateChange" { + event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_privatekindnet.example.com-InstanceStateChange_event_pattern") + name = "privatekindnet.example.com-InstanceStateChange" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "privatekindnet.example.com-InstanceStateChange" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_cloudwatch_event_rule" "privatekindnet-example-com-SpotInterruption" { + event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_privatekindnet.example.com-SpotInterruption_event_pattern") + name = "privatekindnet.example.com-SpotInterruption" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "privatekindnet.example.com-SpotInterruption" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_cloudwatch_event_target" "privatekindnet-example-com-ASGLifecycle-Target" { + arn = aws_sqs_queue.privatekindnet-example-com-nth.arn + rule = aws_cloudwatch_event_rule.privatekindnet-example-com-ASGLifecycle.id +} + +resource "aws_cloudwatch_event_target" "privatekindnet-example-com-InstanceScheduledChange-Target" { + arn = aws_sqs_queue.privatekindnet-example-com-nth.arn + rule = aws_cloudwatch_event_rule.privatekindnet-example-com-InstanceScheduledChange.id +} + +resource "aws_cloudwatch_event_target" "privatekindnet-example-com-InstanceStateChange-Target" { + arn = aws_sqs_queue.privatekindnet-example-com-nth.arn + rule = aws_cloudwatch_event_rule.privatekindnet-example-com-InstanceStateChange.id +} + +resource "aws_cloudwatch_event_target" "privatekindnet-example-com-SpotInterruption-Target" { + arn = aws_sqs_queue.privatekindnet-example-com-nth.arn + rule = aws_cloudwatch_event_rule.privatekindnet-example-com-SpotInterruption.id +} + +resource "aws_ebs_volume" "us-test-1a-etcd-events-privatekindnet-example-com" { + availability_zone = "us-test-1a" + encrypted = false + iops = 3000 + size = 20 + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "us-test-1a.etcd-events.privatekindnet.example.com" + "k8s.io/etcd/events" = "us-test-1a/us-test-1a" + "k8s.io/role/control-plane" = "1" + "k8s.io/role/master" = "1" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + throughput = 125 + type = "gp3" +} + +resource "aws_ebs_volume" "us-test-1a-etcd-main-privatekindnet-example-com" { + availability_zone = "us-test-1a" + encrypted = false + iops = 3000 + size = 20 + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "us-test-1a.etcd-main.privatekindnet.example.com" + "k8s.io/etcd/main" = "us-test-1a/us-test-1a" + "k8s.io/role/control-plane" = "1" + "k8s.io/role/master" = "1" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + throughput = 125 + type = "gp3" +} + +resource "aws_eip" "us-test-1a-privatekindnet-example-com" { + domain = "vpc" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "us-test-1a.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_elb" "api-privatekindnet-example-com" { + connection_draining = true + connection_draining_timeout = 300 + cross_zone_load_balancing = false + health_check { + healthy_threshold = 2 + interval = 10 + target = "SSL:443" + timeout = 5 + unhealthy_threshold = 2 + } + idle_timeout = 300 + listener { + instance_port = 443 + instance_protocol = "TCP" + lb_port = 443 + lb_protocol = "TCP" + } + name = "api-privatekindnet-exampl-c5d4tf" + security_groups = [aws_security_group.api-elb-privatekindnet-example-com.id] + subnets = [aws_subnet.utility-us-test-1a-privatekindnet-example-com.id] + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "api.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_iam_instance_profile" "bastions-privatekindnet-example-com" { + name = "bastions.privatekindnet.example.com" + role = aws_iam_role.bastions-privatekindnet-example-com.name + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "bastions.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_iam_instance_profile" "masters-privatekindnet-example-com" { + name = "masters.privatekindnet.example.com" + role = aws_iam_role.masters-privatekindnet-example-com.name + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "masters.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_iam_instance_profile" "nodes-privatekindnet-example-com" { + name = "nodes.privatekindnet.example.com" + role = aws_iam_role.nodes-privatekindnet-example-com.name + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "nodes.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_iam_role" "bastions-privatekindnet-example-com" { + assume_role_policy = file("${path.module}/data/aws_iam_role_bastions.privatekindnet.example.com_policy") + name = "bastions.privatekindnet.example.com" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "bastions.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_iam_role" "masters-privatekindnet-example-com" { + assume_role_policy = file("${path.module}/data/aws_iam_role_masters.privatekindnet.example.com_policy") + name = "masters.privatekindnet.example.com" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "masters.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_iam_role" "nodes-privatekindnet-example-com" { + assume_role_policy = file("${path.module}/data/aws_iam_role_nodes.privatekindnet.example.com_policy") + name = "nodes.privatekindnet.example.com" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "nodes.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_iam_role_policy" "bastions-privatekindnet-example-com" { + name = "bastions.privatekindnet.example.com" + policy = file("${path.module}/data/aws_iam_role_policy_bastions.privatekindnet.example.com_policy") + role = aws_iam_role.bastions-privatekindnet-example-com.name +} + +resource "aws_iam_role_policy" "masters-privatekindnet-example-com" { + name = "masters.privatekindnet.example.com" + policy = file("${path.module}/data/aws_iam_role_policy_masters.privatekindnet.example.com_policy") + role = aws_iam_role.masters-privatekindnet-example-com.name +} + +resource "aws_iam_role_policy" "nodes-privatekindnet-example-com" { + name = "nodes.privatekindnet.example.com" + policy = file("${path.module}/data/aws_iam_role_policy_nodes.privatekindnet.example.com_policy") + role = aws_iam_role.nodes-privatekindnet-example-com.name +} + +resource "aws_internet_gateway" "privatekindnet-example-com" { + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + vpc_id = aws_vpc.privatekindnet-example-com.id +} + +resource "aws_key_pair" "kubernetes-privatekindnet-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157" { + key_name = "kubernetes.privatekindnet.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57" + public_key = file("${path.module}/data/aws_key_pair_kubernetes.privatekindnet.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key") + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_launch_template" "bastion-privatekindnet-example-com" { + block_device_mappings { + device_name = "/dev/xvda" + ebs { + delete_on_termination = true + encrypted = true + iops = 3000 + throughput = 125 + volume_size = 32 + volume_type = "gp3" + } + } + iam_instance_profile { + name = aws_iam_instance_profile.bastions-privatekindnet-example-com.id + } + image_id = "ami-12345678" + instance_type = "t2.micro" + key_name = aws_key_pair.kubernetes-privatekindnet-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id + lifecycle { + create_before_destroy = true + } + metadata_options { + http_endpoint = "enabled" + http_protocol_ipv6 = "disabled" + http_put_response_hop_limit = 1 + http_tokens = "required" + } + monitoring { + enabled = false + } + name = "bastion.privatekindnet.example.com" + network_interfaces { + associate_public_ip_address = true + delete_on_termination = true + ipv6_address_count = 0 + security_groups = [aws_security_group.bastion-privatekindnet-example-com.id] + } + tag_specifications { + resource_type = "instance" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "bastion.privatekindnet.example.com" + "aws-node-termination-handler/managed" = "" + "k8s.io/role/bastion" = "1" + "kops.k8s.io/instancegroup" = "bastion" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + } + tag_specifications { + resource_type = "volume" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "bastion.privatekindnet.example.com" + "aws-node-termination-handler/managed" = "" + "k8s.io/role/bastion" = "1" + "kops.k8s.io/instancegroup" = "bastion" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + } + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "bastion.privatekindnet.example.com" + "aws-node-termination-handler/managed" = "" + "k8s.io/role/bastion" = "1" + "kops.k8s.io/instancegroup" = "bastion" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_launch_template" "master-us-test-1a-masters-privatekindnet-example-com" { + block_device_mappings { + device_name = "/dev/xvda" + ebs { + delete_on_termination = true + encrypted = true + iops = 3000 + throughput = 125 + volume_size = 64 + volume_type = "gp3" + } + } + block_device_mappings { + device_name = "/dev/sdc" + virtual_name = "ephemeral0" + } + iam_instance_profile { + name = aws_iam_instance_profile.masters-privatekindnet-example-com.id + } + image_id = "ami-12345678" + instance_type = "m3.medium" + key_name = aws_key_pair.kubernetes-privatekindnet-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id + lifecycle { + create_before_destroy = true + } + metadata_options { + http_endpoint = "enabled" + http_protocol_ipv6 = "disabled" + http_put_response_hop_limit = 1 + http_tokens = "required" + } + monitoring { + enabled = false + } + name = "master-us-test-1a.masters.privatekindnet.example.com" + network_interfaces { + associate_public_ip_address = false + delete_on_termination = true + ipv6_address_count = 0 + security_groups = [aws_security_group.masters-privatekindnet-example-com.id] + } + tag_specifications { + resource_type = "instance" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "master-us-test-1a.masters.privatekindnet.example.com" + "aws-node-termination-handler/managed" = "" + "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" + "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" + "k8s.io/role/control-plane" = "1" + "k8s.io/role/master" = "1" + "kops.k8s.io/instancegroup" = "master-us-test-1a" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + } + tag_specifications { + resource_type = "volume" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "master-us-test-1a.masters.privatekindnet.example.com" + "aws-node-termination-handler/managed" = "" + "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" + "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" + "k8s.io/role/control-plane" = "1" + "k8s.io/role/master" = "1" + "kops.k8s.io/instancegroup" = "master-us-test-1a" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + } + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "master-us-test-1a.masters.privatekindnet.example.com" + "aws-node-termination-handler/managed" = "" + "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" + "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" + "k8s.io/role/control-plane" = "1" + "k8s.io/role/master" = "1" + "kops.k8s.io/instancegroup" = "master-us-test-1a" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + user_data = filebase64("${path.module}/data/aws_launch_template_master-us-test-1a.masters.privatekindnet.example.com_user_data") +} + +resource "aws_launch_template" "nodes-privatekindnet-example-com" { + block_device_mappings { + device_name = "/dev/xvda" + ebs { + delete_on_termination = true + encrypted = true + iops = 3000 + throughput = 125 + volume_size = 128 + volume_type = "gp3" + } + } + iam_instance_profile { + name = aws_iam_instance_profile.nodes-privatekindnet-example-com.id + } + image_id = "ami-12345678" + instance_type = "t2.medium" + key_name = aws_key_pair.kubernetes-privatekindnet-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id + lifecycle { + create_before_destroy = true + } + metadata_options { + http_endpoint = "enabled" + http_protocol_ipv6 = "disabled" + http_put_response_hop_limit = 1 + http_tokens = "required" + } + monitoring { + enabled = false + } + name = "nodes.privatekindnet.example.com" + network_interfaces { + associate_public_ip_address = false + delete_on_termination = true + ipv6_address_count = 0 + security_groups = [aws_security_group.nodes-privatekindnet-example-com.id] + } + tag_specifications { + resource_type = "instance" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "nodes.privatekindnet.example.com" + "aws-node-termination-handler/managed" = "" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "nodes" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + } + tag_specifications { + resource_type = "volume" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "nodes.privatekindnet.example.com" + "aws-node-termination-handler/managed" = "" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "nodes" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + } + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "nodes.privatekindnet.example.com" + "aws-node-termination-handler/managed" = "" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "nodes" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + user_data = filebase64("${path.module}/data/aws_launch_template_nodes.privatekindnet.example.com_user_data") +} + +resource "aws_lb" "bastion-privatekindnet-example-com" { + enable_cross_zone_load_balancing = false + internal = false + load_balancer_type = "network" + name = "bastion-privatekindnet-ex-70mv5p" + security_groups = [aws_security_group.bastion-elb-privatekindnet-example-com.id] + subnet_mapping { + subnet_id = aws_subnet.utility-us-test-1a-privatekindnet-example-com.id + } + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "bastion.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_lb_listener" "bastion-privatekindnet-example-com-22" { + default_action { + target_group_arn = aws_lb_target_group.bastion-privatekindnet-ex-70mv5p.id + type = "forward" + } + load_balancer_arn = aws_lb.bastion-privatekindnet-example-com.id + port = 22 + protocol = "TCP" +} + +resource "aws_lb_target_group" "bastion-privatekindnet-ex-70mv5p" { + connection_termination = "true" + deregistration_delay = "30" + health_check { + healthy_threshold = 2 + interval = 10 + protocol = "TCP" + unhealthy_threshold = 2 + } + name = "bastion-privatekindnet-ex-70mv5p" + port = 22 + protocol = "TCP" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "bastion-privatekindnet-ex-70mv5p" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + vpc_id = aws_vpc.privatekindnet-example-com.id +} + +resource "aws_nat_gateway" "us-test-1a-privatekindnet-example-com" { + allocation_id = aws_eip.us-test-1a-privatekindnet-example-com.id + subnet_id = aws_subnet.utility-us-test-1a-privatekindnet-example-com.id + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "us-test-1a.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_route" "route-0-0-0-0--0" { + destination_cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.privatekindnet-example-com.id + route_table_id = aws_route_table.privatekindnet-example-com.id +} + +resource "aws_route" "route-__--0" { + destination_ipv6_cidr_block = "::/0" + gateway_id = aws_internet_gateway.privatekindnet-example-com.id + route_table_id = aws_route_table.privatekindnet-example-com.id +} + +resource "aws_route" "route-private-us-test-1a-0-0-0-0--0" { + destination_cidr_block = "0.0.0.0/0" + nat_gateway_id = aws_nat_gateway.us-test-1a-privatekindnet-example-com.id + route_table_id = aws_route_table.private-us-test-1a-privatekindnet-example-com.id +} + +resource "aws_route53_record" "api-privatekindnet-example-com" { + alias { + evaluate_target_health = false + name = aws_elb.api-privatekindnet-example-com.dns_name + zone_id = aws_elb.api-privatekindnet-example-com.zone_id + } + name = "api.privatekindnet.example.com" + type = "A" + zone_id = "/hostedzone/Z1AFAKE1ZON3YO" +} + +resource "aws_route53_record" "api-privatekindnet-example-com-AAAA" { + alias { + evaluate_target_health = false + name = aws_elb.api-privatekindnet-example-com.dns_name + zone_id = aws_elb.api-privatekindnet-example-com.zone_id + } + name = "api.privatekindnet.example.com" + type = "AAAA" + zone_id = "/hostedzone/Z1AFAKE1ZON3YO" +} + +resource "aws_route_table" "private-us-test-1a-privatekindnet-example-com" { + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "private-us-test-1a.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + "kubernetes.io/kops/role" = "private-us-test-1a" + } + vpc_id = aws_vpc.privatekindnet-example-com.id +} + +resource "aws_route_table" "privatekindnet-example-com" { + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + "kubernetes.io/kops/role" = "public" + } + vpc_id = aws_vpc.privatekindnet-example-com.id +} + +resource "aws_route_table_association" "private-us-test-1a-privatekindnet-example-com" { + route_table_id = aws_route_table.private-us-test-1a-privatekindnet-example-com.id + subnet_id = aws_subnet.us-test-1a-privatekindnet-example-com.id +} + +resource "aws_route_table_association" "utility-us-test-1a-privatekindnet-example-com" { + route_table_id = aws_route_table.privatekindnet-example-com.id + subnet_id = aws_subnet.utility-us-test-1a-privatekindnet-example-com.id +} + +resource "aws_s3_object" "cluster-completed-spec" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_cluster-completed.spec_content") + key = "clusters.example.com/privatekindnet.example.com/cluster-completed.spec" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "etcd-cluster-spec-events" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_etcd-cluster-spec-events_content") + key = "clusters.example.com/privatekindnet.example.com/backups/etcd/events/control/etcd-cluster-spec" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "etcd-cluster-spec-main" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_etcd-cluster-spec-main_content") + key = "clusters.example.com/privatekindnet.example.com/backups/etcd/main/control/etcd-cluster-spec" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "kops-version-txt" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_kops-version.txt_content") + key = "clusters.example.com/privatekindnet.example.com/kops-version.txt" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "manifests-etcdmanager-events-master-us-test-1a" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-events-master-us-test-1a_content") + key = "clusters.example.com/privatekindnet.example.com/manifests/etcd/events-master-us-test-1a.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "manifests-etcdmanager-main-master-us-test-1a" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-main-master-us-test-1a_content") + key = "clusters.example.com/privatekindnet.example.com/manifests/etcd/main-master-us-test-1a.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "manifests-static-kube-apiserver-healthcheck" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_manifests-static-kube-apiserver-healthcheck_content") + key = "clusters.example.com/privatekindnet.example.com/manifests/static/kube-apiserver-healthcheck.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "nodeupconfig-master-us-test-1a" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_nodeupconfig-master-us-test-1a_content") + key = "clusters.example.com/privatekindnet.example.com/igconfig/control-plane/master-us-test-1a/nodeupconfig.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "nodeupconfig-nodes" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_nodeupconfig-nodes_content") + key = "clusters.example.com/privatekindnet.example.com/igconfig/node/nodes/nodeupconfig.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "privatekindnet-example-com-addons-aws-cloud-controller-addons-k8s-io-k8s-1-18" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_privatekindnet.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content") + key = "clusters.example.com/privatekindnet.example.com/addons/aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "privatekindnet-example-com-addons-aws-ebs-csi-driver-addons-k8s-io-k8s-1-17" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_privatekindnet.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content") + key = "clusters.example.com/privatekindnet.example.com/addons/aws-ebs-csi-driver.addons.k8s.io/k8s-1.17.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "privatekindnet-example-com-addons-bootstrap" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_privatekindnet.example.com-addons-bootstrap_content") + key = "clusters.example.com/privatekindnet.example.com/addons/bootstrap-channel.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "privatekindnet-example-com-addons-coredns-addons-k8s-io-k8s-1-12" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_privatekindnet.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content") + key = "clusters.example.com/privatekindnet.example.com/addons/coredns.addons.k8s.io/k8s-1.12.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "privatekindnet-example-com-addons-dns-controller-addons-k8s-io-k8s-1-12" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_privatekindnet.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content") + key = "clusters.example.com/privatekindnet.example.com/addons/dns-controller.addons.k8s.io/k8s-1.12.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "privatekindnet-example-com-addons-kops-controller-addons-k8s-io-k8s-1-16" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_privatekindnet.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content") + key = "clusters.example.com/privatekindnet.example.com/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "privatekindnet-example-com-addons-kubelet-api-rbac-addons-k8s-io-k8s-1-9" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_privatekindnet.example.com-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content") + key = "clusters.example.com/privatekindnet.example.com/addons/kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "privatekindnet-example-com-addons-limit-range-addons-k8s-io" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_privatekindnet.example.com-addons-limit-range.addons.k8s.io_content") + key = "clusters.example.com/privatekindnet.example.com/addons/limit-range.addons.k8s.io/v1.5.0.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "privatekindnet-example-com-addons-networking-kindnet-k8s-1-32" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_privatekindnet.example.com-addons-networking.kindnet-k8s-1.32_content") + key = "clusters.example.com/privatekindnet.example.com/addons/networking.kindnet/k8s-1.32.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "privatekindnet-example-com-addons-node-termination-handler-aws-k8s-1-11" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_privatekindnet.example.com-addons-node-termination-handler.aws-k8s-1.11_content") + key = "clusters.example.com/privatekindnet.example.com/addons/node-termination-handler.aws/k8s-1.11.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "privatekindnet-example-com-addons-storage-aws-addons-k8s-io-v1-15-0" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_privatekindnet.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content") + key = "clusters.example.com/privatekindnet.example.com/addons/storage-aws.addons.k8s.io/v1.15.0.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_security_group" "api-elb-privatekindnet-example-com" { + description = "Security group for api ELB" + name = "api-elb.privatekindnet.example.com" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "api-elb.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + vpc_id = aws_vpc.privatekindnet-example-com.id +} + +resource "aws_security_group" "bastion-elb-privatekindnet-example-com" { + description = "Security group for bastion ELB" + name = "bastion-elb.privatekindnet.example.com" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "bastion-elb.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + vpc_id = aws_vpc.privatekindnet-example-com.id +} + +resource "aws_security_group" "bastion-privatekindnet-example-com" { + description = "Security group for bastion" + name = "bastion.privatekindnet.example.com" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "bastion.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + vpc_id = aws_vpc.privatekindnet-example-com.id +} + +resource "aws_security_group" "masters-privatekindnet-example-com" { + description = "Security group for masters" + name = "masters.privatekindnet.example.com" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "masters.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + vpc_id = aws_vpc.privatekindnet-example-com.id +} + +resource "aws_security_group" "nodes-privatekindnet-example-com" { + description = "Security group for nodes" + name = "nodes.privatekindnet.example.com" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "nodes.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + vpc_id = aws_vpc.privatekindnet-example-com.id +} + +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-elb-privatekindnet-example-com" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.bastion-elb-privatekindnet-example-com.id + to_port = 22 + type = "ingress" +} + +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-443to443-api-elb-privatekindnet-example-com" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 443 + protocol = "tcp" + security_group_id = aws_security_group.api-elb-privatekindnet-example-com.id + to_port = 443 + type = "ingress" +} + +resource "aws_security_group_rule" "from-172-20-4-0--22-ingress-tcp-22to22-bastion-elb-privatekindnet-example-com" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.bastion-elb-privatekindnet-example-com.id + to_port = 22 + type = "ingress" +} + +resource "aws_security_group_rule" "from-api-elb-privatekindnet-example-com-egress-all-0to0-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.api-elb-privatekindnet-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-api-elb-privatekindnet-example-com-egress-all-0to0-__--0" { + from_port = 0 + ipv6_cidr_blocks = ["::/0"] + protocol = "-1" + security_group_id = aws_security_group.api-elb-privatekindnet-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-bastion-elb-privatekindnet-example-com-egress-all-0to0-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.bastion-elb-privatekindnet-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-bastion-elb-privatekindnet-example-com-egress-all-0to0-__--0" { + from_port = 0 + ipv6_cidr_blocks = ["::/0"] + protocol = "-1" + security_group_id = aws_security_group.bastion-elb-privatekindnet-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-bastion-elb-privatekindnet-example-com-ingress-icmp-3to4-bastion-privatekindnet-example-com" { + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-privatekindnet-example-com.id + source_security_group_id = aws_security_group.bastion-elb-privatekindnet-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "from-bastion-elb-privatekindnet-example-com-ingress-tcp-22to22-bastion-privatekindnet-example-com" { + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.bastion-privatekindnet-example-com.id + source_security_group_id = aws_security_group.bastion-elb-privatekindnet-example-com.id + to_port = 22 + type = "ingress" +} + +resource "aws_security_group_rule" "from-bastion-privatekindnet-example-com-egress-all-0to0-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.bastion-privatekindnet-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-bastion-privatekindnet-example-com-egress-all-0to0-__--0" { + from_port = 0 + ipv6_cidr_blocks = ["::/0"] + protocol = "-1" + security_group_id = aws_security_group.bastion-privatekindnet-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-bastion-privatekindnet-example-com-ingress-icmp-3to4-bastion-elb-privatekindnet-example-com" { + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-elb-privatekindnet-example-com.id + source_security_group_id = aws_security_group.bastion-privatekindnet-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "from-bastion-privatekindnet-example-com-ingress-tcp-22to22-masters-privatekindnet-example-com" { + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.masters-privatekindnet-example-com.id + source_security_group_id = aws_security_group.bastion-privatekindnet-example-com.id + to_port = 22 + type = "ingress" +} + +resource "aws_security_group_rule" "from-bastion-privatekindnet-example-com-ingress-tcp-22to22-nodes-privatekindnet-example-com" { + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.nodes-privatekindnet-example-com.id + source_security_group_id = aws_security_group.bastion-privatekindnet-example-com.id + to_port = 22 + type = "ingress" +} + +resource "aws_security_group_rule" "from-masters-privatekindnet-example-com-egress-all-0to0-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.masters-privatekindnet-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-masters-privatekindnet-example-com-egress-all-0to0-__--0" { + from_port = 0 + ipv6_cidr_blocks = ["::/0"] + protocol = "-1" + security_group_id = aws_security_group.masters-privatekindnet-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-masters-privatekindnet-example-com-ingress-all-0to0-masters-privatekindnet-example-com" { + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.masters-privatekindnet-example-com.id + source_security_group_id = aws_security_group.masters-privatekindnet-example-com.id + to_port = 0 + type = "ingress" +} + +resource "aws_security_group_rule" "from-masters-privatekindnet-example-com-ingress-all-0to0-nodes-privatekindnet-example-com" { + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.nodes-privatekindnet-example-com.id + source_security_group_id = aws_security_group.masters-privatekindnet-example-com.id + to_port = 0 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-privatekindnet-example-com-egress-all-0to0-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.nodes-privatekindnet-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-nodes-privatekindnet-example-com-egress-all-0to0-__--0" { + from_port = 0 + ipv6_cidr_blocks = ["::/0"] + protocol = "-1" + security_group_id = aws_security_group.nodes-privatekindnet-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-nodes-privatekindnet-example-com-ingress-all-0to0-nodes-privatekindnet-example-com" { + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.nodes-privatekindnet-example-com.id + source_security_group_id = aws_security_group.nodes-privatekindnet-example-com.id + to_port = 0 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-privatekindnet-example-com-ingress-tcp-1to2379-masters-privatekindnet-example-com" { + from_port = 1 + protocol = "tcp" + security_group_id = aws_security_group.masters-privatekindnet-example-com.id + source_security_group_id = aws_security_group.nodes-privatekindnet-example-com.id + to_port = 2379 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-privatekindnet-example-com-ingress-tcp-2382to4000-masters-privatekindnet-example-com" { + from_port = 2382 + protocol = "tcp" + security_group_id = aws_security_group.masters-privatekindnet-example-com.id + source_security_group_id = aws_security_group.nodes-privatekindnet-example-com.id + to_port = 4000 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-privatekindnet-example-com-ingress-tcp-4003to65535-masters-privatekindnet-example-com" { + from_port = 4003 + protocol = "tcp" + security_group_id = aws_security_group.masters-privatekindnet-example-com.id + source_security_group_id = aws_security_group.nodes-privatekindnet-example-com.id + to_port = 65535 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-privatekindnet-example-com-ingress-udp-1to65535-masters-privatekindnet-example-com" { + from_port = 1 + protocol = "udp" + security_group_id = aws_security_group.masters-privatekindnet-example-com.id + source_security_group_id = aws_security_group.nodes-privatekindnet-example-com.id + to_port = 65535 + type = "ingress" +} + +resource "aws_security_group_rule" "https-elb-to-master" { + from_port = 443 + protocol = "tcp" + security_group_id = aws_security_group.masters-privatekindnet-example-com.id + source_security_group_id = aws_security_group.api-elb-privatekindnet-example-com.id + to_port = 443 + type = "ingress" +} + +resource "aws_security_group_rule" "icmp-pmtu-api-elb-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.api-elb-privatekindnet-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "icmp-pmtu-cp-to-elb" { + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.api-elb-privatekindnet-example-com.id + source_security_group_id = aws_security_group.masters-privatekindnet-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "icmp-pmtu-elb-to-cp" { + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.masters-privatekindnet-example-com.id + source_security_group_id = aws_security_group.api-elb-privatekindnet-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-elb-privatekindnet-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-172-20-4-0--22" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-elb-privatekindnet-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_sqs_queue" "privatekindnet-example-com-nth" { + message_retention_seconds = 300 + name = "privatekindnet-example-com-nth" + policy = file("${path.module}/data/aws_sqs_queue_privatekindnet-example-com-nth_policy") + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "privatekindnet-example-com-nth" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_subnet" "us-test-1a-privatekindnet-example-com" { + availability_zone = "us-test-1a" + cidr_block = "172.20.32.0/19" + enable_resource_name_dns_a_record_on_launch = true + private_dns_hostname_type_on_launch = "resource-name" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "us-test-1a.privatekindnet.example.com" + "SubnetType" = "Private" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + "kubernetes.io/role/internal-elb" = "1" + } + vpc_id = aws_vpc.privatekindnet-example-com.id +} + +resource "aws_subnet" "utility-us-test-1a-privatekindnet-example-com" { + availability_zone = "us-test-1a" + cidr_block = "172.20.4.0/22" + enable_resource_name_dns_a_record_on_launch = true + private_dns_hostname_type_on_launch = "resource-name" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "utility-us-test-1a.privatekindnet.example.com" + "SubnetType" = "Utility" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + } + vpc_id = aws_vpc.privatekindnet-example-com.id +} + +resource "aws_vpc" "privatekindnet-example-com" { + assign_generated_ipv6_cidr_block = true + cidr_block = "172.20.0.0/16" + enable_dns_hostnames = true + enable_dns_support = true + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_vpc_dhcp_options" "privatekindnet-example-com" { + domain_name = "us-test-1.compute.internal" + domain_name_servers = ["AmazonProvidedDNS"] + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_vpc_dhcp_options_association" "privatekindnet-example-com" { + dhcp_options_id = aws_vpc_dhcp_options.privatekindnet-example-com.id + vpc_id = aws_vpc.privatekindnet-example-com.id +} + +terraform { + required_version = ">= 0.15.0" + required_providers { + aws = { + "configuration_aliases" = [aws.files] + "source" = "hashicorp/aws" + "version" = ">= 5.0.0" + } + } +} diff --git a/upup/models/cloudup/resources/addons/networking.kindnet/k8s-1.32.yaml.template b/upup/models/cloudup/resources/addons/networking.kindnet/k8s-1.32.yaml.template new file mode 100644 index 0000000000000..21498dac04246 --- /dev/null +++ b/upup/models/cloudup/resources/addons/networking.kindnet/k8s-1.32.yaml.template @@ -0,0 +1,2299 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kindnet +rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch + - patch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - apiGroups: + - "" + resources: + - pods + - namespaces + verbs: + - list + - watch + - apiGroups: + - "networking.k8s.io" + resources: + - networkpolicies + verbs: + - list + - watch +{{- if or .Networking.Kindnet.AdminNetworkPolicies .Networking.Kindnet.BaselineAdminNetworkPolicies}} + - apiGroups: + - "policy.networking.k8s.io" + resources: + {{- if .Networking.Kindnet.AdminNetworkPolicies }} + - adminnetworkpolicies + {{- end }} + {{- if .Networking.Kindnet.BaselineAdminNetworkPolicies }} + - baselineadminnetworkpolicies + {{- end }} + verbs: + - list + - watch +{{- end }} +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kindnet +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kindnet +subjects: +- kind: ServiceAccount + name: kindnet + namespace: kube-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kindnet + namespace: kube-system +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kindnet + namespace: kube-system + labels: + tier: node + app: kindnet + k8s-app: kindnet +spec: + selector: + matchLabels: + app: kindnet + template: + metadata: + labels: + tier: node + app: kindnet + k8s-app: kindnet + spec: + hostNetwork: true + tolerations: + - operator: Exists + effect: NoSchedule + serviceAccountName: kindnet + initContainers: + - name: install-cni-bin + image: "ghcr.io/aojea/kindnetd:{{ .Networking.Kindnet.Version }}" + command: ['sh', '-c', 'cat /opt/cni/bin/cni-kindnet > /cni/cni-kindnet ; chmod +x /cni/cni-kindnet'] + volumeMounts: + - name: cni-bin + mountPath: /cni + containers: + - name: kindnet-cni + image: "ghcr.io/aojea/kindnetd:{{ .Networking.Kindnet.Version }}" + args: + - /bin/kindnetd + - --hostname-override=$(NODE_NAME) + {{- if .Networking.Kindnet.LogLevel }} + - --v={{ .Networking.Kindnet.LogLevel }} + {{- end }} + {{- if .Networking.Kindnet.NetworkPolicies }} + - --network-policy={{ .Networking.Kindnet.NetworkPolicies }} + {{- end }} + {{- if .Networking.Kindnet.AdminNetworkPolicies }} + - --admin-network-policy={{ .Networking.Kindnet.AdminNetworkPolicies }} + {{- end }} + {{- if .Networking.Kindnet.BaselineAdminNetworkPolicies }} + - --admin-network-policy={{ .Networking.Kindnet.BaselineAdminNetworkPolicies }} + {{- end }} + {{- if .Networking.Kindnet.DNSCaching }} + - --dns-caching={{ .Networking.Kindnet.DNSCaching }} + {{- end }} + {{- if .Networking.Kindnet.NAT64 }} + - --nat64={{ .Networking.Kindnet.NAT64 }} + {{- end }} + {{- if .Networking.Kindnet.FastPathThreshold }} + - --fastpath-threshold={{ .Networking.Kindnet.FastPathThreshold }} + {{- end }} + {{- if .Networking.Kindnet.Masquerade }} + {{- if .Networking.Kindnet.Masquerade.Enabled }} + - --masquerading={{ .Networking.Kindnet.Masquerade.Enabled }} + {{- end }} + {{- if .Networking.Kindnet.Masquerade.NonMasqueradeCIDRs }} + - --no-masquerade-cidr={{ range $index, $element := .Networking.Kindnet.Masquerade.NonMasqueradeCIDRs }}{{if $index}},{{end}}{{$element}}{{end}} + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: cni-cfg + mountPath: /etc/cni/net.d + - name: var-lib-kindnet + mountPath: /var/lib/cni-kindnet + resources: + requests: + cpu: "100m" + memory: "50Mi" + securityContext: + privileged: true + volumes: + - name: cni-bin + hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + - name: cni-cfg + hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + - name: var-lib-kindnet + hostPath: + path: /var/lib/cni-kindnet + type: DirectoryOrCreate +{{- if .Networking.Kindnet.AdminNetworkPolicies }} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/network-policy-api/pull/30 + policy.networking.k8s.io/bundle-version: v0.1.1 + policy.networking.k8s.io/channel: experimental + creationTimestamp: null + name: adminnetworkpolicies.policy.networking.k8s.io +spec: + group: policy.networking.k8s.io + names: + kind: AdminNetworkPolicy + listKind: AdminNetworkPolicyList + plural: adminnetworkpolicies + shortNames: + - anp + singular: adminnetworkpolicy + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.priority + name: Priority + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + AdminNetworkPolicy is a cluster level resource that is part of the + AdminNetworkPolicy API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specification of the desired behavior of AdminNetworkPolicy. + properties: + egress: + description: |- + Egress is the list of Egress rules to be applied to the selected pods. + A total of 100 rules will be allowed in each ANP instance. + The relative precedence of egress rules within a single ANP object (all of + which share the priority) will be determined by the order in which the rule + is written. Thus, a rule that appears at the top of the egress rules + would take the highest precedence. + ANPs with no egress rules do not affect egress traffic. + + Support: Core + items: + description: |- + AdminNetworkPolicyEgressRule describes an action to take on a particular + set of traffic originating from pods selected by a AdminNetworkPolicy's + Subject field. + + properties: + action: + description: |- + Action specifies the effect this rule will have on matching traffic. + Currently the following actions are supported: + Allow: allows the selected traffic (even if it would otherwise have been denied by NetworkPolicy) + Deny: denies the selected traffic + Pass: instructs the selected traffic to skip any remaining ANP rules, and + then pass execution to any NetworkPolicies that select the pod. + If the pod is not selected by any NetworkPolicies then execution + is passed to any BaselineAdminNetworkPolicies that select the pod. + + Support: Core + enum: + - Allow + - Deny + - Pass + type: string + name: + description: |- + Name is an identifier for this rule, that may be no more than 100 characters + in length. This field should be used by the implementation to help + improve observability, readability and error-reporting for any applied + AdminNetworkPolicies. + + Support: Core + maxLength: 100 + type: string + ports: + description: |- + Ports allows for matching traffic based on port and protocols. + This field is a list of destination ports for the outgoing egress traffic. + If Ports is not set then the rule does not filter traffic via port. + + Support: Core + items: + description: |- + AdminNetworkPolicyPort describes how to select network ports on pod(s). + Exactly one field must be set. + maxProperties: 1 + minProperties: 1 + properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + Support: Extended + + + type: string + portNumber: + description: |- + Port selects a port on a pod(s) based on number. + + Support: Core + properties: + port: + description: |- + Number defines a network port value. + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + default: TCP + description: |- + Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must + match. If not specified, this field defaults to TCP. + + Support: Core + type: string + required: + - port + - protocol + type: object + portRange: + description: |- + PortRange selects a port range on a pod(s) based on provided start and end + values. + + Support: Core + properties: + end: + description: |- + End defines a network port that is the end of a port range, the End value + must be greater than Start. + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + default: TCP + description: |- + Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must + match. If not specified, this field defaults to TCP. + + Support: Core + type: string + start: + description: |- + Start defines a network port that is the start of a port range, the Start + value must be less than End. + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - end + - start + type: object + type: object + maxItems: 100 + minItems: 1 + type: array + to: + description: |- + To is the List of destinations whose traffic this rule applies to. + If any AdminNetworkPolicyEgressPeer matches the destination of outgoing + traffic then the specified action is applied. + This field must be defined and contain at least one item. + + Support: Core + items: + description: |- + AdminNetworkPolicyEgressPeer defines a peer to allow traffic to. + Exactly one of the selector pointers must be set for a given peer. If a + consumer observes none of its fields are set, they must assume an unknown + option has been specified and fail closed. + maxProperties: 1 + minProperties: 1 + properties: + domainNames: + description: |- + DomainNames provides a way to specify domain names as peers. + + DomainNames is only supported for ALLOW rules. In order to control + access, DomainNames ALLOW rules should be used with a lower priority + egress deny -- this allows the admin to maintain an explicit "allowlist" + of reachable domains. + + DomainNames can have up to 25 domain names specified in one rule. + + Support: Extended + + + items: + description: |- + DomainName describes one or more domain names to be used as a peer. + + DomainName can be an exact match, or use the wildcard specifier '*' to match + one or more labels. + + '*', the wildcard specifier, matches one or more entire labels. It does not + support partial matches. '*' may only be specified as a prefix. + + Examples: + - `kubernetes.io` matches only `kubernetes.io`. + It does not match "www.kubernetes.io", "blog.kubernetes.io", + "my-kubernetes.io", or "wikipedia.org". + - `blog.kubernetes.io` matches only "blog.kubernetes.io". + It does not match "www.kubernetes.io" or "kubernetes.io". + - `*.kubernetes.io` matches subdomains of kubernetes.io. + "www.kubernetes.io", "blog.kubernetes.io", and + "latest.blog.kubernetes.io" match, however "kubernetes.io", and + "wikipedia.org" do not. + pattern: ^(\*\.)?([a-zA-z0-9]([-a-zA-Z0-9_]*[a-zA-Z0-9])?\.)+[a-zA-z0-9]([-a-zA-Z0-9_]*[a-zA-Z0-9])?\.?$ + type: string + maxItems: 25 + minItems: 1 + type: array + x-kubernetes-list-type: set + namespaces: + description: |- + Namespaces defines a way to select all pods within a set of Namespaces. + Note that host-networked pods are not included in this type of peer. + + Support: Core + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + networks: + description: |- + Networks defines a way to select peers via CIDR blocks. + This is intended for representing entities that live outside the cluster, + which can't be selected by pods, namespaces and nodes peers, but note + that cluster-internal traffic will be checked against the rule as + well. So if you Allow or Deny traffic to `"0.0.0.0/0"`, that will allow + or deny all IPv4 pod-to-pod traffic as well. If you don't want that, + add a rule that Passes all pod traffic before the Networks rule. + + Each item in Networks should be provided in the CIDR format and should be + IPv4 or IPv6, for example "10.0.0.0/8" or "fd00::/8". + + Networks can have upto 25 CIDRs specified. + + Support: Extended + + + items: + description: CIDR is an IP address range in CIDR notation + (for example, "10.0.0.0/8" or "fd00::/8"). + maxLength: 43 + type: string + x-kubernetes-validations: + - message: Invalid CIDR format provided + rule: isCIDR(self) + maxItems: 25 + minItems: 1 + type: array + x-kubernetes-list-type: set + nodes: + description: |- + Nodes defines a way to select a set of nodes in + the cluster (based on the node's labels). It selects + the nodeIPs as the peer type by matching on the IPs + present in the node.Status.Addresses field of the node. + This field follows standard label selector + semantics; if present but empty, it selects all Nodes. + + Support: Extended + + + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + pods: + description: |- + Pods defines a way to select a set of pods in + a set of namespaces. Note that host-networked pods + are not included in this type of peer. + + Support: Core + properties: + namespaceSelector: + description: |- + NamespaceSelector follows standard label selector semantics; if empty, + it selects all Namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + podSelector: + description: |- + PodSelector is used to explicitly select pods within a namespace; if empty, + it selects all Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - namespaceSelector + - podSelector + type: object + type: object + maxItems: 100 + minItems: 1 + type: array + required: + - action + - to + type: object + x-kubernetes-validations: + - message: networks/nodes peer cannot be set with namedPorts since + there are no namedPorts for networks/nodes + rule: '!(self.to.exists(peer, has(peer.networks) || has(peer.nodes)) + && has(self.ports) && self.ports.exists(port, has(port.namedPort)))' + maxItems: 100 + type: array + ingress: + description: |- + Ingress is the list of Ingress rules to be applied to the selected pods. + A total of 100 rules will be allowed in each ANP instance. + The relative precedence of ingress rules within a single ANP object (all of + which share the priority) will be determined by the order in which the rule + is written. Thus, a rule that appears at the top of the ingress rules + would take the highest precedence. + ANPs with no ingress rules do not affect ingress traffic. + + Support: Core + items: + description: |- + AdminNetworkPolicyIngressRule describes an action to take on a particular + set of traffic destined for pods selected by an AdminNetworkPolicy's + Subject field. + properties: + action: + description: |- + Action specifies the effect this rule will have on matching traffic. + Currently the following actions are supported: + Allow: allows the selected traffic (even if it would otherwise have been denied by NetworkPolicy) + Deny: denies the selected traffic + Pass: instructs the selected traffic to skip any remaining ANP rules, and + then pass execution to any NetworkPolicies that select the pod. + If the pod is not selected by any NetworkPolicies then execution + is passed to any BaselineAdminNetworkPolicies that select the pod. + + Support: Core + enum: + - Allow + - Deny + - Pass + type: string + from: + description: |- + From is the list of sources whose traffic this rule applies to. + If any AdminNetworkPolicyIngressPeer matches the source of incoming + traffic then the specified action is applied. + This field must be defined and contain at least one item. + + Support: Core + items: + description: |- + AdminNetworkPolicyIngressPeer defines an in-cluster peer to allow traffic from. + Exactly one of the selector pointers must be set for a given peer. If a + consumer observes none of its fields are set, they must assume an unknown + option has been specified and fail closed. + maxProperties: 1 + minProperties: 1 + properties: + namespaces: + description: |- + Namespaces defines a way to select all pods within a set of Namespaces. + Note that host-networked pods are not included in this type of peer. + + Support: Core + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + pods: + description: |- + Pods defines a way to select a set of pods in + a set of namespaces. Note that host-networked pods + are not included in this type of peer. + + Support: Core + properties: + namespaceSelector: + description: |- + NamespaceSelector follows standard label selector semantics; if empty, + it selects all Namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + podSelector: + description: |- + PodSelector is used to explicitly select pods within a namespace; if empty, + it selects all Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - namespaceSelector + - podSelector + type: object + type: object + maxItems: 100 + minItems: 1 + type: array + name: + description: |- + Name is an identifier for this rule, that may be no more than 100 characters + in length. This field should be used by the implementation to help + improve observability, readability and error-reporting for any applied + AdminNetworkPolicies. + + Support: Core + maxLength: 100 + type: string + ports: + description: |- + Ports allows for matching traffic based on port and protocols. + This field is a list of ports which should be matched on + the pods selected for this policy i.e the subject of the policy. + So it matches on the destination port for the ingress traffic. + If Ports is not set then the rule does not filter traffic via port. + + Support: Core + items: + description: |- + AdminNetworkPolicyPort describes how to select network ports on pod(s). + Exactly one field must be set. + maxProperties: 1 + minProperties: 1 + properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + Support: Extended + + + type: string + portNumber: + description: |- + Port selects a port on a pod(s) based on number. + + Support: Core + properties: + port: + description: |- + Number defines a network port value. + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + default: TCP + description: |- + Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must + match. If not specified, this field defaults to TCP. + + Support: Core + type: string + required: + - port + - protocol + type: object + portRange: + description: |- + PortRange selects a port range on a pod(s) based on provided start and end + values. + + Support: Core + properties: + end: + description: |- + End defines a network port that is the end of a port range, the End value + must be greater than Start. + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + default: TCP + description: |- + Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must + match. If not specified, this field defaults to TCP. + + Support: Core + type: string + start: + description: |- + Start defines a network port that is the start of a port range, the Start + value must be less than End. + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - end + - start + type: object + type: object + maxItems: 100 + minItems: 1 + type: array + required: + - action + - from + type: object + maxItems: 100 + type: array + priority: + description: |- + Priority is a value from 0 to 1000. Policies with lower priority values have + higher precedence, and are checked before policies with higher priority values. + All AdminNetworkPolicy rules have higher precedence than NetworkPolicy or + BaselineAdminNetworkPolicy rules + Every AdminNetworkPolicy should have a unique priority value; if two (or more) + policies with the same priority could both match a connection, then the + implementation can apply any of the matching policies to the connection, and + there is no way for the user to reliably determine which one it will choose. + + Support: Core + format: int32 + maximum: 1000 + minimum: 0 + type: integer + subject: + description: |- + Subject defines the pods to which this AdminNetworkPolicy applies. + Note that host-networked pods are not included in subject selection. + + Support: Core + maxProperties: 1 + minProperties: 1 + properties: + namespaces: + description: Namespaces is used to select pods via namespace selectors. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + pods: + description: Pods is used to select pods via namespace AND pod + selectors. + properties: + namespaceSelector: + description: |- + NamespaceSelector follows standard label selector semantics; if empty, + it selects all Namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + podSelector: + description: |- + PodSelector is used to explicitly select pods within a namespace; if empty, + it selects all Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - namespaceSelector + - podSelector + type: object + type: object + required: + - priority + - subject + type: object + status: + description: Status is the status to be reported by the implementation. + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + required: + - conditions + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: null + storedVersions: null +{{- end }} +{{- if .Networking.Kindnet.BaselineAdminNetworkPolicies }} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/network-policy-api/pull/30 + policy.networking.k8s.io/bundle-version: v0.1.1 + policy.networking.k8s.io/channel: experimental + creationTimestamp: null + name: baselineadminnetworkpolicies.policy.networking.k8s.io +spec: + group: policy.networking.k8s.io + names: + kind: BaselineAdminNetworkPolicy + listKind: BaselineAdminNetworkPolicyList + plural: baselineadminnetworkpolicies + shortNames: + - banp + singular: baselineadminnetworkpolicy + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + BaselineAdminNetworkPolicy is a cluster level resource that is part of the + AdminNetworkPolicy API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specification of the desired behavior of BaselineAdminNetworkPolicy. + properties: + egress: + description: |- + Egress is the list of Egress rules to be applied to the selected pods if + they are not matched by any AdminNetworkPolicy or NetworkPolicy rules. + A total of 100 Egress rules will be allowed in each BANP instance. + The relative precedence of egress rules within a single BANP object + will be determined by the order in which the rule is written. + Thus, a rule that appears at the top of the egress rules + would take the highest precedence. + BANPs with no egress rules do not affect egress traffic. + + Support: Core + items: + description: |- + BaselineAdminNetworkPolicyEgressRule describes an action to take on a particular + set of traffic originating from pods selected by a BaselineAdminNetworkPolicy's + Subject field. + + properties: + action: + description: |- + Action specifies the effect this rule will have on matching traffic. + Currently the following actions are supported: + Allow: allows the selected traffic + Deny: denies the selected traffic + + Support: Core + enum: + - Allow + - Deny + type: string + name: + description: |- + Name is an identifier for this rule, that may be no more than 100 characters + in length. This field should be used by the implementation to help + improve observability, readability and error-reporting for any applied + BaselineAdminNetworkPolicies. + + Support: Core + maxLength: 100 + type: string + ports: + description: |- + Ports allows for matching traffic based on port and protocols. + This field is a list of destination ports for the outgoing egress traffic. + If Ports is not set then the rule does not filter traffic via port. + items: + description: |- + AdminNetworkPolicyPort describes how to select network ports on pod(s). + Exactly one field must be set. + maxProperties: 1 + minProperties: 1 + properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + Support: Extended + + + type: string + portNumber: + description: |- + Port selects a port on a pod(s) based on number. + + Support: Core + properties: + port: + description: |- + Number defines a network port value. + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + default: TCP + description: |- + Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must + match. If not specified, this field defaults to TCP. + + Support: Core + type: string + required: + - port + - protocol + type: object + portRange: + description: |- + PortRange selects a port range on a pod(s) based on provided start and end + values. + + Support: Core + properties: + end: + description: |- + End defines a network port that is the end of a port range, the End value + must be greater than Start. + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + default: TCP + description: |- + Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must + match. If not specified, this field defaults to TCP. + + Support: Core + type: string + start: + description: |- + Start defines a network port that is the start of a port range, the Start + value must be less than End. + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - end + - start + type: object + type: object + maxItems: 100 + minItems: 1 + type: array + to: + description: |- + To is the list of destinations whose traffic this rule applies to. + If any BaselineAdminNetworkPolicyEgressPeer matches the destination of outgoing + traffic then the specified action is applied. + This field must be defined and contain at least one item. + + Support: Core + items: + description: |- + BaselineAdminNetworkPolicyEgressPeer defines a peer to allow traffic to. + Exactly one of the selector pointers must be set for a given peer. If a + consumer observes none of its fields are set, they must assume an unknown + option has been specified and fail closed. + maxProperties: 1 + minProperties: 1 + properties: + namespaces: + description: |- + Namespaces defines a way to select all pods within a set of Namespaces. + Note that host-networked pods are not included in this type of peer. + + Support: Core + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + networks: + description: |- + Networks defines a way to select peers via CIDR blocks. + This is intended for representing entities that live outside the cluster, + which can't be selected by pods, namespaces and nodes peers, but note + that cluster-internal traffic will be checked against the rule as + well. So if you Allow or Deny traffic to `"0.0.0.0/0"`, that will allow + or deny all IPv4 pod-to-pod traffic as well. If you don't want that, + add a rule that Passes all pod traffic before the Networks rule. + + Each item in Networks should be provided in the CIDR format and should be + IPv4 or IPv6, for example "10.0.0.0/8" or "fd00::/8". + + Networks can have upto 25 CIDRs specified. + + Support: Extended + + + items: + description: CIDR is an IP address range in CIDR notation + (for example, "10.0.0.0/8" or "fd00::/8"). + maxLength: 43 + type: string + x-kubernetes-validations: + - message: Invalid CIDR format provided + rule: isCIDR(self) + maxItems: 25 + minItems: 1 + type: array + x-kubernetes-list-type: set + nodes: + description: |- + Nodes defines a way to select a set of nodes in + the cluster (based on the node's labels). It selects + the nodeIPs as the peer type by matching on the IPs + present in the node.Status.Addresses field of the node. + This field follows standard label selector + semantics; if present but empty, it selects all Nodes. + + Support: Extended + + + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + pods: + description: |- + Pods defines a way to select a set of pods in + a set of namespaces. Note that host-networked pods + are not included in this type of peer. + + Support: Core + properties: + namespaceSelector: + description: |- + NamespaceSelector follows standard label selector semantics; if empty, + it selects all Namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + podSelector: + description: |- + PodSelector is used to explicitly select pods within a namespace; if empty, + it selects all Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - namespaceSelector + - podSelector + type: object + type: object + maxItems: 100 + minItems: 1 + type: array + required: + - action + - to + type: object + x-kubernetes-validations: + - message: networks/nodes peer cannot be set with namedPorts since + there are no namedPorts for networks/nodes + rule: '!(self.to.exists(peer, has(peer.networks) || has(peer.nodes)) + && has(self.ports) && self.ports.exists(port, has(port.namedPort)))' + maxItems: 100 + type: array + ingress: + description: |- + Ingress is the list of Ingress rules to be applied to the selected pods + if they are not matched by any AdminNetworkPolicy or NetworkPolicy rules. + A total of 100 Ingress rules will be allowed in each BANP instance. + The relative precedence of ingress rules within a single BANP object + will be determined by the order in which the rule is written. + Thus, a rule that appears at the top of the ingress rules + would take the highest precedence. + BANPs with no ingress rules do not affect ingress traffic. + + Support: Core + items: + description: |- + BaselineAdminNetworkPolicyIngressRule describes an action to take on a particular + set of traffic destined for pods selected by a BaselineAdminNetworkPolicy's + Subject field. + properties: + action: + description: |- + Action specifies the effect this rule will have on matching traffic. + Currently the following actions are supported: + Allow: allows the selected traffic + Deny: denies the selected traffic + + Support: Core + enum: + - Allow + - Deny + type: string + from: + description: |- + From is the list of sources whose traffic this rule applies to. + If any AdminNetworkPolicyIngressPeer matches the source of incoming + traffic then the specified action is applied. + This field must be defined and contain at least one item. + + Support: Core + items: + description: |- + AdminNetworkPolicyIngressPeer defines an in-cluster peer to allow traffic from. + Exactly one of the selector pointers must be set for a given peer. If a + consumer observes none of its fields are set, they must assume an unknown + option has been specified and fail closed. + maxProperties: 1 + minProperties: 1 + properties: + namespaces: + description: |- + Namespaces defines a way to select all pods within a set of Namespaces. + Note that host-networked pods are not included in this type of peer. + + Support: Core + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + pods: + description: |- + Pods defines a way to select a set of pods in + a set of namespaces. Note that host-networked pods + are not included in this type of peer. + + Support: Core + properties: + namespaceSelector: + description: |- + NamespaceSelector follows standard label selector semantics; if empty, + it selects all Namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + podSelector: + description: |- + PodSelector is used to explicitly select pods within a namespace; if empty, + it selects all Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - namespaceSelector + - podSelector + type: object + type: object + maxItems: 100 + minItems: 1 + type: array + name: + description: |- + Name is an identifier for this rule, that may be no more than 100 characters + in length. This field should be used by the implementation to help + improve observability, readability and error-reporting for any applied + BaselineAdminNetworkPolicies. + + Support: Core + maxLength: 100 + type: string + ports: + description: |- + Ports allows for matching traffic based on port and protocols. + This field is a list of ports which should be matched on + the pods selected for this policy i.e the subject of the policy. + So it matches on the destination port for the ingress traffic. + If Ports is not set then the rule does not filter traffic via port. + + Support: Core + items: + description: |- + AdminNetworkPolicyPort describes how to select network ports on pod(s). + Exactly one field must be set. + maxProperties: 1 + minProperties: 1 + properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + Support: Extended + + + type: string + portNumber: + description: |- + Port selects a port on a pod(s) based on number. + + Support: Core + properties: + port: + description: |- + Number defines a network port value. + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + default: TCP + description: |- + Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must + match. If not specified, this field defaults to TCP. + + Support: Core + type: string + required: + - port + - protocol + type: object + portRange: + description: |- + PortRange selects a port range on a pod(s) based on provided start and end + values. + + Support: Core + properties: + end: + description: |- + End defines a network port that is the end of a port range, the End value + must be greater than Start. + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + default: TCP + description: |- + Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must + match. If not specified, this field defaults to TCP. + + Support: Core + type: string + start: + description: |- + Start defines a network port that is the start of a port range, the Start + value must be less than End. + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - end + - start + type: object + type: object + maxItems: 100 + minItems: 1 + type: array + required: + - action + - from + type: object + maxItems: 100 + type: array + subject: + description: |- + Subject defines the pods to which this BaselineAdminNetworkPolicy applies. + Note that host-networked pods are not included in subject selection. + + Support: Core + maxProperties: 1 + minProperties: 1 + properties: + namespaces: + description: Namespaces is used to select pods via namespace selectors. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + pods: + description: Pods is used to select pods via namespace AND pod + selectors. + properties: + namespaceSelector: + description: |- + NamespaceSelector follows standard label selector semantics; if empty, + it selects all Namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + podSelector: + description: |- + PodSelector is used to explicitly select pods within a namespace; if empty, + it selects all Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - namespaceSelector + - podSelector + type: object + type: object + required: + - subject + type: object + status: + description: Status is the status to be reported by the implementation. + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + required: + - conditions + type: object + required: + - metadata + - spec + type: object + x-kubernetes-validations: + - message: Only one baseline admin network policy with metadata.name="default" + can be created in the cluster + rule: self.metadata.name == 'default' + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: null + storedVersions: null +{{- end }} \ No newline at end of file diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go index 4446fbf685ef4..00b38ec884710 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go @@ -1018,6 +1018,23 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext) } } + if b.Cluster.Spec.Networking.Kindnet != nil { + key := "networking.kindnet" + + { + id := "k8s-1.32" + location := key + "/" + id + ".yaml" + + addons.Add(&channelsapi.AddonSpec{ + Name: fi.PtrTo(key), + Selector: networkingSelector(), + Manifest: fi.PtrTo(location), + Id: id, + NeedsRollingUpdate: channelsapi.NeedsRollingUpdateAll, + }) + } + } + err := addCiliumAddon(b, addons) if err != nil { return nil, nil, fmt.Errorf("failed to add cilium addon: %w", err) diff --git a/upup/pkg/fi/cloudup/new_cluster.go b/upup/pkg/fi/cloudup/new_cluster.go index e028b63f82cee..38b1f2a0f83f7 100644 --- a/upup/pkg/fi/cloudup/new_cluster.go +++ b/upup/pkg/fi/cloudup/new_cluster.go @@ -1262,6 +1262,8 @@ func setupNetworking(opt *NewClusterOptions, cluster *api.Cluster) error { cluster.Spec.Networking.Cilium.IPAM = "eni" case "gcp", "gce": cluster.Spec.Networking.GCP = &api.GCPNetworkingSpec{} + case "kindnet": + cluster.Spec.Networking.Kindnet = &api.KindnetNetworkingSpec{} default: return fmt.Errorf("unknown networking mode %q", opt.Networking) } diff --git a/upup/pkg/fi/cloudup/new_cluster_test.go b/upup/pkg/fi/cloudup/new_cluster_test.go index 9fef00651e15d..24c25c1d9e9d2 100644 --- a/upup/pkg/fi/cloudup/new_cluster_test.go +++ b/upup/pkg/fi/cloudup/new_cluster_test.go @@ -313,6 +313,18 @@ func TestSetupNetworking(t *testing.T) { }, }, }, + { + options: NewClusterOptions{ + Networking: "kindnet", + }, + expected: api.Cluster{ + Spec: api.ClusterSpec{ + Networking: api.NetworkingSpec{ + Kindnet: &api.KindnetNetworkingSpec{}, + }, + }, + }, + }, } for _, test := range tests { diff --git a/upup/pkg/fi/cloudup/populate_cluster_spec.go b/upup/pkg/fi/cloudup/populate_cluster_spec.go index c56608600e28f..2bc88ac846efc 100644 --- a/upup/pkg/fi/cloudup/populate_cluster_spec.go +++ b/upup/pkg/fi/cloudup/populate_cluster_spec.go @@ -318,6 +318,7 @@ func (c *populateClusterSpec) run(ctx context.Context, clientset simple.Clientse codeModels = append(codeModels, &components.CloudConfigurationOptionsBuilder{Context: optionsContext}) codeModels = append(codeModels, &components.CalicoOptionsBuilder{Context: optionsContext}) codeModels = append(codeModels, &components.CiliumOptionsBuilder{Context: optionsContext}) + codeModels = append(codeModels, &components.KindnetOptionsBuilder{Context: optionsContext}) codeModels = append(codeModels, &components.OpenStackOptionsBuilder{Context: optionsContext}) codeModels = append(codeModels, &components.DiscoveryOptionsBuilder{OptionsContext: optionsContext}) codeModels = append(codeModels, &components.ClusterAutoscalerOptionsBuilder{OptionsContext: optionsContext}) diff --git a/upup/pkg/fi/nodeup/command.go b/upup/pkg/fi/nodeup/command.go index 5590166776e69..2c05b3fe63b70 100644 --- a/upup/pkg/fi/nodeup/command.go +++ b/upup/pkg/fi/nodeup/command.go @@ -322,6 +322,7 @@ func (c *NodeUpCommand) Run(out io.Writer) error { loader.Builders = append(loader.Builders, &networking.CommonBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &networking.CalicoBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &networking.CiliumBuilder{NodeupModelContext: modelContext}) + loader.Builders = append(loader.Builders, &networking.KindnetBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &networking.AmazonVPCRoutedENIBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &networking.KuberouterBuilder{NodeupModelContext: modelContext}) @@ -559,12 +560,20 @@ func modprobe(module string) error { } // loadKernelModules is a hack to force br_netfilter to be loaded +// and used by some components to load its recommended modules. // TODO: Move to tasks architecture func loadKernelModules(context *model.NodeupModelContext) error { - err := modprobe("br_netfilter") - if err != nil { - // TODO: Return error in 1.11 (too risky for 1.10) - klog.Warningf("error loading br_netfilter module: %v", err) + if context.NodeupConfig.Networking.Kindnet != nil { + err := modprobe("nfnetlink_queue") + if err != nil { + klog.Warningf("error loading nfnetlink_queue module: %v", err) + } + } else { + err := modprobe("br_netfilter") + if err != nil { + // TODO: Return error in 1.11 (too risky for 1.10) + klog.Warningf("error loading br_netfilter module: %v", err) + } } // TODO: Add to /etc/modules-load.d/ ? return nil