feat: CN pooling support

.github/env

@@ -1,4 +1,4 @@
-golang-version=1.20
+golang-version=1.21
 kind-version=v0.11.1
 kind-image=kindest/node:v1.23.0
 helm-version=v3.8.1

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.20 as builder
+FROM golang:1.21 as builder
 
 ARG GOPROXY="https://proxy.golang.org,direct"
 

api/core/v1alpha1/br_types.go

@@ -1,4 +1,4 @@
-// Copyright 2023 Matrix Origin
+// Copyright 2024 Matrix Origin
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

api/core/v1alpha1/bucketclaim_helpers.go

@@ -1,4 +1,4 @@
-// Copyright 2023 Matrix Origin
+// Copyright 2024 Matrix Origin
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

api/core/v1alpha1/bucketclaim_types.go

@@ -1,4 +1,4 @@
-// Copyright 2023 Matrix Origin
+// Copyright 2024 Matrix Origin
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

api/core/v1alpha1/cnclaim_types.go

@@ -0,0 +1,156 @@
+// Copyright 2024 Matrix Origin
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+type CNClaimPhase string
+
+const (
+	CNClaimPhasePending CNClaimPhase = "Pending"
+	CNClaimPhaseBound   CNClaimPhase = "Bound"
+	CNClaimPhaseLost    CNClaimPhase = "Lost"
+
+	ClaimOwnerNameLabel = "matrixorigin.io/claim-owner"
+)
+
+type CNClaimSpec struct {
+	Selector *metav1.LabelSelector `json:"selector"`
+
+	// +optional
+	CNLabels []CNLabel `json:"cnLabels,omitempty"`
+
+	// +optional
+	OwnerName *string `json:"ownerName,omitempty"`
+
+	// +optional
+	// PodName is usually populated by controller and would be part of the claim spec
+	// that must be persisted once bound
+	PodName string `json:"podName,omitempty"`
+
+	// +optional
+	// PoolName is usually populated by controller that which pool the claim is nominated
+	PoolName string `json:"poolName,omitempty"`
+}
+
+type CNClaimStatus struct {
+	Phase CNClaimPhase  `json:"phase,omitempty"`
+	Store CNStoreStatus `json:"store,omitempty"`
+}
+
+type CNStoreStatus struct {
+	ServiceID              string    `json:"serviceID,omitempty"`
+	LockServiceAddress     string    `json:"lockServiceAddress,omitempty"`
+	PipelineServiceAddress string    `json:"pipelineServiceAddress,omitempty"`
+	SQLAddress             string    `json:"sqlAddress,omitempty"`
+	QueryAddress           string    `json:"queryAddress,omitempty"`
+	WorkState              int32     `json:"workState,omitempty"`
+	Labels                 []CNLabel `json:"labels,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// CNClaim claim a CN to use
+// +kubebuilder:subresource:status
+// +kubebuilder:printcolumn:name="Pod",type="string",JSONPath=".spec.podName"
+// +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase"
+// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
+// +kubebuilder:resource:scope="Namespaced"
+type CNClaim struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec CNClaimSpec `json:"spec,omitempty"`
+
+	// +optional
+	Status CNClaimStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// CNClaimList contains a list of CNClaims
+type CNClaimList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+
+	Items []CNClaim `json:"items"`
+}
+
+type CNClaimSetSpec struct {
+	Replicas int32           `json:"replicas"`
+	Template CNClaimTemplate `json:"template"`
+
+	Selector *metav1.LabelSelector `json:"selector,omitempty"`
+}
+
+type CNClaimTemplate struct {
+	EmbeddedMetadata `json:"metadata,omitempty"`
+
+	Spec CNClaimSpec `json:"spec,omitempty"`
+}
+
+type EmbeddedMetadata struct {
+	// +optional
+	Name string `json:"name,omitempty"`
+
+	// +optional
+	Labels map[string]string `json:"labels,omitempty"`
+
+	// +optional
+	Annotations map[string]string `json:"annotations,omitempty"`
+}
+
+type CNClaimSetStatus struct {
+	Replicas int32           `json:"replicas"`
+	Claims   []CNClaimStatus `json:"claims,omitempty"`
+
+	PodSelector string `json:"podSelector,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// CNClaimSet orchestrates a set of CNClaims
+// +kubebuilder:subresource:status
+// +kubebuilder:printcolumn:name="Replicas",type="integer",JSONPath=".spec.replicas"
+// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
+// +kubebuilder:resource:scope="Namespaced"
+// +kubebuilder:subresource:status
+// +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas,selectorpath=.status.labelSelector
+type CNClaimSet struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec CNClaimSetSpec `json:"spec,omitempty"`
+
+	// +optional
+	Status CNClaimSetStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// CNClaimSetList contains a list of CNClaimSet
+type CNClaimSetList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+
+	Items []CNClaimSet `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&CNClaim{}, &CNClaimList{})
+	SchemeBuilder.Register(&CNClaimSet{}, &CNClaimSetList{})
+}

api/core/v1alpha1/cnpool_helper.go

@@ -0,0 +1,21 @@
+// Copyright 2024 Matrix Origin
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package v1alpha1
+
+import corev1 "k8s.io/api/core/v1"
+
+func IsPoolingPolicy(pod *corev1.Pod) bool {
+	return pod.Annotations[PodManagementPolicyAnno] == PodManagementPolicyPooling
+}

api/core/v1alpha1/cnpool_types.go

@@ -0,0 +1,117 @@
+// Copyright 2024 Matrix Origin
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"math"
+)
+
+const (
+	CNPodPhaseBound       = "Bound"
+	CNPodPhaseIdle        = "Idle"
+	CNPodPhaseDraining    = "Draining"
+	CNPodPhaseUnknown     = "Unknown"
+	CNPodPhaseTerminating = "Terminating"
+)
+
+const (
+	PodManagementPolicyPooling = "Pooling"
+)
+
+const (
+	// PodClaimedByLabel is a Pod label records the claim the claims the Pod
+	PodClaimedByLabel = "pool.matrixorigin.io/claimed-by"
+	// CNPodPhaseLabel is the pod phase in Pool
+	CNPodPhaseLabel = "pool.matrixorigin.io/phase"
+	// PoolNameLabel is the pool of CN claim or CN Pod
+	PoolNameLabel = "pool.matrixorigin.io/pool-name"
+
+	// PodManagementPolicyAnno denotes the management policy of a Pod
+	PodManagementPolicyAnno = "pool.matrixorigin.io/management-policy"
+)
+
+type CNPoolSpec struct {
+	// Template is the CNSet template of the Pool
+	Template CNSetSpec `json:"template"`
+
+	// PodLabels is the Pod labels of the CN in Pool
+	PodLabels map[string]string `json:"podLabels,omitempty"`
+
+	// Deps is the dependencies of the Pool
+	Deps CNSetDeps `json:"deps"`
+
+	Strategy PoolStrategy `json:"strategy"`
+}
+
+type PoolStrategy struct {
+	// UpdateStrategy defines the strategy for pool updating
+	UpdateStrategy PoolUpdateStrategy `json:"updateStrategy"`
+
+	// UpdateStrategy defines the strategy for pool scaling
+	ScaleStrategy PoolScaleStrategy `json:"scaleStrategy"`
+}
+
+type PoolUpdateStrategy struct {
+	// +optional
+	ReclaimTimeout *metav1.Duration `json:"reclaimTimeout,omitempty"`
+}
+
+type PoolScaleStrategy struct {
+	MaxIdle int32 `json:"maxIdle"`
+
+	// +optional
+	// MaxPods allowed in this Pool, nil means no limit
+	MaxPods *int32 `json:"maxPods,omitempty"`
+}
+
+func (s *PoolScaleStrategy) GetMaxPods() int32 {
+	if s.MaxPods == nil {
+		return math.MaxInt32
+	}
+	return *s.MaxPods
+}
+
+type CNPoolStatus struct {
+}
+
+// +kubebuilder:object:root=true
+
+// CNPool maintains a pool of CN Pods
+// +kubebuilder:subresource:status
+// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
+// +kubebuilder:resource:scope="Namespaced"
+type CNPool struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec CNPoolSpec `json:"spec"`
+
+	// +optional
+	Status CNPoolStatus `json:"status,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+// CNPoolList contains a list of CNPool
+type CNPoolList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []CNPool `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&CNPool{}, &CNPoolList{})
+}

api/core/v1alpha1/cnset_types.go

@@ -1,4 +1,4 @@
-// Copyright 2023 Matrix Origin
+// Copyright 2024 Matrix Origin
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -85,6 +85,15 @@ type CNSetSpec struct {
 
 	// PythonUdfSidecar is the python udf server in CN
 	PythonUdfSidecar PythonUdfSidecar `json:"pythonUdfSidecar,omitempty"`
+
+	// PodManagementPolicy is the pod management policy of the Pod in this Set
+	PodManagementPolicy *string `json:"podManagementPolicy,omitempty"`
+
+	// PodsToDelete are the Pods to delete in the CNSet
+	PodsToDelete []string `json:"podsToDelete,omitempty"`
+
+	// PauseUpdate means the CNSet should pause rolling-update
+	PauseUpdate bool `json:"pauseUpdate,omitempty"`
 }
 
 type ScalingConfig struct {

api/core/v1alpha1/cnset_webhook.go

@@ -1,4 +1,4 @@
-// Copyright 2023 Matrix Origin
+// Copyright 2024 Matrix Origin
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

api/core/v1alpha1/cnset_webhook_test.go

@@ -1,4 +1,4 @@
-// Copyright 2023 Matrix Origin
+// Copyright 2024 Matrix Origin
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

api/core/v1alpha1/common_helpers.go

@@ -1,4 +1,4 @@
-// Copyright 2023 Matrix Origin
+// Copyright 2024 Matrix Origin
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

api/core/v1alpha1/common_helpers_test.go

@@ -1,4 +1,4 @@
-// Copyright 2023 Matrix Origin
+// Copyright 2024 Matrix Origin
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

api/core/v1alpha1/common_types.go

@@ -1,4 +1,4 @@
-// Copyright 2023 Matrix Origin
+// Copyright 2024 Matrix Origin
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -58,7 +58,8 @@ type PodSet struct {
 	Overlay *Overlay `json:"overlay,omitempty"`
 
 	// Replicas is the desired number of pods of this set
-	Replicas int32 `json:"replicas"`
+	// +optional
+	Replicas int32 `json:"replicas,omitempty"`
 
 	// TopologyEvenSpread specifies what topology domains the Pods in set should be
 	// evenly spread in.

api/core/v1alpha1/dnset_types.go

@@ -1,4 +1,4 @@
-// Copyright 2023 Matrix Origin
+// Copyright 2024 Matrix Origin
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.