diff --git a/Dockerfile b/Dockerfile index af79c7f1..a7b06bc3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20 as builder +FROM golang:1.21 as builder ARG GOPROXY="https://proxy.golang.org,direct" diff --git a/api/core/v1alpha1/cnclaim_types.go b/api/core/v1alpha1/cnclaim_types.go new file mode 100644 index 00000000..783a376b --- /dev/null +++ b/api/core/v1alpha1/cnclaim_types.go @@ -0,0 +1,134 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type CNClaimPhase string + +const ( + CNClaimPhasePending CNClaimPhase = "Pending" + CNClaimPhaseBound CNClaimPhase = "Bound" + CNClaimPhaseLost CNClaimPhase = "Lost" +) + +type CNClaimSpec struct { + Selector *metav1.LabelSelector `json:"selector"` + // +optional + CNLabels []CNLabel `json:"cnLabels,omitempty"` + + // +optional + // PodName is usually populated by controller and would be part of the claim spec + // that must be persisted once bound + PodName string `json:"podName,omitempty"` + + // +optional + // PoolName is usually populated by controller that which pool the claim is nominated + PoolName string `json:"poolName,omitempty"` +} + +type CNClaimStatus struct { + Phase CNClaimPhase `json:"phase,omitempty"` + Store CNStoreStatus `json:"store,omitempty"` +} + +type CNStoreStatus struct { + ServiceID string `json:"serviceID,omitempty"` + LockServiceAddress string `json:"lockServiceAddress,omitempty"` + PipelineServiceAddress string `json:"pipelineServiceAddress,omitempty"` + SQLAddress string `json:"sqlAddress,omitempty"` + QueryAddress string `json:"queryAddress,omitempty"` + WorkState int32 `json:"workState,omitempty"` + Labels []CNLabel `json:"labels,omitempty"` +} + +// +kubebuilder:object:root=true + +// CNClaim claim a CN to use +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Pod",type="string",JSONPath=".spec.podName" +// +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" +// +kubebuilder:resource:scope="Namespaced" +type CNClaim struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CNClaimSpec `json:"spec,omitempty"` + + // +optional + Status CNClaimStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// CNClaimList contains a list of CNClaims +type CNClaimList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + Items []CNClaim `json:"items"` +} + +type CNClaimSetSpec struct { + Replicas int32 `json:"replicas"` + Template CNClaimTemplate `json:"template"` + + Selector *metav1.LabelSelector `json:"selector,omitempty"` +} + +type CNClaimTemplate struct { + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CNClaimSpec `json:"spec,omitempty"` +} + +type CNClaimSetStatus struct { + Replicas int32 `json:"replicas"` + Claims []CNClaimStatus `json:"claims,omitempty"` +} + +// +kubebuilder:object:root=true + +// CNClaimSet orchestrates a set of CNClaims +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" +// +kubebuilder:resource:scope="Namespaced" +type CNClaimSet struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CNClaimSetSpec `json:"spec,omitempty"` + + // +optional + Status CNClaimSetStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// CNClaimSetList contains a list of CNClaimSet +type CNClaimSetList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + Items []CNClaimSet `json:"items"` +} + +func init() { + SchemeBuilder.Register(&CNClaim{}, &CNClaimList{}) + SchemeBuilder.Register(&CNClaimSet{}, &CNClaimSetList{}) +} diff --git a/api/core/v1alpha1/cnpool_helper.go b/api/core/v1alpha1/cnpool_helper.go new file mode 100644 index 00000000..692c9ac1 --- /dev/null +++ b/api/core/v1alpha1/cnpool_helper.go @@ -0,0 +1,21 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1 + +import corev1 "k8s.io/api/core/v1" + +func IsPoolingPolicy(pod *corev1.Pod) bool { + return pod.Annotations[PodManagementPolicyAnno] == PodManagementPolicyPooling +} diff --git a/api/core/v1alpha1/cnpool_types.go b/api/core/v1alpha1/cnpool_types.go new file mode 100644 index 00000000..e03c243a --- /dev/null +++ b/api/core/v1alpha1/cnpool_types.go @@ -0,0 +1,117 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "math" +) + +const ( + CNPodPhaseBound = "Bound" + CNPodPhaseIdle = "Idle" + CNPodPhaseDraining = "Draining" + CNPodPhaseUnknown = "Unknown" + CNPodPhaseTerminating = "Terminating" +) + +const ( + PodManagementPolicyPooling = "Pooling" +) + +const ( + // PodClaimedByLabel is a Pod label records the claim the claims the Pod + PodClaimedByLabel = "pool.matrixorigin.io/claimed-by" + // CNPodPhaseLabel is the pod phase in Pool + CNPodPhaseLabel = "pool.matrixorigin.io/phase" + // PoolNameLabel is the pool of CN claim or CN Pod + PoolNameLabel = "pool.matrixorigin.io/pool-name" + + // PodManagementPolicyAnno denotes the management policy of a Pod + PodManagementPolicyAnno = "pool.matrixorigin.io/management-policy" +) + +type CNPoolSpec struct { + // Template is the CNSet template of the Pool + Template CNSetSpec `json:"template"` + + // PodLabels is the Pod labels of the CN in Pool + PodLabels map[string]string `json:"podLabels,omitempty"` + + // Deps is the dependencies of the Pool + Deps CNSetDeps `json:"deps"` + + Strategy PoolStrategy `json:"strategy"` +} + +type PoolStrategy struct { + // UpdateStrategy defines the strategy for pool updating + UpdateStrategy PoolUpdateStrategy `json:"updateStrategy"` + + // UpdateStrategy defines the strategy for pool scaling + ScaleStrategy PoolScaleStrategy `json:"scaleStrategy"` +} + +type PoolUpdateStrategy struct { + // +optional + ReclaimTimeout *metav1.Duration `json:"reclaimTimeout,omitempty"` +} + +type PoolScaleStrategy struct { + MaxIdle int32 `json:"maxIdle"` + + // +optional + // MaxPods allowed in this Pool, nil means no limit + MaxPods *int32 `json:"maxPods,omitempty"` +} + +func (s *PoolScaleStrategy) GetMaxPods() int32 { + if s.MaxPods == nil { + return math.MaxInt32 + } + return *s.MaxPods +} + +type CNPoolStatus struct { +} + +// +kubebuilder:object:root=true + +// CNPool maintains a pool of CN Pods +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" +// +kubebuilder:resource:scope="Namespaced" +type CNPool struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CNPoolSpec `json:"spec"` + + // +optional + Status CNPoolStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// CNPoolList contains a list of CNPool +type CNPoolList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CNPool `json:"items"` +} + +func init() { + SchemeBuilder.Register(&CNPool{}, &CNPoolList{}) +} diff --git a/api/core/v1alpha1/cnset_types.go b/api/core/v1alpha1/cnset_types.go index c7b8015a..a8ecbe45 100644 --- a/api/core/v1alpha1/cnset_types.go +++ b/api/core/v1alpha1/cnset_types.go @@ -74,10 +74,6 @@ type CNSetSpec struct { // Labels are the CN labels for all the CN stores managed by this CNSet Labels []CNLabel `json:"cnLabels,omitempty"` - // ExternalStoreControl indicates mo-operator should not sync CN store state and label - // so that the store state can be safely managed externally - ExternalStoreControl bool `json:"externalStoreControl,omitempty"` - // ScalingConfig declares the CN scaling behavior ScalingConfig ScalingConfig `json:"scalingConfig,omitempty"` @@ -89,6 +85,15 @@ type CNSetSpec struct { // PythonUdfSidecar is the python udf server in CN PythonUdfSidecar PythonUdfSidecar `json:"pythonUdfSidecar,omitempty"` + + // PodManagementPolicy is the pod management policy of the Pod in this Set + PodManagementPolicy *string `json:"podManagementPolicy,omitempty"` + + // PodsToDelete are the Pods to delete in the CNSet + PodsToDelete []string `json:"podsToDelete,omitempty"` + + // PauseUpdate means the CNSet should pause rolling-update + PauseUpdate bool `json:"pauseUpdate,omitempty"` } type ScalingConfig struct { diff --git a/api/core/v1alpha1/store_helper.go b/api/core/v1alpha1/store_helper.go index 87ece658..1ea5ce9b 100644 --- a/api/core/v1alpha1/store_helper.go +++ b/api/core/v1alpha1/store_helper.go @@ -23,8 +23,4 @@ const ( // StoreCordonAnno cordons a CN store StoreCordonAnno = "matrixorigin.io/store-cordon" - - // StoreExternalControlledAnno marks the CN Pod as externally controlled so that - // CN state and labels performed by mo-operator will be skipped - StoreExternalControlledAnno = "matrixorigin.io/external-controlled" ) diff --git a/api/core/v1alpha1/zz_generated.deepcopy.go b/api/core/v1alpha1/zz_generated.deepcopy.go index 79f7088a..6de4b473 100644 --- a/api/core/v1alpha1/zz_generated.deepcopy.go +++ b/api/core/v1alpha1/zz_generated.deepcopy.go @@ -335,6 +335,227 @@ func (in *BucketClaimStatus) DeepCopy() *BucketClaimStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CNClaim) DeepCopyInto(out *CNClaim) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CNClaim. +func (in *CNClaim) DeepCopy() *CNClaim { + if in == nil { + return nil + } + out := new(CNClaim) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CNClaim) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CNClaimList) DeepCopyInto(out *CNClaimList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CNClaim, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CNClaimList. +func (in *CNClaimList) DeepCopy() *CNClaimList { + if in == nil { + return nil + } + out := new(CNClaimList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CNClaimList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CNClaimSet) DeepCopyInto(out *CNClaimSet) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CNClaimSet. +func (in *CNClaimSet) DeepCopy() *CNClaimSet { + if in == nil { + return nil + } + out := new(CNClaimSet) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CNClaimSet) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CNClaimSetList) DeepCopyInto(out *CNClaimSetList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CNClaimSet, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CNClaimSetList. +func (in *CNClaimSetList) DeepCopy() *CNClaimSetList { + if in == nil { + return nil + } + out := new(CNClaimSetList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CNClaimSetList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CNClaimSetSpec) DeepCopyInto(out *CNClaimSetSpec) { + *out = *in + in.Template.DeepCopyInto(&out.Template) + if in.Selector != nil { + in, out := &in.Selector, &out.Selector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CNClaimSetSpec. +func (in *CNClaimSetSpec) DeepCopy() *CNClaimSetSpec { + if in == nil { + return nil + } + out := new(CNClaimSetSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CNClaimSetStatus) DeepCopyInto(out *CNClaimSetStatus) { + *out = *in + if in.Claims != nil { + in, out := &in.Claims, &out.Claims + *out = make([]CNClaimStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CNClaimSetStatus. +func (in *CNClaimSetStatus) DeepCopy() *CNClaimSetStatus { + if in == nil { + return nil + } + out := new(CNClaimSetStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CNClaimSpec) DeepCopyInto(out *CNClaimSpec) { + *out = *in + if in.Selector != nil { + in, out := &in.Selector, &out.Selector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } + if in.CNLabels != nil { + in, out := &in.CNLabels, &out.CNLabels + *out = make([]CNLabel, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CNClaimSpec. +func (in *CNClaimSpec) DeepCopy() *CNClaimSpec { + if in == nil { + return nil + } + out := new(CNClaimSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CNClaimStatus) DeepCopyInto(out *CNClaimStatus) { + *out = *in + in.Store.DeepCopyInto(&out.Store) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CNClaimStatus. +func (in *CNClaimStatus) DeepCopy() *CNClaimStatus { + if in == nil { + return nil + } + out := new(CNClaimStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CNClaimTemplate) DeepCopyInto(out *CNClaimTemplate) { + *out = *in + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CNClaimTemplate. +func (in *CNClaimTemplate) DeepCopy() *CNClaimTemplate { + if in == nil { + return nil + } + out := new(CNClaimTemplate) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CNGroup) DeepCopyInto(out *CNGroup) { *out = *in @@ -406,6 +627,105 @@ func (in *CNLabel) DeepCopy() *CNLabel { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CNPool) DeepCopyInto(out *CNPool) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CNPool. +func (in *CNPool) DeepCopy() *CNPool { + if in == nil { + return nil + } + out := new(CNPool) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CNPool) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CNPoolList) DeepCopyInto(out *CNPoolList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CNPool, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CNPoolList. +func (in *CNPoolList) DeepCopy() *CNPoolList { + if in == nil { + return nil + } + out := new(CNPoolList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CNPoolList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CNPoolSpec) DeepCopyInto(out *CNPoolSpec) { + *out = *in + in.Template.DeepCopyInto(&out.Template) + if in.PodLabels != nil { + in, out := &in.PodLabels, &out.PodLabels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + in.Deps.DeepCopyInto(&out.Deps) + in.Strategy.DeepCopyInto(&out.Strategy) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CNPoolSpec. +func (in *CNPoolSpec) DeepCopy() *CNPoolSpec { + if in == nil { + return nil + } + out := new(CNPoolSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CNPoolStatus) DeepCopyInto(out *CNPoolStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CNPoolStatus. +func (in *CNPoolStatus) DeepCopy() *CNPoolStatus { + if in == nil { + return nil + } + out := new(CNPoolStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CNSet) DeepCopyInto(out *CNSet) { *out = *in @@ -524,6 +844,16 @@ func (in *CNSetSpec) DeepCopyInto(out *CNSetSpec) { } in.UpdateStrategy.DeepCopyInto(&out.UpdateStrategy) in.PythonUdfSidecar.DeepCopyInto(&out.PythonUdfSidecar) + if in.PodManagementPolicy != nil { + in, out := &in.PodManagementPolicy, &out.PodManagementPolicy + *out = new(string) + **out = **in + } + if in.PodsToDelete != nil { + in, out := &in.PodsToDelete, &out.PodsToDelete + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CNSetSpec. @@ -572,6 +902,28 @@ func (in *CNStore) DeepCopy() *CNStore { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CNStoreStatus) DeepCopyInto(out *CNStoreStatus) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make([]CNLabel, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CNStoreStatus. +func (in *CNStoreStatus) DeepCopy() *CNStoreStatus { + if in == nil { + return nil + } + out := new(CNStoreStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CertificateRef) DeepCopyInto(out *CertificateRef) { *out = *in @@ -1488,6 +1840,63 @@ func (in *PodSet) DeepCopy() *PodSet { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PoolScaleStrategy) DeepCopyInto(out *PoolScaleStrategy) { + *out = *in + if in.MaxPods != nil { + in, out := &in.MaxPods, &out.MaxPods + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PoolScaleStrategy. +func (in *PoolScaleStrategy) DeepCopy() *PoolScaleStrategy { + if in == nil { + return nil + } + out := new(PoolScaleStrategy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PoolStrategy) DeepCopyInto(out *PoolStrategy) { + *out = *in + in.UpdateStrategy.DeepCopyInto(&out.UpdateStrategy) + in.ScaleStrategy.DeepCopyInto(&out.ScaleStrategy) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PoolStrategy. +func (in *PoolStrategy) DeepCopy() *PoolStrategy { + if in == nil { + return nil + } + out := new(PoolStrategy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PoolUpdateStrategy) DeepCopyInto(out *PoolUpdateStrategy) { + *out = *in + if in.ReclaimTimeout != nil { + in, out := &in.ReclaimTimeout, &out.ReclaimTimeout + *out = new(v1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PoolUpdateStrategy. +func (in *PoolUpdateStrategy) DeepCopy() *PoolUpdateStrategy { + if in == nil { + return nil + } + out := new(PoolUpdateStrategy) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProxySet) DeepCopyInto(out *ProxySet) { *out = *in diff --git a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnclaims.yaml b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnclaims.yaml new file mode 100644 index 00000000..34cdce8e --- /dev/null +++ b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnclaims.yaml @@ -0,0 +1,157 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: cnclaims.core.matrixorigin.io +spec: + group: core.matrixorigin.io + names: + kind: CNClaim + listKind: CNClaimList + plural: cnclaims + singular: cnclaim + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.podName + name: Pod + type: string + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: CNClaim claim a CN to use + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + cnLabels: + items: + properties: + key: + description: Key is the store label key + type: string + values: + description: Values are the store label values + items: + type: string + type: array + type: object + type: array + podName: + description: PodName is usually populated by controller and would + be part of the claim spec that must be persisted once bound + type: string + poolName: + description: PoolName is usually populated by controller that which + pool the claim is nominated + type: string + selector: + description: A label selector is a label query over a set of resources. + The result of matchLabels and matchExpressions are ANDed. An empty + label selector matches all objects. A null label selector matches + no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - selector + type: object + status: + properties: + phase: + type: string + store: + properties: + labels: + items: + properties: + key: + description: Key is the store label key + type: string + values: + description: Values are the store label values + items: + type: string + type: array + type: object + type: array + lockServiceAddress: + type: string + pipelineServiceAddress: + type: string + queryAddress: + type: string + serviceID: + type: string + sqlAddress: + type: string + workState: + format: int32 + type: integer + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnclaimsets.yaml b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnclaimsets.yaml new file mode 100644 index 00000000..a689ccc8 --- /dev/null +++ b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnclaimsets.yaml @@ -0,0 +1,225 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: cnclaimsets.core.matrixorigin.io +spec: + group: core.matrixorigin.io + names: + kind: CNClaimSet + listKind: CNClaimSetList + plural: cnclaimsets + singular: cnclaimset + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: CNClaimSet orchestrates a set of CNClaims + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + replicas: + format: int32 + type: integer + selector: + description: A label selector is a label query over a set of resources. + The result of matchLabels and matchExpressions are ANDed. An empty + label selector matches all objects. A null label selector matches + no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + template: + properties: + metadata: + type: object + spec: + properties: + cnLabels: + items: + properties: + key: + description: Key is the store label key + type: string + values: + description: Values are the store label values + items: + type: string + type: array + type: object + type: array + podName: + description: PodName is usually populated by controller and + would be part of the claim spec that must be persisted once + bound + type: string + poolName: + description: PoolName is usually populated by controller that + which pool the claim is nominated + type: string + selector: + description: A label selector is a label query over a set + of resources. The result of matchLabels and matchExpressions + are ANDed. An empty label selector matches all objects. + A null label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is + "key", the operator is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - selector + type: object + type: object + required: + - replicas + - template + type: object + status: + properties: + claims: + items: + properties: + phase: + type: string + store: + properties: + labels: + items: + properties: + key: + description: Key is the store label key + type: string + values: + description: Values are the store label values + items: + type: string + type: array + type: object + type: array + lockServiceAddress: + type: string + pipelineServiceAddress: + type: string + queryAddress: + type: string + serviceID: + type: string + sqlAddress: + type: string + workState: + format: int32 + type: integer + type: object + type: object + type: array + replicas: + format: int32 + type: integer + required: + - replicas + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnpools.yaml b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnpools.yaml new file mode 100644 index 00000000..bdd5d6f7 --- /dev/null +++ b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnpools.yaml @@ -0,0 +1,418 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: cnpools.core.matrixorigin.io +spec: + group: core.matrixorigin.io + names: + kind: CNPool + listKind: CNPoolList + plural: cnpools + singular: cnpool + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: CNPool maintains a pool of CN Pods + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + deps: + description: Deps is the dependencies of the Pool + properties: + dnSet: + description: The DNSet it depends on + type: object + x-kubernetes-preserve-unknown-fields: true + externalLogSet: + description: 'An external LogSet the CNSet should connected to, + mutual exclusive with LogSet TODO: rethink the schema of ExternalLogSet' + properties: + haKeeperEndpoint: + description: HAKeeperEndpoint of the ExternalLogSet + type: string + type: object + logSet: + description: The LogSet it depends on, mutual exclusive with ExternalLogSet + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + podLabels: + additionalProperties: + type: string + description: PodLabels is the Pod labels of the CN in Pool + type: object + strategy: + properties: + scaleStrategy: + description: UpdateStrategy defines the strategy for pool scaling + properties: + maxIdle: + format: int32 + type: integer + maxPods: + description: MaxPods allowed in this Pool, nil means no limit + format: int32 + type: integer + required: + - maxIdle + type: object + updateStrategy: + description: UpdateStrategy defines the strategy for pool updating + properties: + reclaimTimeout: + type: string + type: object + required: + - scaleStrategy + - updateStrategy + type: object + template: + description: Template is the CNSet template of the Pool + properties: + cacheVolume: + description: CacheVolume is the desired local cache volume for + CNSet, node storage will be used if not specified + properties: + memoryCacheSize: + anyOf: + - type: integer + - type: string + description: 'Deprecated: use SharedStorageCache instead' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + size: + anyOf: + - type: integer + - type: string + description: Size is the desired storage size of the volume + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + storageClassName: + description: StorageClassName reference to the storageclass + of the desired volume, the default storageclass of the cluster + would be used if no specified. + type: string + type: object + clusterDomain: + description: ClusterDomain is the cluster-domain of current kubernetes + cluster, refer https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/ + for details + type: string + cnLabels: + description: Labels are the CN labels for all the CN stores managed + by this CNSet + items: + properties: + key: + description: Key is the store label key + type: string + values: + description: Values are the store label values + items: + type: string + type: array + type: object + type: array + config: + description: Config is the raw config for pods + type: string + dnsBasedIdentity: + description: If enabled, use the Pod dns name as the Pod identity + type: boolean + exportToPrometheus: + type: boolean + image: + description: Image is the docker image of the main container + type: string + memoryLimitPercent: + description: MemoryLimitPercent is percent used to set GOMEMLIMIT + env, its value must be in interval (0, 100]. GOMEMLIMIT = limits.memory + * MemoryLimitPercent / 100 + type: integer + metricsSecretRef: + description: MetricsSecretRef is the secret reference for the + operator to access CN metrics + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + nodePort: + description: NodePort specifies the node port to use when ServiceType + is NodePort or LoadBalancer, reconciling will fail if the node + port is not available. + format: int32 + type: integer + nodeSelector: + additionalProperties: + type: string + type: object + overlay: + x-kubernetes-preserve-unknown-fields: true + pauseUpdate: + description: PauseUpdate means the CNSet should pause rolling-update + type: boolean + podManagementPolicy: + description: PodManagementPolicy is the pod management policy + of the Pod in this Set + type: string + podsToDelete: + description: PodsToDelete are the Pods to delete in the CNSet + items: + type: string + type: array + pythonUdfSidecar: + description: PythonUdfSidecar is the python udf server in CN + properties: + enabled: + type: boolean + image: + description: Image is the docker image of the python udf server + type: string + overlay: + x-kubernetes-preserve-unknown-fields: true + port: + type: integer + resources: + description: Resources is the resource requirement of the + python udf server + properties: + claims: + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. \n This field + is immutable. It can only be set for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry + in pod.spec.resourceClaims of the Pod where this + field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of + compute resources required. If Requests is omitted for + a container, it defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined value. + Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + type: object + replicas: + description: Replicas is the desired number of pods of this set + format: int32 + type: integer + resources: + description: Resources is the resource requirement of the main + conainer + properties: + claims: + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. It can only be + set for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry in + pod.spec.resourceClaims of the Pod where this field + is used. It makes that resource available inside a + container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. Requests cannot exceed + Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + role: + description: '[TP, AP], default to TP Deprecated: use labels instead' + type: string + scalingConfig: + description: ScalingConfig declares the CN scaling behavior + properties: + storeDrainEnabled: + description: StoreDrainEnabled is the flag to enable store + draining + type: boolean + storeDrainTimeout: + description: StoreDrainTimeout is the timeout for draining + a CN store + type: string + type: object + serviceAnnotations: + additionalProperties: + type: string + description: ServiceAnnotations are the annotations for the cn + service + type: object + serviceArgs: + description: 'ServiceArgs define command line options for process, + used by logset/cnset/dnset service. NOTE: user should not define + "-cfg" argument in this field, which is defined default by controller' + items: + type: string + type: array + serviceType: + default: ClusterIP + description: ServiceType is the service type of cn service + enum: + - ClusterIP + - NodePort + - LoadBalancer + type: string + sharedStorageCache: + description: SharedStorageCache is the configuration of the S3 + sharedStorageCache + properties: + diskCacheSize: + anyOf: + - type: integer + - type: string + description: 'DiskCacheSize specifies how much disk space + can be used to cache the object in shared storage, the default + size would be 90% of the cacheVolume size to reserve some + space to the filesystem metadata and avoid disk space exhaustion + DiskCache would be disabled if CacheVolume is not set for + DN/CN, and if DiskCacheSize is set while the CacheVolume + is not set for DN/CN, an error would be raised to indicate + the misconfiguration. NOTE: Unless there is a specific reason + not to set this field, it is usually more reasonable to + let the operator set the available disk cache size according + to the actual size of the cacheVolume.' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memoryCacheSize: + anyOf: + - type: integer + - type: string + description: MemoryCacheSize specifies how much memory would + be used to cache the object in shared storage, the default + size would be 50% of the container memory request MemoryCache + cannot be completely disabled due to MO limitation currently, + you can set MemoryCacheSize to 1B to achieve an effect similar + to disabling + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + topologySpread: + description: TopologyEvenSpread specifies what topology domains + the Pods in set should be evenly spread in. This will be overridden + by .overlay.TopologySpreadConstraints + items: + type: string + type: array + updateStrategy: + description: UpdateStrategy is the rolling-update strategy of + CN + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: MaxSurge is an optional field that specifies + the maximum number of Pods that can be created over the + desired number of Pods. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: MaxUnavailable an optional field that specifies + the maximum number of Pods that can be unavailable during + the update process. + x-kubernetes-int-or-string: true + type: object + type: object + required: + - deps + - strategy + - template + type: object + status: + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnsets.yaml b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnsets.yaml index e20373f8..980775fb 100644 --- a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnsets.yaml +++ b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnsets.yaml @@ -117,11 +117,6 @@ spec: type: boolean exportToPrometheus: type: boolean - externalStoreControl: - description: ExternalStoreControl indicates mo-operator should not - sync CN store state and label so that the store state can be safely - managed externally - type: boolean image: description: Image is the docker image of the main container type: string @@ -153,6 +148,18 @@ spec: type: object overlay: x-kubernetes-preserve-unknown-fields: true + pauseUpdate: + description: PauseUpdate means the CNSet should pause rolling-update + type: boolean + podManagementPolicy: + description: PodManagementPolicy is the pod management policy of the + Pod in this Set + type: string + podsToDelete: + description: PodsToDelete are the Pods to delete in the CNSet + items: + type: string + type: array pythonUdfSidecar: description: PythonUdfSidecar is the python udf server in CN properties: diff --git a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_matrixoneclusters.yaml b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_matrixoneclusters.yaml index aab879b7..92bba64d 100644 --- a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_matrixoneclusters.yaml +++ b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_matrixoneclusters.yaml @@ -110,11 +110,6 @@ spec: type: boolean exportToPrometheus: type: boolean - externalStoreControl: - description: ExternalStoreControl indicates mo-operator should - not sync CN store state and label so that the store state can - be safely managed externally - type: boolean image: description: Image is the docker image of the main container type: string @@ -146,6 +141,18 @@ spec: type: object overlay: x-kubernetes-preserve-unknown-fields: true + pauseUpdate: + description: PauseUpdate means the CNSet should pause rolling-update + type: boolean + podManagementPolicy: + description: PodManagementPolicy is the pod management policy + of the Pod in this Set + type: string + podsToDelete: + description: PodsToDelete are the Pods to delete in the CNSet + items: + type: string + type: array pythonUdfSidecar: description: PythonUdfSidecar is the python udf server in CN properties: @@ -419,11 +426,6 @@ spec: type: boolean exportToPrometheus: type: boolean - externalStoreControl: - description: ExternalStoreControl indicates mo-operator should - not sync CN store state and label so that the store state - can be safely managed externally - type: boolean image: description: Image is the docker image of the main container type: string @@ -459,6 +461,18 @@ spec: type: object overlay: x-kubernetes-preserve-unknown-fields: true + pauseUpdate: + description: PauseUpdate means the CNSet should pause rolling-update + type: boolean + podManagementPolicy: + description: PodManagementPolicy is the pod management policy + of the Pod in this Set + type: string + podsToDelete: + description: PodsToDelete are the Pods to delete in the CNSet + items: + type: string + type: array pythonUdfSidecar: description: PythonUdfSidecar is the python udf server in CN properties: @@ -1437,11 +1451,6 @@ spec: type: boolean exportToPrometheus: type: boolean - externalStoreControl: - description: ExternalStoreControl indicates mo-operator should - not sync CN store state and label so that the store state can - be safely managed externally - type: boolean image: description: Image is the docker image of the main container type: string @@ -1473,6 +1482,18 @@ spec: type: object overlay: x-kubernetes-preserve-unknown-fields: true + pauseUpdate: + description: PauseUpdate means the CNSet should pause rolling-update + type: boolean + podManagementPolicy: + description: PodManagementPolicy is the pod management policy + of the Pod in this Set + type: string + podsToDelete: + description: PodsToDelete are the Pods to delete in the CNSet + items: + type: string + type: array pythonUdfSidecar: description: PythonUdfSidecar is the python udf server in CN properties: diff --git a/charts/matrixone-operator/templates/deployment.yaml b/charts/matrixone-operator/templates/deployment.yaml index acba39ce..397c9383 100644 --- a/charts/matrixone-operator/templates/deployment.yaml +++ b/charts/matrixone-operator/templates/deployment.yaml @@ -46,6 +46,8 @@ spec: image: {{ include "matrixone-operator.image" . }} command: - /manager + args: + - -leader-elect imagePullPolicy: {{ .Values.image.pullPolicy }} env: {{- range $key, $value := .Values.env }} diff --git a/charts/matrixone-operator/templates/role.yaml b/charts/matrixone-operator/templates/role.yaml index f2ab2c10..1bb69f99 100644 --- a/charts/matrixone-operator/templates/role.yaml +++ b/charts/matrixone-operator/templates/role.yaml @@ -62,6 +62,12 @@ rules: - '*' verbs: - '*' + - apiGroups: + - coordination.k8s.io + resources: + - '*' + verbs: + - '*' - apiGroups: - admissionregistration.k8s.io resources: diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 1437faa3..d50979e6 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -19,6 +19,8 @@ import ( "fmt" "github.com/go-logr/zapr" "github.com/matrixorigin/matrixone-operator/pkg/controllers/br" + "github.com/matrixorigin/matrixone-operator/pkg/controllers/cnclaim" + "github.com/matrixorigin/matrixone-operator/pkg/controllers/cnpool" "github.com/matrixorigin/matrixone-operator/pkg/querycli" "os" @@ -206,14 +208,23 @@ func main() { qc, err := querycli.New(zapLogger) exitIf(err, "unable to create query client") + haCliMgr := hacli.NewManager(mgr.GetClient(), mgr.GetLogger()) if features.DefaultFeatureGate.Enabled(features.CNLabel) { - cnLabelController := cnstore.NewController(hacli.NewManager(mgr.GetClient(), mgr.GetLogger()), qc) + cnLabelController := cnstore.NewController(haCliMgr, qc) err = cnLabelController.Reconcile(mgr) exitIf(err, "unable to set up cnlabel controller") } else { setupLog.Info(fmt.Sprintf("cn label not enabled, skip setup cnlabel")) } + poolController := cnpool.Actor{Logger: mgr.GetLogger().WithName("pool-controller")} + err = poolController.Start(mgr) + exitIf(err, "error running pool controller") + + claimController := cnclaim.NewActor(haCliMgr) + err = claimController.Start(mgr) + exitIf(err, "error running claim controller") + err = mgr.AddHealthzCheck("healthz", healthz.Ping) exitIf(err, "unable to set up health check") err = mgr.AddReadyzCheck("readyz", healthz.Ping) diff --git a/deploy/crds/core.matrixorigin.io_cnclaims.yaml b/deploy/crds/core.matrixorigin.io_cnclaims.yaml new file mode 100644 index 00000000..34cdce8e --- /dev/null +++ b/deploy/crds/core.matrixorigin.io_cnclaims.yaml @@ -0,0 +1,157 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: cnclaims.core.matrixorigin.io +spec: + group: core.matrixorigin.io + names: + kind: CNClaim + listKind: CNClaimList + plural: cnclaims + singular: cnclaim + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.podName + name: Pod + type: string + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: CNClaim claim a CN to use + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + cnLabels: + items: + properties: + key: + description: Key is the store label key + type: string + values: + description: Values are the store label values + items: + type: string + type: array + type: object + type: array + podName: + description: PodName is usually populated by controller and would + be part of the claim spec that must be persisted once bound + type: string + poolName: + description: PoolName is usually populated by controller that which + pool the claim is nominated + type: string + selector: + description: A label selector is a label query over a set of resources. + The result of matchLabels and matchExpressions are ANDed. An empty + label selector matches all objects. A null label selector matches + no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - selector + type: object + status: + properties: + phase: + type: string + store: + properties: + labels: + items: + properties: + key: + description: Key is the store label key + type: string + values: + description: Values are the store label values + items: + type: string + type: array + type: object + type: array + lockServiceAddress: + type: string + pipelineServiceAddress: + type: string + queryAddress: + type: string + serviceID: + type: string + sqlAddress: + type: string + workState: + format: int32 + type: integer + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/crds/core.matrixorigin.io_cnclaimsets.yaml b/deploy/crds/core.matrixorigin.io_cnclaimsets.yaml new file mode 100644 index 00000000..a689ccc8 --- /dev/null +++ b/deploy/crds/core.matrixorigin.io_cnclaimsets.yaml @@ -0,0 +1,225 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: cnclaimsets.core.matrixorigin.io +spec: + group: core.matrixorigin.io + names: + kind: CNClaimSet + listKind: CNClaimSetList + plural: cnclaimsets + singular: cnclaimset + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: CNClaimSet orchestrates a set of CNClaims + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + replicas: + format: int32 + type: integer + selector: + description: A label selector is a label query over a set of resources. + The result of matchLabels and matchExpressions are ANDed. An empty + label selector matches all objects. A null label selector matches + no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + template: + properties: + metadata: + type: object + spec: + properties: + cnLabels: + items: + properties: + key: + description: Key is the store label key + type: string + values: + description: Values are the store label values + items: + type: string + type: array + type: object + type: array + podName: + description: PodName is usually populated by controller and + would be part of the claim spec that must be persisted once + bound + type: string + poolName: + description: PoolName is usually populated by controller that + which pool the claim is nominated + type: string + selector: + description: A label selector is a label query over a set + of resources. The result of matchLabels and matchExpressions + are ANDed. An empty label selector matches all objects. + A null label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is + "key", the operator is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - selector + type: object + type: object + required: + - replicas + - template + type: object + status: + properties: + claims: + items: + properties: + phase: + type: string + store: + properties: + labels: + items: + properties: + key: + description: Key is the store label key + type: string + values: + description: Values are the store label values + items: + type: string + type: array + type: object + type: array + lockServiceAddress: + type: string + pipelineServiceAddress: + type: string + queryAddress: + type: string + serviceID: + type: string + sqlAddress: + type: string + workState: + format: int32 + type: integer + type: object + type: object + type: array + replicas: + format: int32 + type: integer + required: + - replicas + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/crds/core.matrixorigin.io_cnpools.yaml b/deploy/crds/core.matrixorigin.io_cnpools.yaml new file mode 100644 index 00000000..bdd5d6f7 --- /dev/null +++ b/deploy/crds/core.matrixorigin.io_cnpools.yaml @@ -0,0 +1,418 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: cnpools.core.matrixorigin.io +spec: + group: core.matrixorigin.io + names: + kind: CNPool + listKind: CNPoolList + plural: cnpools + singular: cnpool + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: CNPool maintains a pool of CN Pods + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + deps: + description: Deps is the dependencies of the Pool + properties: + dnSet: + description: The DNSet it depends on + type: object + x-kubernetes-preserve-unknown-fields: true + externalLogSet: + description: 'An external LogSet the CNSet should connected to, + mutual exclusive with LogSet TODO: rethink the schema of ExternalLogSet' + properties: + haKeeperEndpoint: + description: HAKeeperEndpoint of the ExternalLogSet + type: string + type: object + logSet: + description: The LogSet it depends on, mutual exclusive with ExternalLogSet + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + podLabels: + additionalProperties: + type: string + description: PodLabels is the Pod labels of the CN in Pool + type: object + strategy: + properties: + scaleStrategy: + description: UpdateStrategy defines the strategy for pool scaling + properties: + maxIdle: + format: int32 + type: integer + maxPods: + description: MaxPods allowed in this Pool, nil means no limit + format: int32 + type: integer + required: + - maxIdle + type: object + updateStrategy: + description: UpdateStrategy defines the strategy for pool updating + properties: + reclaimTimeout: + type: string + type: object + required: + - scaleStrategy + - updateStrategy + type: object + template: + description: Template is the CNSet template of the Pool + properties: + cacheVolume: + description: CacheVolume is the desired local cache volume for + CNSet, node storage will be used if not specified + properties: + memoryCacheSize: + anyOf: + - type: integer + - type: string + description: 'Deprecated: use SharedStorageCache instead' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + size: + anyOf: + - type: integer + - type: string + description: Size is the desired storage size of the volume + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + storageClassName: + description: StorageClassName reference to the storageclass + of the desired volume, the default storageclass of the cluster + would be used if no specified. + type: string + type: object + clusterDomain: + description: ClusterDomain is the cluster-domain of current kubernetes + cluster, refer https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/ + for details + type: string + cnLabels: + description: Labels are the CN labels for all the CN stores managed + by this CNSet + items: + properties: + key: + description: Key is the store label key + type: string + values: + description: Values are the store label values + items: + type: string + type: array + type: object + type: array + config: + description: Config is the raw config for pods + type: string + dnsBasedIdentity: + description: If enabled, use the Pod dns name as the Pod identity + type: boolean + exportToPrometheus: + type: boolean + image: + description: Image is the docker image of the main container + type: string + memoryLimitPercent: + description: MemoryLimitPercent is percent used to set GOMEMLIMIT + env, its value must be in interval (0, 100]. GOMEMLIMIT = limits.memory + * MemoryLimitPercent / 100 + type: integer + metricsSecretRef: + description: MetricsSecretRef is the secret reference for the + operator to access CN metrics + properties: + name: + type: string + namespace: + type: string + required: + - name + type: object + nodePort: + description: NodePort specifies the node port to use when ServiceType + is NodePort or LoadBalancer, reconciling will fail if the node + port is not available. + format: int32 + type: integer + nodeSelector: + additionalProperties: + type: string + type: object + overlay: + x-kubernetes-preserve-unknown-fields: true + pauseUpdate: + description: PauseUpdate means the CNSet should pause rolling-update + type: boolean + podManagementPolicy: + description: PodManagementPolicy is the pod management policy + of the Pod in this Set + type: string + podsToDelete: + description: PodsToDelete are the Pods to delete in the CNSet + items: + type: string + type: array + pythonUdfSidecar: + description: PythonUdfSidecar is the python udf server in CN + properties: + enabled: + type: boolean + image: + description: Image is the docker image of the python udf server + type: string + overlay: + x-kubernetes-preserve-unknown-fields: true + port: + type: integer + resources: + description: Resources is the resource requirement of the + python udf server + properties: + claims: + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. \n This field + is immutable. It can only be set for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry + in pod.spec.resourceClaims of the Pod where this + field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of + compute resources required. If Requests is omitted for + a container, it defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined value. + Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + type: object + replicas: + description: Replicas is the desired number of pods of this set + format: int32 + type: integer + resources: + description: Resources is the resource requirement of the main + conainer + properties: + claims: + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. It can only be + set for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry in + pod.spec.resourceClaims of the Pod where this field + is used. It makes that resource available inside a + container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. Requests cannot exceed + Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + role: + description: '[TP, AP], default to TP Deprecated: use labels instead' + type: string + scalingConfig: + description: ScalingConfig declares the CN scaling behavior + properties: + storeDrainEnabled: + description: StoreDrainEnabled is the flag to enable store + draining + type: boolean + storeDrainTimeout: + description: StoreDrainTimeout is the timeout for draining + a CN store + type: string + type: object + serviceAnnotations: + additionalProperties: + type: string + description: ServiceAnnotations are the annotations for the cn + service + type: object + serviceArgs: + description: 'ServiceArgs define command line options for process, + used by logset/cnset/dnset service. NOTE: user should not define + "-cfg" argument in this field, which is defined default by controller' + items: + type: string + type: array + serviceType: + default: ClusterIP + description: ServiceType is the service type of cn service + enum: + - ClusterIP + - NodePort + - LoadBalancer + type: string + sharedStorageCache: + description: SharedStorageCache is the configuration of the S3 + sharedStorageCache + properties: + diskCacheSize: + anyOf: + - type: integer + - type: string + description: 'DiskCacheSize specifies how much disk space + can be used to cache the object in shared storage, the default + size would be 90% of the cacheVolume size to reserve some + space to the filesystem metadata and avoid disk space exhaustion + DiskCache would be disabled if CacheVolume is not set for + DN/CN, and if DiskCacheSize is set while the CacheVolume + is not set for DN/CN, an error would be raised to indicate + the misconfiguration. NOTE: Unless there is a specific reason + not to set this field, it is usually more reasonable to + let the operator set the available disk cache size according + to the actual size of the cacheVolume.' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memoryCacheSize: + anyOf: + - type: integer + - type: string + description: MemoryCacheSize specifies how much memory would + be used to cache the object in shared storage, the default + size would be 50% of the container memory request MemoryCache + cannot be completely disabled due to MO limitation currently, + you can set MemoryCacheSize to 1B to achieve an effect similar + to disabling + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + topologySpread: + description: TopologyEvenSpread specifies what topology domains + the Pods in set should be evenly spread in. This will be overridden + by .overlay.TopologySpreadConstraints + items: + type: string + type: array + updateStrategy: + description: UpdateStrategy is the rolling-update strategy of + CN + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: MaxSurge is an optional field that specifies + the maximum number of Pods that can be created over the + desired number of Pods. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: MaxUnavailable an optional field that specifies + the maximum number of Pods that can be unavailable during + the update process. + x-kubernetes-int-or-string: true + type: object + type: object + required: + - deps + - strategy + - template + type: object + status: + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/crds/core.matrixorigin.io_cnsets.yaml b/deploy/crds/core.matrixorigin.io_cnsets.yaml index e20373f8..980775fb 100644 --- a/deploy/crds/core.matrixorigin.io_cnsets.yaml +++ b/deploy/crds/core.matrixorigin.io_cnsets.yaml @@ -117,11 +117,6 @@ spec: type: boolean exportToPrometheus: type: boolean - externalStoreControl: - description: ExternalStoreControl indicates mo-operator should not - sync CN store state and label so that the store state can be safely - managed externally - type: boolean image: description: Image is the docker image of the main container type: string @@ -153,6 +148,18 @@ spec: type: object overlay: x-kubernetes-preserve-unknown-fields: true + pauseUpdate: + description: PauseUpdate means the CNSet should pause rolling-update + type: boolean + podManagementPolicy: + description: PodManagementPolicy is the pod management policy of the + Pod in this Set + type: string + podsToDelete: + description: PodsToDelete are the Pods to delete in the CNSet + items: + type: string + type: array pythonUdfSidecar: description: PythonUdfSidecar is the python udf server in CN properties: diff --git a/deploy/crds/core.matrixorigin.io_matrixoneclusters.yaml b/deploy/crds/core.matrixorigin.io_matrixoneclusters.yaml index aab879b7..92bba64d 100644 --- a/deploy/crds/core.matrixorigin.io_matrixoneclusters.yaml +++ b/deploy/crds/core.matrixorigin.io_matrixoneclusters.yaml @@ -110,11 +110,6 @@ spec: type: boolean exportToPrometheus: type: boolean - externalStoreControl: - description: ExternalStoreControl indicates mo-operator should - not sync CN store state and label so that the store state can - be safely managed externally - type: boolean image: description: Image is the docker image of the main container type: string @@ -146,6 +141,18 @@ spec: type: object overlay: x-kubernetes-preserve-unknown-fields: true + pauseUpdate: + description: PauseUpdate means the CNSet should pause rolling-update + type: boolean + podManagementPolicy: + description: PodManagementPolicy is the pod management policy + of the Pod in this Set + type: string + podsToDelete: + description: PodsToDelete are the Pods to delete in the CNSet + items: + type: string + type: array pythonUdfSidecar: description: PythonUdfSidecar is the python udf server in CN properties: @@ -419,11 +426,6 @@ spec: type: boolean exportToPrometheus: type: boolean - externalStoreControl: - description: ExternalStoreControl indicates mo-operator should - not sync CN store state and label so that the store state - can be safely managed externally - type: boolean image: description: Image is the docker image of the main container type: string @@ -459,6 +461,18 @@ spec: type: object overlay: x-kubernetes-preserve-unknown-fields: true + pauseUpdate: + description: PauseUpdate means the CNSet should pause rolling-update + type: boolean + podManagementPolicy: + description: PodManagementPolicy is the pod management policy + of the Pod in this Set + type: string + podsToDelete: + description: PodsToDelete are the Pods to delete in the CNSet + items: + type: string + type: array pythonUdfSidecar: description: PythonUdfSidecar is the python udf server in CN properties: @@ -1437,11 +1451,6 @@ spec: type: boolean exportToPrometheus: type: boolean - externalStoreControl: - description: ExternalStoreControl indicates mo-operator should - not sync CN store state and label so that the store state can - be safely managed externally - type: boolean image: description: Image is the docker image of the main container type: string @@ -1473,6 +1482,18 @@ spec: type: object overlay: x-kubernetes-preserve-unknown-fields: true + pauseUpdate: + description: PauseUpdate means the CNSet should pause rolling-update + type: boolean + podManagementPolicy: + description: PodManagementPolicy is the pod management policy + of the Pod in this Set + type: string + podsToDelete: + description: PodsToDelete are the Pods to delete in the CNSet + items: + type: string + type: array pythonUdfSidecar: description: PythonUdfSidecar is the python udf server in CN properties: diff --git a/docs/reference/api-reference.md b/docs/reference/api-reference.md index 1c90f9fe..391d65a1 100644 --- a/docs/reference/api-reference.md +++ b/docs/reference/api-reference.md @@ -18,6 +18,12 @@ Refer to https://docs.matrixorigin.io/ for more information about MatrixOne data - [BackupList](#backuplist) - [BucketClaim](#bucketclaim) - [BucketClaimList](#bucketclaimlist) +- [CNClaim](#cnclaim) +- [CNClaimList](#cnclaimlist) +- [CNClaimSet](#cnclaimset) +- [CNClaimSetList](#cnclaimsetlist) +- [CNPool](#cnpool) +- [CNPoolList](#cnpoollist) - [CNSet](#cnset) - [DNSet](#dnset) - [LogSet](#logset) @@ -201,6 +207,138 @@ _Appears in:_ +#### CNClaim + + + +CNClaim claim a CN to use + +_Appears in:_ +- [CNClaimList](#cnclaimlist) + +| Field | Description | +| --- | --- | +| `apiVersion` _string_ | `core.matrixorigin.io/v1alpha1` +| `kind` _string_ | `CNClaim` +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[CNClaimSpec](#cnclaimspec)_ | | + + +#### CNClaimList + + + +CNClaimList contains a list of CNClaims + + + +| Field | Description | +| --- | --- | +| `apiVersion` _string_ | `core.matrixorigin.io/v1alpha1` +| `kind` _string_ | `CNClaimList` +| `metadata` _[ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#listmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | +| `items` _[CNClaim](#cnclaim) array_ | | + + +#### CNClaimSet + + + +CNClaimSet orchestrates a set of CNClaims + +_Appears in:_ +- [CNClaimSetList](#cnclaimsetlist) + +| Field | Description | +| --- | --- | +| `apiVersion` _string_ | `core.matrixorigin.io/v1alpha1` +| `kind` _string_ | `CNClaimSet` +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[CNClaimSetSpec](#cnclaimsetspec)_ | | + + +#### CNClaimSetList + + + +CNClaimSetList contains a list of CNClaimSet + + + +| Field | Description | +| --- | --- | +| `apiVersion` _string_ | `core.matrixorigin.io/v1alpha1` +| `kind` _string_ | `CNClaimSetList` +| `metadata` _[ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#listmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | +| `items` _[CNClaimSet](#cnclaimset) array_ | | + + +#### CNClaimSetSpec + + + + + +_Appears in:_ +- [CNClaimSet](#cnclaimset) + +| Field | Description | +| --- | --- | +| `replicas` _integer_ | | +| `template` _[CNClaimTemplate](#cnclaimtemplate)_ | | +| `selector` _[LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#labelselector-v1-meta)_ | | + + + + +#### CNClaimSpec + + + + + +_Appears in:_ +- [CNClaim](#cnclaim) +- [CNClaimTemplate](#cnclaimtemplate) + +| Field | Description | +| --- | --- | +| `selector` _[LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#labelselector-v1-meta)_ | | +| `cnLabels` _[CNLabel](#cnlabel) array_ | | +| `podName` _string_ | PodName is usually populated by controller and would be part of the claim spec that must be persisted once bound | +| `poolName` _string_ | PoolName is usually populated by controller that which pool the claim is nominated | + + +#### CNClaimStatus + + + + + +_Appears in:_ +- [CNClaimSetStatus](#cnclaimsetstatus) + +| Field | Description | +| --- | --- | +| `phase` _CNClaimPhase_ | | +| `store` _[CNStoreStatus](#cnstorestatus)_ | | + + +#### CNClaimTemplate + + + + + +_Appears in:_ +- [CNClaimSetSpec](#cnclaimsetspec) + +| Field | Description | +| --- | --- | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[CNClaimSpec](#cnclaimspec)_ | | + + #### CNGroup @@ -242,7 +380,9 @@ _Appears in:_ _Appears in:_ +- [CNClaimSpec](#cnclaimspec) - [CNSetSpec](#cnsetspec) +- [CNStoreStatus](#cnstorestatus) | Field | Description | | --- | --- | @@ -250,6 +390,69 @@ _Appears in:_ | `values` _string array_ | Values are the store label values | +#### CNPool + + + +CNPool maintains a pool of CN Pods + +_Appears in:_ +- [CNPoolList](#cnpoollist) + +| Field | Description | +| --- | --- | +| `apiVersion` _string_ | `core.matrixorigin.io/v1alpha1` +| `kind` _string_ | `CNPool` +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[CNPoolSpec](#cnpoolspec)_ | | + + +#### CNPoolList + + + +CNPoolList contains a list of CNPool + + + +| Field | Description | +| --- | --- | +| `apiVersion` _string_ | `core.matrixorigin.io/v1alpha1` +| `kind` _string_ | `CNPoolList` +| `metadata` _[ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#listmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | +| `items` _[CNPool](#cnpool) array_ | | + + +#### CNPoolSpec + + + + + +_Appears in:_ +- [CNPool](#cnpool) + +| Field | Description | +| --- | --- | +| `template` _[CNSetSpec](#cnsetspec)_ | Template is the CNSet template of the Pool | +| `podLabels` _object (keys:string, values:string)_ | PodLabels is the Pod labels of the CN in Pool | +| `deps` _[CNSetDeps](#cnsetdeps)_ | Deps is the dependencies of the Pool | +| `strategy` _[PoolStrategy](#poolstrategy)_ | | + + + + +#### CNRole + +_Underlying type:_ `string` + + + +_Appears in:_ +- [CNSetSpec](#cnsetspec) + + + #### CNSet @@ -275,6 +478,7 @@ _Appears in:_ _Appears in:_ +- [CNPoolSpec](#cnpoolspec) - [CNSet](#cnset) | Field | Description | @@ -291,6 +495,7 @@ _Appears in:_ _Appears in:_ - [CNGroup](#cngroup) +- [CNPoolSpec](#cnpoolspec) - [CNSet](#cnset) - [MatrixOneClusterSpec](#matrixoneclusterspec) @@ -302,17 +507,39 @@ _Appears in:_ | `nodePort` _integer_ | NodePort specifies the node port to use when ServiceType is NodePort or LoadBalancer, reconciling will fail if the node port is not available. | | `cacheVolume` _[Volume](#volume)_ | CacheVolume is the desired local cache volume for CNSet, node storage will be used if not specified | | `sharedStorageCache` _[SharedStorageCache](#sharedstoragecache)_ | SharedStorageCache is the configuration of the S3 sharedStorageCache | -| `role` _CNRole_ | [TP, AP], default to TP Deprecated: use labels instead | +| `role` _[CNRole](#cnrole)_ | [TP, AP], default to TP Deprecated: use labels instead | | `cnLabels` _[CNLabel](#cnlabel) array_ | Labels are the CN labels for all the CN stores managed by this CNSet | -| `externalStoreControl` _boolean_ | ExternalStoreControl indicates mo-operator should not sync CN store state and label so that the store state can be safely managed externally | | `scalingConfig` _[ScalingConfig](#scalingconfig)_ | ScalingConfig declares the CN scaling behavior | | `metricsSecretRef` _[ObjectRef](#objectref)_ | MetricsSecretRef is the secret reference for the operator to access CN metrics | | `updateStrategy` _[RollingUpdateStrategy](#rollingupdatestrategy)_ | UpdateStrategy is the rolling-update strategy of CN | | `pythonUdfSidecar` _[PythonUdfSidecar](#pythonudfsidecar)_ | PythonUdfSidecar is the python udf server in CN | +| `podManagementPolicy` _string_ | PodManagementPolicy is the pod management policy of the Pod in this Set | +| `podsToDelete` _string array_ | PodsToDelete are the Pods to delete in the CNSet | +| `pauseUpdate` _boolean_ | PauseUpdate means the CNSet should pause rolling-update | + + + + +#### CNStoreStatus + +_Appears in:_ +- [CNClaimStatus](#cnclaimstatus) + +| Field | Description | +| --- | --- | +| `serviceID` _string_ | | +| `lockServiceAddress` _string_ | | +| `pipelineServiceAddress` _string_ | | +| `sqlAddress` _string_ | | +| `queryAddress` _string_ | | +| `workState` _integer_ | | +| `labels` _[CNLabel](#cnlabel) array_ | | + + #### CertificateRef @@ -450,10 +677,10 @@ _Appears in:_ | Field | Description | | --- | --- | -| `logShards` _integer_ | LogShards is the initial number of log shards, cannot be tuned after cluster creation currently. default to 1 | -| `dnShards` _integer_ | DNShards is the initial number of DN shards, cannot be tuned after cluster creation currently. default to 1 | -| `logShardReplicas` _integer_ | LogShardReplicas is the replica numbers of each log shard, cannot be tuned after cluster creation currently. default to 3 if LogSet replicas >= 3, to 1 otherwise | -| `restoreFrom` _string_ | RestoreFrom declares the HAKeeper data should be restored from the given path when hakeeper is bootstrapped | +| `logShards` _[int](#int)_ | LogShards is the initial number of log shards, cannot be tuned after cluster creation currently. default to 1 | +| `dnShards` _[int](#int)_ | DNShards is the initial number of DN shards, cannot be tuned after cluster creation currently. default to 1 | +| `logShardReplicas` _[int](#int)_ | LogShardReplicas is the replica numbers of each log shard, cannot be tuned after cluster creation currently. default to 3 if LogSet replicas >= 3, to 1 otherwise | +| `restoreFrom` _[string](#string)_ | RestoreFrom declares the HAKeeper data should be restored from the given path when hakeeper is bootstrapped | #### LogSet @@ -684,6 +911,50 @@ _Appears in:_ | `exportToPrometheus` _boolean_ | | +#### PoolScaleStrategy + + + + + +_Appears in:_ +- [PoolStrategy](#poolstrategy) + +| Field | Description | +| --- | --- | +| `maxIdle` _integer_ | | +| `maxPods` _integer_ | MaxPods allowed in this Pool, nil means no limit | + + +#### PoolStrategy + + + + + +_Appears in:_ +- [CNPoolSpec](#cnpoolspec) + +| Field | Description | +| --- | --- | +| `updateStrategy` _[PoolUpdateStrategy](#poolupdatestrategy)_ | UpdateStrategy defines the strategy for pool updating | +| `scaleStrategy` _[PoolScaleStrategy](#poolscalestrategy)_ | UpdateStrategy defines the strategy for pool scaling | + + +#### PoolUpdateStrategy + + + + + +_Appears in:_ +- [PoolStrategy](#poolstrategy) + +| Field | Description | +| --- | --- | +| `reclaimTimeout` _[Duration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#duration-v1-meta)_ | | + + #### ProxySet diff --git a/examples/pool/claim.yaml b/examples/pool/claim.yaml new file mode 100644 index 00000000..59089dc3 --- /dev/null +++ b/examples/pool/claim.yaml @@ -0,0 +1,13 @@ +apiVersion: core.matrixorigin.io/v1alpha1 +kind: CNClaim +metadata: + name: test + namespace: default +spec: + selector: + matchLabels: + pool: default + cnLabels: + - key: role + values: + - test diff --git a/examples/pool/cn-pool.yaml b/examples/pool/cn-pool.yaml new file mode 100644 index 00000000..ecd8fc8e --- /dev/null +++ b/examples/pool/cn-pool.yaml @@ -0,0 +1,20 @@ +apiVersion: core.matrixorigin.io/v1alpha1 +kind: CNPool +metadata: + name: test + namespace: default +spec: + deps: + logSet: + metadata: + name: mo + namespace: default + strategy: + scaleStrategy: + maxIdle: 2 + updateStrategy: + reclaimTimeout: 24h + podLabels: + pool: default + template: + image: matrixorigin/matrixone:1.1.0 diff --git a/examples/pool/cnset.yaml b/examples/pool/cnset.yaml new file mode 100644 index 00000000..d62f9dba --- /dev/null +++ b/examples/pool/cnset.yaml @@ -0,0 +1,14 @@ +apiVersion: core.matrixorigin.io/v1alpha1 +deps: + logSet: + metadata: + name: mo + namespace: default +kind: CNSet +metadata: + name: mo + namespace: default +spec: + dnsBasedIdentity: true + image: matrixorigin/matrixone:1.1.0 + replicas: 1 diff --git a/examples/pool/dnset.yaml b/examples/pool/dnset.yaml new file mode 100644 index 00000000..83c45bf5 --- /dev/null +++ b/examples/pool/dnset.yaml @@ -0,0 +1,13 @@ +apiVersion: core.matrixorigin.io/v1alpha1 +deps: + logSet: + metadata: + name: mo + namespace: default +kind: DNSet +metadata: + name: mo + namespace: default +spec: + image: matrixorigin/matrixone:1.1.0 + replicas: 1 diff --git a/examples/pool/logset.yaml b/examples/pool/logset.yaml new file mode 100644 index 00000000..fce14ef5 --- /dev/null +++ b/examples/pool/logset.yaml @@ -0,0 +1,18 @@ +apiVersion: core.matrixorigin.io/v1alpha1 +kind: LogSet +metadata: + name: mo + namespace: default +spec: + image: matrixorigin/matrixone:1.1.0 + replicas: 3 + sharedStorage: + s3: + type: minio + path: matrixone + endpoint: http://minio.default:9000 + secretRef: + name: minio + storeFailureTimeout: 30m0s + volume: + size: 10Gi diff --git a/go.mod b/go.mod index 12a97ef8..561647ee 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,10 @@ module github.com/matrixorigin/matrixone-operator -go 1.20 +go 1.21 require ( github.com/cespare/xxhash v1.1.0 + github.com/davecgh/go-spew v1.1.1 github.com/go-logr/logr v1.2.4 github.com/go-logr/zapr v1.2.4 github.com/go-sql-driver/mysql v1.7.1 @@ -68,7 +69,6 @@ require ( github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/pebble v0.0.0-20220407171941-2120d145e292 // indirect github.com/cockroachdb/redact v1.1.3 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect diff --git a/go.sum b/go.sum index 75e9e7b4..3a4b8cb2 100644 --- a/go.sum +++ b/go.sum @@ -3,12 +3,14 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= +github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo= github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= @@ -38,6 +40,7 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aws/aws-sdk-go v1.45.5 h1:bxilnhv9FngUgdPNJmOIv2bk+2sP0dpqX3e4olhWcGM= github.com/aws/aws-sdk-go v1.45.5/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v1.18.0 h1:882kkTpSFhdgYRKVZ/VCgf7sd0ru57p2JCxz4/oN5RY= @@ -243,6 +246,7 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v1.12.80 h1:aC68NT6VK715WeUapxcPSFq/a3gZdS32HdtghdOIgAo= +github.com/gopherjs/gopherjs v1.12.80/go.mod h1:d55Q4EjGQHeJVms+9LGtXul6ykz5Xzx1E1gaXQXdimY= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -400,11 +404,13 @@ github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OS github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= @@ -426,6 +432,7 @@ github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20201029093017-5a7df2af2ac7 h1:wQKuKP2HUtej2gSvx1cZmY4DENUH6tlOxRkfvPT8EBU= +github.com/pingcap/errors v0.11.5-0.20201029093017-5a7df2af2ac7/go.mod h1:G7x87le1poQzLB/TqvTJI2ILrSgobnq4Ut7luOwvfvI= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -437,6 +444,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= +github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -474,8 +482,10 @@ github.com/smartystreets-prototypes/go-disruptor v0.0.0-20200316140655-c96477fd7 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/assertions v1.13.1 h1:Ef7KhSmjZcK6AVf9YbJdvPYG9avaF0ZxudX+ThRdWfU= +github.com/smartystreets/assertions v1.13.1/go.mod h1:cXr/IwVfSo/RbCSPhoAPv73p3hlSdrBH/b3SdnW/LMY= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= +github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= @@ -504,6 +514,7 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg= github.com/tidwall/btree v1.6.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= @@ -539,6 +550,7 @@ go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= @@ -617,6 +629,7 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/pkg/controllers/cnclaim/controller.go b/pkg/controllers/cnclaim/controller.go new file mode 100644 index 00000000..c4e5075e --- /dev/null +++ b/pkg/controllers/cnclaim/controller.go @@ -0,0 +1,290 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cnclaim + +import ( + "cmp" + "context" + recon "github.com/matrixorigin/controller-runtime/pkg/reconciler" + "github.com/matrixorigin/matrixone-operator/api/core/v1alpha1" + "github.com/matrixorigin/matrixone-operator/pkg/controllers/common" + "github.com/matrixorigin/matrixone-operator/pkg/hacli" + logpb "github.com/matrixorigin/matrixone/pkg/pb/logservice" + "github.com/matrixorigin/matrixone/pkg/pb/metadata" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + "slices" + "time" +) + +// Actor reconciles CN Claim +type Actor struct { + clientMgr *hacli.HAKeeperClientManager +} + +func NewActor(mgr *hacli.HAKeeperClientManager) *Actor { + return &Actor{clientMgr: mgr} +} + +func (r *Actor) Observe(ctx *recon.Context[*v1alpha1.CNClaim]) (recon.Action[*v1alpha1.CNClaim], error) { + if ctx.Obj.Spec.PodName == "" { + return r.Bind, nil + } + return nil, r.Sync(ctx) +} + +func (r *Actor) Bind(ctx *recon.Context[*v1alpha1.CNClaim]) error { + c := ctx.Obj + + // collect orphan CNs left by former broken reconciliation + orphanCNs, err := common.ListPods(ctx, client.InNamespace(c.Namespace), client.MatchingLabels{ + v1alpha1.CNPodPhaseLabel: v1alpha1.CNPodPhaseBound, + v1alpha1.PodClaimedByLabel: c.Name, + }) + if err != nil && !apierrors.IsNotFound(err) { + return errors.Wrap(err, "error get potential orphan CNs") + } + + // claim CN + claimedPod, err := r.claimCN(ctx, orphanCNs) + if err != nil { + return errors.Wrap(err, "error claim idle CN") + } + + // no pod available, bound to a certain Pool (maybe we can loosen this constrain) + if claimedPod == nil && c.Spec.PoolName == "" { + poolList := &v1alpha1.CNPoolList{} + if err := ctx.List(poolList, client.InNamespace(c.Namespace)); err != nil { + return errors.Wrap(err, "error get list CN pools") + } + // TODO: multiple matching support (prioritize) + var pool *v1alpha1.CNPool + sl := common.MustAsSelector(c.Spec.Selector) + for i := range poolList.Items { + if sl.Matches(labels.Set(poolList.Items[i].Spec.PodLabels)) { + pool = &poolList.Items[i] + break + } + } + if pool == nil { + return errors.Wrapf(err, "no matching pool for claim %s/%s", c.Namespace, c.Name) + } + } + // re-bound later + // TODO: configurable + return recon.ErrReSync("wait pod to bound", 5*time.Second) +} + +func (r *Actor) claimCN(ctx *recon.Context[*v1alpha1.CNClaim], orphans []corev1.Pod) (*corev1.Pod, error) { + c := ctx.Obj + claimed, err := r.doClaimCN(ctx, orphans) + if err != nil { + return nil, errors.Wrap(err, "error claim CN") + } + // claim failed, wait + if claimed == nil { + return nil, nil + } + // alter CN label and working state + store, err := r.patchStore(ctx, claimed, logpb.CNStateLabel{ + State: metadata.WorkState_Working, + Labels: common.ToStoreLabels(c.Spec.CNLabels), + }) + if err != nil { + return nil, errors.Wrapf(err, "error patch Store state of claimed CN %s/%s", claimed.Namespace, claimed.Name) + } + if err := r.bindPod(ctx, claimed, store); err != nil { + return nil, errors.Wrap(err, "error bind pod") + } + return claimed, nil +} + +func (r *Actor) doClaimCN(ctx *recon.Context[*v1alpha1.CNClaim], orphans []corev1.Pod) (*corev1.Pod, error) { + c := ctx.Obj + + // bound orphan CN first + if len(orphans) > 0 { + if len(orphans) > 1 { + ctx.Log.Info("multiple orphan CN bound to 1 claim", "count", len(orphans), "claimName", c.Name) + } + return &orphans[0], nil + } + + // ordinary case: no orphans, try claim an idle CN + baseSelector := common.MustAsSelector(c.Spec.Selector) + podSelector := baseSelector.Add(common.MustEqual(v1alpha1.CNPodPhaseLabel, v1alpha1.CNPodPhaseIdle)) + if c.Spec.PoolName != "" { + podSelector = podSelector.Add(common.MustEqual(v1alpha1.PoolNameLabel, c.Spec.PodName)) + } + idleCNs, err := common.ListPods(ctx, client.InNamespace(c.Namespace), client.MatchingLabelsSelector{Selector: podSelector}) + if err != nil && !apierrors.IsNotFound(err) { + return nil, errors.Wrap(err, "error list idle Pods") + } + + slices.SortFunc(idleCNs, priorityFunc(c)) + for i := range idleCNs { + pod := &idleCNs[i] + pod.Labels[v1alpha1.CNPodPhaseLabel] = v1alpha1.CNPodPhaseBound + pod.Labels[v1alpha1.PodClaimedByLabel] = c.Name + // atomic operation with optimistic concurrency control, succeed means claimed + if err := ctx.Update(pod); err != nil { + if apierrors.IsConflict(err) { + ctx.Log.Info("CN pod is not up to date, try next", "podName", pod.Name) + } else { + ctx.Log.Error(err, "error claim Pod", "podName", pod.Name) + } + } else { + return pod, nil + } + } + return nil, nil +} + +func (r *Actor) bindPod(ctx *recon.Context[*v1alpha1.CNClaim], pod *corev1.Pod, store *metadata.CNService) error { + c := ctx.Obj + c.Spec.PodName = pod.Name + c.Spec.PoolName = pod.Labels[v1alpha1.PoolNameLabel] + if c.Labels == nil { + c.Labels = map[string]string{} + } + c.Labels[v1alpha1.PoolNameLabel] = c.Spec.PoolName + if err := ctx.Update(c); err != nil { + return errors.Wrap(err, "error bound pod to claim") + } + + c.Status.Phase = v1alpha1.CNPodPhaseBound + c.Status.Store = toStoreStatus(store) + // if we failed to update status here, observe would help fulfill the status later + if err := ctx.UpdateStatus(c); err != nil { + return errors.Wrap(err, "error update claim status") + } + return nil +} + +func (r *Actor) Sync(ctx *recon.Context[*v1alpha1.CNClaim]) error { + // TODO: monitor pod health + return nil +} + +func (r *Actor) Finalize(ctx *recon.Context[*v1alpha1.CNClaim]) (bool, error) { + c := ctx.Obj + ownedCNs, err := common.ListPods(ctx, client.InNamespace(c.Namespace), client.MatchingLabels{ + v1alpha1.CNPodPhaseLabel: v1alpha1.CNPodPhaseBound, + v1alpha1.PodClaimedByLabel: c.Name, + }) + if err != nil && !apierrors.IsNotFound(err) { + return false, errors.Wrap(err, "error get owned CNs") + } + if len(ownedCNs) == 0 { + return true, nil + } + for i := range ownedCNs { + cn := ownedCNs[i] + // TODO(aylei): this will overwrite all labels, keep the base label if necessary + _, err := r.patchStore(ctx, &cn, logpb.CNStateLabel{ + State: metadata.WorkState_Draining, + // FIXME(aylei): HAKeeper does not support patch labels to empty yet, use a dummy one + Labels: map[string]metadata.LabelList{ + "dummy": {Labels: []string{"pool"}}, + }, + }) + if err != nil { + return false, errors.Wrapf(err, "error drain CN %s/%s", cn.Namespace, cn.Name) + } + // set the CN Pod to draining phase and let the draining process handle recycling + if err := ctx.Patch(&cn, func() error { + cn.Labels[v1alpha1.CNPodPhaseLabel] = v1alpha1.CNPodPhaseDraining + return nil + }); err != nil { + return false, errors.Wrapf(err, "error reclaim CN %s/%s", cn.Namespace, cn.Name) + } + } + return false, nil +} + +func (r *Actor) patchStore(ctx *recon.Context[*v1alpha1.CNClaim], pod *corev1.Pod, req logpb.CNStateLabel) (*metadata.CNService, error) { + cs, err := common.ResolveCNSet(ctx, pod) + if err != nil { + return nil, errors.Wrap(err, "error resolve CNSet") + } + ls, err := common.ResolveLogSet(ctx, cs) + if err != nil { + return nil, errors.Wrap(err, "error resolve LogSet") + } + hc, err := r.clientMgr.GetClient(ls) + if err != nil { + return nil, errors.Wrap(err, "error get HAKeeper client") + } + timeout, cancel := context.WithTimeout(ctx, hacli.HAKeeperTimeout) + defer cancel() + uid := v1alpha1.GetCNPodUUID(pod) + req.UUID = uid + err = hc.Client.PatchCNStore(timeout, req) + if err != nil { + return nil, errors.Wrap(err, "error patch CNStore") + } + cn, ok := hc.StoreCache.GetCN(uid) + if !ok { + return nil, errors.Errorf("store not found in cache: %s", uid) + } + // the cache may be stale, update it locally + cn.Labels = req.Labels + cn.WorkState = req.State + return &cn, nil +} + +func (r *Actor) Start(mgr manager.Manager) error { + return recon.Setup[*v1alpha1.CNClaim](&v1alpha1.CNClaim{}, "cn-claim-manager", mgr, r) +} + +func toStoreStatus(cn *metadata.CNService) v1alpha1.CNStoreStatus { + var ls []v1alpha1.CNLabel + for k, v := range cn.Labels { + ls = append(ls, v1alpha1.CNLabel{ + Key: k, + Values: v.Labels, + }) + } + slices.SortFunc(ls, func(a, b v1alpha1.CNLabel) int { + return cmp.Compare(a.Key, b.Key) + }) + return v1alpha1.CNStoreStatus{ + ServiceID: cn.ServiceID, + LockServiceAddress: cn.LockServiceAddress, + PipelineServiceAddress: cn.PipelineServiceAddress, + SQLAddress: cn.SQLAddress, + QueryAddress: cn.QueryAddress, + WorkState: int32(cn.WorkState), + Labels: ls, + } +} + +// TODO: label similarity +func priorityFunc(c *v1alpha1.CNClaim) func(a, b corev1.Pod) int { + return func(a, b corev1.Pod) int { + return getScore(c, a) - getScore(c, b) + } +} + +func getScore(c *v1alpha1.CNClaim, p corev1.Pod) int { + if c.Labels[v1alpha1.PodClaimedByLabel] == p.Name { + return -1 + } + return 0 +} diff --git a/pkg/controllers/cnclaimset/controller.go b/pkg/controllers/cnclaimset/controller.go new file mode 100644 index 00000000..c943c4b3 --- /dev/null +++ b/pkg/controllers/cnclaimset/controller.go @@ -0,0 +1,305 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cnclaimset + +import ( + recon "github.com/matrixorigin/controller-runtime/pkg/reconciler" + "github.com/matrixorigin/controller-runtime/pkg/util" + "github.com/matrixorigin/matrixone-operator/api/core/v1alpha1" + "github.com/matrixorigin/matrixone-operator/pkg/controllers/common" + "github.com/pkg/errors" + "golang.org/x/exp/slices" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + "strconv" +) + +const ( + ClaimInstanceIDLabel = "claim.matrixorigin.io/instance-id" + + LengthOfInstanceID = 5 +) + +var podPhaseToOrdinal = map[corev1.PodPhase]int{ + corev1.PodPending: 0, + corev1.PodFailed: 1, + corev1.PodRunning: 2, +} + +// Actor reconciles CNClaimSet +type Actor struct { +} + +func NewActor() *Actor { + return &Actor{} +} + +func (r *Actor) Observe(ctx *recon.Context[*v1alpha1.CNClaimSet]) (recon.Action[*v1alpha1.CNClaimSet], error) { + return nil, r.Sync(ctx) +} + +func (r *Actor) Sync(ctx *recon.Context[*v1alpha1.CNClaimSet]) error { + s := ctx.Obj + filteredClaims, filterOutClaims, err := listClaims(ctx, s) + if err != nil { + return errors.Wrap(err, "error filter claims") + } + if s.Spec.Replicas > int32(len(filteredClaims)) { + if err := r.scaleOut(ctx, filteredClaims, filterOutClaims, int(s.Spec.Replicas)-len(filterOutClaims)); err != nil { + return errors.Wrap(err, "error scale out cn claim set") + } + } else if s.Spec.Replicas < int32(len(filteredClaims)) { + if err := r.scaleIn(ctx, filteredClaims, len(filterOutClaims)-int(s.Spec.Replicas)); err != nil { + return errors.Wrap(err, "error scale out cn claim set") + } + } + // clean lost claims + if err := cleanClaims(ctx, filterOutClaims); err != nil { + return errors.Wrap(err, "clean filter out claims") + } + // collect status + s.Status.Replicas = int32(len(filteredClaims)) + var claimStatuses []v1alpha1.CNClaimStatus + for _, c := range filteredClaims { + claimStatuses = append(claimStatuses, c.Status) + } + return nil +} + +func (r *Actor) scaleOut(ctx *recon.Context[*v1alpha1.CNClaimSet], filtered []v1alpha1.CNClaim, filteredOut []v1alpha1.CNClaim, count int) error { + used := sets.New[string]() + for _, c := range filtered { + used.Insert(c.Labels[ClaimInstanceIDLabel]) + } + for _, c := range filteredOut { + used.Insert(c.Labels[ClaimInstanceIDLabel]) + } + ids := genAvailableIds(count, used) + for _, id := range ids { + err := ctx.Create(makeClaim(ctx.Obj, id)) + if err != nil { + return errors.Wrap(err, "error create new claim") + } + } + return nil +} + +func makeClaim(cs *v1alpha1.CNClaimSet, id string) *v1alpha1.CNClaim { + return &v1alpha1.CNClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: cs.Namespace, + Name: cs.Name + "-" + id, + Labels: map[string]string{ + ClaimInstanceIDLabel: id, + }, + }, + Spec: *cs.Spec.Template.Spec.DeepCopy(), + } +} + +type claimAndPod struct { + claim *v1alpha1.CNClaim + pod *corev1.Pod +} + +func (c *claimAndPod) scoreHasPod() int { + if c.pod != nil { + return 1 + } + return 0 +} + +func (r *Actor) scaleIn(ctx *recon.Context[*v1alpha1.CNClaimSet], filtered []v1alpha1.CNClaim, count int) error { + var cps []claimAndPod + for i := range filtered { + c := filtered[i] + pod, err := getClaimedPod(ctx, &c) + if err != nil { + return errors.Wrap(err, "error get claimed pod") + } + cps = append(cps, claimAndPod{ + claim: &c, + pod: pod, + }) + } + if count >= len(cps) { + // simply delete all claims + count = len(cps) + } else { + slices.SortFunc(cps, claimDeletionOrder) + } + for i := 0; i < count; i++ { + c := cps[i].claim + if err := ctx.Delete(c); err != nil { + if !apierrors.IsNotFound(err) { + return errors.Wrapf(err, "error scale-in claim %s", c.Name) + } + } + } + return nil +} + +func (r *Actor) Finalize(ctx *recon.Context[*v1alpha1.CNClaimSet]) (bool, error) { + c1, c2, err := listClaims(ctx, ctx.Obj) + if err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, errors.Wrap(err, "error list claims") + } + if len(c1) == 0 && len(c2) == 0 { + return true, nil + } + for _, c := range append(c1, c2...) { + if err := ctx.Delete(&c); err != nil && !apierrors.IsNotFound(err) { + return false, errors.Wrapf(err, "error delete claim %s", c.Name) + } + } + return false, nil +} + +func getClaimedPod(cli recon.KubeClient, c *v1alpha1.CNClaim) (*corev1.Pod, error) { + if c.Spec.PodName == "" || c.Status.Phase == v1alpha1.CNClaimPhaseLost { + return nil, nil + } + pod := &corev1.Pod{} + err := cli.Get(types.NamespacedName{Namespace: c.Namespace, Name: c.Spec.PodName}, pod) + if err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + return nil, errors.Wrap(err, "error get claimed pod") + } + return pod, nil +} + +func claimDeletionOrder(a, b claimAndPod) int { + // 1. delete pending/lost claim first + hasPod := a.scoreHasPod() - b.scoreHasPod() + if hasPod != 0 { + return hasPod + } + // 2. there is a tie, two cases: + if a.scoreHasPod() == 1 { + // has pod, compare pod deletion order + return podDeletionOrder(a.pod, b.pod) + } + // both no pod, compare creation time, delete newer one first + return -(a.claim.CreationTimestamp.Second() - b.claim.CreationTimestamp.Second()) +} + +// TODO(aylei): pod deletion order should be fine-tuned +func podDeletionOrder(a, b *corev1.Pod) int { + // 1. UnScheduled < Scheduled + if a.Spec.NodeName != b.Spec.NodeName && (a.Spec.NodeName == "" || b.Spec.NodeName == "") { + return len(a.Spec.NodeName) - len(b.Spec.NodeName) + } + // 2. Pending < Failed < Running + if podPhaseToOrdinal[a.Status.Phase] != podPhaseToOrdinal[b.Status.Phase] { + return podPhaseToOrdinal[a.Status.Phase] - podPhaseToOrdinal[b.Status.Phase] + } + // 3. NotReady < Ready + if util.IsPodReady(a) != util.IsPodReady(b) { + if util.IsPodReady(a) { + return 1 + } + return -1 + } + // 4. deletion-cost + aCost := podDeletionCost(a) + bCost := podDeletionCost(b) + if aCost != bCost { + if aCost-bCost < 0 { + return -1 + } + return 1 + } + return -(a.CreationTimestamp.Second() - b.CreationTimestamp.Second()) +} + +func genAvailableIds(num int, used sets.Set[string]) []string { + var ret []string + for i := 0; i < num; i++ { + id := genAvailableID(used) + ret = append(ret, id) + used.Insert(id) + } + return ret +} + +func genAvailableID(used sets.Set[string]) string { + var id string + for { + id = rand.String(LengthOfInstanceID) + if !used.Has(id) { + break + } + } + return id +} + +func podDeletionCost(pod *corev1.Pod) int64 { + if value, exist := pod.Annotations[common.DeletionCostAnno]; exist { + i, err := strconv.ParseInt(value, 10, 32) + if err != nil { + return 0 + } + return i + } + return 0 +} + +func cleanClaims(ctx recon.KubeClient, cs []v1alpha1.CNClaim) error { + for i := range cs { + if err := ctx.Delete(&cs[i]); err != nil { + if !apierrors.IsNotFound(err) { + return errors.Wrap(err, "error delete lost claim") + } + } + } + return nil +} + +func listClaims(cli recon.KubeClient, s *v1alpha1.CNClaimSet) (filtered []v1alpha1.CNClaim, filterOut []v1alpha1.CNClaim, err error) { + cList := &v1alpha1.CNClaimList{} + err = cli.List(cList, client.InNamespace(s.Namespace), client.MatchingLabelsSelector{ + Selector: common.MustAsSelector(s.Spec.Selector), + }) + if err != nil { + return + } + for i := range cList.Items { + c := cList.Items[i] + if c.Status.Phase == v1alpha1.CNClaimPhaseLost || c.Labels[ClaimInstanceIDLabel] == "" { + filterOut = append(filterOut, c) + } else { + filtered = append(filtered, c) + } + } + return +} + +func (r *Actor) Start(mgr manager.Manager) error { + return recon.Setup[*v1alpha1.CNClaimSet](&v1alpha1.CNClaimSet{}, "cn-claimset-manager", mgr, r, recon.WithBuildFn(func(b *builder.Builder) { + b.Owns(&v1alpha1.CNClaim{}) + })) +} diff --git a/pkg/controllers/cnpool/controller.go b/pkg/controllers/cnpool/controller.go new file mode 100644 index 00000000..fb0129d1 --- /dev/null +++ b/pkg/controllers/cnpool/controller.go @@ -0,0 +1,368 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cnpool + +import ( + "context" + "fmt" + "github.com/go-logr/logr" + recon "github.com/matrixorigin/controller-runtime/pkg/reconciler" + "github.com/matrixorigin/matrixone-operator/api/core/v1alpha1" + "github.com/matrixorigin/matrixone-operator/pkg/controllers/common" + "github.com/pkg/errors" + "golang.org/x/exp/slices" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "time" +) + +// Actor reconciles CN Pool +type Actor struct { + PodWatched int + PoolReconciled int + Logger logr.Logger +} + +func (r *Actor) Observe(ctx *recon.Context[*v1alpha1.CNPool]) (recon.Action[*v1alpha1.CNPool], error) { + r.PoolReconciled++ + r.Logger.Info("watched pool", "name", ctx.Obj.Name, "watched", r.PoolReconciled) + return nil, r.Sync(ctx) +} + +func (r *Actor) Sync(ctx *recon.Context[*v1alpha1.CNPool]) error { + p := ctx.Obj + desired := buildCNSet(p) + + ls := ownedLabels(p) + csList := &v1alpha1.CNSetList{} + if err := ctx.List(csList, client.InNamespace(p.Namespace), client.MatchingLabels(ls)); err != nil { + return errors.Wrap(err, "error list current sets") + } + + var totalPods int32 + maxPods := p.Spec.Strategy.ScaleStrategy.GetMaxPods() + var current *v1alpha1.CNSet + for i := range csList.Items { + cs := csList.Items[i] + if cs.Name == desired.Name { + current = &cs + continue + } + legacyReplicas, err := r.syncLegacySet(ctx, &cs) + if err != nil { + return errors.Wrapf(err, "error sync legacy CNSet %s", cs.Name) + } + totalPods += legacyReplicas + } + + var inUse int32 + var idlePods []*corev1.Pod + var terminatingPods []*corev1.Pod + if current != nil { + pods, err := listCNSetPods(ctx, current) + if err != nil { + return errors.Wrapf(err, "error list CN pods for %s", current.Name) + } + for i := range pods { + if podInUse(&pods[i]) { + inUse++ + } else if pods[i].Labels[v1alpha1.CNPodPhaseLabel] == v1alpha1.CNPodPhaseTerminating { + ctx.Log.Info("find pod still in terminating state", "pod", pods[i].Name) + terminatingPods = append(terminatingPods, &pods[i]) + } else { + idlePods = append(idlePods, &pods[i]) + } + } + } + + claims, err := listNominatedClaims(ctx, p) + if err != nil { + return errors.Wrapf(err, "error list nominated claims for pool %s", p.Name) + } + var pendingClaims int32 + for _, claim := range claims { + if claim.Status.Phase == v1alpha1.CNClaimPhasePending { + pendingClaims++ + } + } + + desiredReplicas := inUse + pendingClaims + p.Spec.Strategy.ScaleStrategy.MaxIdle + totalPods += desiredReplicas + if totalPods > maxPods { + return recon.ErrReSync(fmt.Sprintf("Pool %s has reached MaxPods limit %d, total Pods: %d, requeue", p.Name, totalPods, maxPods), time.Minute) + } + // ensure and scale desired CNSet to provide enough CN pods + err = recon.CreateOwnedOrUpdate(ctx, desired, func() error { + ctx.Log.Info("scale cnset", "cnset", desired.Name, "replicas", desiredReplicas) + // sync terminating pods to delete + desired.Spec.PodsToDelete = podNames(terminatingPods) + if desired.Spec.Replicas > desiredReplicas { + // CNSet is going to be scaled-in + if pendingClaims > 0 { + // don't scale-in if we still have pending claims + ctx.Log.Info("pool has enough pods but there's still pending claims, pause scale-in", + "current replicas", desired.Spec.Replicas, + "desired replicas", desiredReplicas, + "pending claims", pendingClaims, + "in use pods", inUse) + return nil + } + if len(terminatingPods) > 0 { + // wait former termination complete first + ctx.Log.Info("pool has pods terminating, wait", + "current replicas", desired.Spec.Replicas, + "desired replicas", desiredReplicas, + "terminating pods", terminatingPods) + return nil + } + scaleInCount := desired.Spec.Replicas - desiredReplicas + slices.SortFunc(idlePods, deletionOrder) + if int32(len(idlePods)) > scaleInCount { + // pick first N to scale-in + idlePods = idlePods[0:scaleInCount] + } + ctx.Log.Info("try scale-in CN Pool", "pods", len(idlePods)) + // set pod to terminating phase, optimistic lock prevents race + var deleted []*corev1.Pod + for i := range idlePods { + pod := idlePods[i] + idlePods[i].Labels[v1alpha1.CNPodPhaseLabel] = v1alpha1.CNPodPhaseTerminating + if err := ctx.Update(idlePods[i]); err != nil { + ctx.Log.Error(err, "error set pod terminating", "pod", idlePods[i].Name) + continue + } + deleted = append(deleted, pod) + } + ctx.Log.Info("scale-in CN Pool complete", "deleted", len(deleted)) + desired.Spec.Replicas = desired.Spec.Replicas - int32(len(deleted)) + desired.Spec.PodsToDelete = podNames(deleted) + } else { + // scale-out, if we have terminating pods left, replace them + desired.Spec.Replicas = desiredReplicas + } + // GC the pods that have already been deleted from podsToDelete + return nil + }) + if err != nil { + return errors.Wrapf(err, "error sync desired CNSet %s/%s", desired.Namespace, desired.Name) + } + return nil +} + +func (r *Actor) syncLegacySet(ctx *recon.Context[*v1alpha1.CNPool], cnSet *v1alpha1.CNSet) (int32, error) { + var replicas int32 + pods, err := listCNSetPods(ctx, cnSet) + if err != nil { + return replicas, errors.Wrapf(err, "error list CNSet Pods for %s", cnSet.Name) + } + var toDelete []string + for i := range pods { + pod := pods[i] + // TODO(aylei): reclaim timeout logic + if podInUse(&pod) { + // keep in-use CN + replicas++ + } else { + // recalim other CN + toDelete = append(toDelete, pods[i].Name) + } + } + // clean unused CNSet + if err := recon.CreateOwnedOrUpdate(ctx, cnSet, func() error { + cnSet.Spec.Replicas = replicas + cnSet.Spec.PodsToDelete = toDelete + return nil + }); err != nil { + return replicas, errors.Wrapf(err, "error patch CNSet replicas, CNSet %s", cnSet.Name) + } + // legacy CNSet is scaled to zero the scaling has been done, GC it + //if cnSet.Spec.Replicas == 0 && cnSet.Status.Replicas == 0 { + // if err := ctx.Delete(cnSet); err != nil { + // return replicas, errors.Wrapf(err, "error GC legacy CNSet %s", cnSet.Name) + // } + //} + return replicas, nil +} + +// listNominatedClaims list all claims that are nominated to this pool +func listNominatedClaims(cli recon.KubeClient, pool *v1alpha1.CNPool) ([]v1alpha1.CNClaim, error) { + claimList := &v1alpha1.CNClaimList{} + if err := cli.List(claimList, client.InNamespace(pool.Namespace), client.MatchingLabels(map[string]string{ + v1alpha1.PoolNameLabel: pool.Name, + })); err != nil { + return nil, errors.Wrapf(err, "error list claims for pool %s", pool.Name) + } + return claimList.Items, nil +} + +func listCNSetPods(cli recon.KubeClient, cnSet *v1alpha1.CNSet) ([]corev1.Pod, error) { + if cnSet.Status.LabelSelector == "" { + return nil, nil + } + ls, err := metav1.ParseToLabelSelector(cnSet.Status.LabelSelector) + if err != nil { + return nil, errors.Wrapf(err, "error parsing label selector of CNSet, name: %s, selector: %s", cnSet.Name, cnSet.Status.LabelSelector) + } + podList := &corev1.PodList{} + if err := cli.List(podList, client.InNamespace(cnSet.Namespace), client.MatchingLabels(ls.MatchLabels)); err != nil { + return nil, errors.Wrapf(err, "error list pods of CNSet %s", cnSet.Name) + } + return podList.Items, nil +} + +func buildCNSet(p *v1alpha1.CNPool) *v1alpha1.CNSet { + csSpec := p.Spec.Template.DeepCopy() + + // override fields that managed by Pool, we expect the webhook will reject these fields if + // they are set by user, so that this process would not silently change users' expectation. + csSpec.PodManagementPolicy = pointer.String(v1alpha1.PodManagementPolicyPooling) + // pause update, cn pool don't rolling-update a single set, instead, we roll-out new sets if spec changes + csSpec.PauseUpdate = true + csSpec.ScalingConfig.StoreDrainEnabled = pointer.Bool(true) + csSpec.Labels = nil + csSpec.PodSet.Replicas = 0 + csSpec.ServiceType = "" + csSpec.ServiceAnnotations = nil + csSpec.NodePort = nil + if csSpec.Overlay == nil { + csSpec.Overlay = &v1alpha1.Overlay{} + } + if csSpec.Overlay.PodLabels == nil { + csSpec.Overlay.PodLabels = map[string]string{} + } + for k, v := range p.Spec.PodLabels { + csSpec.Overlay.PodLabels[k] = v + } + csSpec.Overlay.PodLabels[v1alpha1.CNPodPhaseLabel] = v1alpha1.CNPodPhaseUnknown + csSpec.Overlay.PodLabels[v1alpha1.PoolNameLabel] = p.Name + + // generate the controller revision hash + hash := common.HashControllerRevision(csSpec) + name := fmt.Sprintf("%s-%s", p.Name, hash) + + labels := ownedLabels(p) + labels[appsv1.ControllerRevisionHashLabelKey] = hash + cs := &v1alpha1.CNSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: p.Namespace, + Labels: labels, + }, + Spec: *csSpec, + Deps: p.Spec.Deps, + } + return cs +} + +func (r *Actor) Finalize(ctx *recon.Context[*v1alpha1.CNPool]) (bool, error) { + + ls := ownedLabels(ctx.Obj) + if err := ctx.Client.DeleteAllOf(ctx, &v1alpha1.CNSet{}, client.InNamespace(ctx.Obj.Namespace), client.MatchingLabels(ls)); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, errors.Wrapf(err, "error delete CNSets owned by Pool %s", ctx.Obj.Name) + } + csList := &v1alpha1.CNSetList{} + if err := ctx.List(csList, client.InNamespace(ctx.Obj.Namespace), client.MatchingLabels(ls)); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, errors.Wrapf(err, "error check CNSets owned by Pool %s", ctx.Obj.Name) + } + return len(csList.Items) < 1, nil +} + +func ownedLabels(p *v1alpha1.CNPool) map[string]string { + return map[string]string{ + common.LabelManagedBy: fmt.Sprintf("%s-%s", p.Kind, p.Name), + common.LabelOwnerUID: string(p.UID), + } +} + +func podInUse(pod *corev1.Pod) bool { + return pod.Labels[v1alpha1.CNPodPhaseLabel] == v1alpha1.CNPodPhaseBound || + pod.Labels[v1alpha1.CNPodPhaseLabel] == v1alpha1.CNPodPhaseDraining +} + +func deletionOrder(a, b *corev1.Pod) int { + c := deletionCost(a) - deletionCost(b) + if c == 0 { + // if two pods have same deletion cost, delete the newly created one first + return -(a.CreationTimestamp.Second() - b.CreationTimestamp.Second()) + } + return c +} + +func podNames(pods []*corev1.Pod) []string { + var ss []string + for _, pod := range pods { + ss = append(ss, pod.Name) + } + return ss +} + +func deletionCost(pod *corev1.Pod) int { + score := 0 + switch pod.Labels[v1alpha1.CNPodPhaseLabel] { + case v1alpha1.CNPodPhaseTerminating: + score += 0 + case "", v1alpha1.CNPodPhaseUnknown: + score += 1 << 8 + case v1alpha1.CNPodPhaseIdle: + score += 1 << 9 + default: + score += 1 << 10 + } + // scale un-used pod first in same phase + if pod.Labels[v1alpha1.PodClaimedByLabel] != "" { + score++ + } + return score +} + +func (r *Actor) Start(mgr manager.Manager) error { + return recon.Setup[*v1alpha1.CNPool](&v1alpha1.CNPool{}, "cn-pool-manager", mgr, r, recon.WithBuildFn(func(b *builder.Builder) { + // TODO: watch Pods in Pool + b.Watches(&corev1.Pod{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, object client.Object) []reconcile.Request { + pod, ok := object.(*corev1.Pod) + if !ok { + return nil + } + r.PodWatched++ + r.Logger.Info("watched pod", "name", pod.Name, "watched", r.PodWatched) + poolName, ok := pod.Labels[v1alpha1.PoolNameLabel] + if !ok { + return nil + } + return []reconcile.Request{{ + NamespacedName: types.NamespacedName{ + Namespace: pod.Namespace, + Name: poolName, + }, + }} + })) + b.Owns(&v1alpha1.CNSet{}) + })) +} diff --git a/pkg/controllers/cnset/controller.go b/pkg/controllers/cnset/controller.go index 957894ea..dc4b102d 100644 --- a/pkg/controllers/cnset/controller.go +++ b/pkg/controllers/cnset/controller.go @@ -95,7 +95,11 @@ func (c *Actor) Observe(ctx *recon.Context[*v1alpha1.CNSet]) (recon.Action[*v1al return nil, errors.Wrap(err, "dry run update cnset") } if !equality.Semantic.DeepEqual(origin, cs) { - return c.with(cs).Update, nil + if cn.Spec.PauseUpdate { + ctx.Log.Info("CNSet does not reach desired state, but update is paused") + } else { + return c.with(cs).Update, nil + } } // calculate status var stores []v1alpha1.CNStore @@ -105,6 +109,7 @@ func (c *Actor) Observe(ctx *recon.Context[*v1alpha1.CNSet]) (recon.Action[*v1al if err != nil { return nil, errors.Wrap(err, "list cn pods") } + livePods := map[string]bool{} for _, pod := range podList.Items { uid := v1alpha1.GetCNPodUUID(&pod) cnState := pod.Annotations[common.CNStateAnno] @@ -116,6 +121,7 @@ func (c *Actor) Observe(ctx *recon.Context[*v1alpha1.CNSet]) (recon.Action[*v1al PodName: pod.Name, State: cnState, }) + livePods[pod.Name] = true } cn.Status.Stores = stores cn.Status.Replicas = cs.Status.Replicas @@ -141,7 +147,8 @@ func (c *Actor) Observe(ctx *recon.Context[*v1alpha1.CNSet]) (recon.Action[*v1al } } } - if cn.Spec.Replicas != *cs.Spec.Replicas { + if cn.Spec.Replicas != *cs.Spec.Replicas || + !equality.Semantic.DeepEqual(cn.Spec.PodsToDelete, cs.Spec.ScaleStrategy.PodsToDelete) { return c.with(cs).Scale, nil } @@ -156,7 +163,7 @@ func (c *Actor) Observe(ctx *recon.Context[*v1alpha1.CNSet]) (recon.Action[*v1al func (c *WithResources) Scale(ctx *recon.Context[*v1alpha1.CNSet]) error { return ctx.Patch(c.cs, func() error { - syncReplicas(ctx.Obj, c.cs) + scaleSet(ctx.Obj, c.cs) return nil }) } @@ -219,7 +226,7 @@ func (c *Actor) Create(ctx *recon.Context[*v1alpha1.CNSet]) error { hSvc := buildHeadlessSvc(cn) cnSet := buildCNSet(cn, hSvc) svc := buildSvc(cn) - syncReplicas(cn, cnSet) + scaleSet(cn, cnSet) if err := syncCloneSet(ctx, cnSet); err != nil { return errors.Wrap(err, "sync clone set") } diff --git a/pkg/controllers/cnset/resource.go b/pkg/controllers/cnset/resource.go index 3eb381b2..e6fc12e6 100644 --- a/pkg/controllers/cnset/resource.go +++ b/pkg/controllers/cnset/resource.go @@ -110,8 +110,9 @@ func syncPersistentVolumeClaim(cn *v1alpha1.CNSet, cs *kruisev1alpha1.CloneSet) } } -func syncReplicas(cn *v1alpha1.CNSet, cs *kruisev1alpha1.CloneSet) { +func scaleSet(cn *v1alpha1.CNSet, cs *kruisev1alpha1.CloneSet) { cs.Spec.Replicas = &cn.Spec.Replicas + cs.Spec.ScaleStrategy.PodsToDelete = cn.Spec.PodsToDelete } func syncService(cn *v1alpha1.CNSet, svc *corev1.Service) { @@ -146,8 +147,8 @@ func syncPodMeta(cn *v1alpha1.CNSet, cs *kruisev1alpha1.CloneSet) error { return err } meta.Annotations[common.CNLabelAnnotation] = string(s) - if cn.Spec.ExternalStoreControl { - meta.Annotations[v1alpha1.StoreExternalControlledAnno] = string(metav1.ConditionTrue) + if cn.Spec.PodManagementPolicy != nil { + meta.Annotations[v1alpha1.PodManagementPolicyAnno] = *cn.Spec.PodManagementPolicy } cn.Spec.Overlay.OverlayPodMeta(&cs.Spec.Template.ObjectMeta) return nil diff --git a/pkg/controllers/cnstore/controller.go b/pkg/controllers/cnstore/controller.go index 2541507c..259e013e 100644 --- a/pkg/controllers/cnstore/controller.go +++ b/pkg/controllers/cnstore/controller.go @@ -27,10 +27,8 @@ import ( "github.com/matrixorigin/matrixone/pkg/pb/metadata" "github.com/openkruise/kruise-api/apps/pub" "github.com/pkg/errors" - "go.uber.org/zap" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "reflect" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -44,8 +42,6 @@ import ( ) const ( - deletionCostAnno = "controller.kubernetes.io/pod-deletion-cost" - messageCNCordon = "CNStoreCordon" messageCNPrepareStop = "CNStorePrepareStop" messageCNStoreReady = "CNStoreReady" @@ -151,13 +147,9 @@ func (c *withCNSet) OnPreparingStop(ctx *recon.Context[*corev1.Pod]) error { return errors.Wrap(err, "error set CN state draining") } - connectionStr, ok := pod.Annotations[v1alpha1.StoreConnectionAnno] - if !ok { - return errors.Errorf("cannot find connection count for CN pod %s/%s, connection annotation is empty", pod.Namespace, pod.Name) - } - count, err := strconv.Atoi(connectionStr) + count, err := common.GetStoreConnection(pod) if err != nil { - return errors.Wrap(err, "error parsing connection count") + return errors.Wrap(err, "error get store connection count") } if count == 0 { return c.completeDraining(ctx) @@ -179,17 +171,15 @@ func (c *withCNSet) completeDraining(ctx *recon.Context[*corev1.Pod]) error { // OnNormal ensure CNStore labels and transit CN store to UP state func (c *withCNSet) OnNormal(ctx *recon.Context[*corev1.Pod]) error { pod := ctx.Obj - uid := v1alpha1.GetCNPodUUID(pod) - // 1. ensure finalizers - if err := ctx.Patch(ctx.Obj, func() error { + // ensure finalizers + if err := ctx.Patch(pod, func() error { controllerutil.AddFinalizer(ctx.Obj, common.CNDrainingFinalizer) return nil }); err != nil { return errors.Wrap(err, "ensure finalizers for CNStore Pod") } - - // 2 remove draining start time in case we regret formal deletion decision + // remove draining start time in case we regret formal deletion decision if pod.Annotations == nil { pod.Annotations = map[string]string{} } @@ -200,12 +190,18 @@ func (c *withCNSet) OnNormal(ctx *recon.Context[*corev1.Pod]) error { return errors.Wrap(err, "removing CN draining start time") } - // 3. if CN label is controlled externally, skip the following sync - if _, ok := pod.Annotations[v1alpha1.StoreExternalControlledAnno]; ok { - return c.markCNReady(ctx) + // policy based reconciliation + if v1alpha1.IsPoolingPolicy(ctx.Obj) { + return c.poolingCNReconcile(ctx) } + return c.defaultCNNormalReconcile(ctx) +} + +func (c *withCNSet) defaultCNNormalReconcile(ctx *recon.Context[*corev1.Pod]) error { + pod := ctx.Obj + uid := v1alpha1.GetCNPodUUID(pod) - // 4. sync CN labels for store and mark store as UP state + // sync CN labels for store and mark store as UP state var cnLabels []v1alpha1.CNLabel labelStr, ok := pod.Annotations[common.CNLabelAnnotation] if ok { @@ -221,14 +217,14 @@ func (c *withCNSet) OnNormal(ctx *recon.Context[*corev1.Pod]) error { return h.Client.PatchCNStore(timeout, logpb.CNStateLabel{ UUID: uid, State: metadata.WorkState_Working, - Labels: toStoreLabels(cnLabels), + Labels: common.ToStoreLabels(cnLabels), }) }) } else { err = c.withHAKeeperClient(ctx, func(timeout context.Context, h *hacli.Handler) error { return h.Client.UpdateCNLabel(timeout, logpb.CNStoreLabel{ UUID: uid, - Labels: toStoreLabels(cnLabels), + Labels: common.ToStoreLabels(cnLabels), }) }) } @@ -302,32 +298,22 @@ func (c *Controller) observe(ctx *recon.Context[*corev1.Pod]) error { return c.OnDeleted(ctx) } - // 2. pre-check - if component, ok := pod.Labels[common.ComponentLabelKey]; !ok || component != "CNSet" { - ctx.Log.V(4).Info("pod is not a CN pod, skip", zap.String("namespace", pod.Namespace), zap.String("name", pod.Name)) - return nil - } - cnName, ok := pod.Labels[common.InstanceLabelKey] - if !ok || cnName == "" { - return errors.Errorf("cannot find CNSet for CN pod %s/%s, instance label is empty", pod.Namespace, pod.Name) - } - - // 3. build dep - cn := &v1alpha1.CNSet{} - if err := ctx.Get(types.NamespacedName{Namespace: pod.Namespace, Name: cnName}, cn); err != nil { - return errors.Wrap(err, "get CNSet") + // 2. resolve CNSet + cnSet, err := common.ResolveCNSet(ctx, pod) + if err != nil { + return errors.Wrap(err, "error resolve CNSet") } wc := &withCNSet{ Controller: c, - cn: cn, + cn: cnSet, } - // 4. sync stats, including connections and deletion cost + // 3. sync stats, including connections and deletion cost if err := wc.syncStats(ctx); err != nil { return errors.Wrap(err, fmt.Sprintf("error sync stats for CN pod %s/%s", pod.Namespace, pod.Name)) } - // 5. optionally, store is asked to be cordoned + // 4. optionally, store is asked to be cordoned if _, ok := pod.Annotations[v1alpha1.StoreCordonAnno]; ok { return wc.OnCordon(ctx) } @@ -365,12 +351,13 @@ func (c *withCNSet) syncStats(ctx *recon.Context[*corev1.Pod]) error { // cannot get session, sync next time return nil } - // sync deletion cost - return ctx.Patch(pod, func() error { + // cost is equal to connection count by default + cost := count + err = ctx.Patch(pod, func() error { if pod.Annotations == nil { pod.Annotations = map[string]string{} } - pod.Annotations[deletionCostAnno] = strconv.Itoa(count) + pod.Annotations[common.DeletionCostAnno] = strconv.Itoa(cost) pod.Annotations[v1alpha1.StoreConnectionAnno] = strconv.Itoa(count) if pod.Labels == nil { pod.Labels = map[string]string{} @@ -378,6 +365,10 @@ func (c *withCNSet) syncStats(ctx *recon.Context[*corev1.Pod]) error { pod.Labels[common.CNUUIDLabelKey] = uid return nil }) + if err != nil { + return errors.Wrap(err, "error patch stats to pod anno") + } + return nil } func (c *Controller) getSessionCount(pod *corev1.Pod, queryAddress string) (int, error) { @@ -399,15 +390,9 @@ func (c *Controller) getSessionCount(pod *corev1.Pod, queryAddress string) (int, func (c *withCNSet) withHAKeeperClient(ctx *recon.Context[*corev1.Pod], fn func(context.Context, *hacli.Handler) error) error { pod := ctx.Obj - cn := c.cn - // TODO: consider edge cluster federation scenario - if cn.Deps.LogSet == nil { - return errors.Errorf("cannot get logset of CN pod %s/%s, logset dep is nil", pod.Namespace, pod.Name) - } - ls := &v1alpha1.LogSet{} - // refresh logset status - if err := ctx.Get(client.ObjectKeyFromObject(cn.Deps.LogSet), ls); err != nil { - return errors.Wrap(err, "error get logset") + ls, err := common.ResolveLogSet(ctx, c.cn) + if err != nil { + return errors.Wrap(err, "error resolve logset") } if !recon.IsReady(ls) { return recon.ErrReSync(fmt.Sprintf("logset is not ready for Pod %s, cannot update CN labels", pod.Name), retryInterval) @@ -431,16 +416,6 @@ func (c *Controller) setCNState(pod *corev1.Pod, state string) { pod.Annotations[common.CNStateAnno] = state } -func toStoreLabels(labels []v1alpha1.CNLabel) map[string]metadata.LabelList { - lm := make(map[string]metadata.LabelList, len(labels)) - for _, l := range labels { - lm[l.Key] = metadata.LabelList{ - Labels: l.Values, - } - } - return lm -} - func (c *Controller) Observe(ctx *recon.Context[*corev1.Pod]) (recon.Action[*corev1.Pod], error) { return nil, c.observe(ctx) } @@ -488,7 +463,7 @@ func (annotationChangedExcludeStats) Update(e event.UpdateEvent) bool { } oldAnnos := e.ObjectOld.GetAnnotations() newAnnos := e.ObjectNew.GetAnnotations() - for _, key := range []string{deletionCostAnno, v1alpha1.StoreCordonAnno} { + for _, key := range []string{common.DeletionCostAnno, v1alpha1.StoreCordonAnno} { delete(oldAnnos, key) delete(newAnnos, key) } diff --git a/pkg/controllers/cnstore/pooling.go b/pkg/controllers/cnstore/pooling.go new file mode 100644 index 00000000..f12dc8c1 --- /dev/null +++ b/pkg/controllers/cnstore/pooling.go @@ -0,0 +1,74 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cnstore + +import ( + "context" + recon "github.com/matrixorigin/controller-runtime/pkg/reconciler" + "github.com/matrixorigin/matrixone-operator/api/core/v1alpha1" + "github.com/matrixorigin/matrixone-operator/pkg/controllers/common" + "github.com/matrixorigin/matrixone-operator/pkg/hacli" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" +) + +func (c *withCNSet) poolingCNReconcile(ctx *recon.Context[*corev1.Pod]) error { + pod := ctx.Obj + uid := v1alpha1.GetCNPodUUID(pod) + + var ready bool + if err := c.withHAKeeperClient(ctx, func(ctx context.Context, h *hacli.Handler) error { + _, ready = h.StoreCache.GetCN(uid) + return nil + }); err != nil { + return errors.Wrap(err, "error call hakeeper") + } + + switch pod.Labels[v1alpha1.CNPodPhaseLabel] { + case v1alpha1.CNPodPhaseDraining: + // recycle the pod + // TODO(aylei): timeout + count, err := common.GetStoreConnection(pod) + if err != nil { + return errors.Wrap(err, "error get store connection count") + } + if count == 0 { + err := ctx.Patch(pod, func() error { + pod.Labels[v1alpha1.CNPodPhaseLabel] = v1alpha1.CNPodPhaseIdle + return nil + }) + if err != nil { + return errors.Wrap(err, "error patch CN phase idle") + } + return nil + } + return recon.ErrReSync("store is still draining", retryInterval) + case v1alpha1.CNPodPhaseBound, v1alpha1.CNPodPhaseIdle: + // noop + case v1alpha1.CNPodPhaseUnknown: + if ready { + err := ctx.Patch(pod, func() error { + pod.Labels[v1alpha1.CNPodPhaseLabel] = v1alpha1.CNPodPhaseIdle + return nil + }) + if err != nil { + return errors.Wrap(err, "error patch CN phase idle") + } + } + default: + return errors.Errorf("unkown CN phase %s", pod.Labels[v1alpha1.CNPodPhaseLabel]) + } + return nil +} diff --git a/pkg/controllers/common/cnstore.go b/pkg/controllers/common/cnstore.go index e1fa6e1e..004749e6 100644 --- a/pkg/controllers/common/cnstore.go +++ b/pkg/controllers/common/cnstore.go @@ -15,8 +15,15 @@ package common import ( + recon "github.com/matrixorigin/controller-runtime/pkg/reconciler" + "github.com/matrixorigin/matrixone-operator/api/core/v1alpha1" + "github.com/matrixorigin/matrixone/pkg/pb/metadata" + "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "strconv" ) const ( @@ -59,3 +66,88 @@ func NewCNReadinessCondition(status corev1.ConditionStatus, msg string) corev1.P LastTransitionTime: metav1.Now(), } } + +type objectWithDependency interface { + client.Object + recon.Dependant +} + +func ResolveLogSet(cli recon.KubeClient, cs *v1alpha1.CNSet) (*v1alpha1.LogSet, error) { + if cs.Deps.LogSet == nil { + return nil, errors.Errorf("cannot get logset of CNSet %s/%s, logset dep is nil", cs.Namespace, cs.Name) + } + ls := &v1alpha1.LogSet{} + // refresh logset status + if err := cli.Get(client.ObjectKeyFromObject(cs.Deps.LogSet), ls); err != nil { + return nil, errors.Wrap(err, "error get logset") + } + return ls, nil +} + +// ResolveCNSet resoles the CNSet of an CN Pod +func ResolveCNSet(cli recon.KubeClient, pod *corev1.Pod) (*v1alpha1.CNSet, error) { + owner, err := ResolveOwner(cli, pod) + if err != nil { + return nil, errors.Wrap(err, "error resolve CNSet") + } + cnSet, ok := owner.(*v1alpha1.CNSet) + if !ok { + return nil, errors.Wrap(err, "pod is not a CN Pod") + } + return cnSet, nil +} + +// ResolveOwner resolves the owner set of an MO Pod +func ResolveOwner(cli recon.KubeClient, pod *corev1.Pod) (client.Object, error) { + comp, ok := pod.Labels[ComponentLabelKey] + if !ok { + return nil, errors.New("cannot resolve logset of non-mo pod") + } + instanceName, ok := pod.Labels[InstanceLabelKey] + if !ok || instanceName == "" { + return nil, errors.Errorf("cannot find isstance name for pod %s/%s, instance label is empty", pod.Namespace, pod.Name) + } + + var o client.Object + switch comp { + case "CNSet": + o = &v1alpha1.CNSet{} + case "DNSet": + o = &v1alpha1.DNSet{} + case "LogSet": + o = &v1alpha1.LogSet{} + case "ProxySet": + o = &v1alpha1.ProxySet{} + default: + return nil, errors.Errorf("unknown component %s", comp) + } + + if err := cli.Get(types.NamespacedName{Namespace: pod.Namespace, Name: instanceName}, o); err != nil { + return nil, errors.Wrap(err, "error get owner set") + } + return o, nil +} + +// ToStoreLabels transform a list of CNLabel to CNStore Label +func ToStoreLabels(labels []v1alpha1.CNLabel) map[string]metadata.LabelList { + lm := make(map[string]metadata.LabelList, len(labels)) + for _, l := range labels { + lm[l.Key] = metadata.LabelList{ + Labels: l.Values, + } + } + return lm +} + +// GetStoreConnection get the store connection count from Pod anno +func GetStoreConnection(pod *corev1.Pod) (int, error) { + connectionStr, ok := pod.Annotations[v1alpha1.StoreConnectionAnno] + if !ok { + return 0, errors.Errorf("cannot find connection count for CN pod %s/%s, connection annotation is empty", pod.Namespace, pod.Name) + } + count, err := strconv.Atoi(connectionStr) + if err != nil { + return 0, errors.Wrap(err, "error parsing connection count") + } + return count, nil +} diff --git a/pkg/controllers/common/consts.go b/pkg/controllers/common/consts.go index e017c226..af3bd141 100644 --- a/pkg/controllers/common/consts.go +++ b/pkg/controllers/common/consts.go @@ -20,10 +20,15 @@ const ( MetricsPort = 7001 + DeletionCostAnno = "controller.kubernetes.io/pod-deletion-cost" + CNUUIDLabelKey = "matrixone.cloud/cn-uuid" CNLabelAnnotation = "matrixone.cloud/cn-label" PrometheusScrapeAnno = "prometheus.io/scrape" PrometheusPortAnno = "prometheus.io/port" PrometheusPathAnno = "prometheus.io/path" + + LabelManagedBy = "matrixorigin.io/managed-by" + LabelOwnerUID = "matrixorigin.io/owner-uid" ) diff --git a/pkg/controllers/common/history.go b/pkg/controllers/common/history.go new file mode 100644 index 00000000..fe4cb2c8 --- /dev/null +++ b/pkg/controllers/common/history.go @@ -0,0 +1,50 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "fmt" + "github.com/davecgh/go-spew/spew" + "hash" + "hash/fnv" + "k8s.io/apimachinery/pkg/util/rand" +) + +var prettyPrintConfigForHash = &spew.ConfigState{ + Indent: " ", + SortKeys: true, + DisableMethods: true, + SpewKeys: true, + DisablePointerAddresses: true, + DisableCapacities: true, +} + +func HashControllerRevision(obj any) string { + hf := fnv.New32() + if obj != nil { + DeepHashObject(hf, obj) + } + return rand.SafeEncodeString(fmt.Sprint(hf.Sum32())) +} + +func DeepHashObject(hasher hash.Hash, objectToWrite interface{}) { + hasher.Reset() + fmt.Fprintf(hasher, "%v", ForHash(objectToWrite)) +} + +// ForHash keeps the original Spew.Sprintf format to ensure the same checksum +func ForHash(a interface{}) string { + return prettyPrintConfigForHash.Sprintf("%#v", a) +} diff --git a/pkg/controllers/common/selector.go b/pkg/controllers/common/selector.go new file mode 100644 index 00000000..ae890d27 --- /dev/null +++ b/pkg/controllers/common/selector.go @@ -0,0 +1,54 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + recon "github.com/matrixorigin/controller-runtime/pkg/reconciler" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/util/validation/field" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func ListPods(cli recon.KubeClient, opts ...client.ListOption) ([]corev1.Pod, error) { + podList := &corev1.PodList{} + if err := cli.List(podList, opts...); err != nil { + return nil, err + } + return podList.Items, nil +} + +func MustAsSelector(ps *metav1.LabelSelector) labels.Selector { + ls, err := metav1.LabelSelectorAsSelector(ps) + if err != nil { + panic(errors.Wrap(err, "impossible path: LabelSelectorAsSelector failed")) + } + return ls +} + +func MustNewRequirement(key string, op selection.Operator, vals []string, opts ...field.PathOption) labels.Requirement { + r, err := labels.NewRequirement(key, op, vals) + if err != nil { + panic(errors.Wrap(err, "impossible path: new requirement failed")) + } + return *r +} + +func MustEqual(key string, value string) labels.Requirement { + return MustNewRequirement(key, selection.Equals, []string{value}) +} diff --git a/pkg/controllers/webhook/controller.go b/pkg/controllers/webhook/controller.go index b375021f..d285e669 100644 --- a/pkg/controllers/webhook/controller.go +++ b/pkg/controllers/webhook/controller.go @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + package webhook import ( diff --git a/pkg/hook/cnclaim.go b/pkg/hook/cnclaim.go new file mode 100644 index 00000000..6afc0e37 --- /dev/null +++ b/pkg/hook/cnclaim.go @@ -0,0 +1,91 @@ +// Copyright 2024 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hook + +import ( + "context" + "github.com/go-logr/logr" + "github.com/matrixorigin/matrixone-operator/api/core/v1alpha1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +type CNClaimHook struct { + Client client.Client + Logger logr.Logger +} + +func NewCNClaimHook(cli client.Client, logger logr.Logger) *CNClaimHook { + return &CNClaimHook{ + Client: cli, + Logger: logger, + } +} + +func (h *CNClaimHook) Default(ctx context.Context, obj runtime.Object) error { + c, ok := obj.(*v1alpha1.CNClaim) + if !ok { + return nil + } + // claim has not been reconciled, shortcut to take an CN + if c.Spec.PodName == "" && c.Spec.PoolName == "" { + + } + return nil +} + +func (h *CNClaimHook) ValidateCreate(ctx context.Context, obj runtime.Object) (warnings admission.Warnings, err error) { + return nil, nil +} + +func (h *CNClaimHook) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (warnings admission.Warnings, err error) { + var errs field.ErrorList + oldClaim, ok := oldObj.(*v1alpha1.CNClaim) + if !ok { + return nil, nil + } + newClaim, ok := newObj.(*v1alpha1.CNClaim) + if !ok { + return nil, nil + } + if oldClaim.Spec.PodName != "" && oldClaim.Spec.PodName != newClaim.Spec.PodName { + errs = append(errs, field.Invalid(field.NewPath("spec", "podName"), newClaim.Spec.PodName, "podName is immutable")) + } + return nil, invalidOrNil(errs, newClaim) +} + +func (h *CNClaimHook) ValidateDelete(ctx context.Context, obj runtime.Object) (warnings admission.Warnings, err error) { + return nil, nil +} + +func (h *CNClaimHook) Setup(mgr manager.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(&v1alpha1.CNClaim{}). + WithDefaulter(h). + WithValidator(h). + Complete() +} + +func invalidOrNil(allErrs field.ErrorList, r client.Object) error { + if len(allErrs) == 0 { + return nil + } + return apierrors.NewInvalid(r.GetObjectKind().GroupVersionKind().GroupKind(), r.GetName(), allErrs) +}