From 0a2d28729c69159f4eef3b044c49b9e29c3ada16 Mon Sep 17 00:00:00 2001 From: cyberchen98 Date: Sat, 12 Oct 2024 14:14:55 +0800 Subject: [PATCH 01/11] feat: support InPlaceUpdate when only configuration changes are made --- pkg/controllers/cnset/controller.go | 3 +- pkg/controllers/cnset/resource.go | 16 ++++--- pkg/controllers/cnset/resource_test.go | 7 +-- pkg/controllers/common/configmap.go | 61 +++++++++++++++--------- pkg/controllers/common/configmap_test.go | 1 - pkg/controllers/common/consts.go | 2 + pkg/controllers/common/podset.go | 6 ++- pkg/controllers/common/template.go | 2 + pkg/controllers/dnset/controller.go | 4 +- pkg/controllers/dnset/resource.go | 20 ++++---- pkg/controllers/dnset/resource_test.go | 7 +-- pkg/controllers/logset/configmap.go | 16 +++---- pkg/controllers/logset/controller.go | 6 ++- pkg/controllers/logset/sts.go | 1 + pkg/controllers/proxyset/resource.go | 18 +++---- 15 files changed, 102 insertions(+), 68 deletions(-) diff --git a/pkg/controllers/cnset/controller.go b/pkg/controllers/cnset/controller.go index 3f51dd0b..b7cde7dd 100644 --- a/pkg/controllers/cnset/controller.go +++ b/pkg/controllers/cnset/controller.go @@ -324,10 +324,11 @@ func syncCloneSet(ctx *recon.Context[*v1alpha1.CNSet], cs *kruisev1alpha1.CloneS // ref: https://openkruise.io/zh/docs/next/user-manuals/cloneset/#%E6%94%AF%E6%8C%81-pvc-%E6%A8%A1%E6%9D%BF syncPersistentVolumeClaim(cn, cs) - cm, err := buildCNSetConfigMap(ctx.Obj, ctx.Dep.Deps.LogSet) + cm, configSuffix, err := buildCNSetConfigMap(ctx.Obj, ctx.Dep.Deps.LogSet) if err != nil { return err } + cs.Spec.Template.Annotations[common.ConfigSuffixAnno] = configSuffix return common.SyncConfigMap(ctx, &cs.Spec.Template.Spec, cm) } diff --git a/pkg/controllers/cnset/resource.go b/pkg/controllers/cnset/resource.go index 3196400a..5ec7db1e 100644 --- a/pkg/controllers/cnset/resource.go +++ b/pkg/controllers/cnset/resource.go @@ -54,7 +54,7 @@ sql-address = "${POD_IP}:{{ .CNSQLPort }}" service-host = "${POD_IP}" EOF # build instance config -sed "/\[cn\]/r ${bc}" {{ .ConfigFilePath }} > ${conf} +sed "/\[cn\]/r ${bc}" {{ .ConfigFilePath }}-${CONFIG_SUFFIX} > ${conf} # append lock-service configs lsc=$(mktemp) @@ -196,6 +196,7 @@ func syncPodSpec(cn *v1alpha1.CNSet, cs *kruisev1alpha1.CloneSet, sp v1alpha1.Sh mainRef.Env = []corev1.EnvVar{ util.FieldRefEnv(common.PodNameEnvKey, "metadata.name"), util.FieldRefEnv(common.NamespaceEnvKey, "metadata.namespace"), + util.FieldRefEnv(common.ConfigSuffixEnvKey, fmt.Sprintf("metadata.annotations['%s']", common.ConfigSuffixAnno)), {Name: common.HeadlessSvcEnvKey, Value: headlessSvcName(cn)}, util.FieldRefEnv(common.PodIPEnvKey, "status.podIP"), } @@ -249,9 +250,9 @@ func syncPodSpec(cn *v1alpha1.CNSet, cs *kruisev1alpha1.CloneSet, sp v1alpha1.Sh } } -func buildCNSetConfigMap(cn *v1alpha1.CNSet, ls *v1alpha1.LogSet) (*corev1.ConfigMap, error) { +func buildCNSetConfigMap(cn *v1alpha1.CNSet, ls *v1alpha1.LogSet) (*corev1.ConfigMap, string, error) { if ls.Status.Discovery == nil { - return nil, errors.New("logset had not yet exposed HAKeeper discovery address") + return nil, "", errors.New("logset had not yet exposed HAKeeper discovery address") } cfg := cn.Spec.Config if cfg == nil { @@ -285,7 +286,7 @@ func buildCNSetConfigMap(cn *v1alpha1.CNSet, ls *v1alpha1.LogSet) (*corev1.Confi } s, err := cfg.ToString() if err != nil { - return nil, err + return nil, "", err } buff := new(bytes.Buffer) err = startScriptTpl.Execute(buff, &model{ @@ -295,9 +296,10 @@ func buildCNSetConfigMap(cn *v1alpha1.CNSet, ls *v1alpha1.LogSet) (*corev1.Confi LockServicePort: common.LockServicePort, }) if err != nil { - return nil, err + return nil, "", err } + configSuffix := common.DataDigest([]byte(s)) return &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: configMapName(cn), @@ -305,8 +307,8 @@ func buildCNSetConfigMap(cn *v1alpha1.CNSet, ls *v1alpha1.LogSet) (*corev1.Confi Labels: common.SubResourceLabels(cn), }, Data: map[string]string{ - common.ConfigFile: s, + fmt.Sprintf("%s-%s", common.ConfigFile, configSuffix): s, common.Entrypoint: buff.String(), }, - }, nil + }, configSuffix, nil } diff --git a/pkg/controllers/cnset/resource_test.go b/pkg/controllers/cnset/resource_test.go index f2fadeba..2199e16a 100644 --- a/pkg/controllers/cnset/resource_test.go +++ b/pkg/controllers/cnset/resource_test.go @@ -227,13 +227,14 @@ service-addresses = [] for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { g := NewGomegaWithT(t) - got, err := buildCNSetConfigMap(tt.args.cn, tt.args.ls) + got, configSuffix, err := buildCNSetConfigMap(tt.args.cn, tt.args.ls) if (err != nil) != tt.wantErr { t.Errorf("buildDNSetConfigMap() error = %v, wantErr %v", err, tt.wantErr) return } - g.Expect(got.Data["config.toml"]).NotTo(BeNil()) - g.Expect(cmp.Diff(tt.wantConfig, got.Data["config.toml"])).To(BeEmpty()) + configKey := "config.toml-" + configSuffix + g.Expect(got.Data[configKey]).NotTo(BeNil()) + g.Expect(cmp.Diff(tt.wantConfig, got.Data[configKey])).To(BeEmpty()) }) } } diff --git a/pkg/controllers/common/configmap.go b/pkg/controllers/common/configmap.go index db1bf53f..8b4a1190 100644 --- a/pkg/controllers/common/configmap.go +++ b/pkg/controllers/common/configmap.go @@ -15,14 +15,14 @@ package common import ( - "encoding/json" "fmt" "github.com/cespare/xxhash" "github.com/go-errors/errors" recon "github.com/matrixorigin/controller-runtime/pkg/reconciler" "github.com/matrixorigin/controller-runtime/pkg/util" corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" + "strings" ) const ( @@ -39,13 +39,8 @@ const ( // SyncConfigMap syncs the desired configmap for pods, which will cause rolling-update if the // data of the configmap is changed func SyncConfigMap(kubeCli recon.KubeClient, podSpec *corev1.PodSpec, cm *corev1.ConfigMap) error { - var currentCmName string vp := util.FindFirst(podSpec.Volumes, util.WithVolumeName("config")) - if vp != nil { - currentCmName = vp.Name - } - // TODO(aylei): GC stale configmaps (maybe in another worker?) - desiredName, err := ensureConfigMap(kubeCli, currentCmName, cm) + desiredName, err := ensureConfigMap(kubeCli, cm) if err != nil { return err } @@ -66,30 +61,50 @@ func SyncConfigMap(kubeCli recon.KubeClient, podSpec *corev1.PodSpec, cm *corev1 } // ensureConfigMap ensures the configmap exist in k8s -func ensureConfigMap(kubeCli recon.KubeClient, currentCm string, desired *corev1.ConfigMap) (string, error) { +func ensureConfigMap(kubeCli recon.KubeClient, desired *corev1.ConfigMap) (string, error) { c := desired.DeepCopy() - if err := addConfigMapDigest(c); err != nil { + old := &corev1.ConfigMap{} + exist, err := kubeCli.Exist(client.ObjectKeyFromObject(c), old) + if err != nil { return "", err } - // config digest not changed - if c.Name == currentCm { - return currentCm, nil + if exist { + podList := &corev1.PodList{} + err = kubeCli.List(podList, client.InNamespace(c.Namespace)) + if err != nil { + return "", err + } + for key, v := range old.Data { + if withDigest(key, v) && configInUse(key, podList.Items) { + // append item that is still in use + c.Data[key] = v + } + } + err = kubeCli.Update(c) + } else { + err = kubeCli.CreateOwned(c) } - // otherwise ensure the configmap exists - err := util.Ignore(apierrors.IsAlreadyExists, kubeCli.CreateOwned(c)) if err != nil { return "", err } return c.Name, nil } -func addConfigMapDigest(cm *corev1.ConfigMap) error { - s, err := json.Marshal(cm.Data) - if err != nil { - return err +func withDigest(key string, v string) bool { + return strings.Contains(key, DataDigest([]byte(v))) +} + +func configInUse(key string, podList []corev1.Pod) bool { + for _, pod := range podList { + s := pod.Annotations[ConfigSuffixAnno] + if len(s) > 0 && strings.Contains(key, s) { + return true + } } - sum := xxhash.Sum64(s) - suffix := fmt.Sprintf("%x", sum)[0:7] - cm.Name = fmt.Sprintf("%s-%s", cm.Name, suffix) - return nil + return false +} + +func DataDigest(data []byte) string { + sum := xxhash.Sum64(data) + return fmt.Sprintf("%x", sum)[0:7] } diff --git a/pkg/controllers/common/configmap_test.go b/pkg/controllers/common/configmap_test.go index cd6c9db0..13e671d0 100644 --- a/pkg/controllers/common/configmap_test.go +++ b/pkg/controllers/common/configmap_test.go @@ -33,7 +33,6 @@ func TestAddConfigMapDigest(t *testing.T) { } g := NewGomegaWithT(t) for _, cm := range cmList { - g.Expect(addConfigMapDigest(cm)).To(Succeed()) g.Expect(utf8string.NewString(cm.Name).IsASCII()).To(BeTrue()) } } diff --git a/pkg/controllers/common/consts.go b/pkg/controllers/common/consts.go index af3bd141..de356915 100644 --- a/pkg/controllers/common/consts.go +++ b/pkg/controllers/common/consts.go @@ -31,4 +31,6 @@ const ( LabelManagedBy = "matrixorigin.io/managed-by" LabelOwnerUID = "matrixorigin.io/owner-uid" + + ConfigSuffixAnno = "matrixorigin.io/config-suffix" ) diff --git a/pkg/controllers/common/podset.go b/pkg/controllers/common/podset.go index e245c555..2a9c7545 100644 --- a/pkg/controllers/common/podset.go +++ b/pkg/controllers/common/podset.go @@ -35,7 +35,7 @@ type SyncMOPodTask struct { ConfigMap *corev1.ConfigMap KubeCli recon.KubeClient StorageProvider *v1alpha1.SharedStorageProvider - + ConfigSuffix string // optional MutateContainer func(c *corev1.Container) MutatePod func(p *corev1.PodTemplateSpec) @@ -106,6 +106,9 @@ func syncPodTemplate(t *SyncMOPodTask) { if t.MutatePod != nil { t.MutatePod(t.TargetTemplate) } + if t.ConfigSuffix != "" { + t.TargetTemplate.ObjectMeta.Annotations[ConfigSuffixAnno] = t.ConfigSuffix + } p.Overlay.OverlayPodMeta(&t.TargetTemplate.ObjectMeta) p.Overlay.OverlayPodSpec(specRef) @@ -119,6 +122,7 @@ func syncMainContainer(p *v1alpha1.PodSet, c *corev1.Container, mutateFn func(c c.Env = []corev1.EnvVar{ util.FieldRefEnv(PodNameEnvKey, "metadata.name"), util.FieldRefEnv(NamespaceEnvKey, "metadata.namespace"), + util.FieldRefEnv(ConfigSuffixEnvKey, fmt.Sprintf("metadata.annotations['%s']", ConfigSuffixAnno)), } memLimitEnv := GoMemLimitEnv(p.MemoryLimitPercent, c.Resources.Limits.Memory(), p.Overlay) if memLimitEnv != nil { diff --git a/pkg/controllers/common/template.go b/pkg/controllers/common/template.go index dd299809..b1b35c5a 100644 --- a/pkg/controllers/common/template.go +++ b/pkg/controllers/common/template.go @@ -59,6 +59,8 @@ const ( NamespaceEnvKey = "NAMESPACE" // PodIPEnvKey is the container environment variable to reflect the IP of the Pod that runs the container PodIPEnvKey = "POD_IP" + // ConfigSuffixEnvKey is the container environment variable to reflect the config suffix + ConfigSuffixEnvKey = "CONFIG_SUFFIX" ) // SubResourceLabels generate labels for sub-resources diff --git a/pkg/controllers/dnset/controller.go b/pkg/controllers/dnset/controller.go index 4745452f..2fc5a388 100644 --- a/pkg/controllers/dnset/controller.go +++ b/pkg/controllers/dnset/controller.go @@ -186,11 +186,11 @@ func (d *Actor) Create(ctx *recon.Context[*v1alpha1.DNSet]) error { syncPodSpec(dn, dnSet, ctx.Dep.Deps.LogSet.Spec.SharedStorage) syncPersistentVolumeClaim(dn, dnSet) - configMap, err := buildDNSetConfigMap(dn, ctx.Dep.Deps.LogSet) + configMap, configSuffix, err := buildDNSetConfigMap(dn, ctx.Dep.Deps.LogSet) if err != nil { return err } - + dnSet.Spec.Template.Annotations[common.ConfigSuffixAnno] = configSuffix if err := common.SyncConfigMap(ctx, &dnSet.Spec.Template.Spec, configMap); err != nil { return err } diff --git a/pkg/controllers/dnset/resource.go b/pkg/controllers/dnset/resource.go index 7abfd875..0daa0413 100644 --- a/pkg/controllers/dnset/resource.go +++ b/pkg/controllers/dnset/resource.go @@ -62,7 +62,7 @@ service-address = "${ADDR}:{{ .DNServicePort }}" service-host = "${ADDR}" EOF # build instance config -sed "/\[{{ .ConfigAlias }}\]/r ${bc}" {{ .ConfigFilePath }} > ${conf} +sed "/\[{{ .ConfigAlias }}\]/r ${bc}" {{ .ConfigFilePath }}-${CONFIG_SUFFIX} > ${conf} # append lock-service configs lsc=$(mktemp) @@ -152,6 +152,7 @@ func syncPodSpec(dn *v1alpha1.DNSet, sts *kruise.StatefulSet, sp v1alpha1.Shared mainRef.Env = []corev1.EnvVar{ util.FieldRefEnv(common.PodNameEnvKey, "metadata.name"), util.FieldRefEnv(common.NamespaceEnvKey, "metadata.namespace"), + util.FieldRefEnv(common.ConfigSuffixEnvKey, fmt.Sprintf("metadata.annotations['%s']", common.ConfigSuffixAnno)), {Name: common.HeadlessSvcEnvKey, Value: headlessSvcName(dn)}, } memLimitEnv := common.GoMemLimitEnv(dn.Spec.MemoryLimitPercent, dn.Spec.Resources.Limits.Memory(), dn.Spec.Overlay) @@ -177,9 +178,9 @@ func syncPodSpec(dn *v1alpha1.DNSet, sts *kruise.StatefulSet, sp v1alpha1.Shared } // buildDNSetConfigMap return dn set configmap -func buildDNSetConfigMap(dn *v1alpha1.DNSet, ls *v1alpha1.LogSet) (*corev1.ConfigMap, error) { +func buildDNSetConfigMap(dn *v1alpha1.DNSet, ls *v1alpha1.LogSet) (*corev1.ConfigMap, string, error) { if ls.Status.Discovery == nil { - return nil, errors.New("HAKeeper discovery address not ready") + return nil, "", errors.New("HAKeeper discovery address not ready") } conf := dn.Spec.Config if conf == nil { @@ -208,7 +209,7 @@ func buildDNSetConfigMap(dn *v1alpha1.DNSet, ls *v1alpha1.LogSet) (*corev1.Confi } s, err := conf.ToString() if err != nil { - return nil, err + return nil, "", err } buff := new(bytes.Buffer) @@ -220,16 +221,17 @@ func buildDNSetConfigMap(dn *v1alpha1.DNSet, ls *v1alpha1.LogSet) (*corev1.Confi ConfigAlias: configAlias, }) if err != nil { - return nil, err + return nil, "", err } + configSuffix := common.DataDigest([]byte(s)) return &corev1.ConfigMap{ ObjectMeta: common.ObjMetaTemplate(dn, configMapName(dn)), Data: map[string]string{ - common.ConfigFile: s, + fmt.Sprintf("%s-%s", common.ConfigFile, configSuffix): s, common.Entrypoint: buff.String(), }, - }, nil + }, configSuffix, nil } func buildHeadlessSvc(dn *v1alpha1.DNSet) *corev1.Service { @@ -250,12 +252,12 @@ func syncPersistentVolumeClaim(dn *v1alpha1.DNSet, sts *kruise.StatefulSet) { } func syncPods(ctx *recon.Context[*v1alpha1.DNSet], sts *kruise.StatefulSet) error { - cm, err := buildDNSetConfigMap(ctx.Obj, ctx.Dep.Deps.LogSet) + cm, configSuffix, err := buildDNSetConfigMap(ctx.Obj, ctx.Dep.Deps.LogSet) if err != nil { return err } - syncPodMeta(ctx.Obj, sts) + sts.Spec.Template.Annotations[common.ConfigSuffixAnno] = configSuffix if ctx.Dep != nil { syncPodSpec(ctx.Obj, sts, ctx.Dep.Deps.LogSet.Spec.SharedStorage) diff --git a/pkg/controllers/dnset/resource_test.go b/pkg/controllers/dnset/resource_test.go index e4a45358..52ff6de3 100644 --- a/pkg/controllers/dnset/resource_test.go +++ b/pkg/controllers/dnset/resource_test.go @@ -245,13 +245,14 @@ discovery-address = "test:6001" for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { g := NewGomegaWithT(t) - got, err := buildDNSetConfigMap(tt.args.dn, tt.args.ls) + got, configSuffix, err := buildDNSetConfigMap(tt.args.dn, tt.args.ls) if (err != nil) != tt.wantErr { t.Errorf("buildDNSetConfigMap() error = %v, wantErr %v", err, tt.wantErr) return } - g.Expect(got.Data["config.toml"]).NotTo(BeNil()) - g.Expect(cmp.Diff(tt.wantConfig, got.Data["config.toml"])).To(BeEmpty()) + configKey := "config.toml-" + configSuffix + g.Expect(got.Data[configKey]).NotTo(BeNil()) + g.Expect(cmp.Diff(tt.wantConfig, got.Data[configKey])).To(BeEmpty()) }) } } diff --git a/pkg/controllers/logset/configmap.go b/pkg/controllers/logset/configmap.go index b881982f..a1381254 100644 --- a/pkg/controllers/logset/configmap.go +++ b/pkg/controllers/logset/configmap.go @@ -68,7 +68,7 @@ gossip-address-v2 = "${ADDR}:{{ .GossipPort }}" EOF # build instance config -sed "/\[logservice\]/r ${bc}" {{ .ConfigFilePath }} > ${conf} +sed "/\[logservice\]/r ${bc}" {{ .ConfigFilePath }}-${CONFIG_SUFFIX} > ${conf} # insert gossip config gossipTmp=$(mktemp) @@ -124,7 +124,7 @@ gossip-port = {{ .GossipPort }} EOF # build instance config -sed "/\[logservice\]/r ${bc}" {{ .ConfigFilePath }} > ${conf} +sed "/\[logservice\]/r ${bc}" {{ .ConfigFilePath }}-${CONFIG_SUFFIX} > ${conf} # insert gossip config gossipTmp=$(mktemp) @@ -185,7 +185,7 @@ func buildGossipSeedsConfigMap(ls *v1alpha1.LogSet, sts *kruisev1.StatefulSet) ( } // buildConfigMap build the configmap for log service -func buildConfigMap(ls *v1alpha1.LogSet) (*corev1.ConfigMap, error) { +func buildConfigMap(ls *v1alpha1.LogSet) (*corev1.ConfigMap, string, error) { conf := ls.Spec.Config if conf == nil { conf = v1alpha1.NewTomlConfig(map[string]interface{}{}) @@ -210,7 +210,7 @@ func buildConfigMap(ls *v1alpha1.LogSet) (*corev1.ConfigMap, error) { } s, err := conf.ToString() if err != nil { - return nil, err + return nil, "", err } // 2. build the start script @@ -228,11 +228,11 @@ func buildConfigMap(ls *v1alpha1.LogSet) (*corev1.ConfigMap, error) { BootstrapFilePath: fmt.Sprintf("%s/%s", bootstrapPath, bootstrapFile), GossipFilePath: fmt.Sprintf("%s/%s", gossipPath, gossipFile), }) - if err != nil { - return nil, err + return nil, "", err } + configSuffix := common.DataDigest([]byte(s)) return &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: ls.Namespace, @@ -240,10 +240,10 @@ func buildConfigMap(ls *v1alpha1.LogSet) (*corev1.ConfigMap, error) { Labels: common.SubResourceLabels(ls), }, Data: map[string]string{ - configFile: s, + fmt.Sprintf("%s-%s", configFile, configSuffix): s, entrypoint: buff.String(), }, - }, nil + }, configSuffix, nil } func HaKeeperAdds(ls *v1alpha1.LogSet) []string { diff --git a/pkg/controllers/logset/controller.go b/pkg/controllers/logset/controller.go index 24748846..36ac4917 100644 --- a/pkg/controllers/logset/controller.go +++ b/pkg/controllers/logset/controller.go @@ -164,10 +164,11 @@ func (r *Actor) Create(ctx *recon.Context[*v1alpha1.LogSet]) error { return err } // sync the config - cm, err := buildConfigMap(ls) + cm, configSuffix, err := buildConfigMap(ls) if err != nil { return err } + sts.Spec.Template.Annotations[common.ConfigSuffixAnno] = configSuffix if err := common.SyncConfigMap(ctx, &sts.Spec.Template.Spec, cm); err != nil { return err } @@ -367,11 +368,12 @@ func updateGossipConfig(ctx *recon.Context[*v1alpha1.LogSet], sts *kruisev1.Stat } func syncPods(ctx *recon.Context[*v1alpha1.LogSet], sts *kruisev1.StatefulSet) error { - cm, err := buildConfigMap(ctx.Obj) + cm, configSuffix, err := buildConfigMap(ctx.Obj) if err != nil { return err } syncPodMeta(ctx.Obj, sts) + sts.Spec.Template.Annotations[common.ConfigSuffixAnno] = configSuffix syncPodSpec(ctx.Obj, &sts.Spec.Template.Spec) return common.SyncConfigMap(ctx, &sts.Spec.Template.Spec, cm) } diff --git a/pkg/controllers/logset/sts.go b/pkg/controllers/logset/sts.go index dd25998e..7c7548d5 100644 --- a/pkg/controllers/logset/sts.go +++ b/pkg/controllers/logset/sts.go @@ -78,6 +78,7 @@ func syncPodSpec(ls *v1alpha1.LogSet, specRef *corev1.PodSpec) { mainRef.Env = []corev1.EnvVar{ util.FieldRefEnv(PodNameEnvKey, "metadata.name"), util.FieldRefEnv(NamespaceEnvKey, "metadata.namespace"), + util.FieldRefEnv(common.ConfigSuffixEnvKey, fmt.Sprintf("metadata.annotations['%s']", common.ConfigSuffixAnno)), util.FieldRefEnv(PodIPEnvKey, "status.podIP"), {Name: HeadlessSvcEnvKey, Value: headlessSvcName(ls)}, } diff --git a/pkg/controllers/proxyset/resource.go b/pkg/controllers/proxyset/resource.go index 929fa73b..a7fff297 100644 --- a/pkg/controllers/proxyset/resource.go +++ b/pkg/controllers/proxyset/resource.go @@ -58,7 +58,7 @@ cat < ${bc} uuid = "${UUID}" EOF # build instance config -sed "/\[proxy\]/r ${bc}" {{ .ConfigFilePath }} > ${conf} +sed "/\[proxy\]/r ${bc}" {{ .ConfigFilePath }}-${CONFIG_SUFFIX} > ${conf} echo "/mo-service -cfg ${conf} $@" exec /mo-service -cfg ${conf} $@ @@ -69,7 +69,7 @@ func buildCloneSet(proxy *v1alpha1.ProxySet) *kruisev1alpha1.CloneSet { } func syncCloneSet(ctx *recon.Context[*v1alpha1.ProxySet], proxy *v1alpha1.ProxySet, cs *kruisev1alpha1.CloneSet) error { - cm, err := buildProxyConfigMap(proxy, ctx.Dep.Deps.LogSet) + cm, configSuffix, err := buildProxyConfigMap(proxy, ctx.Dep.Deps.LogSet) if err != nil { return errors.WrapPrefix(err, "build configmap", 0) } @@ -81,6 +81,7 @@ func syncCloneSet(ctx *recon.Context[*v1alpha1.ProxySet], proxy *v1alpha1.ProxyS ConfigMap: cm, KubeCli: ctx, StorageProvider: &ctx.Dep.Deps.LogSet.Spec.SharedStorage, + ConfigSuffix: configSuffix, MutateContainer: syncMainContainer, }) } @@ -145,9 +146,9 @@ func syncSvc(proxy *v1alpha1.ProxySet, svc *corev1.Service) { } } -func buildProxyConfigMap(proxy *v1alpha1.ProxySet, ls *v1alpha1.LogSet) (*corev1.ConfigMap, error) { +func buildProxyConfigMap(proxy *v1alpha1.ProxySet, ls *v1alpha1.LogSet) (*corev1.ConfigMap, string, error) { if ls.Status.Discovery == nil { - return nil, errors.New("HAKeeper discovery address not ready") + return nil, "", errors.New("HAKeeper discovery address not ready") } conf := proxy.Spec.Config if conf == nil { @@ -162,7 +163,7 @@ func buildProxyConfigMap(proxy *v1alpha1.ProxySet, ls *v1alpha1.LogSet) (*corev1 } s, err := conf.ToString() if err != nil { - return nil, err + return nil, "", err } buff := new(bytes.Buffer) @@ -170,16 +171,17 @@ func buildProxyConfigMap(proxy *v1alpha1.ProxySet, ls *v1alpha1.LogSet) (*corev1 ConfigFilePath: fmt.Sprintf("%s/%s", common.ConfigPath, common.ConfigFile), }) if err != nil { - return nil, err + return nil, "", err } + configSuffix := common.DataDigest([]byte(s)) return &corev1.ConfigMap{ ObjectMeta: configMapKey(proxy), Data: map[string]string{ - common.ConfigFile: s, + fmt.Sprintf("%s-%s", common.ConfigFile, configSuffix): s, common.Entrypoint: buff.String(), }, - }, nil + }, configSuffix, nil } func configMapKey(p *v1alpha1.ProxySet) metav1.ObjectMeta { From 4ec55ce890aa1dacec3e7387c466c3f758edd316 Mon Sep 17 00:00:00 2001 From: cyberchen98 Date: Sat, 12 Oct 2024 14:29:36 +0800 Subject: [PATCH 02/11] chore: remove redundant code --- pkg/controllers/common/podset.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/controllers/common/podset.go b/pkg/controllers/common/podset.go index 2a9c7545..d7857077 100644 --- a/pkg/controllers/common/podset.go +++ b/pkg/controllers/common/podset.go @@ -106,9 +106,7 @@ func syncPodTemplate(t *SyncMOPodTask) { if t.MutatePod != nil { t.MutatePod(t.TargetTemplate) } - if t.ConfigSuffix != "" { - t.TargetTemplate.ObjectMeta.Annotations[ConfigSuffixAnno] = t.ConfigSuffix - } + t.TargetTemplate.ObjectMeta.Annotations[ConfigSuffixAnno] = t.ConfigSuffix p.Overlay.OverlayPodMeta(&t.TargetTemplate.ObjectMeta) p.Overlay.OverlayPodSpec(specRef) From e8ee028fc6d1be6293fba7abf9ea805e65842ba1 Mon Sep 17 00:00:00 2001 From: cyberchen98 Date: Sat, 12 Oct 2024 15:00:39 +0800 Subject: [PATCH 03/11] test: update ut cases --- pkg/controllers/common/configmap_test.go | 5 +++-- pkg/controllers/logset/sts_test.go | 8 ++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/pkg/controllers/common/configmap_test.go b/pkg/controllers/common/configmap_test.go index 13e671d0..3cfe4e71 100644 --- a/pkg/controllers/common/configmap_test.go +++ b/pkg/controllers/common/configmap_test.go @@ -22,7 +22,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func TestAddConfigMapDigest(t *testing.T) { +func TestDataDigest(t *testing.T) { // need fuzz? cmList := []*corev1.ConfigMap{ newCM(""), @@ -33,7 +33,8 @@ func TestAddConfigMapDigest(t *testing.T) { } g := NewGomegaWithT(t) for _, cm := range cmList { - g.Expect(utf8string.NewString(cm.Name).IsASCII()).To(BeTrue()) + digest := DataDigest([]byte(cm.Data["config"])) + g.Expect(utf8string.NewString(digest).IsASCII()).To(BeTrue()) } } diff --git a/pkg/controllers/logset/sts_test.go b/pkg/controllers/logset/sts_test.go index f4bf3f6f..86a83b94 100644 --- a/pkg/controllers/logset/sts_test.go +++ b/pkg/controllers/logset/sts_test.go @@ -116,6 +116,14 @@ func Test_syncPodSpec(t *testing.T) { FieldPath: "metadata.namespace", }, }, + }, { + Name: "CONFIG_SUFFIX", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.annotations['matrixorigin.io/config-suffix']", + }, + }, }, { Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{ From fa86f58d4cc9909a58ea5114903bf20acfc7fcce Mon Sep 17 00:00:00 2001 From: cyberchen98 Date: Tue, 15 Oct 2024 16:11:27 +0800 Subject: [PATCH 04/11] e2e: add case for inplace update --- test/e2e/logset_test.go | 117 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/test/e2e/logset_test.go b/test/e2e/logset_test.go index 05fe4e62..8dadd2d3 100644 --- a/test/e2e/logset_test.go +++ b/test/e2e/logset_test.go @@ -156,6 +156,123 @@ var _ = Describe("MatrixOneCluster test", func() { }, teardownClusterTimeout, pollInterval).Should(Succeed()) }) + It("Should restart log pod inplace when only config changed", func() { + type podInfo struct { + uid types.UID + restart int32 + } + By("Create logset") + pull := corev1.PullIfNotPresent + l := &v1alpha1.LogSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: env.Namespace, + Name: "log-" + rand.String(6), + }, + Spec: v1alpha1.LogSetSpec{ + PodSet: v1alpha1.PodSet{ + Replicas: 3, + MainContainer: v1alpha1.MainContainer{ + Image: fmt.Sprintf("%s:%s", moImageRepo, moVersion), + }, + Overlay: &v1alpha1.Overlay{ + MainContainerOverlay: v1alpha1.MainContainerOverlay{ + ImagePullPolicy: &pull, + }, + }, + }, + Volume: v1alpha1.Volume{ + Size: resource.MustParse("100Mi"), + }, + SharedStorage: v1alpha1.SharedStorageProvider{ + FileSystem: &v1alpha1.FileSystemProvider{ + Path: "/test", + }, + }, + StoreFailureTimeout: &metav1.Duration{Duration: 2 * time.Minute}, + }, + } + Expect(kubeCli.Create(ctx, l)).To(Succeed()) + Eventually(func() error { + if err := kubeCli.Get(ctx, client.ObjectKeyFromObject(l), l); err != nil { + logger.Errorw("error get logset status", "logset", l.Name, "error", err) + return err + } + if !recon.IsReady(&l.Status.ConditionalStatus) { + logger.Infow("wait logset ready", "logset", l.Name) + return errWait + } + return nil + }, createLogSetTimeout, pollInterval).Should(Succeed()) + + By("Logset update config") + oldPodList := &corev1.PodList{} + Expect(kubeCli.List(ctx, oldPodList, client.MatchingLabels(map[string]string{common.InstanceLabelKey: l.Name}))).To(Succeed()) + oldPodNameToUID := map[string]podInfo{} + for _, pod := range oldPodList.Items { + oldPodNameToUID[pod.Name] = podInfo{uid: pod.UID, restart: pod.Status.ContainerStatuses[0].RestartCount} + } + newConfig := l.Spec.Config.DeepCopy() + if newConfig == nil { + newConfig = v1alpha1.NewTomlConfig(map[string]interface{}{}) + } + Expect(e2eutil.Patch(ctx, kubeCli, l, func() error { + newConfig.Set([]string{"observability.labelSelector", "role"}, "__MO_UPDATE_TESTER__") + l.Spec.Config = newConfig + return nil + })).To(Succeed()) + Eventually(func() error { + podList := &corev1.PodList{} + clone := l.DeepCopy() + if err := kubeCli.Get(ctx, client.ObjectKeyFromObject(l), clone); err != nil { + logger.Errorw("error get logset status", "logset", l.Name, "error", err) + } + if !recon.IsReady(&clone.Status.ConditionalStatus) { + logger.Infow("wait logset ready", "logset", l.Name) + return errWait + } + if err := kubeCli.List(ctx, podList, client.MatchingLabels(map[string]string{common.InstanceLabelKey: l.Name})); err != nil { + logger.Errorw("error list pods", "logset", l.Name, "error", err) + return err + } + // expect pods not change, but restarted + for _, pod := range podList.Items { + if v, ok := oldPodNameToUID[pod.Name]; ok && v.uid == pod.UID && v.restart < pod.Status.ContainerStatuses[0].RestartCount { + continue + } + logger.Infow("waiting pod update", "pod", pod.Name) + return errWait + } + return nil + }, createLogSetTimeout, pollInterval).Should(Succeed()) + + By("Teardown logset") + Expect(kubeCli.Delete(ctx, l)).To(Succeed()) + Eventually(func() error { + err := kubeCli.Get(ctx, client.ObjectKeyFromObject(l), l) + if err == nil { + logger.Infow("wait logset teardown", "logset", l.Name) + return errWait + } + if !apierrors.IsNotFound(err) { + logger.Errorw("unexpected error when get logset", "logset", l, "error", err) + return err + } + podList := &corev1.PodList{} + err = kubeCli.List(ctx, podList, client.InNamespace(l.Namespace)) + if err != nil { + logger.Errorw("error list pods", "error", err) + return err + } + for _, pod := range podList.Items { + if strings.HasPrefix(pod.Name, l.Name) { + logger.Infow("Pod that belongs to the logset is not cleaned", "pod", pod.Name) + return errWait + } + } + return nil + }, teardownClusterTimeout, pollInterval).Should(Succeed()) + }) + It("Should start logset service successfully when logset replicas is 1", func() { By("Create logset") l := &v1alpha1.LogSet{ From c14cd2fadd95fd948a4c4426a7e64f69c7b03ee4 Mon Sep 17 00:00:00 2001 From: cyberchen98 Date: Tue, 15 Oct 2024 16:42:51 +0800 Subject: [PATCH 05/11] e2e: compare config suffix --- test/e2e/logset_test.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/test/e2e/logset_test.go b/test/e2e/logset_test.go index 8dadd2d3..ca2bb250 100644 --- a/test/e2e/logset_test.go +++ b/test/e2e/logset_test.go @@ -157,10 +157,6 @@ var _ = Describe("MatrixOneCluster test", func() { }) It("Should restart log pod inplace when only config changed", func() { - type podInfo struct { - uid types.UID - restart int32 - } By("Create logset") pull := corev1.PullIfNotPresent l := &v1alpha1.LogSet{ @@ -207,9 +203,11 @@ var _ = Describe("MatrixOneCluster test", func() { By("Logset update config") oldPodList := &corev1.PodList{} Expect(kubeCli.List(ctx, oldPodList, client.MatchingLabels(map[string]string{common.InstanceLabelKey: l.Name}))).To(Succeed()) - oldPodNameToUID := map[string]podInfo{} + oldPodNameToUID := map[string]types.UID{} + var oldSuffix string for _, pod := range oldPodList.Items { - oldPodNameToUID[pod.Name] = podInfo{uid: pod.UID, restart: pod.Status.ContainerStatuses[0].RestartCount} + oldSuffix = pod.Annotations[common.ConfigSuffixAnno] + oldPodNameToUID[pod.Name] = pod.UID } newConfig := l.Spec.Config.DeepCopy() if newConfig == nil { @@ -236,7 +234,8 @@ var _ = Describe("MatrixOneCluster test", func() { } // expect pods not change, but restarted for _, pod := range podList.Items { - if v, ok := oldPodNameToUID[pod.Name]; ok && v.uid == pod.UID && v.restart < pod.Status.ContainerStatuses[0].RestartCount { + if uid, ok := oldPodNameToUID[pod.Name]; ok && uid == pod.UID && + oldSuffix != pod.Annotations[common.ConfigSuffixAnno] { continue } logger.Infow("waiting pod update", "pod", pod.Name) From abae4ed676766adbca1a269f9e0694bbdaffdca7 Mon Sep 17 00:00:00 2001 From: cyberchen98 Date: Wed, 16 Oct 2024 10:42:41 +0800 Subject: [PATCH 06/11] feat: filter events via OperatorVersion --- api/core/v1alpha1/common_types.go | 1 + .../crds/core.matrixorigin.io_cnpools.yaml | 1 + .../crds/core.matrixorigin.io_cnsets.yaml | 1 + .../crds/core.matrixorigin.io_dnsets.yaml | 1 + .../crds/core.matrixorigin.io_logsets.yaml | 1 + .../core.matrixorigin.io_matrixoneclusters.yaml | 8 ++++++++ .../crds/core.matrixorigin.io_proxysets.yaml | 1 + .../crds/core.matrixorigin.io_webuis.yaml | 1 + deploy/crds/core.matrixorigin.io_cnpools.yaml | 1 + deploy/crds/core.matrixorigin.io_cnsets.yaml | 1 + deploy/crds/core.matrixorigin.io_dnsets.yaml | 1 + deploy/crds/core.matrixorigin.io_logsets.yaml | 1 + .../core.matrixorigin.io_matrixoneclusters.yaml | 8 ++++++++ deploy/crds/core.matrixorigin.io_proxysets.yaml | 1 + deploy/crds/core.matrixorigin.io_webuis.yaml | 1 + docs/reference/api-reference.md | 14 +++++++------- pkg/controllers/cnset/controller.go | 9 +++++++++ pkg/controllers/dnset/controller.go | 9 +++++++++ pkg/controllers/logset/controller.go | 9 +++++++++ pkg/controllers/proxyset/controller.go | 9 +++++++++ pkg/webhook/proxy_webhook.go | 2 ++ pkg/webhook/utils.go | 2 +- 22 files changed, 75 insertions(+), 8 deletions(-) diff --git a/api/core/v1alpha1/common_types.go b/api/core/v1alpha1/common_types.go index 99f23eb2..ff9a49c1 100644 --- a/api/core/v1alpha1/common_types.go +++ b/api/core/v1alpha1/common_types.go @@ -116,6 +116,7 @@ type PodSet struct { // OperatorVersion is the controller version of mo-operator that should be used to // reconcile this set + // +kubebuilder:default="1.2.0" OperatorVersion *string `json:"operatorVersion,omitempty"` } diff --git a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnpools.yaml b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnpools.yaml index a760f13b..55588340 100644 --- a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnpools.yaml +++ b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnpools.yaml @@ -173,6 +173,7 @@ spec: type: string type: object operatorVersion: + default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set diff --git a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnsets.yaml b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnsets.yaml index 929b03d0..bfe5f998 100644 --- a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnsets.yaml +++ b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnsets.yaml @@ -147,6 +147,7 @@ spec: type: string type: object operatorVersion: + default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set diff --git a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_dnsets.yaml b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_dnsets.yaml index 07927bea..86f6c6d8 100644 --- a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_dnsets.yaml +++ b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_dnsets.yaml @@ -122,6 +122,7 @@ spec: type: string type: object operatorVersion: + default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set diff --git a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_logsets.yaml b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_logsets.yaml index 141f5674..a48ec515 100644 --- a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_logsets.yaml +++ b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_logsets.yaml @@ -113,6 +113,7 @@ spec: type: string type: object operatorVersion: + default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set diff --git a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_matrixoneclusters.yaml b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_matrixoneclusters.yaml index d12a1686..9c86eb35 100644 --- a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_matrixoneclusters.yaml +++ b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_matrixoneclusters.yaml @@ -139,6 +139,7 @@ spec: type: string type: object operatorVersion: + default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set @@ -497,6 +498,7 @@ spec: type: string type: object operatorVersion: + default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set @@ -838,6 +840,7 @@ spec: type: string type: object operatorVersion: + default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set @@ -1042,6 +1045,7 @@ spec: type: string type: object operatorVersion: + default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set @@ -1317,6 +1321,7 @@ spec: type: string type: object operatorVersion: + default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set @@ -1498,6 +1503,7 @@ spec: type: string type: object operatorVersion: + default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set @@ -1713,6 +1719,7 @@ spec: type: string type: object operatorVersion: + default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set @@ -2029,6 +2036,7 @@ spec: type: string type: object operatorVersion: + default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set diff --git a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_proxysets.yaml b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_proxysets.yaml index c104d427..85b3a729 100644 --- a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_proxysets.yaml +++ b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_proxysets.yaml @@ -107,6 +107,7 @@ spec: type: string type: object operatorVersion: + default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set diff --git a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_webuis.yaml b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_webuis.yaml index 36643764..ac5a289b 100644 --- a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_webuis.yaml +++ b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_webuis.yaml @@ -91,6 +91,7 @@ spec: type: string type: object operatorVersion: + default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set diff --git a/deploy/crds/core.matrixorigin.io_cnpools.yaml b/deploy/crds/core.matrixorigin.io_cnpools.yaml index a760f13b..55588340 100644 --- a/deploy/crds/core.matrixorigin.io_cnpools.yaml +++ b/deploy/crds/core.matrixorigin.io_cnpools.yaml @@ -173,6 +173,7 @@ spec: type: string type: object operatorVersion: + default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set diff --git a/deploy/crds/core.matrixorigin.io_cnsets.yaml b/deploy/crds/core.matrixorigin.io_cnsets.yaml index 929b03d0..bfe5f998 100644 --- a/deploy/crds/core.matrixorigin.io_cnsets.yaml +++ b/deploy/crds/core.matrixorigin.io_cnsets.yaml @@ -147,6 +147,7 @@ spec: type: string type: object operatorVersion: + default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set diff --git a/deploy/crds/core.matrixorigin.io_dnsets.yaml b/deploy/crds/core.matrixorigin.io_dnsets.yaml index 07927bea..86f6c6d8 100644 --- a/deploy/crds/core.matrixorigin.io_dnsets.yaml +++ b/deploy/crds/core.matrixorigin.io_dnsets.yaml @@ -122,6 +122,7 @@ spec: type: string type: object operatorVersion: + default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set diff --git a/deploy/crds/core.matrixorigin.io_logsets.yaml b/deploy/crds/core.matrixorigin.io_logsets.yaml index 141f5674..a48ec515 100644 --- a/deploy/crds/core.matrixorigin.io_logsets.yaml +++ b/deploy/crds/core.matrixorigin.io_logsets.yaml @@ -113,6 +113,7 @@ spec: type: string type: object operatorVersion: + default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set diff --git a/deploy/crds/core.matrixorigin.io_matrixoneclusters.yaml b/deploy/crds/core.matrixorigin.io_matrixoneclusters.yaml index d12a1686..9c86eb35 100644 --- a/deploy/crds/core.matrixorigin.io_matrixoneclusters.yaml +++ b/deploy/crds/core.matrixorigin.io_matrixoneclusters.yaml @@ -139,6 +139,7 @@ spec: type: string type: object operatorVersion: + default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set @@ -497,6 +498,7 @@ spec: type: string type: object operatorVersion: + default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set @@ -838,6 +840,7 @@ spec: type: string type: object operatorVersion: + default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set @@ -1042,6 +1045,7 @@ spec: type: string type: object operatorVersion: + default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set @@ -1317,6 +1321,7 @@ spec: type: string type: object operatorVersion: + default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set @@ -1498,6 +1503,7 @@ spec: type: string type: object operatorVersion: + default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set @@ -1713,6 +1719,7 @@ spec: type: string type: object operatorVersion: + default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set @@ -2029,6 +2036,7 @@ spec: type: string type: object operatorVersion: + default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set diff --git a/deploy/crds/core.matrixorigin.io_proxysets.yaml b/deploy/crds/core.matrixorigin.io_proxysets.yaml index c104d427..85b3a729 100644 --- a/deploy/crds/core.matrixorigin.io_proxysets.yaml +++ b/deploy/crds/core.matrixorigin.io_proxysets.yaml @@ -107,6 +107,7 @@ spec: type: string type: object operatorVersion: + default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set diff --git a/deploy/crds/core.matrixorigin.io_webuis.yaml b/deploy/crds/core.matrixorigin.io_webuis.yaml index 36643764..ac5a289b 100644 --- a/deploy/crds/core.matrixorigin.io_webuis.yaml +++ b/deploy/crds/core.matrixorigin.io_webuis.yaml @@ -91,6 +91,7 @@ spec: type: string type: object operatorVersion: + default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set diff --git a/docs/reference/api-reference.md b/docs/reference/api-reference.md index 155e4d8d..f52386e7 100644 --- a/docs/reference/api-reference.md +++ b/docs/reference/api-reference.md @@ -421,7 +421,7 @@ _Appears in:_ | `exportToPrometheus` _boolean_ | ExportToPrometheus enables the pod to be discovered scraped by Prometheus | | | | `promDiscoveryScheme` _[PromDiscoveryScheme](#promdiscoveryscheme)_ | PromDiscoveryScheme indicates how the Pod will be discovered by prometheus, options:
- Pod: the pod will be discovered via will-known labels on the Pod
- Service: the pod will be discovered via will-known annotations in the service which expose endpoints to the pods
default to Service | | | | `semanticVersion` _string_ | SemanticVersion override the semantic version of CN if set,
the semantic version of CN will be default to the image tag,
if the semantic version is not set, nor the image tag is a valid semantic version,
operator will treat the MO as unknown version and will not apply any version-specific
reconciliations | | | -| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | | | +| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | 1.2.0 | | | `cacheVolume` _[Volume](#volume)_ | CacheVolume is the desired local cache volume for CNSet,
node storage will be used if not specified | | | | `sharedStorageCache` _[SharedStorageCache](#sharedstoragecache)_ | SharedStorageCache is the configuration of the S3 sharedStorageCache | | | | `pythonUdfSidecar` _[PythonUdfSidecar](#pythonudfsidecar)_ | PythonUdfSidecar is the python udf server in CN | | | @@ -622,7 +622,7 @@ _Appears in:_ | `exportToPrometheus` _boolean_ | ExportToPrometheus enables the pod to be discovered scraped by Prometheus | | | | `promDiscoveryScheme` _[PromDiscoveryScheme](#promdiscoveryscheme)_ | PromDiscoveryScheme indicates how the Pod will be discovered by prometheus, options:
- Pod: the pod will be discovered via will-known labels on the Pod
- Service: the pod will be discovered via will-known annotations in the service which expose endpoints to the pods
default to Service | | | | `semanticVersion` _string_ | SemanticVersion override the semantic version of CN if set,
the semantic version of CN will be default to the image tag,
if the semantic version is not set, nor the image tag is a valid semantic version,
operator will treat the MO as unknown version and will not apply any version-specific
reconciliations | | | -| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | | | +| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | 1.2.0 | | | `cacheVolume` _[Volume](#volume)_ | CacheVolume is the desired local cache volume for CNSet,
node storage will be used if not specified | | | | `sharedStorageCache` _[SharedStorageCache](#sharedstoragecache)_ | SharedStorageCache is the configuration of the S3 sharedStorageCache | | | | `pythonUdfSidecar` _[PythonUdfSidecar](#pythonudfsidecar)_ | PythonUdfSidecar is the python udf server in CN | | | @@ -819,7 +819,7 @@ _Appears in:_ | `exportToPrometheus` _boolean_ | ExportToPrometheus enables the pod to be discovered scraped by Prometheus | | | | `promDiscoveryScheme` _[PromDiscoveryScheme](#promdiscoveryscheme)_ | PromDiscoveryScheme indicates how the Pod will be discovered by prometheus, options:
- Pod: the pod will be discovered via will-known labels on the Pod
- Service: the pod will be discovered via will-known annotations in the service which expose endpoints to the pods
default to Service | | | | `semanticVersion` _string_ | SemanticVersion override the semantic version of CN if set,
the semantic version of CN will be default to the image tag,
if the semantic version is not set, nor the image tag is a valid semantic version,
operator will treat the MO as unknown version and will not apply any version-specific
reconciliations | | | -| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | | | +| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | 1.2.0 | | | `cacheVolume` _[Volume](#volume)_ | CacheVolume is the desired local cache volume for DNSet,
node storage will be used if not specified | | | | `sharedStorageCache` _[SharedStorageCache](#sharedstoragecache)_ | | | | @@ -986,7 +986,7 @@ _Appears in:_ | `exportToPrometheus` _boolean_ | ExportToPrometheus enables the pod to be discovered scraped by Prometheus | | | | `promDiscoveryScheme` _[PromDiscoveryScheme](#promdiscoveryscheme)_ | PromDiscoveryScheme indicates how the Pod will be discovered by prometheus, options:
- Pod: the pod will be discovered via will-known labels on the Pod
- Service: the pod will be discovered via will-known annotations in the service which expose endpoints to the pods
default to Service | | | | `semanticVersion` _string_ | SemanticVersion override the semantic version of CN if set,
the semantic version of CN will be default to the image tag,
if the semantic version is not set, nor the image tag is a valid semantic version,
operator will treat the MO as unknown version and will not apply any version-specific
reconciliations | | | -| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | | | +| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | 1.2.0 | | | `volume` _[Volume](#volume)_ | Volume is the local persistent volume for each LogService instance | | | | `sharedStorage` _[SharedStorageProvider](#sharedstorageprovider)_ | SharedStorage is an external shared storage shared by all LogService instances | | | | `initialConfig` _[InitialConfig](#initialconfig)_ | InitialConfig is the initial configuration of HAKeeper
InitialConfig is immutable | | | @@ -1215,7 +1215,7 @@ _Appears in:_ | `exportToPrometheus` _boolean_ | ExportToPrometheus enables the pod to be discovered scraped by Prometheus | | | | `promDiscoveryScheme` _[PromDiscoveryScheme](#promdiscoveryscheme)_ | PromDiscoveryScheme indicates how the Pod will be discovered by prometheus, options:
- Pod: the pod will be discovered via will-known labels on the Pod
- Service: the pod will be discovered via will-known annotations in the service which expose endpoints to the pods
default to Service | | | | `semanticVersion` _string_ | SemanticVersion override the semantic version of CN if set,
the semantic version of CN will be default to the image tag,
if the semantic version is not set, nor the image tag is a valid semantic version,
operator will treat the MO as unknown version and will not apply any version-specific
reconciliations | | | -| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | | | +| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | 1.2.0 | | #### PoolScaleStrategy @@ -1370,7 +1370,7 @@ _Appears in:_ | `exportToPrometheus` _boolean_ | ExportToPrometheus enables the pod to be discovered scraped by Prometheus | | | | `promDiscoveryScheme` _[PromDiscoveryScheme](#promdiscoveryscheme)_ | PromDiscoveryScheme indicates how the Pod will be discovered by prometheus, options:
- Pod: the pod will be discovered via will-known labels on the Pod
- Service: the pod will be discovered via will-known annotations in the service which expose endpoints to the pods
default to Service | | | | `semanticVersion` _string_ | SemanticVersion override the semantic version of CN if set,
the semantic version of CN will be default to the image tag,
if the semantic version is not set, nor the image tag is a valid semantic version,
operator will treat the MO as unknown version and will not apply any version-specific
reconciliations | | | -| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | | | +| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | 1.2.0 | | | `serviceType` _[ServiceType](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#servicetype-v1-core)_ | ServiceType is the service type of proxy service | ClusterIP | Enum: [ClusterIP NodePort LoadBalancer]
| | `serviceAnnotations` _object (keys:string, values:string)_ | ServiceAnnotations are the annotations for the proxy service | | | | `nodePort` _integer_ | NodePort specifies the node port to use when ServiceType is NodePort or LoadBalancer,
reconciling will fail if the node port is not available. | | | @@ -1713,7 +1713,7 @@ _Appears in:_ | `exportToPrometheus` _boolean_ | ExportToPrometheus enables the pod to be discovered scraped by Prometheus | | | | `promDiscoveryScheme` _[PromDiscoveryScheme](#promdiscoveryscheme)_ | PromDiscoveryScheme indicates how the Pod will be discovered by prometheus, options:
- Pod: the pod will be discovered via will-known labels on the Pod
- Service: the pod will be discovered via will-known annotations in the service which expose endpoints to the pods
default to Service | | | | `semanticVersion` _string_ | SemanticVersion override the semantic version of CN if set,
the semantic version of CN will be default to the image tag,
if the semantic version is not set, nor the image tag is a valid semantic version,
operator will treat the MO as unknown version and will not apply any version-specific
reconciliations | | | -| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | | | +| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | 1.2.0 | | | `serviceType` _[ServiceType](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#servicetype-v1-core)_ | ServiceType is the service type of cn service | ClusterIP | Enum: [ClusterIP NodePort LoadBalancer]
| | `updateStrategy` _[RollingUpdateStrategy](#rollingupdatestrategy)_ | UpdateStrategy rolling update strategy | | | | `imagePullPolicy` _[PullPolicy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#pullpolicy-v1-core)_ | | | | diff --git a/pkg/controllers/cnset/controller.go b/pkg/controllers/cnset/controller.go index b7cde7dd..d98b6ca6 100644 --- a/pkg/controllers/cnset/controller.go +++ b/pkg/controllers/cnset/controller.go @@ -21,6 +21,7 @@ import ( "github.com/openkruise/kruise-api/apps/pub" kruisev1alpha1 "github.com/openkruise/kruise-api/apps/v1alpha1" "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/predicate" "time" "github.com/go-errors/errors" @@ -278,6 +279,14 @@ func (c *Actor) Reconcile(mgr manager.Manager) error { recon.WithBuildFn(func(b *builder.Builder) { b.Owns(&kruisev1alpha1.CloneSet{}). Owns(&corev1.Service{}) + b.WithEventFilter(predicate.NewPredicateFuncs(func(obj client.Object) bool { + set, ok := obj.(*v1alpha1.CNSet) + if ok && !set.Spec.GetOperatorVersion().Equals(v1alpha1.LatestOpVersion) { + // only filter CNSet + return false + } + return true + })) })) if err != nil { return err diff --git a/pkg/controllers/dnset/controller.go b/pkg/controllers/dnset/controller.go index 2fc5a388..1d322659 100644 --- a/pkg/controllers/dnset/controller.go +++ b/pkg/controllers/dnset/controller.go @@ -17,6 +17,7 @@ package dnset import ( "github.com/matrixorigin/matrixone-operator/api/features" "github.com/matrixorigin/matrixone-operator/pkg/utils" + "sigs.k8s.io/controller-runtime/pkg/predicate" "strconv" "time" @@ -255,6 +256,14 @@ func (d *Actor) Reconcile(mgr manager.Manager) error { recon.WithBuildFn(func(b *builder.Builder) { b.Owns(&kruise.StatefulSet{}). Owns(&corev1.Service{}) + b.WithEventFilter(predicate.NewPredicateFuncs(func(obj client.Object) bool { + set, ok := obj.(*v1alpha1.DNSet) + if ok && !set.Spec.GetOperatorVersion().Equals(v1alpha1.LatestOpVersion) { + // only filter DNSet + return false + } + return true + })) })) if err != nil { return err diff --git a/pkg/controllers/logset/controller.go b/pkg/controllers/logset/controller.go index 36ac4917..538dbbfd 100644 --- a/pkg/controllers/logset/controller.go +++ b/pkg/controllers/logset/controller.go @@ -15,6 +15,7 @@ package logset import ( + "sigs.k8s.io/controller-runtime/pkg/predicate" "strconv" "time" @@ -384,5 +385,13 @@ func (r *Actor) Reconcile(mgr manager.Manager) error { // watch all changes on the owned statefulset since we need perform failover if there is a pod failure b.Owns(&kruisev1.StatefulSet{}). Owns(&corev1.Service{}) + b.WithEventFilter(predicate.NewPredicateFuncs(func(obj client.Object) bool { + set, ok := obj.(*v1alpha1.LogSet) + if ok && !set.Spec.GetOperatorVersion().Equals(v1alpha1.LatestOpVersion) { + // only filter LogSet + return false + } + return true + })) })) } diff --git a/pkg/controllers/proxyset/controller.go b/pkg/controllers/proxyset/controller.go index b545bdf8..4f21dd75 100644 --- a/pkg/controllers/proxyset/controller.go +++ b/pkg/controllers/proxyset/controller.go @@ -27,6 +27,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" ) const ( @@ -101,5 +102,13 @@ func (r *Actor) Reconcile(mgr manager.Manager) error { return recon.Setup[*v1alpha1.ProxySet](&v1alpha1.ProxySet{}, "proxyset", mgr, r, recon.WithBuildFn(func(b *builder.Builder) { b.Owns(&kruisev1alpha1.CloneSet{}) + b.WithEventFilter(predicate.NewPredicateFuncs(func(obj client.Object) bool { + set, ok := obj.(*v1alpha1.ProxySet) + if ok && !set.Spec.GetOperatorVersion().Equals(v1alpha1.LatestOpVersion) { + // only filter ProxySet + return false + } + return true + })) })) } diff --git a/pkg/webhook/proxy_webhook.go b/pkg/webhook/proxy_webhook.go index 9fb94adf..ddb034aa 100644 --- a/pkg/webhook/proxy_webhook.go +++ b/pkg/webhook/proxy_webhook.go @@ -16,6 +16,7 @@ package webhook import ( "context" + "k8s.io/utils/pointer" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" @@ -53,6 +54,7 @@ func (p *proxySetDefaulter) Default(_ context.Context, obj runtime.Object) error func (p *proxySetDefaulter) DefaultSpec(spec *v1alpha1.ProxySetSpec) { setDefaultServiceArgs(spec) + spec.OperatorVersion = pointer.String(v1alpha1.LatestOpVersion.String()) } // +kubebuilder:webhook:path=/validate-core-matrixorigin-io-v1alpha1-proxyset,mutating=false,failurePolicy=fail,sideEffects=None,groups=core.matrixorigin.io,resources=proxysets,verbs=create;update,versions=v1alpha1,name=vproxyset.kb.io,admissionReviewVersions={v1,v1beta1} diff --git a/pkg/webhook/utils.go b/pkg/webhook/utils.go index eedc2e47..f7b9674c 100644 --- a/pkg/webhook/utils.go +++ b/pkg/webhook/utils.go @@ -64,7 +64,7 @@ func setPodSetDefaults(s *v1alpha1.PodSet) { s.Overlay.Env = appendIfNotExist(s.Overlay.Env, corev1.EnvVar{Name: v1alpha1.EnvGoDebug, Value: v1alpha1.DefaultGODebug}, func(v corev1.EnvVar) string { return v.Name }) - + s.OperatorVersion = pointer.String(v1alpha1.LatestOpVersion.String()) if s.ExportToPrometheus != nil && *s.ExportToPrometheus { if s.PromDiscoveryScheme == nil { s.PromDiscoveryScheme = (*v1alpha1.PromDiscoveryScheme)(pointer.String(string(v1alpha1.PromDiscoverySchemeService))) From be4796cf293855a38eb464140db0a43c1ed5b233 Mon Sep 17 00:00:00 2001 From: cyberchen98 Date: Wed, 16 Oct 2024 14:01:06 +0800 Subject: [PATCH 07/11] fix: reconcile diffrent code path by OperatorVersion --- api/core/v1alpha1/common_helpers.go | 5 ++- api/core/v1alpha1/common_types.go | 1 - .../crds/core.matrixorigin.io_cnpools.yaml | 1 - .../crds/core.matrixorigin.io_cnsets.yaml | 1 - .../crds/core.matrixorigin.io_dnsets.yaml | 1 - .../crds/core.matrixorigin.io_logsets.yaml | 1 - ...ore.matrixorigin.io_matrixoneclusters.yaml | 8 ---- .../crds/core.matrixorigin.io_proxysets.yaml | 1 - .../crds/core.matrixorigin.io_webuis.yaml | 1 - deploy/crds/core.matrixorigin.io_cnpools.yaml | 1 - deploy/crds/core.matrixorigin.io_cnsets.yaml | 1 - deploy/crds/core.matrixorigin.io_dnsets.yaml | 1 - deploy/crds/core.matrixorigin.io_logsets.yaml | 1 - ...ore.matrixorigin.io_matrixoneclusters.yaml | 8 ---- .../crds/core.matrixorigin.io_proxysets.yaml | 1 - deploy/crds/core.matrixorigin.io_webuis.yaml | 1 - docs/reference/api-reference.md | 14 +++--- pkg/controllers/cnset/controller.go | 15 ++----- pkg/controllers/cnset/resource.go | 6 ++- pkg/controllers/common/configmap.go | 43 ++++++++++++++++++- pkg/controllers/common/podset.go | 6 ++- pkg/controllers/dnset/controller.go | 15 ++----- pkg/controllers/dnset/resource.go | 12 ++++-- pkg/controllers/logset/configmap.go | 12 +++++- pkg/controllers/logset/controller.go | 21 ++++----- pkg/controllers/proxyset/controller.go | 9 ---- pkg/controllers/proxyset/resource.go | 7 ++- pkg/controllers/webui/controller.go | 2 +- pkg/controllers/webui/resource.go | 2 +- pkg/webhook/cnset_webhook.go | 4 +- pkg/webhook/dnset_webhook.go | 4 +- pkg/webhook/logset_webhook.go | 4 +- pkg/webhook/matrixonecluster_webhook.go | 4 +- pkg/webhook/proxy_webhook.go | 4 +- pkg/webhook/utils.go | 18 +++++++- 35 files changed, 129 insertions(+), 107 deletions(-) diff --git a/api/core/v1alpha1/common_helpers.go b/api/core/v1alpha1/common_helpers.go index c5b09065..76171615 100644 --- a/api/core/v1alpha1/common_helpers.go +++ b/api/core/v1alpha1/common_helpers.go @@ -34,6 +34,7 @@ const ( ) var ( + LegacyOpVersion = semver.MustParse("1.2.0") LatestOpVersion = semver.MustParse("1.3.0") ) @@ -300,11 +301,11 @@ func (p *PodSet) GetSemVer() (*semver.Version, bool) { func (p *PodSet) GetOperatorVersion() semver.Version { if p.OperatorVersion == nil { - return LatestOpVersion + return LegacyOpVersion } v, err := semver.ParseTolerant(*p.OperatorVersion) if err != nil { - return LatestOpVersion + return LegacyOpVersion } return v } diff --git a/api/core/v1alpha1/common_types.go b/api/core/v1alpha1/common_types.go index ff9a49c1..99f23eb2 100644 --- a/api/core/v1alpha1/common_types.go +++ b/api/core/v1alpha1/common_types.go @@ -116,7 +116,6 @@ type PodSet struct { // OperatorVersion is the controller version of mo-operator that should be used to // reconcile this set - // +kubebuilder:default="1.2.0" OperatorVersion *string `json:"operatorVersion,omitempty"` } diff --git a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnpools.yaml b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnpools.yaml index 55588340..a760f13b 100644 --- a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnpools.yaml +++ b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnpools.yaml @@ -173,7 +173,6 @@ spec: type: string type: object operatorVersion: - default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set diff --git a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnsets.yaml b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnsets.yaml index bfe5f998..929b03d0 100644 --- a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnsets.yaml +++ b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_cnsets.yaml @@ -147,7 +147,6 @@ spec: type: string type: object operatorVersion: - default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set diff --git a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_dnsets.yaml b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_dnsets.yaml index 86f6c6d8..07927bea 100644 --- a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_dnsets.yaml +++ b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_dnsets.yaml @@ -122,7 +122,6 @@ spec: type: string type: object operatorVersion: - default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set diff --git a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_logsets.yaml b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_logsets.yaml index a48ec515..141f5674 100644 --- a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_logsets.yaml +++ b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_logsets.yaml @@ -113,7 +113,6 @@ spec: type: string type: object operatorVersion: - default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set diff --git a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_matrixoneclusters.yaml b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_matrixoneclusters.yaml index 9c86eb35..d12a1686 100644 --- a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_matrixoneclusters.yaml +++ b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_matrixoneclusters.yaml @@ -139,7 +139,6 @@ spec: type: string type: object operatorVersion: - default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set @@ -498,7 +497,6 @@ spec: type: string type: object operatorVersion: - default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set @@ -840,7 +838,6 @@ spec: type: string type: object operatorVersion: - default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set @@ -1045,7 +1042,6 @@ spec: type: string type: object operatorVersion: - default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set @@ -1321,7 +1317,6 @@ spec: type: string type: object operatorVersion: - default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set @@ -1503,7 +1498,6 @@ spec: type: string type: object operatorVersion: - default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set @@ -1719,7 +1713,6 @@ spec: type: string type: object operatorVersion: - default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set @@ -2036,7 +2029,6 @@ spec: type: string type: object operatorVersion: - default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set diff --git a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_proxysets.yaml b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_proxysets.yaml index 85b3a729..c104d427 100644 --- a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_proxysets.yaml +++ b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_proxysets.yaml @@ -107,7 +107,6 @@ spec: type: string type: object operatorVersion: - default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set diff --git a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_webuis.yaml b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_webuis.yaml index ac5a289b..36643764 100644 --- a/charts/matrixone-operator/templates/crds/core.matrixorigin.io_webuis.yaml +++ b/charts/matrixone-operator/templates/crds/core.matrixorigin.io_webuis.yaml @@ -91,7 +91,6 @@ spec: type: string type: object operatorVersion: - default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set diff --git a/deploy/crds/core.matrixorigin.io_cnpools.yaml b/deploy/crds/core.matrixorigin.io_cnpools.yaml index 55588340..a760f13b 100644 --- a/deploy/crds/core.matrixorigin.io_cnpools.yaml +++ b/deploy/crds/core.matrixorigin.io_cnpools.yaml @@ -173,7 +173,6 @@ spec: type: string type: object operatorVersion: - default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set diff --git a/deploy/crds/core.matrixorigin.io_cnsets.yaml b/deploy/crds/core.matrixorigin.io_cnsets.yaml index bfe5f998..929b03d0 100644 --- a/deploy/crds/core.matrixorigin.io_cnsets.yaml +++ b/deploy/crds/core.matrixorigin.io_cnsets.yaml @@ -147,7 +147,6 @@ spec: type: string type: object operatorVersion: - default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set diff --git a/deploy/crds/core.matrixorigin.io_dnsets.yaml b/deploy/crds/core.matrixorigin.io_dnsets.yaml index 86f6c6d8..07927bea 100644 --- a/deploy/crds/core.matrixorigin.io_dnsets.yaml +++ b/deploy/crds/core.matrixorigin.io_dnsets.yaml @@ -122,7 +122,6 @@ spec: type: string type: object operatorVersion: - default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set diff --git a/deploy/crds/core.matrixorigin.io_logsets.yaml b/deploy/crds/core.matrixorigin.io_logsets.yaml index a48ec515..141f5674 100644 --- a/deploy/crds/core.matrixorigin.io_logsets.yaml +++ b/deploy/crds/core.matrixorigin.io_logsets.yaml @@ -113,7 +113,6 @@ spec: type: string type: object operatorVersion: - default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set diff --git a/deploy/crds/core.matrixorigin.io_matrixoneclusters.yaml b/deploy/crds/core.matrixorigin.io_matrixoneclusters.yaml index 9c86eb35..d12a1686 100644 --- a/deploy/crds/core.matrixorigin.io_matrixoneclusters.yaml +++ b/deploy/crds/core.matrixorigin.io_matrixoneclusters.yaml @@ -139,7 +139,6 @@ spec: type: string type: object operatorVersion: - default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set @@ -498,7 +497,6 @@ spec: type: string type: object operatorVersion: - default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set @@ -840,7 +838,6 @@ spec: type: string type: object operatorVersion: - default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set @@ -1045,7 +1042,6 @@ spec: type: string type: object operatorVersion: - default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set @@ -1321,7 +1317,6 @@ spec: type: string type: object operatorVersion: - default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set @@ -1503,7 +1498,6 @@ spec: type: string type: object operatorVersion: - default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set @@ -1719,7 +1713,6 @@ spec: type: string type: object operatorVersion: - default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set @@ -2036,7 +2029,6 @@ spec: type: string type: object operatorVersion: - default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set diff --git a/deploy/crds/core.matrixorigin.io_proxysets.yaml b/deploy/crds/core.matrixorigin.io_proxysets.yaml index 85b3a729..c104d427 100644 --- a/deploy/crds/core.matrixorigin.io_proxysets.yaml +++ b/deploy/crds/core.matrixorigin.io_proxysets.yaml @@ -107,7 +107,6 @@ spec: type: string type: object operatorVersion: - default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set diff --git a/deploy/crds/core.matrixorigin.io_webuis.yaml b/deploy/crds/core.matrixorigin.io_webuis.yaml index ac5a289b..36643764 100644 --- a/deploy/crds/core.matrixorigin.io_webuis.yaml +++ b/deploy/crds/core.matrixorigin.io_webuis.yaml @@ -91,7 +91,6 @@ spec: type: string type: object operatorVersion: - default: 1.2.0 description: |- OperatorVersion is the controller version of mo-operator that should be used to reconcile this set diff --git a/docs/reference/api-reference.md b/docs/reference/api-reference.md index f52386e7..155e4d8d 100644 --- a/docs/reference/api-reference.md +++ b/docs/reference/api-reference.md @@ -421,7 +421,7 @@ _Appears in:_ | `exportToPrometheus` _boolean_ | ExportToPrometheus enables the pod to be discovered scraped by Prometheus | | | | `promDiscoveryScheme` _[PromDiscoveryScheme](#promdiscoveryscheme)_ | PromDiscoveryScheme indicates how the Pod will be discovered by prometheus, options:
- Pod: the pod will be discovered via will-known labels on the Pod
- Service: the pod will be discovered via will-known annotations in the service which expose endpoints to the pods
default to Service | | | | `semanticVersion` _string_ | SemanticVersion override the semantic version of CN if set,
the semantic version of CN will be default to the image tag,
if the semantic version is not set, nor the image tag is a valid semantic version,
operator will treat the MO as unknown version and will not apply any version-specific
reconciliations | | | -| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | 1.2.0 | | +| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | | | | `cacheVolume` _[Volume](#volume)_ | CacheVolume is the desired local cache volume for CNSet,
node storage will be used if not specified | | | | `sharedStorageCache` _[SharedStorageCache](#sharedstoragecache)_ | SharedStorageCache is the configuration of the S3 sharedStorageCache | | | | `pythonUdfSidecar` _[PythonUdfSidecar](#pythonudfsidecar)_ | PythonUdfSidecar is the python udf server in CN | | | @@ -622,7 +622,7 @@ _Appears in:_ | `exportToPrometheus` _boolean_ | ExportToPrometheus enables the pod to be discovered scraped by Prometheus | | | | `promDiscoveryScheme` _[PromDiscoveryScheme](#promdiscoveryscheme)_ | PromDiscoveryScheme indicates how the Pod will be discovered by prometheus, options:
- Pod: the pod will be discovered via will-known labels on the Pod
- Service: the pod will be discovered via will-known annotations in the service which expose endpoints to the pods
default to Service | | | | `semanticVersion` _string_ | SemanticVersion override the semantic version of CN if set,
the semantic version of CN will be default to the image tag,
if the semantic version is not set, nor the image tag is a valid semantic version,
operator will treat the MO as unknown version and will not apply any version-specific
reconciliations | | | -| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | 1.2.0 | | +| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | | | | `cacheVolume` _[Volume](#volume)_ | CacheVolume is the desired local cache volume for CNSet,
node storage will be used if not specified | | | | `sharedStorageCache` _[SharedStorageCache](#sharedstoragecache)_ | SharedStorageCache is the configuration of the S3 sharedStorageCache | | | | `pythonUdfSidecar` _[PythonUdfSidecar](#pythonudfsidecar)_ | PythonUdfSidecar is the python udf server in CN | | | @@ -819,7 +819,7 @@ _Appears in:_ | `exportToPrometheus` _boolean_ | ExportToPrometheus enables the pod to be discovered scraped by Prometheus | | | | `promDiscoveryScheme` _[PromDiscoveryScheme](#promdiscoveryscheme)_ | PromDiscoveryScheme indicates how the Pod will be discovered by prometheus, options:
- Pod: the pod will be discovered via will-known labels on the Pod
- Service: the pod will be discovered via will-known annotations in the service which expose endpoints to the pods
default to Service | | | | `semanticVersion` _string_ | SemanticVersion override the semantic version of CN if set,
the semantic version of CN will be default to the image tag,
if the semantic version is not set, nor the image tag is a valid semantic version,
operator will treat the MO as unknown version and will not apply any version-specific
reconciliations | | | -| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | 1.2.0 | | +| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | | | | `cacheVolume` _[Volume](#volume)_ | CacheVolume is the desired local cache volume for DNSet,
node storage will be used if not specified | | | | `sharedStorageCache` _[SharedStorageCache](#sharedstoragecache)_ | | | | @@ -986,7 +986,7 @@ _Appears in:_ | `exportToPrometheus` _boolean_ | ExportToPrometheus enables the pod to be discovered scraped by Prometheus | | | | `promDiscoveryScheme` _[PromDiscoveryScheme](#promdiscoveryscheme)_ | PromDiscoveryScheme indicates how the Pod will be discovered by prometheus, options:
- Pod: the pod will be discovered via will-known labels on the Pod
- Service: the pod will be discovered via will-known annotations in the service which expose endpoints to the pods
default to Service | | | | `semanticVersion` _string_ | SemanticVersion override the semantic version of CN if set,
the semantic version of CN will be default to the image tag,
if the semantic version is not set, nor the image tag is a valid semantic version,
operator will treat the MO as unknown version and will not apply any version-specific
reconciliations | | | -| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | 1.2.0 | | +| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | | | | `volume` _[Volume](#volume)_ | Volume is the local persistent volume for each LogService instance | | | | `sharedStorage` _[SharedStorageProvider](#sharedstorageprovider)_ | SharedStorage is an external shared storage shared by all LogService instances | | | | `initialConfig` _[InitialConfig](#initialconfig)_ | InitialConfig is the initial configuration of HAKeeper
InitialConfig is immutable | | | @@ -1215,7 +1215,7 @@ _Appears in:_ | `exportToPrometheus` _boolean_ | ExportToPrometheus enables the pod to be discovered scraped by Prometheus | | | | `promDiscoveryScheme` _[PromDiscoveryScheme](#promdiscoveryscheme)_ | PromDiscoveryScheme indicates how the Pod will be discovered by prometheus, options:
- Pod: the pod will be discovered via will-known labels on the Pod
- Service: the pod will be discovered via will-known annotations in the service which expose endpoints to the pods
default to Service | | | | `semanticVersion` _string_ | SemanticVersion override the semantic version of CN if set,
the semantic version of CN will be default to the image tag,
if the semantic version is not set, nor the image tag is a valid semantic version,
operator will treat the MO as unknown version and will not apply any version-specific
reconciliations | | | -| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | 1.2.0 | | +| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | | | #### PoolScaleStrategy @@ -1370,7 +1370,7 @@ _Appears in:_ | `exportToPrometheus` _boolean_ | ExportToPrometheus enables the pod to be discovered scraped by Prometheus | | | | `promDiscoveryScheme` _[PromDiscoveryScheme](#promdiscoveryscheme)_ | PromDiscoveryScheme indicates how the Pod will be discovered by prometheus, options:
- Pod: the pod will be discovered via will-known labels on the Pod
- Service: the pod will be discovered via will-known annotations in the service which expose endpoints to the pods
default to Service | | | | `semanticVersion` _string_ | SemanticVersion override the semantic version of CN if set,
the semantic version of CN will be default to the image tag,
if the semantic version is not set, nor the image tag is a valid semantic version,
operator will treat the MO as unknown version and will not apply any version-specific
reconciliations | | | -| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | 1.2.0 | | +| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | | | | `serviceType` _[ServiceType](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#servicetype-v1-core)_ | ServiceType is the service type of proxy service | ClusterIP | Enum: [ClusterIP NodePort LoadBalancer]
| | `serviceAnnotations` _object (keys:string, values:string)_ | ServiceAnnotations are the annotations for the proxy service | | | | `nodePort` _integer_ | NodePort specifies the node port to use when ServiceType is NodePort or LoadBalancer,
reconciling will fail if the node port is not available. | | | @@ -1713,7 +1713,7 @@ _Appears in:_ | `exportToPrometheus` _boolean_ | ExportToPrometheus enables the pod to be discovered scraped by Prometheus | | | | `promDiscoveryScheme` _[PromDiscoveryScheme](#promdiscoveryscheme)_ | PromDiscoveryScheme indicates how the Pod will be discovered by prometheus, options:
- Pod: the pod will be discovered via will-known labels on the Pod
- Service: the pod will be discovered via will-known annotations in the service which expose endpoints to the pods
default to Service | | | | `semanticVersion` _string_ | SemanticVersion override the semantic version of CN if set,
the semantic version of CN will be default to the image tag,
if the semantic version is not set, nor the image tag is a valid semantic version,
operator will treat the MO as unknown version and will not apply any version-specific
reconciliations | | | -| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | 1.2.0 | | +| `operatorVersion` _string_ | OperatorVersion is the controller version of mo-operator that should be used to
reconcile this set | | | | `serviceType` _[ServiceType](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#servicetype-v1-core)_ | ServiceType is the service type of cn service | ClusterIP | Enum: [ClusterIP NodePort LoadBalancer]
| | `updateStrategy` _[RollingUpdateStrategy](#rollingupdatestrategy)_ | UpdateStrategy rolling update strategy | | | | `imagePullPolicy` _[PullPolicy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#pullpolicy-v1-core)_ | | | | diff --git a/pkg/controllers/cnset/controller.go b/pkg/controllers/cnset/controller.go index d98b6ca6..fd84669d 100644 --- a/pkg/controllers/cnset/controller.go +++ b/pkg/controllers/cnset/controller.go @@ -21,7 +21,6 @@ import ( "github.com/openkruise/kruise-api/apps/pub" kruisev1alpha1 "github.com/openkruise/kruise-api/apps/v1alpha1" "k8s.io/utils/pointer" - "sigs.k8s.io/controller-runtime/pkg/predicate" "time" "github.com/go-errors/errors" @@ -279,14 +278,6 @@ func (c *Actor) Reconcile(mgr manager.Manager) error { recon.WithBuildFn(func(b *builder.Builder) { b.Owns(&kruisev1alpha1.CloneSet{}). Owns(&corev1.Service{}) - b.WithEventFilter(predicate.NewPredicateFuncs(func(obj client.Object) bool { - set, ok := obj.(*v1alpha1.CNSet) - if ok && !set.Spec.GetOperatorVersion().Equals(v1alpha1.LatestOpVersion) { - // only filter CNSet - return false - } - return true - })) })) if err != nil { return err @@ -337,8 +328,10 @@ func syncCloneSet(ctx *recon.Context[*v1alpha1.CNSet], cs *kruisev1alpha1.CloneS if err != nil { return err } - cs.Spec.Template.Annotations[common.ConfigSuffixAnno] = configSuffix - return common.SyncConfigMap(ctx, &cs.Spec.Template.Spec, cm) + if cn.Spec.GetOperatorVersion().Equals(v1alpha1.LatestOpVersion) { + cs.Spec.Template.Annotations[common.ConfigSuffixAnno] = configSuffix + } + return common.SyncConfigMap(ctx, &cs.Spec.Template.Spec, cm, cn.Spec.GetOperatorVersion()) } func setReady(cn *v1alpha1.CNSet) { diff --git a/pkg/controllers/cnset/resource.go b/pkg/controllers/cnset/resource.go index 5ec7db1e..4a7aa6cb 100644 --- a/pkg/controllers/cnset/resource.go +++ b/pkg/controllers/cnset/resource.go @@ -54,7 +54,11 @@ sql-address = "${POD_IP}:{{ .CNSQLPort }}" service-host = "${POD_IP}" EOF # build instance config -sed "/\[cn\]/r ${bc}" {{ .ConfigFilePath }}-${CONFIG_SUFFIX} > ${conf} +if [ -n "${CONFIG_SUFFIX}" ]; then + sed "/\[cn\]/r ${bc}" {{ .ConfigFilePath }}-${CONFIG_SUFFIX} > ${conf} +else + sed "/\[cn\]/r ${bc}" {{ .ConfigFilePath }} > ${conf} +fi # append lock-service configs lsc=$(mktemp) diff --git a/pkg/controllers/common/configmap.go b/pkg/controllers/common/configmap.go index 8b4a1190..f06a3795 100644 --- a/pkg/controllers/common/configmap.go +++ b/pkg/controllers/common/configmap.go @@ -15,12 +15,16 @@ package common import ( + "encoding/json" "fmt" + "github.com/blang/semver/v4" "github.com/cespare/xxhash" "github.com/go-errors/errors" recon "github.com/matrixorigin/controller-runtime/pkg/reconciler" "github.com/matrixorigin/controller-runtime/pkg/util" + "github.com/matrixorigin/matrixone-operator/api/core/v1alpha1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" "strings" ) @@ -38,9 +42,15 @@ const ( // SyncConfigMap syncs the desired configmap for pods, which will cause rolling-update if the // data of the configmap is changed -func SyncConfigMap(kubeCli recon.KubeClient, podSpec *corev1.PodSpec, cm *corev1.ConfigMap) error { +func SyncConfigMap(kubeCli recon.KubeClient, podSpec *corev1.PodSpec, cm *corev1.ConfigMap, operatorVersion semver.Version) error { vp := util.FindFirst(podSpec.Volumes, util.WithVolumeName("config")) - desiredName, err := ensureConfigMap(kubeCli, cm) + var desiredName string + var err error + if operatorVersion.Equals(v1alpha1.LatestOpVersion) { + desiredName, err = ensureConfigMap(kubeCli, cm) + } else { + desiredName, err = ensureConfigMapLegacy(kubeCli, vp.ConfigMap.Name, cm) + } if err != nil { return err } @@ -90,6 +100,24 @@ func ensureConfigMap(kubeCli recon.KubeClient, desired *corev1.ConfigMap) (strin return c.Name, nil } +// Deprecated: use ensureConfigMap instead +func ensureConfigMapLegacy(kubeCli recon.KubeClient, currentCm string, desired *corev1.ConfigMap) (string, error) { + c := desired.DeepCopy() + if err := addConfigMapDigest(c); err != nil { + return "", err + } + // config digest not changed + if c.Name == currentCm { + return currentCm, nil + } + // otherwise ensure the configmap exists + err := util.Ignore(apierrors.IsAlreadyExists, kubeCli.CreateOwned(c)) + if err != nil { + return "", err + } + return c.Name, nil +} + func withDigest(key string, v string) bool { return strings.Contains(key, DataDigest([]byte(v))) } @@ -108,3 +136,14 @@ func DataDigest(data []byte) string { sum := xxhash.Sum64(data) return fmt.Sprintf("%x", sum)[0:7] } + +func addConfigMapDigest(cm *corev1.ConfigMap) error { + s, err := json.Marshal(cm.Data) + if err != nil { + return err + } + sum := xxhash.Sum64(s) + suffix := fmt.Sprintf("%x", sum)[0:7] + cm.Name = fmt.Sprintf("%s-%s", cm.Name, suffix) + return nil +} diff --git a/pkg/controllers/common/podset.go b/pkg/controllers/common/podset.go index d7857077..b9504ee6 100644 --- a/pkg/controllers/common/podset.go +++ b/pkg/controllers/common/podset.go @@ -74,7 +74,7 @@ func GetSemanticVersion(meta *metav1.ObjectMeta) semver.Version { // SyncMOPod execute the given SyncMOPodTask which keeps the pod spec update to date func SyncMOPod(t *SyncMOPodTask) error { syncPodTemplate(t) - if err := SyncConfigMap(t.KubeCli, &t.TargetTemplate.Spec, t.ConfigMap); err != nil { + if err := SyncConfigMap(t.KubeCli, &t.TargetTemplate.Spec, t.ConfigMap, t.PodSet.GetOperatorVersion()); err != nil { return errors.WrapPrefix(err, "sync configmap", 0) } return nil @@ -106,7 +106,9 @@ func syncPodTemplate(t *SyncMOPodTask) { if t.MutatePod != nil { t.MutatePod(t.TargetTemplate) } - t.TargetTemplate.ObjectMeta.Annotations[ConfigSuffixAnno] = t.ConfigSuffix + if t.PodSet.GetOperatorVersion().Equals(v1alpha1.LatestOpVersion) { + t.TargetTemplate.ObjectMeta.Annotations[ConfigSuffixAnno] = t.ConfigSuffix + } p.Overlay.OverlayPodMeta(&t.TargetTemplate.ObjectMeta) p.Overlay.OverlayPodSpec(specRef) diff --git a/pkg/controllers/dnset/controller.go b/pkg/controllers/dnset/controller.go index 1d322659..fea66f99 100644 --- a/pkg/controllers/dnset/controller.go +++ b/pkg/controllers/dnset/controller.go @@ -17,7 +17,6 @@ package dnset import ( "github.com/matrixorigin/matrixone-operator/api/features" "github.com/matrixorigin/matrixone-operator/pkg/utils" - "sigs.k8s.io/controller-runtime/pkg/predicate" "strconv" "time" @@ -191,8 +190,10 @@ func (d *Actor) Create(ctx *recon.Context[*v1alpha1.DNSet]) error { if err != nil { return err } - dnSet.Spec.Template.Annotations[common.ConfigSuffixAnno] = configSuffix - if err := common.SyncConfigMap(ctx, &dnSet.Spec.Template.Spec, configMap); err != nil { + if dn.Spec.GetOperatorVersion().Equals(v1alpha1.LatestOpVersion) { + dnSet.Spec.Template.Annotations[common.ConfigSuffixAnno] = configSuffix + } + if err := common.SyncConfigMap(ctx, &dnSet.Spec.Template.Spec, configMap, dn.Spec.GetOperatorVersion()); err != nil { return err } @@ -256,14 +257,6 @@ func (d *Actor) Reconcile(mgr manager.Manager) error { recon.WithBuildFn(func(b *builder.Builder) { b.Owns(&kruise.StatefulSet{}). Owns(&corev1.Service{}) - b.WithEventFilter(predicate.NewPredicateFuncs(func(obj client.Object) bool { - set, ok := obj.(*v1alpha1.DNSet) - if ok && !set.Spec.GetOperatorVersion().Equals(v1alpha1.LatestOpVersion) { - // only filter DNSet - return false - } - return true - })) })) if err != nil { return err diff --git a/pkg/controllers/dnset/resource.go b/pkg/controllers/dnset/resource.go index 0daa0413..9fe92f44 100644 --- a/pkg/controllers/dnset/resource.go +++ b/pkg/controllers/dnset/resource.go @@ -62,7 +62,11 @@ service-address = "${ADDR}:{{ .DNServicePort }}" service-host = "${ADDR}" EOF # build instance config -sed "/\[{{ .ConfigAlias }}\]/r ${bc}" {{ .ConfigFilePath }}-${CONFIG_SUFFIX} > ${conf} +if [ -n "${CONFIG_SUFFIX}" ]; then + sed "/\[{{ .ConfigAlias }}\]/r ${bc}" {{ .ConfigFilePath }}-${CONFIG_SUFFIX} > ${conf} +else + sed "/\[{{ .ConfigAlias }}\]/r ${bc}" {{ .ConfigFilePath }} > ${conf} +fi # append lock-service configs lsc=$(mktemp) @@ -257,11 +261,13 @@ func syncPods(ctx *recon.Context[*v1alpha1.DNSet], sts *kruise.StatefulSet) erro return err } syncPodMeta(ctx.Obj, sts) - sts.Spec.Template.Annotations[common.ConfigSuffixAnno] = configSuffix + if ctx.Obj.Spec.GetOperatorVersion().Equals(v1alpha1.LatestOpVersion) { + sts.Spec.Template.Annotations[common.ConfigSuffixAnno] = configSuffix + } if ctx.Dep != nil { syncPodSpec(ctx.Obj, sts, ctx.Dep.Deps.LogSet.Spec.SharedStorage) } - return common.SyncConfigMap(ctx, &sts.Spec.Template.Spec, cm) + return common.SyncConfigMap(ctx, &sts.Spec.Template.Spec, cm, ctx.Obj.Spec.GetOperatorVersion()) } diff --git a/pkg/controllers/logset/configmap.go b/pkg/controllers/logset/configmap.go index a1381254..7d34cb7a 100644 --- a/pkg/controllers/logset/configmap.go +++ b/pkg/controllers/logset/configmap.go @@ -68,7 +68,11 @@ gossip-address-v2 = "${ADDR}:{{ .GossipPort }}" EOF # build instance config -sed "/\[logservice\]/r ${bc}" {{ .ConfigFilePath }}-${CONFIG_SUFFIX} > ${conf} +if [ -n "${CONFIG_SUFFIX}" ]; then + sed "/\[logservice\]/r ${bc}" {{ .ConfigFilePath }}-${CONFIG_SUFFIX} > ${conf} +else + sed "/\[logservice\]/r ${bc}" {{ .ConfigFilePath }} > ${conf} +fi # insert gossip config gossipTmp=$(mktemp) @@ -124,7 +128,11 @@ gossip-port = {{ .GossipPort }} EOF # build instance config -sed "/\[logservice\]/r ${bc}" {{ .ConfigFilePath }}-${CONFIG_SUFFIX} > ${conf} +if [ -n "${CONFIG_SUFFIX}" ]; then + sed "/\[logservice\]/r ${bc}" {{ .ConfigFilePath }}-${CONFIG_SUFFIX} > ${conf} +else + sed "/\[logservice\]/r ${bc}" {{ .ConfigFilePath }} > ${conf} +fi # insert gossip config gossipTmp=$(mktemp) diff --git a/pkg/controllers/logset/controller.go b/pkg/controllers/logset/controller.go index 538dbbfd..17b41d7a 100644 --- a/pkg/controllers/logset/controller.go +++ b/pkg/controllers/logset/controller.go @@ -15,7 +15,6 @@ package logset import ( - "sigs.k8s.io/controller-runtime/pkg/predicate" "strconv" "time" @@ -169,8 +168,10 @@ func (r *Actor) Create(ctx *recon.Context[*v1alpha1.LogSet]) error { if err != nil { return err } - sts.Spec.Template.Annotations[common.ConfigSuffixAnno] = configSuffix - if err := common.SyncConfigMap(ctx, &sts.Spec.Template.Spec, cm); err != nil { + if ls.Spec.GetOperatorVersion().Equals(v1alpha1.LatestOpVersion) { + sts.Spec.Template.Annotations[common.ConfigSuffixAnno] = configSuffix + } + if err := common.SyncConfigMap(ctx, &sts.Spec.Template.Spec, cm, ls.Spec.GetOperatorVersion()); err != nil { return err } @@ -374,9 +375,11 @@ func syncPods(ctx *recon.Context[*v1alpha1.LogSet], sts *kruisev1.StatefulSet) e return err } syncPodMeta(ctx.Obj, sts) - sts.Spec.Template.Annotations[common.ConfigSuffixAnno] = configSuffix + if ctx.Obj.Spec.GetOperatorVersion().Equals(v1alpha1.LatestOpVersion) { + sts.Spec.Template.Annotations[common.ConfigSuffixAnno] = configSuffix + } syncPodSpec(ctx.Obj, &sts.Spec.Template.Spec) - return common.SyncConfigMap(ctx, &sts.Spec.Template.Spec, cm) + return common.SyncConfigMap(ctx, &sts.Spec.Template.Spec, cm, ctx.Obj.Spec.GetOperatorVersion()) } func (r *Actor) Reconcile(mgr manager.Manager) error { @@ -385,13 +388,5 @@ func (r *Actor) Reconcile(mgr manager.Manager) error { // watch all changes on the owned statefulset since we need perform failover if there is a pod failure b.Owns(&kruisev1.StatefulSet{}). Owns(&corev1.Service{}) - b.WithEventFilter(predicate.NewPredicateFuncs(func(obj client.Object) bool { - set, ok := obj.(*v1alpha1.LogSet) - if ok && !set.Spec.GetOperatorVersion().Equals(v1alpha1.LatestOpVersion) { - // only filter LogSet - return false - } - return true - })) })) } diff --git a/pkg/controllers/proxyset/controller.go b/pkg/controllers/proxyset/controller.go index 4f21dd75..b545bdf8 100644 --- a/pkg/controllers/proxyset/controller.go +++ b/pkg/controllers/proxyset/controller.go @@ -27,7 +27,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/predicate" ) const ( @@ -102,13 +101,5 @@ func (r *Actor) Reconcile(mgr manager.Manager) error { return recon.Setup[*v1alpha1.ProxySet](&v1alpha1.ProxySet{}, "proxyset", mgr, r, recon.WithBuildFn(func(b *builder.Builder) { b.Owns(&kruisev1alpha1.CloneSet{}) - b.WithEventFilter(predicate.NewPredicateFuncs(func(obj client.Object) bool { - set, ok := obj.(*v1alpha1.ProxySet) - if ok && !set.Spec.GetOperatorVersion().Equals(v1alpha1.LatestOpVersion) { - // only filter ProxySet - return false - } - return true - })) })) } diff --git a/pkg/controllers/proxyset/resource.go b/pkg/controllers/proxyset/resource.go index a7fff297..a1e0b415 100644 --- a/pkg/controllers/proxyset/resource.go +++ b/pkg/controllers/proxyset/resource.go @@ -58,8 +58,11 @@ cat < ${bc} uuid = "${UUID}" EOF # build instance config -sed "/\[proxy\]/r ${bc}" {{ .ConfigFilePath }}-${CONFIG_SUFFIX} > ${conf} - +if [ -n "${CONFIG_SUFFIX}" ]; then + sed "/\[proxy\]/r ${bc}" "{{ .ConfigFilePath }}-${CONFIG_SUFFIX}" > ${conf} +else + sed "/\[proxy\]/r ${bc}" "{{ .ConfigFilePath }}" > ${conf} +fi echo "/mo-service -cfg ${conf} $@" exec /mo-service -cfg ${conf} $@ `)) diff --git a/pkg/controllers/webui/controller.go b/pkg/controllers/webui/controller.go index 9f79815c..e692d443 100644 --- a/pkg/controllers/webui/controller.go +++ b/pkg/controllers/webui/controller.go @@ -160,7 +160,7 @@ func (w *Actor) Create(ctx *recon.Context[*v1alpha1.WebUI]) error { return err } - if err := common.SyncConfigMap(ctx, &wiObj.Spec.Template.Spec, configMap); err != nil { + if err := common.SyncConfigMap(ctx, &wiObj.Spec.Template.Spec, configMap, wi.Spec.GetOperatorVersion()); err != nil { return err } diff --git a/pkg/controllers/webui/resource.go b/pkg/controllers/webui/resource.go index a9f9759e..412f213a 100644 --- a/pkg/controllers/webui/resource.go +++ b/pkg/controllers/webui/resource.go @@ -135,7 +135,7 @@ func syncPods(ctx *recon.Context[*v1alpha1.WebUI], dp *appsv1.Deployment) error syncPodMeta(ctx.Obj, dp) syncPodSpec(ctx.Obj, dp) - return common.SyncConfigMap(ctx, &dp.Spec.Template.Spec, cm) + return common.SyncConfigMap(ctx, &dp.Spec.Template.Spec, cm, ctx.Obj.Spec.GetOperatorVersion()) } func syncServiceType(wi *v1alpha1.WebUI, svc *corev1.Service) { diff --git a/pkg/webhook/cnset_webhook.go b/pkg/webhook/cnset_webhook.go index 54eb8882..a3a4e446 100644 --- a/pkg/webhook/cnset_webhook.go +++ b/pkg/webhook/cnset_webhook.go @@ -54,7 +54,7 @@ type cnSetDefaulter struct{} var _ webhook.CustomDefaulter = &cnSetDefaulter{} -func (c *cnSetDefaulter) Default(_ context.Context, obj runtime.Object) error { +func (c *cnSetDefaulter) Default(ctx context.Context, obj runtime.Object) error { cnSet, ok := obj.(*v1alpha1.CNSet) if !ok { return unexpectedKindError("CNSet", obj) @@ -64,7 +64,7 @@ func (c *cnSetDefaulter) Default(_ context.Context, obj runtime.Object) error { if cnSet.Spec.Role == "" { cnSet.Spec.Role = v1alpha1.CNRoleTP } - return nil + return setDefaultOperatorVersion(ctx, &cnSet.Spec.PodSet) } func (c *cnSetDefaulter) DefaultSpec(spec *v1alpha1.CNSetSpec) { diff --git a/pkg/webhook/dnset_webhook.go b/pkg/webhook/dnset_webhook.go index 9be19259..eea9a2f1 100644 --- a/pkg/webhook/dnset_webhook.go +++ b/pkg/webhook/dnset_webhook.go @@ -43,13 +43,13 @@ type dnSetDefaulter struct{} var _ webhook.CustomDefaulter = &dnSetDefaulter{} -func (d *dnSetDefaulter) Default(_ context.Context, obj runtime.Object) error { +func (d *dnSetDefaulter) Default(ctx context.Context, obj runtime.Object) error { dnSet, ok := obj.(*v1alpha1.DNSet) if !ok { return unexpectedKindError("DNSet", obj) } d.DefaultSpec(&dnSet.Spec) - return nil + return setDefaultOperatorVersion(ctx, &dnSet.Spec.PodSet) } func (d *dnSetDefaulter) DefaultSpec(spec *v1alpha1.DNSetSpec) { diff --git a/pkg/webhook/logset_webhook.go b/pkg/webhook/logset_webhook.go index c2566fbf..ddcfb23d 100644 --- a/pkg/webhook/logset_webhook.go +++ b/pkg/webhook/logset_webhook.go @@ -61,13 +61,13 @@ type logSetDefaulter struct{} var _ webhook.CustomDefaulter = &logSetDefaulter{} -func (l *logSetDefaulter) Default(_ context.Context, obj runtime.Object) error { +func (l *logSetDefaulter) Default(ctx context.Context, obj runtime.Object) error { logSet, ok := obj.(*v1alpha1.LogSet) if !ok { return unexpectedKindError("LogSet", obj) } l.DefaultSpec(&logSet.Spec) - return nil + return setDefaultOperatorVersion(ctx, &logSet.Spec.PodSet) } func (l *logSetDefaulter) DefaultSpec(spec *v1alpha1.LogSetSpec) { diff --git a/pkg/webhook/matrixonecluster_webhook.go b/pkg/webhook/matrixonecluster_webhook.go index 5668f026..c1bb3808 100644 --- a/pkg/webhook/matrixonecluster_webhook.go +++ b/pkg/webhook/matrixonecluster_webhook.go @@ -68,12 +68,12 @@ type matrixOneClusterDefaulter struct { var _ webhook.CustomDefaulter = &matrixOneClusterDefaulter{} -func (m *matrixOneClusterDefaulter) Default(_ context.Context, obj runtime.Object) error { +func (m *matrixOneClusterDefaulter) Default(ctx context.Context, obj runtime.Object) error { moc, ok := obj.(*v1alpha1.MatrixOneCluster) if !ok { return unexpectedKindError("MatrixOneCluster", obj) } - m.logService.DefaultSpec(&moc.Spec.LogService) + m.logService.DefaultSpec(ctx, &moc.Spec.LogService) if moc.Spec.DN != nil { m.dn.DefaultSpec(moc.Spec.DN) } diff --git a/pkg/webhook/proxy_webhook.go b/pkg/webhook/proxy_webhook.go index ddb034aa..3502f237 100644 --- a/pkg/webhook/proxy_webhook.go +++ b/pkg/webhook/proxy_webhook.go @@ -43,13 +43,13 @@ type proxySetDefaulter struct{} var _ webhook.CustomDefaulter = &proxySetDefaulter{} -func (p *proxySetDefaulter) Default(_ context.Context, obj runtime.Object) error { +func (p *proxySetDefaulter) Default(ctx context.Context, obj runtime.Object) error { proxySet, ok := obj.(*v1alpha1.ProxySet) if !ok { return unexpectedKindError("ProxySet", obj) } p.DefaultSpec(&proxySet.Spec) - return nil + return setDefaultOperatorVersion(ctx, &proxySet.Spec.PodSet) } func (p *proxySetDefaulter) DefaultSpec(spec *v1alpha1.ProxySetSpec) { diff --git a/pkg/webhook/utils.go b/pkg/webhook/utils.go index f7b9674c..32e69675 100644 --- a/pkg/webhook/utils.go +++ b/pkg/webhook/utils.go @@ -15,13 +15,18 @@ package webhook import ( + "context" "fmt" + "github.com/go-errors/errors" - "github.com/matrixorigin/matrixone-operator/api/core/v1alpha1" + admissionv1 "k8s.io/api/admission/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/runtime" "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/matrixorigin/matrixone-operator/api/core/v1alpha1" ) // DefaultArgs alias to v1alpha1.DefaultArgs @@ -90,3 +95,14 @@ func defaultDiskCacheSize(total *resource.Quantity) *resource.Quantity { func unexpectedKindError(expected string, obj runtime.Object) error { return errors.Errorf("expected %s but received %T", expected, obj) } + +func setDefaultOperatorVersion(ctx context.Context, podSet *v1alpha1.PodSet) error { + req, err := admission.RequestFromContext(ctx) + if err != nil { + return err + } + if req.AdmissionRequest.Operation == admissionv1.Create && podSet.OperatorVersion == nil { + podSet.OperatorVersion = pointer.String(v1alpha1.LatestOpVersion.String()) + } + return nil +} From 933b6f14e1e169f4284abd37e43acfd71365c94d Mon Sep 17 00:00:00 2001 From: cyberchen98 Date: Wed, 16 Oct 2024 14:44:29 +0800 Subject: [PATCH 08/11] fix: ut & webhook --- pkg/controllers/common/configmap.go | 12 ++++++++---- pkg/webhook/matrixonecluster_webhook.go | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pkg/controllers/common/configmap.go b/pkg/controllers/common/configmap.go index f06a3795..e1b2a4c6 100644 --- a/pkg/controllers/common/configmap.go +++ b/pkg/controllers/common/configmap.go @@ -43,13 +43,17 @@ const ( // SyncConfigMap syncs the desired configmap for pods, which will cause rolling-update if the // data of the configmap is changed func SyncConfigMap(kubeCli recon.KubeClient, podSpec *corev1.PodSpec, cm *corev1.ConfigMap, operatorVersion semver.Version) error { - vp := util.FindFirst(podSpec.Volumes, util.WithVolumeName("config")) + var currentCmName string var desiredName string var err error + vp := util.FindFirst(podSpec.Volumes, util.WithVolumeName("config")) + if vp != nil { + currentCmName = vp.Name + } if operatorVersion.Equals(v1alpha1.LatestOpVersion) { desiredName, err = ensureConfigMap(kubeCli, cm) } else { - desiredName, err = ensureConfigMapLegacy(kubeCli, vp.ConfigMap.Name, cm) + desiredName, err = ensureConfigMapLegacy(kubeCli, currentCmName, cm) } if err != nil { return err @@ -104,7 +108,7 @@ func ensureConfigMap(kubeCli recon.KubeClient, desired *corev1.ConfigMap) (strin func ensureConfigMapLegacy(kubeCli recon.KubeClient, currentCm string, desired *corev1.ConfigMap) (string, error) { c := desired.DeepCopy() if err := addConfigMapDigest(c); err != nil { - return "", err + return "", errors.Wrap(err, 0) } // config digest not changed if c.Name == currentCm { @@ -113,7 +117,7 @@ func ensureConfigMapLegacy(kubeCli recon.KubeClient, currentCm string, desired * // otherwise ensure the configmap exists err := util.Ignore(apierrors.IsAlreadyExists, kubeCli.CreateOwned(c)) if err != nil { - return "", err + return "", errors.Wrap(err, 0) } return c.Name, nil } diff --git a/pkg/webhook/matrixonecluster_webhook.go b/pkg/webhook/matrixonecluster_webhook.go index c1bb3808..5668f026 100644 --- a/pkg/webhook/matrixonecluster_webhook.go +++ b/pkg/webhook/matrixonecluster_webhook.go @@ -68,12 +68,12 @@ type matrixOneClusterDefaulter struct { var _ webhook.CustomDefaulter = &matrixOneClusterDefaulter{} -func (m *matrixOneClusterDefaulter) Default(ctx context.Context, obj runtime.Object) error { +func (m *matrixOneClusterDefaulter) Default(_ context.Context, obj runtime.Object) error { moc, ok := obj.(*v1alpha1.MatrixOneCluster) if !ok { return unexpectedKindError("MatrixOneCluster", obj) } - m.logService.DefaultSpec(ctx, &moc.Spec.LogService) + m.logService.DefaultSpec(&moc.Spec.LogService) if moc.Spec.DN != nil { m.dn.DefaultSpec(moc.Spec.DN) } From d00f3b37b4053aa3faf0f6a900db8a8420c2d28a Mon Sep 17 00:00:00 2001 From: cyberchen98 Date: Wed, 16 Oct 2024 15:18:13 +0800 Subject: [PATCH 09/11] chore: remove legacy --- pkg/webhook/proxy_webhook.go | 3 --- pkg/webhook/utils.go | 1 - 2 files changed, 4 deletions(-) diff --git a/pkg/webhook/proxy_webhook.go b/pkg/webhook/proxy_webhook.go index 3502f237..de2d55b4 100644 --- a/pkg/webhook/proxy_webhook.go +++ b/pkg/webhook/proxy_webhook.go @@ -16,8 +16,6 @@ package webhook import ( "context" - "k8s.io/utils/pointer" - "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -54,7 +52,6 @@ func (p *proxySetDefaulter) Default(ctx context.Context, obj runtime.Object) err func (p *proxySetDefaulter) DefaultSpec(spec *v1alpha1.ProxySetSpec) { setDefaultServiceArgs(spec) - spec.OperatorVersion = pointer.String(v1alpha1.LatestOpVersion.String()) } // +kubebuilder:webhook:path=/validate-core-matrixorigin-io-v1alpha1-proxyset,mutating=false,failurePolicy=fail,sideEffects=None,groups=core.matrixorigin.io,resources=proxysets,verbs=create;update,versions=v1alpha1,name=vproxyset.kb.io,admissionReviewVersions={v1,v1beta1} diff --git a/pkg/webhook/utils.go b/pkg/webhook/utils.go index 32e69675..ddd969ed 100644 --- a/pkg/webhook/utils.go +++ b/pkg/webhook/utils.go @@ -69,7 +69,6 @@ func setPodSetDefaults(s *v1alpha1.PodSet) { s.Overlay.Env = appendIfNotExist(s.Overlay.Env, corev1.EnvVar{Name: v1alpha1.EnvGoDebug, Value: v1alpha1.DefaultGODebug}, func(v corev1.EnvVar) string { return v.Name }) - s.OperatorVersion = pointer.String(v1alpha1.LatestOpVersion.String()) if s.ExportToPrometheus != nil && *s.ExportToPrometheus { if s.PromDiscoveryScheme == nil { s.PromDiscoveryScheme = (*v1alpha1.PromDiscoveryScheme)(pointer.String(string(v1alpha1.PromDiscoverySchemeService))) From b8c6b6293c0f0ac28aac430cd90f04cac1d56e20 Mon Sep 17 00:00:00 2001 From: cyberchen98 Date: Wed, 16 Oct 2024 16:42:11 +0800 Subject: [PATCH 10/11] chore: add moc default --- pkg/webhook/matrixonecluster_webhook.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/webhook/matrixonecluster_webhook.go b/pkg/webhook/matrixonecluster_webhook.go index 5668f026..60a175fe 100644 --- a/pkg/webhook/matrixonecluster_webhook.go +++ b/pkg/webhook/matrixonecluster_webhook.go @@ -17,6 +17,8 @@ package webhook import ( "context" "fmt" + admissionv1 "k8s.io/api/admission/v1" + "k8s.io/utils/pointer" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation" @@ -68,7 +70,7 @@ type matrixOneClusterDefaulter struct { var _ webhook.CustomDefaulter = &matrixOneClusterDefaulter{} -func (m *matrixOneClusterDefaulter) Default(_ context.Context, obj runtime.Object) error { +func (m *matrixOneClusterDefaulter) Default(ctx context.Context, obj runtime.Object) error { moc, ok := obj.(*v1alpha1.MatrixOneCluster) if !ok { return unexpectedKindError("MatrixOneCluster", obj) @@ -89,6 +91,13 @@ func (m *matrixOneClusterDefaulter) Default(_ context.Context, obj runtime.Objec for i := range moc.Spec.CNGroups { m.cn.DefaultSpec(&moc.Spec.CNGroups[i].CNSetSpec) } + req, err := admission.RequestFromContext(ctx) + if err != nil { + return err + } + if req.AdmissionRequest.Operation == admissionv1.Create && moc.Spec.OperatorVersion == nil { + moc.Spec.OperatorVersion = pointer.String(v1alpha1.LatestOpVersion.String()) + } return nil } From e948c377d866dd8d2aa559ccca964eb19db6bb38 Mon Sep 17 00:00:00 2001 From: cyberchen98 Date: Wed, 16 Oct 2024 17:20:50 +0800 Subject: [PATCH 11/11] chore: fix pool --- test/e2e/claim_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/e2e/claim_test.go b/test/e2e/claim_test.go index bf857b0d..ff4b33fd 100644 --- a/test/e2e/claim_test.go +++ b/test/e2e/claim_test.go @@ -24,6 +24,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "time" @@ -116,6 +117,7 @@ var _ = Describe("CNClaim and CNPool test", func() { MainContainer: v1alpha1.MainContainer{ Image: fmt.Sprintf("%s:%s", moImageRepo, moVersion), }, + OperatorVersion: pointer.String(v1alpha1.LatestOpVersion.String()), }, }, Deps: v1alpha1.CNSetDeps{