Merge pull request #26 from polardb/release/v1.5.0-beta

Release/v1.5.0 beta
polardb · Oct 24, 2023 · 9ffd781 · 9ffd781
2 parents 996e35e + f24a5cb
commit 9ffd781
Show file tree

Hide file tree

Showing 124 changed files with 9,359 additions and 474 deletions.
diff --git a/.gitignore b/.gitignore
@@ -317,9 +317,9 @@ flycheck_*.el
 
 ### VisualStudioCode template
 .vscode/*
-!.vscode/settings.json
-!.vscode/tasks.json
-!.vscode/launch.json
+# !.vscode/settings.json
+# !.vscode/tasks.json
+# !.vscode/launch.json
 !.vscode/extensions.json
 *.code-workspace
 

diff --git a/CHANGELOG/README.md b/CHANGELOG/README.md
@@ -1,5 +1,22 @@
 # Changelog
 
+## 2023-10-24
+
+Release v1.5.0-beta
+
++ Enhancement & New Features
+  + Support PolarDB-X Standard(XStore) dashboard
+  + Remove *Cluster IP*  dependency on the paxos communication improving some performance
+  + Support cross-cluster point-in-time recovery(PITR)
+  + Support collecting DN logs including sql, error, slow logs
+  + Support uploading and downloading backupset by Simple Storage Service (S3)
++ Bug Fix
+  + Fix failing to delete the cluster when its gms is not available
+  + Fix failing to delele rebuild job when the *fromPod* does not exist
+  + Fix disk util alert issue. [#21](https://github.com/polardb/polardbx-operator/issues/21)
+  + Fix helm template issue when set tolerations. [#25](https://github.com/polardb/polardbx-operator/issues/25)
+  + Fix failing to create backup when *serviceName* of PolarDBXCluster differs from its *name*
+
 ## 2023-04-12
 
 Release v1.4.1

diff --git a/api/v1/common/version.go b/api/v1/common/version.go
@@ -0,0 +1,9 @@
+package common
+
+const (
+	AnnotationOperatorCreateVersion = "polardbx-operator/create-version"
+)
+
+func NeedCheckPodClusterIp(version string) bool {
+	return version == ""
+}
diff --git a/api/v1/polardbx/backup_storage.go b/api/v1/polardbx/backup_storage.go
@@ -35,8 +35,9 @@ type BackupStorageProvider struct {
 type BackupStorage string
 
 const (
-	OSS  BackupStorage = "oss"
-	SFTP BackupStorage = "sftp"
+	OSS   BackupStorage = "oss"
+	SFTP  BackupStorage = "sftp"
+	MINIO BackupStorage = "s3"
 )
 
 // BackupStorageFilestreamAction records filestream actions related to specified backup storage
@@ -60,6 +61,12 @@ func NewBackupStorageFilestreamAction(storage BackupStorage) (*BackupStorageFile
 			Upload:   filestream.UploadSsh,
 			List:     filestream.ListSsh,
 		}, nil
+	case MINIO:
+		return &BackupStorageFilestreamAction{
+			Download: filestream.DownloadMinio,
+			Upload:   filestream.UploadMinio,
+			List:     filestream.ListMinio,
+		}, nil
 	default:
 		return nil, errors.New("invalid storage: " + string(storage))
 	}

diff --git a/api/v1/polardbx/config.go b/api/v1/polardbx/config.go
@@ -24,7 +24,8 @@ import (
 type CNStaticConfig struct {
 	AttendHtap bool `json:"AttendHtap,omitempty"`
 	// +kubebuilder:default=true
-	EnableCoroutine      bool                          `json:"EnableCoroutine,omitempty"`
+	// +optional
+	EnableCoroutine      bool                          `json:"EnableCoroutine"`
 	EnableReplicaRead    bool                          `json:"EnableReplicaRead,omitempty"`
 	EnableJvmRemoteDebug bool                          `json:"EnableJvmRemoteDebug,omitempty"`
 	ServerProperties     map[string]intstr.IntOrString `json:"ServerProperties,omitempty"`

diff --git a/api/v1/polardbx/parameter.go b/api/v1/polardbx/parameter.go
@@ -24,6 +24,8 @@ const (
 type ParameterTemplate struct {
 	// parameter template
 	// +optional
+	Namespace string `json:"namespace,omitempty"`
+	// +optional
 	Name string `json:"name,omitempty"`
 }
 

diff --git a/api/v1/polardbx/topology.go b/api/v1/polardbx/topology.go
@@ -37,6 +37,7 @@ type NodeSelectorReference struct {
 
 type StatelessTopologyRuleItem struct {
 	Name         string                 `json:"name,omitempty"`
+	ExtraName    string                 `json:"extraName,omitempty"`
 	Replicas     *intstr.IntOrString    `json:"replicas,omitempty"`
 	NodeSelector *NodeSelectorReference `json:"selector,omitempty"`
 }
@@ -210,17 +211,31 @@ type TopologyNodeCN struct {
 
 type TopologyNodeCDC struct {
 	// +kubebuilder:default=2
-	// +kubebuilder:validation:Minimum=0
 
-	Replicas int32 `json:"replicas,omitempty"`
+	Replicas intstr.IntOrString `json:"replicas,omitempty"`
 
 	// +kubebuilder:validation:Minimum=0
 
-	XReplicas int32 `json:"xReplicas,omitempty"`
+	XReplicas int `json:"xReplicas,omitempty"`
 
 	// +kubebuilder:default={resources: {limits: {cpu: 4, memory: "8Gi"}}}
 
 	Template CDCTemplate `json:"template,omitempty"`
+
+	Groups []*CdcGroup `json:"groups,omitempty"`
+}
+
+type CdcGroup struct {
+	Name string `json:"name,omitempty"`
+
+	// +kubebuilder:default=2
+	// +kubebuilder:validation:Minimum=0
+
+	Replicas int32 `json:"replicas,omitempty"`
+
+	Template *CDCTemplate `json:"template,omitempty"`
+
+	Config CDCConfig `json:"config,omitempty"`
 }
 
 type TopologyNodeColumnar struct {

diff --git a/api/v1/polardbx/zz_generated.deepcopy.go b/api/v1/polardbx/zz_generated.deepcopy.go
diff --git a/api/v1/polardbxcluster_types.go b/api/v1/polardbxcluster_types.go
@@ -131,6 +131,9 @@ type PolarDBXClusterStatus struct {
 
 	//PitrStatus represents the status of the pitr restore
 	PitrStatus *polardbx.PitrStatus `json:"pitrStatus,omitempty"`
+
+	//LatestSyncReadonlyTs represents the lastest time sync readonly storage info to metadb
+	ReadonlyStorageInfoHash string `json:"readonlyStorageInfoHash,omitempty"`
 }
 
 // +kubebuilder:object:root=true

diff --git a/api/v1/xstore/parameter.go b/api/v1/xstore/parameter.go
@@ -19,6 +19,8 @@ package xstore
 type ParameterTemplate struct {
 	// Parameter template
 	// +optional
+	Namespace string `json:"namespace,omitempty"`
+	// +optional
 	Name string `json:"name,omitempty"`
 }
 

diff --git a/api/v1/xstore/topology.go b/api/v1/xstore/topology.go
@@ -37,7 +37,7 @@ type NodeSpec struct {
 
 	// +kubebuilder:default=false
 
-	// HostNetwork defines whether the node uses the host network. Default is true.
+	// HostNetwork defines whether the node uses the host network. Default is false.
 	HostNetwork *bool `json:"hostNetwork,omitempty"`
 
 	// Affinity defines the affinity of the node (pod).

diff --git a/api/v1/xstore_follower_types.go b/api/v1/xstore_follower_types.go
@@ -75,9 +75,6 @@ type XStoreFollowerStatus struct {
 	//CurrentJobTask represents the task name of the current job
 	CurrentJobTask string `json:"currentJobTask,omitempty"`
 
-	//ElectionWeight represents the election weight of target pod
-	ElectionWeight int `json:"electionWeight,omitempty"`
-
 	//TargetPodName represents the target pod name
 	TargetPodName string `json:"targetPodName,omitempty"`
 

diff --git a/build/images/xstore-tools/Dockerfile b/build/images/xstore-tools/Dockerfile
@@ -65,7 +65,7 @@ RUN CGO_ENABLED=0 GOOS=linux GO111MODULE=on go build -tags polardbx -o polardbx-
 
 
 # Build the image with scripts
-FROM polardbx/xstore-tools-base:v1.4.1
+FROM polardbx/xstore-tools-base:ts20231018152345
 
 ARG VERSION=test
 

diff --git a/charts/polardbx-logcollector/Chart.yaml b/charts/polardbx-logcollector/Chart.yaml
@@ -2,8 +2,8 @@ apiVersion: v2
 name: polardbx-logcollector
 description: Helm chart of polardbx-operator logcollector plugin
 type: application
-version: 1.4.1
-appVersion: v1.4.1
+version: 1.5.0-beta
+appVersion: v1.5.0-beta
 keywords:
   - polardb-x
   - operator

diff --git a/charts/polardbx-logcollector/templates/configmap/filebeat-configmap.yaml b/charts/polardbx-logcollector/templates/configmap/filebeat-configmap.yaml
@@ -50,14 +50,82 @@ data:
                     pattern: '^\[[0-9]{4}-[0-9]{2}-[0-9]{2}'
                     negate: true
                     match: after
+            - condition:
+                and:
+                  - equals:
+                      kubernetes.labels.polardbx/enableAuditLog: "true"
+                  - or:
+                      - equals:
+                          kubernetes.labels.polardbx/role: dn
+                      - equals:
+                          kubernetes.labels.polardbx/role: gms
+              config:
+                - type: filestream
+                  id: dn-audit-log-filestream
+                  prospector.scanner.check_interval: 1s
+                  buffer_size: 1048576
+                  # close.reader.after_interval: 50s
+                  fields:
+                    log_type: dn_audit_log
+                    instance_id: ${data.kubernetes.labels.polardbx/name}
+                    dn_instance_id: ${data.kubernetes.labels.xstore/name}
+                    pod_name: ${data.kubernetes.pod.name}
+                    node_name: ${data.kubernetes.node.name}
+                  paths:
+                    #alog must in mysql/*/*.alog 
+                    - {{ .Values.filebeat.dnDataHostPathPrefix }}/${data.kubernetes.namespace}/${data.kubernetes.pod.name}/tmp/*.alog
+                  parsers:
+                  - multiline:
+                      type: pattern
+                      pattern: '.*\x01$'
+                      negate: true
+                      match: before
+            - condition:
+                or:
+                  - equals:
+                      kubernetes.labels.polardbx/role: dn
+                  - equals:
+                      kubernetes.labels.polardbx/role: gms
+              config:
+                - type: filestream
+                  id: dn-slow-log-filestream
+                  prospector.scanner.check_interval: 1s
+                  fields:
+                    log_type: dn_slow_log
+                    instance_id: ${data.kubernetes.labels.polardbx/name}
+                    dn_instance_id: ${data.kubernetes.labels.xstore/name}
+                    pod_name: ${data.kubernetes.pod.name}
+                    node_name: ${data.kubernetes.node.name}
+                  paths:
+                    - {{ .Values.filebeat.dnDataHostPathPrefix }}/${data.kubernetes.namespace}/${data.kubernetes.pod.name}/data/mysql/slow_log_*.CSV
+                - type: filestream
+                  id: dn-error-log-filestream
+                  prospector.scanner.check_interval: 1s
+                  fields:
+                    log_type: dn_error_log
+                    instance_id: ${data.kubernetes.labels.polardbx/name}
+                    dn_instance_id: ${data.kubernetes.labels.xstore/name}
+                    pod_name: ${data.kubernetes.pod.name}
+                    node_name: ${data.kubernetes.node.name}
+                  paths:
+                    - {{ .Values.filebeat.dnDataHostPathPrefix }}/${data.kubernetes.namespace}/${data.kubernetes.pod.name}/log/alert.log
+                  parsers:
+                  - multiline:
+                      type: pattern
+                      pattern: '^\d{4}-\d{2}-\d{2}|^\[\d{4}-\d{2}-\d{2}'
+                      negate: true
+                      match: after
     
     processors:
       - drop_fields:
           fields: ["input","log","ecs","container","kubernetes","@metadata","agent"]
           ignore_missing: false
     
     
-    
+    # output.file:
+    #   path: "/tmp/filebeat"
+    #   filename: filebeat
+
     #output.console:
     #  enable: false
     #  codec.json: