Merge pull request #7056 from SungJin1212/Add-changelog-for-utf8-support

Author: yeya24

.github/workflows/test-build-deploy.yml

@@ -213,7 +213,7 @@ jobs:
             docker pull quay.io/cortexproject/cortex:v1.18.1
           elif [ "$TEST_TAGS" = "integration_query_fuzz" ]; then
             docker pull quay.io/cortexproject/cortex:v1.18.1 
-            docker pull quay.io/prometheus/prometheus:v3.5.0
+            docker pull quay.io/prometheus/prometheus:v3.6.0
           fi
           docker pull memcached:1.6.1
           docker pull redis:7.0.4-alpine

CHANGELOG.md

@@ -24,6 +24,7 @@
 * [FEATURE] Querier: Support for configuring query optimizers and enabling XFunctions in the Thanos engine. #6873
 * [FEATURE] Query Frontend: Add support /api/v1/format_query API for formatting queries. #6893
 * [FEATURE] Query Frontend: Add support for /api/v1/parse_query API (experimental) to parse a PromQL expression and return it as a JSON-formatted AST (abstract syntax tree). #6978
+* [ENHANCEMENT] Upgrade the Prometheus version to 3.6.0 and add a `-name-validation-scheme` flag to support UTF-8. #7040 #7056
 * [ENHANCEMENT] Distributor: Emit an error with a 400 status code when empty labels are found before the relabelling or label dropping process. #7052
 * [ENHANCEMENT] Parquet Storage: Add support for additional sort columns during Parquet file generation #7003
 * [ENHANCEMENT] Modernizes the entire codebase by using go modernize tool. #7005

docs/configuration/config-file-reference.md

@@ -68,6 +68,11 @@ Where default_value is the value to use if the environment variable is undefined
 # CLI flag: -http.prefix
 [http_prefix: <string> | default = "/api/prom"]
 
+# Name validation scheme for metric names and label names, Support values are:
+# legacy, utf8.
+# CLI flag: -name-validation-scheme
+[name_validation_scheme: <int> | default = legacy]
+
 resource_monitor:
   # Comma-separated list of resources to monitor. Supported values are cpu and
   # heap, which tracks metrics from github.com/prometheus/procfs and
@@ -4344,11 +4349,6 @@ query_rejection:
 
 # list of rule groups to disable
 [disabled_rule_groups: <list of DisabledRuleGroup> | default = []]
-
-# Name validation scheme for metric names and label names, Support values are:
-# legacy, utf8.
-# CLI flag: -validation.name-validation-scheme
-[name_validation_scheme: <int> | default = legacy]
 ```
 
 ### `memberlist_config`

integration/e2e/images/images.go

@@ -11,5 +11,5 @@ var (
 	Minio      = "minio/minio:RELEASE.2024-05-28T17-19-04Z"
 	Consul     = "consul:1.8.4"
 	ETCD       = "gcr.io/etcd-development/etcd:v3.4.7"
-	Prometheus = "quay.io/prometheus/prometheus:v3.5.0"
+	Prometheus = "quay.io/prometheus/prometheus:v3.6.0"
 )

integration/utf8_test.go

@@ -4,11 +4,17 @@
 package integration
 
 import (
+	"context"
 	"fmt"
 	"path/filepath"
 	"testing"
 	"time"
 
+	amlabels "github.com/prometheus/alertmanager/pkg/labels"
+	"github.com/prometheus/alertmanager/types"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/model/rulefmt"
 	"github.com/prometheus/prometheus/prompb"
 	"github.com/stretchr/testify/require"
 
@@ -17,52 +23,156 @@ import (
 	"github.com/cortexproject/cortex/integration/e2ecortex"
 )
 
-func Test_RulerExternalLabels_UTF8Validation(t *testing.T) {
+const utf8AlertmanagerConfig = `route:
+  receiver: dummy
+  group_by: [group.test.🙂]
+  routes:
+    - matchers:
+      - foo.🙂=bar.🙂
+receivers:
+  - name: dummy`
+
+func Test_Alertmanager_UTF8(t *testing.T) {
 	s, err := e2e.NewScenario(networkName)
 	require.NoError(t, err)
 	defer s.Close()
 
-	// Start dependencies.
-	minio := e2edb.NewMinio(9000, bucketName)
+	flags := mergeFlags(AlertmanagerFlags(), AlertmanagerS3Flags(),
+		map[string]string{
+			"-name-validation-scheme": "utf8",
+		},
+	)
+
+	minio := e2edb.NewMinio(9000, alertsBucketName)
 	require.NoError(t, s.StartAndWaitReady(minio))
 
+	alertmanager := e2ecortex.NewAlertmanager(
+		"alertmanager",
+		flags,
+		"",
+	)
+
+	require.NoError(t, s.StartAndWaitReady(alertmanager))
+
+	c, err := e2ecortex.NewClient("", "", alertmanager.HTTPEndpoint(), "", "user-1")
+	require.NoError(t, err)
+
+	ctx := context.Background()
+
+	err = c.SetAlertmanagerConfig(ctx, utf8AlertmanagerConfig, map[string]string{})
+	require.NoError(t, err)
+
+	require.NoError(t, alertmanager.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"cortex_alertmanager_config_last_reload_successful"}, e2e.WaitMissingMetrics))
+	require.NoError(t, alertmanager.WaitSumMetricsWithOptions(e2e.Greater(0), []string{"cortex_alertmanager_config_hash"}, e2e.WaitMissingMetrics))
+
+	silenceId, err := c.CreateSilence(ctx, types.Silence{
+		Matchers: amlabels.Matchers{
+			{Name: "silences.name.🙂", Value: "silences.value.🙂"},
+		},
+		Comment:  "test silences",
+		StartsAt: time.Now(),
+		EndsAt:   time.Now().Add(time.Minute),
+	})
+	require.NoError(t, err)
+	require.NotEmpty(t, silenceId)
+	require.NoError(t, alertmanager.WaitSumMetrics(e2e.Equals(1), "cortex_alertmanager_silences"))
+
+	err = c.SendAlertToAlermanager(ctx, &model.Alert{
+		Labels: model.LabelSet{
+			"alert.name.🙂": "alert.value.🙂",
+		},
+		StartsAt: time.Now(),
+		EndsAt:   time.Now().Add(time.Minute),
+	})
+	require.NoError(t, err)
+	require.NoError(t, alertmanager.WaitSumMetrics(e2e.Equals(1), "cortex_alertmanager_alerts_received_total"))
+}
+
+func Test_Ruler_UTF8(t *testing.T) {
+	s, err := e2e.NewScenario(networkName)
+	require.NoError(t, err)
+	defer s.Close()
+
+	// Start dependencies.
+	consul := e2edb.NewConsul()
+	minio := e2edb.NewMinio(9000, rulestoreBucketName, bucketName)
+	require.NoError(t, s.StartAndWaitReady(consul, minio))
+
 	runtimeConfigYamlFile := `
 overrides:
-  'user-2':
-    name_validation_scheme: utf8
+  'user-1':
     ruler_external_labels:
       test.utf8.metric: 😄
 `
 	require.NoError(t, writeFileToSharedDir(s, runtimeConfigFile, []byte(runtimeConfigYamlFile)))
 	filePath := filepath.Join(e2e.ContainerSharedDir, runtimeConfigFile)
 
-	// Start Cortex components.
-	require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks.yaml", cortexConfigFile))
+	flags := mergeFlags(
+		BlocksStorageFlags(),
+		RulerFlags(),
+		map[string]string{
+			"-runtime-config.file":    filePath,
+			"-runtime-config.backend": "filesystem",
+			"-name-validation-scheme": "utf8",
 
-	flags := map[string]string{
-		"-auth.enabled":           "true",
-		"-runtime-config.file":    filePath,
-		"-runtime-config.backend": "filesystem",
-		// ingester
-		"-blocks-storage.s3.access-key-id":     e2edb.MinioAccessKey,
-		"-blocks-storage.s3.secret-access-key": e2edb.MinioSecretKey,
-		"-blocks-storage.s3.bucket-name":       bucketName,
-		"-blocks-storage.s3.endpoint":          fmt.Sprintf("%s-minio-9000:9000", networkName),
-		"-blocks-storage.s3.insecure":          "true",
-		// alert manager
-		"-alertmanager.web.external-url":   "http://localhost/alertmanager",
-		"-alertmanager-storage.backend":    "local",
-		"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
-	}
+			"-ring.store":      "consul",
+			"-consul.hostname": consul.NetworkHTTPEndpoint(),
+			// ingester
+			"-blocks-storage.s3.access-key-id":     e2edb.MinioAccessKey,
+			"-blocks-storage.s3.secret-access-key": e2edb.MinioSecretKey,
+			"-blocks-storage.s3.bucket-name":       bucketName,
+			"-blocks-storage.s3.endpoint":          fmt.Sprintf("%s-minio-9000:9000", networkName),
+			"-blocks-storage.s3.insecure":          "true",
+			// alert manager
+			"-alertmanager.web.external-url":   "http://localhost/alertmanager",
+			"-alertmanager-storage.backend":    "local",
+			"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
+		},
+	)
 	// make alert manager config dir
 	require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
 
-	// The external labels validation should be success
-	cortex := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, flags, "", 9009, 9095)
+	// Start Cortex.
+	cortex := e2ecortex.NewSingleBinary("cortex", flags, "")
 	require.NoError(t, s.StartAndWaitReady(cortex))
+
+	groupLabels := map[string]string{
+		"group.label.😄": "group.value",
+	}
+	ruleLabels := map[string]string{
+		"rule.label.😄": "rule.value",
+	}
+
+	interval, _ := model.ParseDuration("1s")
+
+	ruleGroup := rulefmt.RuleGroup{
+		Name:     "rule.utf8.😄",
+		Interval: interval,
+		Rules: []rulefmt.Rule{{
+			Alert:  "alert.rule.😄",
+			Expr:   "up",
+			Labels: ruleLabels,
+		}, {
+			Record: "record.rule.😄",
+			Expr:   "up",
+			Labels: ruleLabels,
+		}},
+		Labels: groupLabels,
+	}
+
+	c, err := e2ecortex.NewClient("", "", "", cortex.HTTPEndpoint(), "user-1")
+	require.NoError(t, err)
+
+	err = c.SetRuleGroup(ruleGroup, "namespace")
+	require.NoError(t, err)
+	require.NoError(t, cortex.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"cortex_ruler_managers_total"}), e2e.WithLabelMatchers(labels.MustNewMatcher(labels.MatchEqual, "user", "user-1")))
+	require.NoError(t, cortex.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"cortex_ruler_rule_groups_in_store"}, e2e.WithLabelMatchers(
+		labels.MustNewMatcher(labels.MatchEqual, "user", "user-1")),
+		e2e.WaitMissingMetrics,
+	))
 }
 
-func Test_Distributor_UTF8ValidationPerTenant(t *testing.T) {
+func Test_PushQuery_UTF8(t *testing.T) {
 	s, err := e2e.NewScenario(networkName)
 	require.NoError(t, err)
 	defer s.Close()
@@ -71,22 +181,12 @@ func Test_Distributor_UTF8ValidationPerTenant(t *testing.T) {
 	minio := e2edb.NewMinio(9000, bucketName)
 	require.NoError(t, s.StartAndWaitReady(minio))
 
-	runtimeConfigYamlFile := `
-overrides:
-  'user-2':
-    name_validation_scheme: utf8
-`
-
-	require.NoError(t, writeFileToSharedDir(s, runtimeConfigFile, []byte(runtimeConfigYamlFile)))
-	filePath := filepath.Join(e2e.ContainerSharedDir, runtimeConfigFile)
-
 	// Start Cortex components.
 	require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks.yaml", cortexConfigFile))
 
 	flags := map[string]string{
 		"-auth.enabled":           "true",
-		"-runtime-config.file":    filePath,
-		"-runtime-config.backend": "filesystem",
+		"-name-validation-scheme": "utf8",
 		// ingester
 		"-blocks-storage.s3.access-key-id":     e2edb.MinioAccessKey,
 		"-blocks-storage.s3.secret-access-key": e2edb.MinioSecretKey,
@@ -104,34 +204,62 @@ overrides:
 	cortex := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, flags, "", 9009, 9095)
 	require.NoError(t, s.StartAndWaitReady(cortex))
 
-	// user-1 uses legacy validation
-	user1Client, err := e2ecortex.NewClient(cortex.HTTPEndpoint(), "", "", "", "user-1")
-	require.NoError(t, err)
-
-	// user-2 uses utf8 validation
-	user2Client, err := e2ecortex.NewClient(cortex.HTTPEndpoint(), "", "", "", "user-2")
+	c, err := e2ecortex.NewClient(cortex.HTTPEndpoint(), cortex.HTTPEndpoint(), "", "", "user-1")
 	require.NoError(t, err)
 
 	now := time.Now()
 
-	utf8Series, _ := generateSeries("series_1", now, prompb.Label{Name: "test.utf8.metric", Value: "😄"})
+	utf8Series, _ := generateSeries("series.1", now, prompb.Label{Name: "test.utf8.metric", Value: "😄"})
 	legacySeries, _ := generateSeries("series_2", now, prompb.Label{Name: "job", Value: "test"})
 
-	res, err := user1Client.Push(legacySeries)
+	metadata := []prompb.MetricMetadata{
+		{
+			MetricFamilyName: "metadata.name",
+			Help:             "metadata.help",
+			Unit:             "metadata.unit",
+		},
+	}
+
+	res, err := c.Push(legacySeries, metadata...)
 	require.NoError(t, err)
 	require.Equal(t, 200, res.StatusCode)
 
-	// utf8Series push should be fail for user-1
-	res, err = user1Client.Push(utf8Series)
+	// utf8Series push should be success
+	res, err = c.Push(utf8Series)
 	require.NoError(t, err)
-	require.Equal(t, 400, res.StatusCode)
+	require.Equal(t, 200, res.StatusCode)
 
-	res, err = user2Client.Push(legacySeries)
+	// utf8 querying
+	// c.f. https://prometheus.io/docs/guides/utf8/#querying
+	query := `{"series.1", "test.utf8.metric"="😄"}`
+	queryResult, err := c.Query(query, now)
 	require.NoError(t, err)
 	require.Equal(t, 200, res.StatusCode)
+	require.Equal(t, model.ValVector, queryResult.Type())
+	vec := queryResult.(model.Vector)
+	require.Equal(t, 1, len(vec))
 
-	// utf8Series push should be success for user-2
-	res, err = user2Client.Push(utf8Series)
+	// label names
+	start := now
+	end := now.Add(time.Minute * 5)
+	labelNames, err := c.LabelNames(start, end)
 	require.NoError(t, err)
-	require.Equal(t, 200, res.StatusCode)
+	require.Equal(t, []string{"__name__", "job", "test.utf8.metric"}, labelNames)
+
+	// series
+	series, err := c.Series([]string{`{"test.utf8.metric"="😄"}`}, start, end)
+	require.NoError(t, err)
+	require.Equal(t, 1, len(series))
+	require.Equal(t, `{__name__="series.1", test.utf8.metric="😄"}`, series[0].String())
+
+	// label values
+	labelValues, err := c.LabelValues("test.utf8.metric", start, end, nil)
+	require.NoError(t, err)
+	require.Equal(t, 1, len(labelValues))
+	require.Equal(t, model.LabelValue("😄"), labelValues[0])
+
+	// metadata
+	metadataResult, err := c.Metadata("metadata.name", "")
+	require.NoError(t, err)
+	require.Equal(t, 1, len(metadataResult))
 }