rpk: add well-known protobuf types to rpk

This is the same set of well-known types that
both Redpanda and Redpanda Console already support

This allows users to use the whole suite and
encode/decode using these well-known types.

One rpk unit tests verify that the embedded types
are the same as the ones in the Redpanda repo,
hence the need to trigger rpk tests on changes
in pandaproxy/schema_registry/protobuf

(cherry picked from commit 05cd928)

Author: r-vasquez

.github/workflows/rpk-build.yml

@@ -13,10 +13,12 @@ on:
     branches: [dev]
     paths:
       - 'src/go/rpk/**'
+      - 'src/v/pandaproxy/schema_registry/protobuf/**'
       - '.github/workflows/rpk-build.yml'
   pull_request:
     paths:
       - 'src/go/rpk/**'
+      - 'src/v/pandaproxy/schema_registry/protobuf/**'
       - '.github/workflows/rpk-build.yml'
 jobs:
   test:

src/go/rpk/pkg/serde/BUILD

@@ -11,6 +11,7 @@ go_library(
     importpath = "github.com/redpanda-data/redpanda/src/go/rpk/pkg/serde",
     visibility = ["//visibility:public"],
     deps = [
+        "//src/go/rpk/pkg/serde/embed",
         "@com_github_bufbuild_protocompile//:protocompile",
         "@com_github_bufbuild_protocompile//linker",
         "@com_github_hamba_avro_v2//:avro",

src/go/rpk/pkg/serde/embed/BUILD

@@ -0,0 +1,38 @@
+load("@rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "embed",
+    srcs = ["embed.go"],
+    embedsrcs = [
+        "protobuf/confluent/meta.proto",
+        "protobuf/confluent/types/decimal.proto",
+        "protobuf/google/type/calendar_period.proto",
+        "protobuf/google/type/color.proto",
+        "protobuf/google/type/date.proto",
+        "protobuf/google/type/datetime.proto",
+        "protobuf/google/type/dayofweek.proto",
+        "protobuf/google/type/decimal.proto",
+        "protobuf/google/type/expr.proto",
+        "protobuf/google/type/fraction.proto",
+        "protobuf/google/type/interval.proto",
+        "protobuf/google/type/latlng.proto",
+        "protobuf/google/type/localized_text.proto",
+        "protobuf/google/type/money.proto",
+        "protobuf/google/type/month.proto",
+        "protobuf/google/type/phone_number.proto",
+        "protobuf/google/type/postal_address.proto",
+        "protobuf/google/type/quaternion.proto",
+        "protobuf/google/type/timeofday.proto",
+    ],
+    importpath = "github.com/redpanda-data/redpanda/src/go/rpk/pkg/serde/embed",
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "embed_test",
+    size = "small",
+    srcs = ["embed_test.go"],
+    data = ["//src/v/pandaproxy:schema_registry_protos"],
+    embed = [":embed"],
+    deps = ["@com_github_stretchr_testify//require"],
+)

src/go/rpk/pkg/serde/embed/embed.go

@@ -0,0 +1,62 @@
+// Copyright 2024 Redpanda Data, Inc.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.md
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0
+
+package embed
+
+import (
+	"embed"
+	"io/fs"
+	"path/filepath"
+	"sync"
+)
+
+//go:embed protobuf/google/type/*.proto protobuf/confluent/*.proto protobuf/confluent/types/*.proto
+var content embed.FS
+
+var (
+	once     sync.Once
+	protoMap map[string]string
+)
+
+// CommonProtoFiles returns the file system representation of the common
+// protobuf types.
+func CommonProtoFiles() (fs.FS, error) {
+	return fs.Sub(content, "protobuf")
+}
+
+// CommonProtoFileMap returns the map representation of the common protobuf
+// types. This is useful for protoreflect parsing.
+func CommonProtoFileMap() (map[string]string, error) {
+	protoFS, err := CommonProtoFiles()
+	if err != nil {
+		return nil, err
+	}
+
+	once.Do(func() {
+		protoMap = make(map[string]string)
+		err = fs.WalkDir(protoFS, ".", func(path string, d fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+			if d.IsDir() || filepath.Ext(path) != ".proto" {
+				return nil
+			}
+			data, err := fs.ReadFile(protoFS, path)
+			if err == nil {
+				protoMap[path] = string(data)
+			}
+			return nil
+		})
+	})
+
+	if err != nil {
+		return nil, err
+	}
+	return protoMap, err
+}

src/go/rpk/pkg/serde/embed/embed_test.go

@@ -0,0 +1,42 @@
+package embed
+
+import (
+	"io/fs"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestEmbeddedFiles(t *testing.T) {
+	t.Run("Test Embedded files in rpk, equal to Redpanda", func(t *testing.T) {
+		// /src/v/pandaproxy/schema_registry/protobuf
+		redpandaProtoFS := os.DirFS("../../../../../v/pandaproxy/schema_registry/protobuf/")
+		redpandaMap := make(map[string]string)
+		err := fs.WalkDir(redpandaProtoFS, ".", func(path string, d fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+			if d.IsDir() || filepath.Ext(path) != ".proto" {
+				return nil
+			}
+			data, err := fs.ReadFile(redpandaProtoFS, path)
+			if err == nil {
+				redpandaMap[path] = string(data)
+			}
+			return nil
+		})
+
+		embeddedMap, err := CommonProtoFileMap()
+		require.NoError(t, err)
+
+		for path, embedContent := range embeddedMap {
+			if rpContent, ok := redpandaMap[path]; ok {
+				require.Equalf(t, rpContent, embedContent, "Contents of %v have changed vs the embedded rpk files", path)
+			} else {
+				t.Fatalf("%s not found in Redpanda files", path)
+			}
+		}
+	})
+}

src/go/rpk/pkg/serde/embed/protobuf/.clang-format

@@ -0,0 +1,2 @@
+Language: Proto
+DisableFormat: true

src/go/rpk/pkg/serde/embed/protobuf/confluent/meta.proto

@@ -0,0 +1,27 @@
+syntax = "proto3";
+package confluent;
+
+import "google/protobuf/descriptor.proto";
+
+option go_package="../confluent";
+
+message Meta {
+  string doc = 1;
+  map<string, string> params = 2;
+}
+
+extend google.protobuf.FileOptions {
+  Meta file_meta = 1088;
+}
+extend google.protobuf.MessageOptions {
+  Meta message_meta = 1088;
+}
+extend google.protobuf.FieldOptions {
+  Meta field_meta = 1088;
+}
+extend google.protobuf.EnumOptions {
+  Meta enum_meta = 1088;
+}
+extend google.protobuf.EnumValueOptions {
+  Meta enum_value_meta = 1088;
+}

src/go/rpk/pkg/serde/embed/protobuf/confluent/types/decimal.proto

@@ -0,0 +1,17 @@
+syntax = "proto3";
+
+package confluent.type;
+
+option go_package="../types";
+
+message Decimal {
+
+  // The two's-complement representation of the unscaled integer value in big-endian byte order
+  bytes value = 1;
+
+  // The precision
+  uint32 precision = 2;
+
+  // The scale
+  int32 scale = 3;
+}

src/go/rpk/pkg/serde/embed/protobuf/google/type/README.md

@@ -0,0 +1,16 @@
+# Google Common Types
+
+This package contains definitions of common types for Google APIs.
+All types defined in this package are suitable for different APIs to
+exchange data, and will never break binary compatibility. They should
+have design quality comparable to major programming languages like
+Java and C#.
+
+NOTE: Some common types are defined in the package `google.protobuf`
+as they are directly supported by Protocol Buffers compiler and
+runtime. Those types are called Well-Known Types.
+
+## Java Utilities
+
+A set of Java utilities for the Common Types are provided in the
+`//java/com/google/type/util/` package.

src/go/rpk/pkg/serde/embed/protobuf/google/type/calendar_period.proto

@@ -0,0 +1,56 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package google.type;
+
+option go_package = "google.golang.org/genproto/googleapis/type/calendarperiod;calendarperiod";
+option java_multiple_files = true;
+option java_outer_classname = "CalendarPeriodProto";
+option java_package = "com.google.type";
+option objc_class_prefix = "GTP";
+
+// A `CalendarPeriod` represents the abstract concept of a time period that has
+// a canonical start. Grammatically, "the start of the current
+// `CalendarPeriod`." All calendar times begin at midnight UTC.
+enum CalendarPeriod {
+  // Undefined period, raises an error.
+  CALENDAR_PERIOD_UNSPECIFIED = 0;
+
+  // A day.
+  DAY = 1;
+
+  // A week. Weeks begin on Monday, following
+  // [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date).
+  WEEK = 2;
+
+  // A fortnight. The first calendar fortnight of the year begins at the start
+  // of week 1 according to
+  // [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date).
+  FORTNIGHT = 3;
+
+  // A month.
+  MONTH = 4;
+
+  // A quarter. Quarters start on dates 1-Jan, 1-Apr, 1-Jul, and 1-Oct of each
+  // year.
+  QUARTER = 5;
+
+  // A half-year. Half-years start on dates 1-Jan and 1-Jul.
+  HALF = 6;
+
+  // A year.
+  YEAR = 7;
+}