Skip to content

Commit b895231

Browse files
r-vasquezr-vasquez
authored
Merge pull request #24699 from vbotbuildovich/backport-pr-24480-v24.3.x-468
[v24.3.x] [UX-53] rpk: add well-known protobuf types to rpk
2 parents cdb3ac2 + aa5f7d0 commit b895231

31 files changed

+1813
-11
lines changed

‎.github/workflows/rpk-build.yml

+2
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,12 @@ on:
1313
branches: [dev]
1414
paths:
1515
- 'src/go/rpk/**'
16+
- 'src/v/pandaproxy/schema_registry/protobuf/**'
1617
- '.github/workflows/rpk-build.yml'
1718
pull_request:
1819
paths:
1920
- 'src/go/rpk/**'
21+
- 'src/v/pandaproxy/schema_registry/protobuf/**'
2022
- '.github/workflows/rpk-build.yml'
2123
jobs:
2224
test:

‎src/go/rpk/pkg/serde/BUILD

+1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ go_library(
1111
importpath = "github.com/redpanda-data/redpanda/src/go/rpk/pkg/serde",
1212
visibility = ["//visibility:public"],
1313
deps = [
14+
"//src/go/rpk/pkg/serde/embed",
1415
"@com_github_bufbuild_protocompile//:protocompile",
1516
"@com_github_bufbuild_protocompile//linker",
1617
"@com_github_hamba_avro_v2//:avro",

‎src/go/rpk/pkg/serde/embed/BUILD

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
load("@rules_go//go:def.bzl", "go_library", "go_test")
2+
3+
go_library(
4+
name = "embed",
5+
srcs = ["embed.go"],
6+
embedsrcs = [
7+
"protobuf/confluent/meta.proto",
8+
"protobuf/confluent/types/decimal.proto",
9+
"protobuf/google/type/calendar_period.proto",
10+
"protobuf/google/type/color.proto",
11+
"protobuf/google/type/date.proto",
12+
"protobuf/google/type/datetime.proto",
13+
"protobuf/google/type/dayofweek.proto",
14+
"protobuf/google/type/decimal.proto",
15+
"protobuf/google/type/expr.proto",
16+
"protobuf/google/type/fraction.proto",
17+
"protobuf/google/type/interval.proto",
18+
"protobuf/google/type/latlng.proto",
19+
"protobuf/google/type/localized_text.proto",
20+
"protobuf/google/type/money.proto",
21+
"protobuf/google/type/month.proto",
22+
"protobuf/google/type/phone_number.proto",
23+
"protobuf/google/type/postal_address.proto",
24+
"protobuf/google/type/quaternion.proto",
25+
"protobuf/google/type/timeofday.proto",
26+
],
27+
importpath = "github.com/redpanda-data/redpanda/src/go/rpk/pkg/serde/embed",
28+
visibility = ["//visibility:public"],
29+
)
30+
31+
go_test(
32+
name = "embed_test",
33+
size = "small",
34+
srcs = ["embed_test.go"],
35+
data = ["//src/v/pandaproxy:schema_registry_protos"],
36+
embed = [":embed"],
37+
deps = ["@com_github_stretchr_testify//require"],
38+
)

‎src/go/rpk/pkg/serde/embed/embed.go

+62
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
// Copyright 2024 Redpanda Data, Inc.
2+
//
3+
// Use of this software is governed by the Business Source License
4+
// included in the file licenses/BSL.md
5+
//
6+
// As of the Change Date specified in that file, in accordance with
7+
// the Business Source License, use of this software will be governed
8+
// by the Apache License, Version 2.0
9+
10+
package embed
11+
12+
import (
13+
"embed"
14+
"io/fs"
15+
"path/filepath"
16+
"sync"
17+
)
18+
19+
//go:embed protobuf/google/type/*.proto protobuf/confluent/*.proto protobuf/confluent/types/*.proto
20+
var content embed.FS
21+
22+
var (
23+
once sync.Once
24+
protoMap map[string]string
25+
)
26+
27+
// CommonProtoFiles returns the file system representation of the common
28+
// protobuf types.
29+
func CommonProtoFiles() (fs.FS, error) {
30+
return fs.Sub(content, "protobuf")
31+
}
32+
33+
// CommonProtoFileMap returns the map representation of the common protobuf
34+
// types. This is useful for protoreflect parsing.
35+
func CommonProtoFileMap() (map[string]string, error) {
36+
protoFS, err := CommonProtoFiles()
37+
if err != nil {
38+
return nil, err
39+
}
40+
41+
once.Do(func() {
42+
protoMap = make(map[string]string)
43+
err = fs.WalkDir(protoFS, ".", func(path string, d fs.DirEntry, err error) error {
44+
if err != nil {
45+
return err
46+
}
47+
if d.IsDir() || filepath.Ext(path) != ".proto" {
48+
return nil
49+
}
50+
data, err := fs.ReadFile(protoFS, path)
51+
if err == nil {
52+
protoMap[path] = string(data)
53+
}
54+
return nil
55+
})
56+
})
57+
58+
if err != nil {
59+
return nil, err
60+
}
61+
return protoMap, err
62+
}

‎src/go/rpk/pkg/serde/embed/embed_test.go

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
package embed
2+
3+
import (
4+
"io/fs"
5+
"os"
6+
"path/filepath"
7+
"testing"
8+
9+
"github.com/stretchr/testify/require"
10+
)
11+
12+
func TestEmbeddedFiles(t *testing.T) {
13+
t.Run("Test Embedded files in rpk, equal to Redpanda", func(t *testing.T) {
14+
// /src/v/pandaproxy/schema_registry/protobuf
15+
redpandaProtoFS := os.DirFS("../../../../../v/pandaproxy/schema_registry/protobuf/")
16+
redpandaMap := make(map[string]string)
17+
err := fs.WalkDir(redpandaProtoFS, ".", func(path string, d fs.DirEntry, err error) error {
18+
if err != nil {
19+
return err
20+
}
21+
if d.IsDir() || filepath.Ext(path) != ".proto" {
22+
return nil
23+
}
24+
data, err := fs.ReadFile(redpandaProtoFS, path)
25+
if err == nil {
26+
redpandaMap[path] = string(data)
27+
}
28+
return nil
29+
})
30+
31+
embeddedMap, err := CommonProtoFileMap()
32+
require.NoError(t, err)
33+
34+
for path, embedContent := range embeddedMap {
35+
if rpContent, ok := redpandaMap[path]; ok {
36+
require.Equalf(t, rpContent, embedContent, "Contents of %v have changed vs the embedded rpk files", path)
37+
} else {
38+
t.Fatalf("%s not found in Redpanda files", path)
39+
}
40+
}
41+
})
42+
}

‎src/go/rpk/pkg/serde/embed/protobuf/.clang-format

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Language: Proto
2+
DisableFormat: true

‎src/go/rpk/pkg/serde/embed/protobuf/confluent/meta.proto

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
syntax = "proto3";
2+
package confluent;
3+
4+
import "google/protobuf/descriptor.proto";
5+
6+
option go_package="../confluent";
7+
8+
message Meta {
9+
string doc = 1;
10+
map<string, string> params = 2;
11+
}
12+
13+
extend google.protobuf.FileOptions {
14+
Meta file_meta = 1088;
15+
}
16+
extend google.protobuf.MessageOptions {
17+
Meta message_meta = 1088;
18+
}
19+
extend google.protobuf.FieldOptions {
20+
Meta field_meta = 1088;
21+
}
22+
extend google.protobuf.EnumOptions {
23+
Meta enum_meta = 1088;
24+
}
25+
extend google.protobuf.EnumValueOptions {
26+
Meta enum_value_meta = 1088;
27+
}

‎src/go/rpk/pkg/serde/embed/protobuf/confluent/types/decimal.proto

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
syntax = "proto3";
2+
3+
package confluent.type;
4+
5+
option go_package="../types";
6+
7+
message Decimal {
8+
9+
// The two's-complement representation of the unscaled integer value in big-endian byte order
10+
bytes value = 1;
11+
12+
// The precision
13+
uint32 precision = 2;
14+
15+
// The scale
16+
int32 scale = 3;
17+
}

‎src/go/rpk/pkg/serde/embed/protobuf/google/type/README.md

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Google Common Types
2+
3+
This package contains definitions of common types for Google APIs.
4+
All types defined in this package are suitable for different APIs to
5+
exchange data, and will never break binary compatibility. They should
6+
have design quality comparable to major programming languages like
7+
Java and C#.
8+
9+
NOTE: Some common types are defined in the package `google.protobuf`
10+
as they are directly supported by Protocol Buffers compiler and
11+
runtime. Those types are called Well-Known Types.
12+
13+
## Java Utilities
14+
15+
A set of Java utilities for the Common Types are provided in the
16+
`//java/com/google/type/util/` package.

‎src/go/rpk/pkg/serde/embed/protobuf/google/type/calendar_period.proto

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
// Copyright 2021 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
syntax = "proto3";
16+
17+
package google.type;
18+
19+
option go_package = "google.golang.org/genproto/googleapis/type/calendarperiod;calendarperiod";
20+
option java_multiple_files = true;
21+
option java_outer_classname = "CalendarPeriodProto";
22+
option java_package = "com.google.type";
23+
option objc_class_prefix = "GTP";
24+
25+
// A `CalendarPeriod` represents the abstract concept of a time period that has
26+
// a canonical start. Grammatically, "the start of the current
27+
// `CalendarPeriod`." All calendar times begin at midnight UTC.
28+
enum CalendarPeriod {
29+
// Undefined period, raises an error.
30+
CALENDAR_PERIOD_UNSPECIFIED = 0;
31+
32+
// A day.
33+
DAY = 1;
34+
35+
// A week. Weeks begin on Monday, following
36+
// [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date).
37+
WEEK = 2;
38+
39+
// A fortnight. The first calendar fortnight of the year begins at the start
40+
// of week 1 according to
41+
// [ISO 8601](https://en.wikipedia.org/wiki/ISO_week_date).
42+
FORTNIGHT = 3;
43+
44+
// A month.
45+
MONTH = 4;
46+
47+
// A quarter. Quarters start on dates 1-Jan, 1-Apr, 1-Jul, and 1-Oct of each
48+
// year.
49+
QUARTER = 5;
50+
51+
// A half-year. Half-years start on dates 1-Jan and 1-Jul.
52+
HALF = 6;
53+
54+
// A year.
55+
YEAR = 7;
56+
}

0 commit comments

Comments
 (0)