From 7ca8009bec4214928fdeb2473b7c04294ae7952e Mon Sep 17 00:00:00 2001 From: Erez Rokah Date: Thu, 19 Dec 2024 17:36:03 +0000 Subject: [PATCH] fix: Use field name in json type schema if json tag is missing (#2011) I'm working on validating some data (internal issue https://github.com/cloudquery/cloudquery-issues/issues/2971) using the json type schema and noticed a bug (noticeable on AWS where structs don't have json tags). We should not use the name transformer to get the name as it defaults to `ToSnake` when a json tag is missing. Instead we should mimic the JSON marshaling default behavior to use the field name as is. A better approach can be to create an instance of the type, marshal it to JSON, then use that for the names. Open to ideas how to do that using reflection, with a caveat that we might need to initialize it using non zero values otherwise those can be omitted --- --- transformers/name.go | 12 ++++++++++++ transformers/options.go | 8 ++++++++ transformers/struct.go | 14 ++++++++------ transformers/struct_test.go | 19 +++++++++++++++++++ 4 files changed, 47 insertions(+), 6 deletions(-) diff --git a/transformers/name.go b/transformers/name.go index 93faad20da..8f47fd2fb8 100644 --- a/transformers/name.go +++ b/transformers/name.go @@ -26,4 +26,16 @@ func DefaultNameTransformer(field reflect.StructField) (string, error) { return defaultCaser.ToSnake(name), nil } +func DefaultJSONColumnSchemaNameTransformer(field reflect.StructField) (string, error) { + name := field.Name + if jsonTag := strings.Split(field.Tag.Get("json"), ",")[0]; len(jsonTag) > 0 { + // return empty string if the field is not related api response + if jsonTag == "-" { + return "", nil + } + return jsonTag, nil + } + return name, nil +} + var _ NameTransformer = DefaultNameTransformer diff --git a/transformers/options.go b/transformers/options.go index 779a5791c0..ffeec142b3 100644 --- a/transformers/options.go +++ b/transformers/options.go @@ -31,6 +31,14 @@ func WithNameTransformer(transformer NameTransformer) StructTransformerOption { } } +// WithJSONSchemaNameTransformer overrides how column name will be determined. +// DefaultJSONColumnSchemaNameTransformer is used as the default. +func WithJSONSchemaNameTransformer(transformer NameTransformer) StructTransformerOption { + return func(t *structTransformer) { + t.jsonSchemaNameTransformer = transformer + } +} + // WithTypeTransformer overrides how column type will be determined. // DefaultTypeTransformer is used as the default. func WithTypeTransformer(transformer TypeTransformer) StructTransformerOption { diff --git a/transformers/struct.go b/transformers/struct.go index a85d22aebf..9435c6098a 100644 --- a/transformers/struct.go +++ b/transformers/struct.go @@ -29,6 +29,7 @@ type structTransformer struct { pkFieldsFound []string pkComponentFields []string pkComponentFieldsFound []string + jsonSchemaNameTransformer NameTransformer maxJSONTypeSchemaDepth int } @@ -194,11 +195,12 @@ func (t *structTransformer) addColumnFromField(field reflect.StructField, parent func TransformWithStruct(st any, opts ...StructTransformerOption) schema.Transform { t := &structTransformer{ - nameTransformer: DefaultNameTransformer, - typeTransformer: DefaultTypeTransformer, - resolverTransformer: DefaultResolverTransformer, - ignoreInTestsTransformer: DefaultIgnoreInTestsTransformer, - maxJSONTypeSchemaDepth: DefaultMaxJSONTypeSchemaDepth, + nameTransformer: DefaultNameTransformer, + typeTransformer: DefaultTypeTransformer, + resolverTransformer: DefaultResolverTransformer, + ignoreInTestsTransformer: DefaultIgnoreInTestsTransformer, + jsonSchemaNameTransformer: DefaultJSONColumnSchemaNameTransformer, + maxJSONTypeSchemaDepth: DefaultMaxJSONTypeSchemaDepth, } for _, opt := range opts { opt(t) @@ -284,7 +286,7 @@ func (t *structTransformer) fieldToJSONSchema(field reflect.StructField, depth i if !structField.IsExported() || isTypeIgnored(structField.Type) { continue } - name, err := t.nameTransformer(structField) + name, err := t.jsonSchemaNameTransformer(structField) if err != nil { continue } diff --git a/transformers/struct_test.go b/transformers/struct_test.go index 81234323ca..281ce34eff 100644 --- a/transformers/struct_test.go +++ b/transformers/struct_test.go @@ -12,6 +12,7 @@ import ( "github.com/cloudquery/plugin-sdk/v4/schema" "github.com/cloudquery/plugin-sdk/v4/types" "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" ) type ( @@ -665,6 +666,23 @@ func TestJSONTypeSchema(t *testing.T) { "item": `{"exported":"utf8"}`, }, }, + { + name: "no json tags", + testStruct: struct { + Tags map[string]string + Item struct { + Name string + Tags map[string]string + FlatItems []string + ComplexItems []struct { + Name string + } + } + }{}, + want: map[string]string{ + "item": `{"ComplexItems":[{"Name":"utf8"}],"FlatItems":["utf8"],"Name":"utf8","Tags":{"utf8":"utf8"}}`, + }, + }, } for _, tt := range tests { @@ -684,6 +702,7 @@ func TestJSONTypeSchema(t *testing.T) { } for col, schema := range tt.want { column := table.Column(col) + require.NotNil(t, column, "column %q not found", col) if diff := cmp.Diff(column.TypeSchema, schema); diff != "" { t.Fatalf("table does not match expected. diff (-got, +want): %v", diff) }