From 163e7476bb2046e190990de1a698bb75739b10b3 Mon Sep 17 00:00:00 2001 From: Quentin Date: Wed, 12 Jun 2024 10:48:30 +0200 Subject: [PATCH] [feat] Expend Subject schemaFile on apply (#42) --- go.mod | 1 + go.sum | 2 + resource/resource.go | 26 ++++++++- resource/resource_test.go | 109 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 137 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index f7399f8..3278a8d 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/spf13/cobra v1.8.0 golang.org/x/exp v0.0.0-20240213143201-ec583247a57a gopkg.in/yaml.v3 v3.0.1 + github.com/Jeffail/gabs/v2 v2.7.0 ) require ( diff --git a/go.sum b/go.sum index 4f36be2..d837efd 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/Jeffail/gabs/v2 v2.7.0 h1:Y2edYaTcE8ZpRsR2AtmPu5xQdFDIthFG0jYhu5PY8kg= +github.com/Jeffail/gabs/v2 v2.7.0/go.mod h1:dp5ocw1FvBBQYssgHsG7I1WYsiLRtkUaB1FEtSwvNUw= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= diff --git a/resource/resource.go b/resource/resource.go index e108607..4cfb724 100644 --- a/resource/resource.go +++ b/resource/resource.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strings" + gabs "github.com/Jeffail/gabs/v2" yamlJson "github.com/ghodss/yaml" yaml "gopkg.in/yaml.v3" ) @@ -116,5 +117,28 @@ func yamlByteToResource(data []byte) (Resource, error) { return Resource{}, err } - return Resource{Json: jsonByte, Kind: yamlRoot.Kind, Name: name, Version: yamlRoot.ApiVersion, Metadata: yamlRoot.Metadata}, nil + resource := Resource{Json: jsonByte, Kind: yamlRoot.Kind, Name: name, Version: yamlRoot.ApiVersion, Metadata: yamlRoot.Metadata} + return expendIncludeFiles(resource) +} + +func expendIncludeFiles(r Resource) (Resource, error) { + // Expend spec.schemaFile into spec.schema and remove spec.schemaFile if kind is Subject + if r.Kind == "Subject" { + jsonData, err := gabs.ParseJSON(r.Json) + if err != nil { + return Resource{}, err + } + schemaFileExist := jsonData.ExistsP("spec.schemaFile") + if schemaFileExist { + filePath := jsonData.Path("spec.schemaFile").Data().(string) + fileContent, err := os.ReadFile(filePath) + if err != nil { + return Resource{}, err + } + jsonData.SetP(string(fileContent), "spec.schema") + jsonData.DeleteP("spec.schemaFile") + r.Json = []byte(jsonData.String()) + } + } + return r, nil } diff --git a/resource/resource_test.go b/resource/resource_test.go index 3b97093..81873ce 100644 --- a/resource/resource_test.go +++ b/resource/resource_test.go @@ -2,6 +2,8 @@ package resource import ( "github.com/davecgh/go-spew/spew" + "log" + "os" "testing" ) @@ -107,3 +109,110 @@ func TestFromFolder(t *testing.T) { Json: []byte(`{"apiVersion":"v1","kind":"b","metadata":{"name":"b"},"spec":{"data":"lo"}}`), }) } + +func TestResourceExpansion(t *testing.T) { + + avroSchema, err := os.CreateTemp("/tmp", "schema.avsc") + if err != nil { + t.Fatal(err) + } + defer avroSchema.Close() + defer os.Remove(avroSchema.Name()) + if _, err := avroSchema.Write([]byte(`{"type":"record","name":"myrecord","fields":[{"name":"f1","type":"string"}]}`)); err != nil { + log.Fatal(err) + } + + jsonSchemaContent := []byte(`{ + "$id": "https://mycompany.com/myrecord", + "$schema": "https://json-schema.org/draft/2019-09/schema", + "type": "object", + "title": "MyRecord", + "description": "Json schema for MyRecord", + "properties": { + "id": { "type": "string" }, + "name": { "type": [ "string", "null" ] } + }, + "required": [ "id" ], + "additionalProperties": false +}`) + jsonSchema, err := os.CreateTemp("/tmp", "schema.json") + if err != nil { + t.Fatal(err) + } + defer jsonSchema.Close() + defer os.Remove(jsonSchema.Name()) + if _, err := jsonSchema.Write(jsonSchemaContent); err != nil { + log.Fatal(err) + } + + yamlByte := []byte(` +# comment +--- +apiVersion: v1 +kind: Subject +metadata: + cluster: cluster-a + name: abc.mySchema +spec: + format: avro + schema: | + { + "type":"record", + "name":"myrecord", + "fields": [{ "name":"f1", "type":"string" }] + } +--- +apiVersion: v1 +kind: Subject +metadata: + cluster: cluster-a + name: abc.mySchemaExtAvro +spec: + format: json + schemaFile: ` + avroSchema.Name() + ` +--- +apiVersion: v1 +kind: Subject +metadata: + cluster: cluster-a + name: abc.mySchemaExtJson +spec: + format: avro + schemaFile: ` + jsonSchema.Name() + ` +`) + + results, err := FromByte(yamlByte) + spew.Dump(results) + if err != nil { + t.Error(err) + } + + if len(results) != 3 { + t.Errorf("results expected of length 3, got length %d", len(results)) + } + + checkResource(t, results[0], Resource{ + Version: "v1", + Kind: "Subject", + Name: "abc.mySchema", + Metadata: map[string]interface{}{"cluster": "cluster-a", "name": "abc.mySchema"}, + Json: []byte(`{"apiVersion":"v1","kind":"Subject","metadata":{"cluster":"cluster-a","name":"abc.mySchema"},"spec":{"format":"avro","schema":"{\n \"type\":\"record\",\n \"name\":\"myrecord\",\n \"fields\": [{ \"name\":\"f1\", \"type\":\"string\" }]\n}\n"}}`), + }) + + checkResource(t, results[1], Resource{ + Version: "v1", + Kind: "Subject", + Name: "abc.mySchemaExtAvro", + Metadata: map[string]interface{}{"cluster": "cluster-a", "name": "abc.mySchemaExtAvro"}, + Json: []byte(`{"apiVersion":"v1","kind":"Subject","metadata":{"cluster":"cluster-a","name":"abc.mySchemaExtAvro"},"spec":{"format":"json","schema":"{\"type\":\"record\",\"name\":\"myrecord\",\"fields\":[{\"name\":\"f1\",\"type\":\"string\"}]}"}}`), // schemaFile is expanded + }) + + checkResource(t, results[2], Resource{ + Version: "v1", + Kind: "Subject", + Name: "abc.mySchemaExtJson", + Metadata: map[string]interface{}{"cluster": "cluster-a", "name": "abc.mySchemaExtJson"}, + Json: []byte(`{"apiVersion":"v1","kind":"Subject","metadata":{"cluster":"cluster-a","name":"abc.mySchemaExtJson"},"spec":{"format":"avro","schema":"{\n\t\"$id\": \"https://mycompany.com/myrecord\",\n\t\"$schema\": \"https://json-schema.org/draft/2019-09/schema\",\n\t\"type\": \"object\",\n\t\"title\": \"MyRecord\",\n\t\"description\": \"Json schema for MyRecord\",\n\t\"properties\": {\n\t\t\"id\": { \"type\": \"string\" },\n\t\t\"name\": { \"type\": [ \"string\", \"null\" ] }\n\t},\n\t\"required\": [ \"id\" ],\n\t\"additionalProperties\": false\n}"}}`), + }) + +}