From 7db2517bcdc2c7799dfb170ef0f33521345748de Mon Sep 17 00:00:00 2001 From: stirante Date: Fri, 25 Oct 2024 10:56:14 +0200 Subject: [PATCH] Implement array templating --- jsonte/json_processor.go | 105 ++++++++++++++++++++++++++------------ jsonte/types/types.go | 67 ++++++++++++++++++++++++- test/json_test.go | 106 +++++++++++++++++++++++++++++++++++++-- 3 files changed, 241 insertions(+), 37 deletions(-) diff --git a/jsonte/json_processor.go b/jsonte/json_processor.go index 40394c1..aa272de 100644 --- a/jsonte/json_processor.go +++ b/jsonte/json_processor.go @@ -120,7 +120,7 @@ func ProcessAssertionsFile(name, input string, globalScope *types.JsonObject, ti } // Process processes a template and returns a map of the processed templates -func Process(name, input string, globalScope *types.JsonObject, modules map[string]JsonModule, timeout int64) (utils.NavigableMap[string, *types.JsonObject], error) { +func Process(name, input string, globalScope *types.JsonObject, modules map[string]JsonModule, timeout int64) (utils.NavigableMap[string, types.JsonType], error) { // Set up the deadline deadline := time.Now().UnixMilli() + timeout if timeout <= 0 { @@ -128,7 +128,7 @@ func Process(name, input string, globalScope *types.JsonObject, modules map[stri } // Parse the input - result := utils.NewNavigableMap[string, *types.JsonObject]() + result := utils.NewNavigableMap[string, types.JsonType]() root, err := types.ParseJsonObject([]byte(input)) if err != nil { return result, burrito.WrapErrorf(err, "Failed to parse JSON") @@ -162,7 +162,7 @@ func Process(name, input string, globalScope *types.JsonObject, modules map[stri isCopy := err == nil tempExtend, err := FindAnyCase[types.JsonType](root, "$extend") isExtend := err == nil - temp, err := FindAnyCase[*types.JsonObject](root, "$template") + temp, err := FindAnyCase[types.JsonType](root, "$template") if err != nil && burrito.HasTag(err, WrongTypeErrTag) { return result, utils.WrapJsonErrorf("$template", err, "Invalid $template") } @@ -181,7 +181,7 @@ func Process(name, input string, globalScope *types.JsonObject, modules map[stri } visitor.pushScope(types.DeepCopyObject(scope)) visitor.pushScope(types.AsObject(map[string]interface{}{"$allModules": modules})) - template := types.NewJsonObject() + var template types.JsonType = types.Null // handle generating multiple files file, err := FindAnyCase[*types.JsonObject](root, "$files") @@ -225,8 +225,13 @@ func Process(name, input string, globalScope *types.JsonObject, modules map[stri } mappedModules := map[string]JsonModule{} if isExtend { + if types.IsNull(template) { + template = types.NewJsonObject() + } else if _, ok := template.(*types.JsonObject); !ok { + return result, utils.WrappedJsonErrorf("$extend", "The extend option is not yet supported for types other than objects") + } var resolvedModules []string - template, resolvedModules, err = extendTemplate(*tempExtend, template, &visitor, modules) + template, resolvedModules, err = extendTemplate(*tempExtend, template.(*types.JsonObject), &visitor, modules) if err != nil { return result, burrito.PassError(err) } @@ -237,7 +242,10 @@ func Process(name, input string, globalScope *types.JsonObject, modules map[stri visitor.pushScope(types.AsObject(map[string]interface{}{"$modules": mappedModules})) } if isCopy && hasTemplate { - template = types.MergeObject(template, *temp, true, "#") + template, err = types.MergeValues(template, *temp, true, "#") + if err != nil { + return result, burrito.PassError(err) + } } fName, err := FindAnyCase[*types.JsonString](*file, "file", "name") if err != nil { @@ -247,9 +255,19 @@ func Process(name, input string, globalScope *types.JsonObject, modules map[stri if err != nil { return result, burrito.WrapErrorf(err, "Failed to evaluate $files.filename") } - f, err := visitor.visitObject(types.DeepCopyObject(template), "$template") - if err != nil { - return result, burrito.PassError(err) + var f types.JsonType + if v, ok := template.(*types.JsonObject); ok { + f, err = visitor.visitObject(types.DeepCopyObject(v), "$template") + if err != nil { + return result, burrito.PassError(err) + } + } else if v, ok := template.(*types.JsonArray); ok { + f, err = visitor.visitArray(types.DeepCopyArray(v), "$template") + if err != nil { + return result, burrito.PassError(err) + } + } else { + return result, utils.WrappedJsonErrorf("$template", "Attempted to template unsupported type. Currently only objects and arrays are supported") } if isExtend { for i := 0; i < len(mappedModules); i++ { @@ -279,8 +297,13 @@ func Process(name, input string, globalScope *types.JsonObject, modules map[stri } mappedModules := map[string]JsonModule{} if isExtend { + if types.IsNull(template) { + template = types.NewJsonObject() + } else if _, ok := template.(*types.JsonObject); !ok { + return result, utils.WrappedJsonErrorf("$extend", "The extend option is not yet supported for types other than objects") + } var resolvedModules []string - template, resolvedModules, err = extendTemplate(*tempExtend, template, &visitor, modules) + template, resolvedModules, err = extendTemplate(*tempExtend, template.(*types.JsonObject), &visitor, modules) if err != nil { return result, burrito.PassError(err) } @@ -291,11 +314,26 @@ func Process(name, input string, globalScope *types.JsonObject, modules map[stri visitor.pushScope(types.AsObject(map[string]interface{}{"$modules": mappedModules})) } if hasTemplate { - template = types.MergeObject(template, *temp, true, "#") + template, err = types.MergeValues(template, *temp, true, "#") + if err != nil { + return result, burrito.PassError(err) + } } - f, err := visitor.visitObject(types.DeepCopyObject(template), "$template") - if err != nil { - return result, burrito.PassError(err) + var f types.JsonType + if v, ok := template.(*types.JsonObject); ok { + f, err = visitor.visitObject(types.DeepCopyObject(v), "$template") + if err != nil { + return result, burrito.PassError(err) + } + result.Put(name, types.MergeObject(types.NewJsonObject(), f.(*types.JsonObject), false, "#")) + } else if v, ok := template.(*types.JsonArray); ok { + f, err = visitor.visitArray(types.DeepCopyArray(v), "$template") + if err != nil { + return result, burrito.PassError(err) + } + result.Put(name, types.MergeArray(types.NewJsonArray(), f.(*types.JsonArray), false, "#")) + } else { + return result, utils.WrappedJsonErrorf("$template", "Attempted to template unsupported type. Currently only objects and arrays are supported") } if isExtend { for i := 0; i < len(mappedModules); i++ { @@ -305,34 +343,33 @@ func Process(name, input string, globalScope *types.JsonObject, modules map[stri if isCopy { visitor.popScope() } - result.Put(name, types.MergeObject(types.NewJsonObject(), f.(*types.JsonObject), false, "#")) result.Put(name, types.DeleteNulls(result.Get(name))) } return result, nil } -func processCopy(c types.JsonType, visitor TemplateVisitor, modules map[string]JsonModule, path string, timeout int64) (*types.JsonObject, error) { - result := types.NewJsonObject() +func processCopy(c types.JsonType, visitor TemplateVisitor, modules map[string]JsonModule, path string, timeout int64) (types.JsonType, error) { + var result types.JsonType = types.Null copies := make([]*types.JsonString, 0) if copyArray, ok := c.(*types.JsonArray); ok { for i, item := range copyArray.Value { if copyPath, ok := item.(*types.JsonString); ok { copies = append(copies, copyPath) } else { - return types.NewJsonObject(), utils.WrappedJsonErrorf(fmt.Sprintf("%s[%d]", path, i), "The copy path is not a string") + return types.Null, utils.WrappedJsonErrorf(fmt.Sprintf("%s[%d]", path, i), "The copy path is not a string") } } } else if copyPath, ok := c.(*types.JsonString); ok { copies = append(copies, copyPath) } else { - return types.NewJsonObject(), utils.WrappedJsonErrorf(path, "The copy path is not a string") + return types.Null, utils.WrappedJsonErrorf(path, "The copy path is not a string") } for i, c := range copies { loopPath := fmt.Sprintf("%s[%d]", path, i) c, err := visitor.visitString(c.StringValue(), path) if err != nil { - return types.NewJsonObject(), utils.WrapJsonErrorf(loopPath, err, "Failed to evaluate $copy") + return types.Null, utils.WrapJsonErrorf(loopPath, err, "Failed to evaluate $copy") } if c == nil || types.IsNull(c) { utils.Logger.Debugf("Skipping null copy path at %s", loopPath) @@ -344,40 +381,46 @@ func processCopy(c types.JsonType, visitor TemplateVisitor, modules map[string]J if insideCopyPath, ok := inside.(*types.JsonString); ok { insideCopies = append(insideCopies, insideCopyPath) } else { - return types.NewJsonObject(), utils.WrappedJsonErrorf(fmt.Sprintf("%s[%d]", loopPath, j), "The copy path inside evaluated copy array is not a string") + return types.Null, utils.WrappedJsonErrorf(fmt.Sprintf("%s[%d]", loopPath, j), "The copy path inside evaluated copy array is not a string") } } } else if copyPath, ok := c.(*types.JsonString); ok { insideCopies = append(insideCopies, copyPath) } else { - return types.NewJsonObject(), utils.WrappedJsonErrorf(loopPath, "The copy path evaluated to a non-string") + return types.Null, utils.WrappedJsonErrorf(loopPath, "The copy path evaluated to a non-string") } for _, copyPath := range insideCopies { resolve, err := safeio.Resolver.Open(copyPath.StringValue()) if err != nil { - return types.NewJsonObject(), utils.WrapJsonErrorf(loopPath, err, "Failed to open %s", copyPath.StringValue()) + return types.Null, utils.WrapJsonErrorf(loopPath, err, "Failed to open %s", copyPath.StringValue()) } all, err := io.ReadAll(resolve) if err != nil { - return types.NewJsonObject(), utils.WrapJsonErrorf(loopPath, err, "Failed to read %s", copyPath.StringValue()) + return types.Null, utils.WrapJsonErrorf(loopPath, err, "Failed to read %s", copyPath.StringValue()) } if strings.HasSuffix(copyPath.StringValue(), ".templ") { processedMap, err := Process("copy", string(all), visitor.globalScope, modules, timeout) if err != nil { - return types.NewJsonObject(), utils.WrapJsonErrorf(loopPath, err, "Failed to process copy template %s", copyPath.StringValue()) + return types.Null, utils.WrapJsonErrorf(loopPath, err, "Failed to process copy template %s", copyPath.StringValue()) } if processedMap.Size() > 1 { - return types.NewJsonObject(), utils.WrappedJsonErrorf(path, "The copy template must compile to a single object") + return types.Null, utils.WrappedJsonErrorf(path, "The copy template must compile to a single object") } template := processedMap.Get("copy") - result = types.MergeObject(result, template, false, "#") + result, err = types.MergeValues(result, template, false, "#") + if err != nil { + return types.Null, utils.WrapJsonErrorf(loopPath, err, "Failed to merge %s", copyPath.StringValue()) + } continue } else { - template, err := types.ParseJsonObject(all) + template, err := types.ParseJsonValue(all) + if err != nil { + return types.Null, utils.WrapJsonErrorf(loopPath, err, "Failed to parse %s", copyPath.StringValue()) + } + result, err = types.MergeValues(result, template, false, "#") if err != nil { - return types.NewJsonObject(), utils.WrapJsonErrorf(loopPath, err, "Failed to parse %s", copyPath.StringValue()) + return types.Null, utils.WrapJsonErrorf(loopPath, err, "Failed to merge %s", copyPath.StringValue()) } - result = types.MergeObject(result, template, false, "#") continue } } @@ -447,7 +490,7 @@ func extendTemplate(extend types.JsonType, template *types.JsonObject, visitor * if err != nil { return types.NewJsonObject(), resolvedModules, burrito.WrapErrorf(err, "Error processing $copy for module %s", mod.Name.StringValue()) } - template = types.MergeObject(object, template, true, "#") + template = types.MergeObject(object.(*types.JsonObject), template, true, "#") } if !mod.Template.IsEmpty() { parent, err := visitor.visitObject(mod.Template, "[Module "+module+"]") diff --git a/jsonte/types/types.go b/jsonte/types/types.go index 6635ce4..706e367 100644 --- a/jsonte/types/types.go +++ b/jsonte/types/types.go @@ -292,8 +292,18 @@ func CreateRange(start, end int32) *JsonArray { return &JsonArray{Value: result} } -// DeleteNulls removes all keys with null values from the given JSON object. -func DeleteNulls(object *JsonObject) *JsonObject { +// DeleteNulls removes all keys with null values from the given JSON value. +func DeleteNulls(object JsonType) JsonType { + if IsObject(object) { + return DeleteNullsFromObject(AsObject(object)) + } else if IsArray(object) { + return DeleteNullsFromArray(AsArray(object)) + } + return object +} + +// DeleteNullsFromObject removes all keys with null values from the given JSON object. +func DeleteNullsFromObject(object *JsonObject) *JsonObject { keys := make([]string, len(object.Keys())) copy(keys, object.Keys()) for _, k := range keys { @@ -321,6 +331,26 @@ func DeleteNullsFromArray(array *JsonArray) *JsonArray { return array } +// ParseJsonValue parses a JSON string into a JSON object. It includes support for comments and detects common syntax errors. +func ParseJsonValue(str []byte) (JsonType, error) { + dat, err := json.UnmarshallJSONC(str) + if err != nil { + return NewJsonObject(), err + } + if IsObject(dat) { + return AsObject(dat), nil + } else if IsArray(dat) { + return AsArray(dat), nil + } else if IsString(dat) { + return AsString(dat), nil + } else if IsNumber(dat) { + return AsNumber(dat), nil + } else if IsBool(dat) { + return AsBool(dat), nil + } + return Null, burrito.WrappedErrorf("JSON must be an object, array, string, number or boolean") +} + // ParseJsonObject parses a JSON string into a JSON object. It includes support for comments and detects common syntax errors. func ParseJsonObject(str []byte) (*JsonObject, error) { dat, err := json.UnmarshallJSONC(str) @@ -345,6 +375,39 @@ func ParseJsonArray(str []byte) (*JsonArray, error) { return AsArray(dat), nil } +// MergeValues merges two JSON values into a new JSON value. +// If the same value, that is not an object or an array exists in both objects, the value from the second object will be used. +func MergeValues(template, parent JsonType, keepOverrides bool, path string) (JsonType, error) { + if IsNull(template) { + return parent, nil + } + if IsNull(parent) { + return template, nil + } + if IsObject(template) { + if !IsObject(parent) { + return template, burrito.WrappedErrorf("Cannot merge object with non-object") + } + return MergeObject(AsObject(template), AsObject(parent), keepOverrides, path), nil + } else if IsArray(template) { + if !IsArray(parent) { + return template, burrito.WrappedErrorf("Cannot merge array with non-array") + } + return MergeArray(AsArray(template), AsArray(parent), keepOverrides, path), nil + } + return template, nil +} + +// DeepCopyValue creates a deep copy of the given JSON value. +func DeepCopyValue(object JsonType) JsonType { + if IsObject(object) { + return DeepCopyObject(AsObject(object)) + } else if IsArray(object) { + return DeepCopyArray(AsArray(object)) + } + return Box(object.Unbox()) +} + // JsonAction is an enum for the different actions that can be performed via jsonte. type JsonAction int diff --git a/test/json_test.go b/test/json_test.go index c96a290..fa7589b 100644 --- a/test/json_test.go +++ b/test/json_test.go @@ -23,6 +23,43 @@ func safeTypeName(v interface{}) string { return n } +func compareJson(t *testing.T, expected types.JsonType, actual types.JsonType, path string, checkExtra bool) { + if types.IsObject(expected) { + if !types.IsObject(actual) { + t.Errorf("Field %s is not an object (expected %s, got %s)", path, types.ToString(expected), types.ToString(actual)) + } + compareJsonObject(t, expected.(*types.JsonObject), actual.(*types.JsonObject), path, checkExtra) + } else if types.IsArray(expected) { + if !types.IsArray(actual) { + t.Errorf("Field %s is not an array (expected %s, got %s)", path, types.ToString(expected), types.ToString(actual)) + } + compareJsonArray(t, expected.(*types.JsonArray), actual.(*types.JsonArray), path) + } else if types.IsString(expected) { + if !types.IsString(actual) { + t.Errorf("Field %s is not a string (expected %s, got %s)", path, types.ToString(expected), types.ToString(actual)) + } + if expected.(*types.JsonString).StringValue() != actual.(*types.JsonString).StringValue() { + t.Errorf("Field %s is not correct (expected %s, got %s)", path, types.ToString(expected), types.ToString(actual)) + } + } else if types.IsNumber(expected) { + if !types.IsNumber(actual) { + t.Errorf("Field %s is not a number (expected %s, got %s)", path, types.ToString(expected), types.ToString(actual)) + } + if expected.(*types.JsonNumber).FloatValue() != actual.(*types.JsonNumber).FloatValue() { + t.Errorf("Field %s is not correct (expected %s, got %s)", path, types.ToString(expected), types.ToString(actual)) + } + } else if types.IsBool(expected) { + if !types.IsBool(actual) { + t.Errorf("Field %s is not a boolean (expected %s, got %s)", path, types.ToString(expected), types.ToString(actual)) + } + if expected.(*types.JsonBool).BoolValue() != actual.(*types.JsonBool).BoolValue() { + t.Errorf("Field %s is not correct (expected %s, got %s)", path, types.ToString(expected), types.ToString(actual)) + } + } else { + t.Errorf("Field %s is not correct (expected %s, got %s)", path, types.ToString(expected), types.ToString(actual)) + } +} + func compareJsonObject(t *testing.T, expected *types.JsonObject, actual *types.JsonObject, path string, checkExtra bool) { for _, key := range expected.Keys() { value1 := expected.Get(key) @@ -141,7 +178,7 @@ func assertTemplateWithModule(t *testing.T, template, expected string, globalSco if err != nil { t.Fatal(burrito.WrapErrorf(err, "Failed to parse expected result")) } - compareJsonObject(t, exp, process.Get("test"), "#", true) + compareJson(t, exp, process.Get("test"), "#", true) } func assertTemplate(t *testing.T, template, expected string) { @@ -149,11 +186,11 @@ func assertTemplate(t *testing.T, template, expected string) { if err != nil { t.Fatal(burrito.WrapErrorf(err, "Failed to process template")) } - exp, err := types.ParseJsonObject([]byte(expected)) + exp, err := types.ParseJsonValue([]byte(expected)) if err != nil { t.Fatal(burrito.WrapErrorf(err, "Failed to parse expected result")) } - compareJsonObject(t, exp, process.Get("test"), "#", true) + compareJson(t, exp, process.Get("test"), "#", true) } func assertTemplateError(t *testing.T, template string, error []string) { @@ -197,7 +234,7 @@ func assertTemplateMultiple(t *testing.T, template, expected string) { t.Errorf("Missing file %s", key) continue } - compareJsonObject(t, value.(*types.JsonObject), process.Get(key), fmt.Sprintf("%s#", key), true) + compareJson(t, value, process.Get(key), fmt.Sprintf("%s#", key), true) } for _, key := range process.Keys() { if !exp.ContainsKey(key) { @@ -1646,3 +1683,64 @@ func TestScopeInheritance(t *testing.T) { assertTemplateWithModule(t, template, expected, types.NewJsonObject(), dataModule, dataModule2, templateModule) } + +func TestCopyArray(t *testing.T) { + file := `[1, 2, 3]` + safeio.Resolver = safeio.CreateFakeFS(map[string]interface{}{ + "file.json": file, + }, false) + template := `{ + "$copy": "file.json", + "$template": [ + "asd" + ] + }` + expected := `[ + 1, + 2, + 3, + "asd" + ]` + assertTemplate(t, template, expected) + safeio.Resolver = safeio.DefaultIOResolver +} + +func TestArrayTemplate(t *testing.T) { + template := `{ + "$template": [ + { + "{{#1..3}}": "{{=value * 2}}" + } + ] + }` + expected := `[ + 2, + 4, + 6 + ]` + assertTemplate(t, template, expected) + safeio.Resolver = safeio.DefaultIOResolver +} + +func TestArrayTemplate2(t *testing.T) { + template := `{ + "$template": [ + { + "{{#1..3}}": "{{=value * 2}}" + }, + { + "{{#1..3}}": "{{=value}}" + } + ] + }` + expected := `[ + 2, + 4, + 6, + 1, + 2, + 3 + ]` + assertTemplate(t, template, expected) + safeio.Resolver = safeio.DefaultIOResolver +}