From 24b6841fb5b3cd0eec12b00e4313c3c47ec2b384 Mon Sep 17 00:00:00 2001 From: jakthom Date: Sat, 29 Apr 2023 15:29:05 -0400 Subject: [PATCH 1/8] use better validator --- cmd/test/main.go | 42 ++++++++++++++++ go.mod | 3 +- go.sum | 8 +--- pkg/registry/registry.go | 10 ++++ pkg/validator/payload.go | 57 ---------------------- pkg/validator/payload_test.go | 90 ----------------------------------- pkg/validator/validator.go | 44 ++++++++++------- 7 files changed, 82 insertions(+), 172 deletions(-) create mode 100644 cmd/test/main.go delete mode 100644 pkg/validator/payload.go delete mode 100644 pkg/validator/payload_test.go diff --git a/cmd/test/main.go b/cmd/test/main.go new file mode 100644 index 00000000..9b414331 --- /dev/null +++ b/cmd/test/main.go @@ -0,0 +1,42 @@ +package main + +import ( + "encoding/hex" + "encoding/json" + "encoding/xml" + "log" + "strings" + + "github.com/santhosh-tekuri/jsonschema/v5" +) + +func main() { + c := jsonschema.NewCompiler() + c.AssertContent = true + c.Decoders["hex"] = hex.DecodeString + c.MediaTypes["application/xml"] = func(b []byte) error { + return xml.Unmarshal(b, new(interface{})) + } + + schema := `{ "$id": "io.silverton/buz/example/gettingStarted/v1.0.json", "title": "io.silverton/buz/example/gettingStarted/v1.0.json", "description": "A getting started event used to bootstrap and demonstrate validation", "type": "object", "properties": { "userId": { "type": "integer", "description": "The id of the user" }, "name": { "type": "string", "description": "The name of the user" }, "action": { "type": "string", "description": "The associated user action" } }, "additionalProperties": false, "required": [ "userId", "name", "action" ] }` + instance := `{"action":"didSomething","name":"jakthom","userId":"10"}` + + if err := c.AddResource("schema.json", strings.NewReader(schema)); err != nil { + log.Fatalf("%v", err) + } + + sch, err := c.Compile("schema.json") + if err != nil { + log.Fatalf("%#v", err) + } + + var v interface{} + if err := json.Unmarshal([]byte(instance), &v); err != nil { + log.Fatal(err) + } + + if err = sch.Validate(v); err != nil { + log.Fatalf("%#v", err) + } + // Output: +} diff --git a/go.mod b/go.mod index ed4fc057..41112cd9 100644 --- a/go.mod +++ b/go.mod @@ -22,8 +22,8 @@ require ( github.com/google/uuid v1.3.0 github.com/minio/minio-go/v7 v7.0.34 github.com/nats-io/nats.go v1.15.0 - github.com/qri-io/jsonschema v0.2.1 github.com/rs/zerolog v1.26.1 + github.com/santhosh-tekuri/jsonschema/v5 v5.3.0 github.com/spf13/viper v1.10.1 github.com/stretchr/testify v1.8.2 github.com/tidwall/gjson v1.13.0 @@ -110,7 +110,6 @@ require ( github.com/pierrec/lz4/v4 v4.1.14 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/qri-io/jsonpointer v0.1.1 // indirect github.com/rs/xid v1.4.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect github.com/sirupsen/logrus v1.9.0 // indirect diff --git a/go.sum b/go.sum index 0f507eba..55d13cad 100644 --- a/go.sum +++ b/go.sum @@ -442,10 +442,6 @@ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qR github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/qri-io/jsonpointer v0.1.1 h1:prVZBZLL6TW5vsSB9fFHFAMBLI4b0ri5vribQlTJiBA= -github.com/qri-io/jsonpointer v0.1.1/go.mod h1:DnJPaYgiKu56EuDp8TU5wFLdZIcAnb/uH9v37ZaMV64= -github.com/qri-io/jsonschema v0.2.1 h1:NNFoKms+kut6ABPf6xiKNM5214jzxAhDBrPHCJ97Wg0= -github.com/qri-io/jsonschema v0.2.1/go.mod h1:g7DPkiOsK1xv6T/Ao5scXRkd+yTFygcANPBaaqW+VrI= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= @@ -459,9 +455,9 @@ github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThC github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc= github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.0 h1:uIkTLo0AGRc8l7h5l9r+GcYi9qfVPt6lD4/bhmzfiKo= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= diff --git a/pkg/registry/registry.go b/pkg/registry/registry.go index 8ea30bdb..8135b126 100644 --- a/pkg/registry/registry.go +++ b/pkg/registry/registry.go @@ -9,12 +9,15 @@ import ( "github.com/coocood/freecache" "github.com/rs/zerolog/log" + "github.com/santhosh-tekuri/jsonschema/v5" + _ "github.com/santhosh-tekuri/jsonschema/v5/httploader" "github.com/silverton-io/buz/pkg/config" ) type Registry struct { Cache *freecache.Cache Backend SchemaCacheBackend + Compiler *jsonschema.Compiler maxSizeBytes int ttlSeconds int } @@ -29,6 +32,8 @@ func (r *Registry) Initialize(conf config.Registry) error { r.Cache = freecache.NewCache(conf.MaxSizeBytes) r.maxSizeBytes = conf.MaxSizeBytes r.ttlSeconds = conf.TtlSeconds + // Initialize default resources + r.Compiler = jsonschema.NewCompiler() return nil } @@ -55,6 +60,11 @@ func (r *Registry) Get(key string) (exists bool, data []byte) { log.Error().Err(err).Msg("🔴 error when setting key " + key) } log.Debug().Msg("🟡 " + key + " cached successfully") + log.Debug().Msg("🟡 adding schema to compiler " + key) + err = r.Compiler.AddResource(key, strings.NewReader(string(schemaContents))) + if err != nil { + log.Error().Err(err).Msg("🔴 error when compiling schema " + key) + } return true, schemaContents // Schema was aquired from remote backed and cached successfully } } diff --git a/pkg/validator/payload.go b/pkg/validator/payload.go deleted file mode 100644 index 42b9bc0c..00000000 --- a/pkg/validator/payload.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) 2022 Silverton Data, Inc. -// You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of -// which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE - -package validator - -import ( - "context" - "encoding/json" - "time" - - "github.com/qri-io/jsonschema" - "github.com/rs/zerolog/log" - "github.com/silverton-io/buz/pkg/envelope" -) - -func validatePayload(payload []byte, schema []byte) (isValid bool, validationError envelope.ValidationError) { - ctx := context.Background() - startTime := time.Now().UTC() - s := &jsonschema.Schema{} - unmarshalErr := json.Unmarshal(schema, s) - if unmarshalErr != nil { - log.Error().Stack().Err(unmarshalErr).Msg("🔴 failed to unmarshal schema") - } - validationErrs, vErr := s.ValidateBytes(ctx, payload) - - if unmarshalErr != nil || vErr != nil { - log.Debug().Msg("🟡 event validated in " + time.Now().UTC().Sub(startTime).String()) - validationError := envelope.ValidationError{ - ErrorType: &InvalidSchema.Type, - ErrorResolution: &InvalidSchema.Resolution, - Errors: nil, - } - return false, validationError - } - if len(validationErrs) == 0 { - log.Debug().Msg("🟡 event validated in " + time.Now().UTC().Sub(startTime).String()) - return true, envelope.ValidationError{} - } else { - var payloadValidationErrors []envelope.PayloadValidationError - for _, validationErr := range validationErrs { - payloadValidationError := envelope.PayloadValidationError{ - Field: validationErr.PropertyPath, - Description: validationErr.Message, - ErrorType: validationErr.Error(), - } - payloadValidationErrors = append(payloadValidationErrors, payloadValidationError) - } - validationError := envelope.ValidationError{ - ErrorType: &InvalidPayload.Type, - ErrorResolution: &InvalidPayload.Resolution, - Errors: payloadValidationErrors, - } - log.Debug().Msg("🟡 event validated in " + time.Now().UTC().Sub(startTime).String()) - return false, validationError - } -} diff --git a/pkg/validator/payload_test.go b/pkg/validator/payload_test.go deleted file mode 100644 index ed8cb21e..00000000 --- a/pkg/validator/payload_test.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (c) 2022 Silverton Data, Inc. -// You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of -// which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE - -package validator - -import ( - "context" - "encoding/json" - "reflect" - "testing" - - "github.com/qri-io/jsonschema" - "github.com/silverton-io/buz/pkg/envelope" -) - -type output struct { - isValid bool - validationError envelope.ValidationError -} - -func generatePayloadValidationErrs(payload []byte, schema []byte) []envelope.PayloadValidationError { - ctx := context.Background() - s := &jsonschema.Schema{} - json.Unmarshal(schema, s) - validationErrs, _ := s.ValidateBytes(ctx, payload) - - var payloadValidationErrors []envelope.PayloadValidationError - for _, validationErr := range validationErrs { - payloadValidationError := envelope.PayloadValidationError{ - Field: validationErr.PropertyPath, - Description: validationErr.Message, - ErrorType: validationErr.Error(), - } - payloadValidationErrors = append(payloadValidationErrors, payloadValidationError) - } - return payloadValidationErrors -} - -func TestValidatePayload(t *testing.T) { - - validPayload := []byte(`{"id": 10, "action": "did"}`) - invalidPayload := []byte(`{"id": 10, "action": "did", "somethingBad": 10}`) - validSchema := []byte(` - { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "did", - "title": "did", - "type": "object", - "properties": { - "id": { - "type": "number" - }, - "action": { - "type": "string" - } - }, - "required": ["id", "action"], - "additionalProperties": false - } - `) - invalidSchema := []byte(`{"something": yup`) - - invalidPayloadValidationErrs := generatePayloadValidationErrs(invalidPayload, validSchema) - - var testCases = []struct { - name string - payload []byte - schema []byte - want output - }{ - {"valid payload valid schema", validPayload, validSchema, output{true, envelope.ValidationError{}}}, - {"valid payload invalid schema", validPayload, invalidSchema, output{false, envelope.ValidationError{ErrorType: &InvalidSchema.Type, ErrorResolution: &InvalidSchema.Resolution, Errors: nil}}}, - {"invalid payload valid schema", invalidPayload, validSchema, output{false, envelope.ValidationError{ErrorType: &InvalidPayload.Type, ErrorResolution: &InvalidPayload.Resolution, Errors: invalidPayloadValidationErrs}}}, - } - - for _, tc := range testCases { - - t.Run(tc.name, func(t *testing.T) { - isValid, vErr := validatePayload(tc.payload, tc.schema) - if isValid != tc.want.isValid { - t.Fatalf(`got %v, want %v`, isValid, tc.want.isValid) - } - errEquiv := reflect.DeepEqual(vErr, tc.want.validationError) - if !errEquiv { - t.Fatalf(`got %v, want %v`, vErr, tc.want.validationError) - } - }) - } -} diff --git a/pkg/validator/validator.go b/pkg/validator/validator.go index 3e0a8994..5b48815b 100644 --- a/pkg/validator/validator.go +++ b/pkg/validator/validator.go @@ -6,6 +6,7 @@ package validator import ( "encoding/json" + "time" "github.com/rs/zerolog/log" "github.com/silverton-io/buz/pkg/constants" @@ -36,25 +37,24 @@ func Validate(e envelope.Envelope, registry *registry.Registry) (isValid bool, v } return false, validationError, nil } else { - var payloadToValidate []byte - var err error - // Snowplow events have to be handled separately, as `self_describing_event` is - // the only portion that is validated according to a jsonschema. - if e.Protocol == protocol.SNOWPLOW { - e := e.Payload["self_describing_event"].(map[string]interface{})["data"] - payloadToValidate, err = json.Marshal(e) - } else { - payloadToValidate, err = e.Payload.AsByte() - } - // If the payload cannot be marshaled it should be considered invalid. + schema, err := registry.Compiler.Compile(e.Schema) if err != nil { - log.Error().Stack().Err(err).Msg("🔴 could not marshal payload") + log.Error().Err(err).Msg("could not compile schema") validationError := envelope.ValidationError{ - ErrorType: &InvalidPayload.Type, - ErrorResolution: &InvalidPayload.Resolution, + ErrorType: &InvalidSchema.Type, + ErrorResolution: &InvalidSchema.Resolution, Errors: nil, } - return false, validationError, nil + return false, validationError, schemaContents + } + var payloadToValidate interface{} + // Snowplow events have to be handled separately, as `self_describing_event` is + // the only portion that is validated according to a jsonschema. + if e.Protocol == protocol.SNOWPLOW { + payloadToValidate = e.Payload["self_describing_event"].(map[string]interface{})["data"] + } else { + contents, _ := e.Payload.AsByte() + json.Unmarshal(contents, &payloadToValidate) } // If the payload is not present at all it should be considered invalid. if payloadToValidate == nil { @@ -65,7 +65,17 @@ func Validate(e envelope.Envelope, registry *registry.Registry) (isValid bool, v } return false, validationError, nil } - isValid, validationError := validatePayload(payloadToValidate, schemaContents) - return isValid, validationError, schemaContents + startTime := time.Now().UTC() + vErr := schema.Validate(payloadToValidate) + log.Debug().Msg("🟡 event validated in " + time.Now().UTC().Sub(startTime).String()) + if vErr != nil { + validationError := envelope.ValidationError{ + ErrorType: &InvalidPayload.Type, + ErrorResolution: &InvalidPayload.Resolution, + Errors: []envelope.PayloadValidationError{}, // FIXME -> append errors + } + return false, validationError, schemaContents + } + return true, envelope.ValidationError{}, schemaContents } } From d65309ba758df215bc94936afc60135a0440f59c Mon Sep 17 00:00:00 2001 From: jakthom Date: Sat, 29 Apr 2023 15:30:01 -0400 Subject: [PATCH 2/8] don't need this --- cmd/test/main.go | 42 ------------------------------------------ 1 file changed, 42 deletions(-) delete mode 100644 cmd/test/main.go diff --git a/cmd/test/main.go b/cmd/test/main.go deleted file mode 100644 index 9b414331..00000000 --- a/cmd/test/main.go +++ /dev/null @@ -1,42 +0,0 @@ -package main - -import ( - "encoding/hex" - "encoding/json" - "encoding/xml" - "log" - "strings" - - "github.com/santhosh-tekuri/jsonschema/v5" -) - -func main() { - c := jsonschema.NewCompiler() - c.AssertContent = true - c.Decoders["hex"] = hex.DecodeString - c.MediaTypes["application/xml"] = func(b []byte) error { - return xml.Unmarshal(b, new(interface{})) - } - - schema := `{ "$id": "io.silverton/buz/example/gettingStarted/v1.0.json", "title": "io.silverton/buz/example/gettingStarted/v1.0.json", "description": "A getting started event used to bootstrap and demonstrate validation", "type": "object", "properties": { "userId": { "type": "integer", "description": "The id of the user" }, "name": { "type": "string", "description": "The name of the user" }, "action": { "type": "string", "description": "The associated user action" } }, "additionalProperties": false, "required": [ "userId", "name", "action" ] }` - instance := `{"action":"didSomething","name":"jakthom","userId":"10"}` - - if err := c.AddResource("schema.json", strings.NewReader(schema)); err != nil { - log.Fatalf("%v", err) - } - - sch, err := c.Compile("schema.json") - if err != nil { - log.Fatalf("%#v", err) - } - - var v interface{} - if err := json.Unmarshal([]byte(instance), &v); err != nil { - log.Fatal(err) - } - - if err = sch.Validate(v); err != nil { - log.Fatalf("%#v", err) - } - // Output: -} From fc89d8535f43aa09b08c5c4526e34aa6a27b1085 Mon Sep 17 00:00:00 2001 From: jakthom Date: Sat, 29 Apr 2023 16:12:19 -0400 Subject: [PATCH 3/8] Populate validation errors --- pkg/registry/registry.go | 1 - pkg/validator/validator.go | 25 ++++++++++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/pkg/registry/registry.go b/pkg/registry/registry.go index 8135b126..b7b09d95 100644 --- a/pkg/registry/registry.go +++ b/pkg/registry/registry.go @@ -32,7 +32,6 @@ func (r *Registry) Initialize(conf config.Registry) error { r.Cache = freecache.NewCache(conf.MaxSizeBytes) r.maxSizeBytes = conf.MaxSizeBytes r.ttlSeconds = conf.TtlSeconds - // Initialize default resources r.Compiler = jsonschema.NewCompiler() return nil } diff --git a/pkg/validator/validator.go b/pkg/validator/validator.go index 5b48815b..dea7e74c 100644 --- a/pkg/validator/validator.go +++ b/pkg/validator/validator.go @@ -6,9 +6,11 @@ package validator import ( "encoding/json" + "errors" "time" "github.com/rs/zerolog/log" + "github.com/santhosh-tekuri/jsonschema/v5" "github.com/silverton-io/buz/pkg/constants" "github.com/silverton-io/buz/pkg/envelope" "github.com/silverton-io/buz/pkg/protocol" @@ -68,13 +70,26 @@ func Validate(e envelope.Envelope, registry *registry.Registry) (isValid bool, v startTime := time.Now().UTC() vErr := schema.Validate(payloadToValidate) log.Debug().Msg("🟡 event validated in " + time.Now().UTC().Sub(startTime).String()) + jsonschemaValidationErr := &jsonschema.ValidationError{} if vErr != nil { - validationError := envelope.ValidationError{ - ErrorType: &InvalidPayload.Type, - ErrorResolution: &InvalidPayload.Resolution, - Errors: []envelope.PayloadValidationError{}, // FIXME -> append errors + if errors.As(vErr, &jsonschemaValidationErr) { + var validationErrs = []envelope.PayloadValidationError{} + for _, cause := range jsonschemaValidationErr.Causes { + validationErr := envelope.PayloadValidationError{ + Field: cause.InstanceLocation, + Description: cause.Message, + ErrorType: cause.KeywordLocation, + } + validationErrs = append(validationErrs, validationErr) + } + validationError := envelope.ValidationError{ + ErrorType: &InvalidPayload.Type, + ErrorResolution: &InvalidPayload.Resolution, + Errors: validationErrs, + } + return false, validationError, schemaContents } - return false, validationError, schemaContents + return false, envelope.ValidationError{}, schemaContents } return true, envelope.ValidationError{}, schemaContents } From 71c1307b7ca1f88ad2a32a45fbfdd67740c283d8 Mon Sep 17 00:00:00 2001 From: jakthom Date: Sat, 29 Apr 2023 16:20:09 -0400 Subject: [PATCH 4/8] scrap, tidy, etc --- go.mod | 2 +- pkg/backend/mysqldb/sink.go | 1 - pkg/validator/payload.go | 57 ++++++++++++++++++++++ pkg/validator/payload_test.go | 90 +++++++++++++++++++++++++++++++++++ 4 files changed, 148 insertions(+), 2 deletions(-) create mode 100644 pkg/validator/payload.go create mode 100644 pkg/validator/payload_test.go diff --git a/go.mod b/go.mod index 41112cd9..1c27ac4c 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,6 @@ require ( github.com/gin-contrib/pprof v1.4.0 github.com/gin-contrib/timeout v0.0.3 github.com/gin-gonic/gin v1.8.1 - github.com/go-sql-driver/mysql v1.6.0 github.com/google/uuid v1.3.0 github.com/minio/minio-go/v7 v7.0.34 github.com/nats-io/nats.go v1.15.0 @@ -68,6 +67,7 @@ require ( github.com/go-playground/locales v0.14.0 // indirect github.com/go-playground/universal-translator v0.18.0 // indirect github.com/go-playground/validator/v10 v10.10.0 // indirect + github.com/go-sql-driver/mysql v1.6.0 // indirect github.com/go-stack/stack v1.8.0 // indirect github.com/goccy/go-json v0.9.7 // indirect github.com/gofrs/uuid v4.2.0+incompatible // indirect diff --git a/pkg/backend/mysqldb/sink.go b/pkg/backend/mysqldb/sink.go index 133559d1..80e61171 100644 --- a/pkg/backend/mysqldb/sink.go +++ b/pkg/backend/mysqldb/sink.go @@ -7,7 +7,6 @@ package mysqldb import ( "context" - _ "github.com/go-sql-driver/mysql" "github.com/rs/zerolog/log" "github.com/silverton-io/buz/pkg/backend/backendutils" "github.com/silverton-io/buz/pkg/config" diff --git a/pkg/validator/payload.go b/pkg/validator/payload.go new file mode 100644 index 00000000..42b9bc0c --- /dev/null +++ b/pkg/validator/payload.go @@ -0,0 +1,57 @@ +// Copyright (c) 2022 Silverton Data, Inc. +// You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of +// which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE + +package validator + +import ( + "context" + "encoding/json" + "time" + + "github.com/qri-io/jsonschema" + "github.com/rs/zerolog/log" + "github.com/silverton-io/buz/pkg/envelope" +) + +func validatePayload(payload []byte, schema []byte) (isValid bool, validationError envelope.ValidationError) { + ctx := context.Background() + startTime := time.Now().UTC() + s := &jsonschema.Schema{} + unmarshalErr := json.Unmarshal(schema, s) + if unmarshalErr != nil { + log.Error().Stack().Err(unmarshalErr).Msg("🔴 failed to unmarshal schema") + } + validationErrs, vErr := s.ValidateBytes(ctx, payload) + + if unmarshalErr != nil || vErr != nil { + log.Debug().Msg("🟡 event validated in " + time.Now().UTC().Sub(startTime).String()) + validationError := envelope.ValidationError{ + ErrorType: &InvalidSchema.Type, + ErrorResolution: &InvalidSchema.Resolution, + Errors: nil, + } + return false, validationError + } + if len(validationErrs) == 0 { + log.Debug().Msg("🟡 event validated in " + time.Now().UTC().Sub(startTime).String()) + return true, envelope.ValidationError{} + } else { + var payloadValidationErrors []envelope.PayloadValidationError + for _, validationErr := range validationErrs { + payloadValidationError := envelope.PayloadValidationError{ + Field: validationErr.PropertyPath, + Description: validationErr.Message, + ErrorType: validationErr.Error(), + } + payloadValidationErrors = append(payloadValidationErrors, payloadValidationError) + } + validationError := envelope.ValidationError{ + ErrorType: &InvalidPayload.Type, + ErrorResolution: &InvalidPayload.Resolution, + Errors: payloadValidationErrors, + } + log.Debug().Msg("🟡 event validated in " + time.Now().UTC().Sub(startTime).String()) + return false, validationError + } +} diff --git a/pkg/validator/payload_test.go b/pkg/validator/payload_test.go new file mode 100644 index 00000000..ed8cb21e --- /dev/null +++ b/pkg/validator/payload_test.go @@ -0,0 +1,90 @@ +// Copyright (c) 2022 Silverton Data, Inc. +// You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of +// which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE + +package validator + +import ( + "context" + "encoding/json" + "reflect" + "testing" + + "github.com/qri-io/jsonschema" + "github.com/silverton-io/buz/pkg/envelope" +) + +type output struct { + isValid bool + validationError envelope.ValidationError +} + +func generatePayloadValidationErrs(payload []byte, schema []byte) []envelope.PayloadValidationError { + ctx := context.Background() + s := &jsonschema.Schema{} + json.Unmarshal(schema, s) + validationErrs, _ := s.ValidateBytes(ctx, payload) + + var payloadValidationErrors []envelope.PayloadValidationError + for _, validationErr := range validationErrs { + payloadValidationError := envelope.PayloadValidationError{ + Field: validationErr.PropertyPath, + Description: validationErr.Message, + ErrorType: validationErr.Error(), + } + payloadValidationErrors = append(payloadValidationErrors, payloadValidationError) + } + return payloadValidationErrors +} + +func TestValidatePayload(t *testing.T) { + + validPayload := []byte(`{"id": 10, "action": "did"}`) + invalidPayload := []byte(`{"id": 10, "action": "did", "somethingBad": 10}`) + validSchema := []byte(` + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "did", + "title": "did", + "type": "object", + "properties": { + "id": { + "type": "number" + }, + "action": { + "type": "string" + } + }, + "required": ["id", "action"], + "additionalProperties": false + } + `) + invalidSchema := []byte(`{"something": yup`) + + invalidPayloadValidationErrs := generatePayloadValidationErrs(invalidPayload, validSchema) + + var testCases = []struct { + name string + payload []byte + schema []byte + want output + }{ + {"valid payload valid schema", validPayload, validSchema, output{true, envelope.ValidationError{}}}, + {"valid payload invalid schema", validPayload, invalidSchema, output{false, envelope.ValidationError{ErrorType: &InvalidSchema.Type, ErrorResolution: &InvalidSchema.Resolution, Errors: nil}}}, + {"invalid payload valid schema", invalidPayload, validSchema, output{false, envelope.ValidationError{ErrorType: &InvalidPayload.Type, ErrorResolution: &InvalidPayload.Resolution, Errors: invalidPayloadValidationErrs}}}, + } + + for _, tc := range testCases { + + t.Run(tc.name, func(t *testing.T) { + isValid, vErr := validatePayload(tc.payload, tc.schema) + if isValid != tc.want.isValid { + t.Fatalf(`got %v, want %v`, isValid, tc.want.isValid) + } + errEquiv := reflect.DeepEqual(vErr, tc.want.validationError) + if !errEquiv { + t.Fatalf(`got %v, want %v`, vErr, tc.want.validationError) + } + }) + } +} From 8da16c3dd1158bd11e7bbbde484f3de7d0c2e61a Mon Sep 17 00:00:00 2001 From: jakthom Date: Sat, 29 Apr 2023 16:27:12 -0400 Subject: [PATCH 5/8] clean up --- .VERSION | 2 +- examples/quickstart/docker-compose.yml | 3 +- pkg/validator/payload.go | 57 ---------------- pkg/validator/payload_test.go | 90 -------------------------- pkg/validator/validator.go | 5 +- 5 files changed, 7 insertions(+), 150 deletions(-) delete mode 100644 pkg/validator/payload.go delete mode 100644 pkg/validator/payload_test.go diff --git a/.VERSION b/.VERSION index 6e557b4e..cfe6c009 100644 --- a/.VERSION +++ b/.VERSION @@ -1 +1 @@ -v0.15.13 +v0.16.0 diff --git a/examples/quickstart/docker-compose.yml b/examples/quickstart/docker-compose.yml index 264f28f3..32210a54 100644 --- a/examples/quickstart/docker-compose.yml +++ b/examples/quickstart/docker-compose.yml @@ -20,7 +20,8 @@ x-dependency: services: buz: container_name: buz - image: ghcr.io/silverton-io/buz:latest + # image: ghcr.io/silverton-io/buz:latest + image: buz:v0.16.0 volumes: - type: bind source: ./buz/quickstart.conf.yml diff --git a/pkg/validator/payload.go b/pkg/validator/payload.go deleted file mode 100644 index 42b9bc0c..00000000 --- a/pkg/validator/payload.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) 2022 Silverton Data, Inc. -// You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of -// which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE - -package validator - -import ( - "context" - "encoding/json" - "time" - - "github.com/qri-io/jsonschema" - "github.com/rs/zerolog/log" - "github.com/silverton-io/buz/pkg/envelope" -) - -func validatePayload(payload []byte, schema []byte) (isValid bool, validationError envelope.ValidationError) { - ctx := context.Background() - startTime := time.Now().UTC() - s := &jsonschema.Schema{} - unmarshalErr := json.Unmarshal(schema, s) - if unmarshalErr != nil { - log.Error().Stack().Err(unmarshalErr).Msg("🔴 failed to unmarshal schema") - } - validationErrs, vErr := s.ValidateBytes(ctx, payload) - - if unmarshalErr != nil || vErr != nil { - log.Debug().Msg("🟡 event validated in " + time.Now().UTC().Sub(startTime).String()) - validationError := envelope.ValidationError{ - ErrorType: &InvalidSchema.Type, - ErrorResolution: &InvalidSchema.Resolution, - Errors: nil, - } - return false, validationError - } - if len(validationErrs) == 0 { - log.Debug().Msg("🟡 event validated in " + time.Now().UTC().Sub(startTime).String()) - return true, envelope.ValidationError{} - } else { - var payloadValidationErrors []envelope.PayloadValidationError - for _, validationErr := range validationErrs { - payloadValidationError := envelope.PayloadValidationError{ - Field: validationErr.PropertyPath, - Description: validationErr.Message, - ErrorType: validationErr.Error(), - } - payloadValidationErrors = append(payloadValidationErrors, payloadValidationError) - } - validationError := envelope.ValidationError{ - ErrorType: &InvalidPayload.Type, - ErrorResolution: &InvalidPayload.Resolution, - Errors: payloadValidationErrors, - } - log.Debug().Msg("🟡 event validated in " + time.Now().UTC().Sub(startTime).String()) - return false, validationError - } -} diff --git a/pkg/validator/payload_test.go b/pkg/validator/payload_test.go deleted file mode 100644 index ed8cb21e..00000000 --- a/pkg/validator/payload_test.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (c) 2022 Silverton Data, Inc. -// You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of -// which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE - -package validator - -import ( - "context" - "encoding/json" - "reflect" - "testing" - - "github.com/qri-io/jsonschema" - "github.com/silverton-io/buz/pkg/envelope" -) - -type output struct { - isValid bool - validationError envelope.ValidationError -} - -func generatePayloadValidationErrs(payload []byte, schema []byte) []envelope.PayloadValidationError { - ctx := context.Background() - s := &jsonschema.Schema{} - json.Unmarshal(schema, s) - validationErrs, _ := s.ValidateBytes(ctx, payload) - - var payloadValidationErrors []envelope.PayloadValidationError - for _, validationErr := range validationErrs { - payloadValidationError := envelope.PayloadValidationError{ - Field: validationErr.PropertyPath, - Description: validationErr.Message, - ErrorType: validationErr.Error(), - } - payloadValidationErrors = append(payloadValidationErrors, payloadValidationError) - } - return payloadValidationErrors -} - -func TestValidatePayload(t *testing.T) { - - validPayload := []byte(`{"id": 10, "action": "did"}`) - invalidPayload := []byte(`{"id": 10, "action": "did", "somethingBad": 10}`) - validSchema := []byte(` - { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "did", - "title": "did", - "type": "object", - "properties": { - "id": { - "type": "number" - }, - "action": { - "type": "string" - } - }, - "required": ["id", "action"], - "additionalProperties": false - } - `) - invalidSchema := []byte(`{"something": yup`) - - invalidPayloadValidationErrs := generatePayloadValidationErrs(invalidPayload, validSchema) - - var testCases = []struct { - name string - payload []byte - schema []byte - want output - }{ - {"valid payload valid schema", validPayload, validSchema, output{true, envelope.ValidationError{}}}, - {"valid payload invalid schema", validPayload, invalidSchema, output{false, envelope.ValidationError{ErrorType: &InvalidSchema.Type, ErrorResolution: &InvalidSchema.Resolution, Errors: nil}}}, - {"invalid payload valid schema", invalidPayload, validSchema, output{false, envelope.ValidationError{ErrorType: &InvalidPayload.Type, ErrorResolution: &InvalidPayload.Resolution, Errors: invalidPayloadValidationErrs}}}, - } - - for _, tc := range testCases { - - t.Run(tc.name, func(t *testing.T) { - isValid, vErr := validatePayload(tc.payload, tc.schema) - if isValid != tc.want.isValid { - t.Fatalf(`got %v, want %v`, isValid, tc.want.isValid) - } - errEquiv := reflect.DeepEqual(vErr, tc.want.validationError) - if !errEquiv { - t.Fatalf(`got %v, want %v`, vErr, tc.want.validationError) - } - }) - } -} diff --git a/pkg/validator/validator.go b/pkg/validator/validator.go index dea7e74c..44c27443 100644 --- a/pkg/validator/validator.go +++ b/pkg/validator/validator.go @@ -56,7 +56,10 @@ func Validate(e envelope.Envelope, registry *registry.Registry) (isValid bool, v payloadToValidate = e.Payload["self_describing_event"].(map[string]interface{})["data"] } else { contents, _ := e.Payload.AsByte() - json.Unmarshal(contents, &payloadToValidate) + err := json.Unmarshal(contents, &payloadToValidate) + if err != nil { + log.Error().Err(err).Msg("could not unmarshal payload") + } } // If the payload is not present at all it should be considered invalid. if payloadToValidate == nil { From 0da322685692ebff46c32944646ae253205bbdcd Mon Sep 17 00:00:00 2001 From: jakthom Date: Sat, 29 Apr 2023 16:56:28 -0400 Subject: [PATCH 6/8] thistoo --- examples/quickstart/docker-compose.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/quickstart/docker-compose.yml b/examples/quickstart/docker-compose.yml index 32210a54..a30069b0 100644 --- a/examples/quickstart/docker-compose.yml +++ b/examples/quickstart/docker-compose.yml @@ -20,8 +20,7 @@ x-dependency: services: buz: container_name: buz - # image: ghcr.io/silverton-io/buz:latest - image: buz:v0.16.0 + image: ghcr.io/silverton-io/buz:v0.16.0 volumes: - type: bind source: ./buz/quickstart.conf.yml From 75116487cf0194b99c499080d614ccea30ae809f Mon Sep 17 00:00:00 2001 From: jakthom Date: Sat, 29 Apr 2023 16:57:04 -0400 Subject: [PATCH 7/8] bumpy --- deploy/terraform/aws/lambda/variables.tf | 2 +- deploy/terraform/gcp/cloud_run/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/terraform/aws/lambda/variables.tf b/deploy/terraform/aws/lambda/variables.tf index 0316c079..9e9c9e6b 100644 --- a/deploy/terraform/aws/lambda/variables.tf +++ b/deploy/terraform/aws/lambda/variables.tf @@ -29,7 +29,7 @@ variable "buz_domain" { variable "buz_version" { description = "The version of Buz to run." type = string - default = "v0.15.13" + default = "v0.16.0" } variable "buz_lambda_memory_limit" { diff --git a/deploy/terraform/gcp/cloud_run/variables.tf b/deploy/terraform/gcp/cloud_run/variables.tf index f3a1979a..0d143b30 100644 --- a/deploy/terraform/gcp/cloud_run/variables.tf +++ b/deploy/terraform/gcp/cloud_run/variables.tf @@ -28,7 +28,7 @@ variable "buz_domain" { variable "buz_version" { description = "The version of Buz to run." type = string - default = "v0.15.13" + default = "v0.16.0" } variable "buz_service_timeout_seconds" { From 6c8fe145eb6ed2d1ecef792299ac337e342cf92a Mon Sep 17 00:00:00 2001 From: jakthom Date: Sat, 29 Apr 2023 16:58:28 -0400 Subject: [PATCH 8/8] this too --- pkg/validator/validator.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/validator/validator.go b/pkg/validator/validator.go index 44c27443..3247bea8 100644 --- a/pkg/validator/validator.go +++ b/pkg/validator/validator.go @@ -59,6 +59,13 @@ func Validate(e envelope.Envelope, registry *registry.Registry) (isValid bool, v err := json.Unmarshal(contents, &payloadToValidate) if err != nil { log.Error().Err(err).Msg("could not unmarshal payload") + // If the payload cannot be unmarshaled it should be considered invalid + validationError := envelope.ValidationError{ + ErrorType: &InvalidPayload.Type, + ErrorResolution: &InvalidPayload.Resolution, + Errors: nil, + } + return false, validationError, nil } } // If the payload is not present at all it should be considered invalid.