Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use a local keyword registry to make validate thread-safe #103

Merged
merged 2 commits into from
Jul 23, 2021
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
151 changes: 85 additions & 66 deletions draft2019_09_keywords.go
Original file line number Diff line number Diff line change
@@ -5,88 +5,107 @@ package jsonschema
// this is also the default keyword set loaded automatically
// if no other is loaded
func LoadDraft2019_09() {
r, release := getGlobalKeywordRegistry()
defer release()

r.LoadDraft2019_09()
}

// DefaultIfEmpty populates the KeywordRegistry with the 2019_09
// jsonschema draft specification only if the registry is empty.
func (r *KeywordRegistry) DefaultIfEmpty() {
if !r.IsRegistryLoaded() {
r.LoadDraft2019_09()
}
}

// LoadDraft2019_09 loads the keywords for schema validation
// based on draft2019_09
// this is also the default keyword set loaded automatically
// if no other is loaded
func (r *KeywordRegistry) LoadDraft2019_09() {
// core keywords
RegisterKeyword("$schema", NewSchemaURI)
RegisterKeyword("$id", NewID)
RegisterKeyword("description", NewDescription)
RegisterKeyword("title", NewTitle)
RegisterKeyword("$comment", NewComment)
RegisterKeyword("examples", NewExamples)
RegisterKeyword("readOnly", NewReadOnly)
RegisterKeyword("writeOnly", NewWriteOnly)
RegisterKeyword("$ref", NewRef)
RegisterKeyword("$recursiveRef", NewRecursiveRef)
RegisterKeyword("$anchor", NewAnchor)
RegisterKeyword("$recursiveAnchor", NewRecursiveAnchor)
RegisterKeyword("$defs", NewDefs)
RegisterKeyword("default", NewDefault)

SetKeywordOrder("$ref", 0)
SetKeywordOrder("$recursiveRef", 0)
r.RegisterKeyword("$schema", NewSchemaURI)
r.RegisterKeyword("$id", NewID)
r.RegisterKeyword("description", NewDescription)
r.RegisterKeyword("title", NewTitle)
r.RegisterKeyword("$comment", NewComment)
r.RegisterKeyword("examples", NewExamples)
r.RegisterKeyword("readOnly", NewReadOnly)
r.RegisterKeyword("writeOnly", NewWriteOnly)
r.RegisterKeyword("$ref", NewRef)
r.RegisterKeyword("$recursiveRef", NewRecursiveRef)
r.RegisterKeyword("$anchor", NewAnchor)
r.RegisterKeyword("$recursiveAnchor", NewRecursiveAnchor)
r.RegisterKeyword("$defs", NewDefs)
r.RegisterKeyword("default", NewDefault)

r.SetKeywordOrder("$ref", 0)
r.SetKeywordOrder("$recursiveRef", 0)

// standard keywords
RegisterKeyword("type", NewType)
RegisterKeyword("enum", NewEnum)
RegisterKeyword("const", NewConst)
r.RegisterKeyword("type", NewType)
r.RegisterKeyword("enum", NewEnum)
r.RegisterKeyword("const", NewConst)

// numeric keywords
RegisterKeyword("multipleOf", NewMultipleOf)
RegisterKeyword("maximum", NewMaximum)
RegisterKeyword("exclusiveMaximum", NewExclusiveMaximum)
RegisterKeyword("minimum", NewMinimum)
RegisterKeyword("exclusiveMinimum", NewExclusiveMinimum)
r.RegisterKeyword("multipleOf", NewMultipleOf)
r.RegisterKeyword("maximum", NewMaximum)
r.RegisterKeyword("exclusiveMaximum", NewExclusiveMaximum)
r.RegisterKeyword("minimum", NewMinimum)
r.RegisterKeyword("exclusiveMinimum", NewExclusiveMinimum)

// string keywords
RegisterKeyword("maxLength", NewMaxLength)
RegisterKeyword("minLength", NewMinLength)
RegisterKeyword("pattern", NewPattern)
r.RegisterKeyword("maxLength", NewMaxLength)
r.RegisterKeyword("minLength", NewMinLength)
r.RegisterKeyword("pattern", NewPattern)

// boolean keywords
RegisterKeyword("allOf", NewAllOf)
RegisterKeyword("anyOf", NewAnyOf)
RegisterKeyword("oneOf", NewOneOf)
RegisterKeyword("not", NewNot)
r.RegisterKeyword("allOf", NewAllOf)
r.RegisterKeyword("anyOf", NewAnyOf)
r.RegisterKeyword("oneOf", NewOneOf)
r.RegisterKeyword("not", NewNot)

// object keywords
RegisterKeyword("properties", NewProperties)
RegisterKeyword("patternProperties", NewPatternProperties)
RegisterKeyword("additionalProperties", NewAdditionalProperties)
RegisterKeyword("required", NewRequired)
RegisterKeyword("propertyNames", NewPropertyNames)
RegisterKeyword("maxProperties", NewMaxProperties)
RegisterKeyword("minProperties", NewMinProperties)
RegisterKeyword("dependentSchemas", NewDependentSchemas)
RegisterKeyword("dependentRequired", NewDependentRequired)
RegisterKeyword("unevaluatedProperties", NewUnevaluatedProperties)

SetKeywordOrder("properties", 2)
SetKeywordOrder("additionalProperties", 3)
SetKeywordOrder("unevaluatedProperties", 4)
r.RegisterKeyword("properties", NewProperties)
r.RegisterKeyword("patternProperties", NewPatternProperties)
r.RegisterKeyword("additionalProperties", NewAdditionalProperties)
r.RegisterKeyword("required", NewRequired)
r.RegisterKeyword("propertyNames", NewPropertyNames)
r.RegisterKeyword("maxProperties", NewMaxProperties)
r.RegisterKeyword("minProperties", NewMinProperties)
r.RegisterKeyword("dependentSchemas", NewDependentSchemas)
r.RegisterKeyword("dependentRequired", NewDependentRequired)
r.RegisterKeyword("unevaluatedProperties", NewUnevaluatedProperties)

r.SetKeywordOrder("properties", 2)
r.SetKeywordOrder("additionalProperties", 3)
r.SetKeywordOrder("unevaluatedProperties", 4)

// array keywords
RegisterKeyword("items", NewItems)
RegisterKeyword("additionalItems", NewAdditionalItems)
RegisterKeyword("maxItems", NewMaxItems)
RegisterKeyword("minItems", NewMinItems)
RegisterKeyword("uniqueItems", NewUniqueItems)
RegisterKeyword("contains", NewContains)
RegisterKeyword("maxContains", NewMaxContains)
RegisterKeyword("minContains", NewMinContains)
RegisterKeyword("unevaluatedItems", NewUnevaluatedItems)

SetKeywordOrder("maxContains", 2)
SetKeywordOrder("minContains", 2)
SetKeywordOrder("additionalItems", 3)
SetKeywordOrder("unevaluatedItems", 4)
r.RegisterKeyword("items", NewItems)
r.RegisterKeyword("additionalItems", NewAdditionalItems)
r.RegisterKeyword("maxItems", NewMaxItems)
r.RegisterKeyword("minItems", NewMinItems)
r.RegisterKeyword("uniqueItems", NewUniqueItems)
r.RegisterKeyword("contains", NewContains)
r.RegisterKeyword("maxContains", NewMaxContains)
r.RegisterKeyword("minContains", NewMinContains)
r.RegisterKeyword("unevaluatedItems", NewUnevaluatedItems)

r.SetKeywordOrder("maxContains", 2)
r.SetKeywordOrder("minContains", 2)
r.SetKeywordOrder("additionalItems", 3)
r.SetKeywordOrder("unevaluatedItems", 4)

// conditional keywords
RegisterKeyword("if", NewIf)
RegisterKeyword("then", NewThen)
RegisterKeyword("else", NewElse)
r.RegisterKeyword("if", NewIf)
r.RegisterKeyword("then", NewThen)
r.RegisterKeyword("else", NewElse)

SetKeywordOrder("then", 2)
SetKeywordOrder("else", 2)
r.SetKeywordOrder("then", 2)
r.SetKeywordOrder("else", 2)

//optional formats
RegisterKeyword("format", NewFormat)
r.RegisterKeyword("format", NewFormat)
}
105 changes: 84 additions & 21 deletions keyword.go
Original file line number Diff line number Diff line change
@@ -5,6 +5,7 @@ import (
"context"
"encoding/json"
"fmt"
"sync"

jptr "github.com/qri-io/jsonpointer"
)
@@ -24,66 +25,128 @@ var notSupported = map[string]bool{
"dependencies": true,
}

var (
keywordRegistry = map[string]KeyMaker{}
keywordOrder = map[string]int{}
keywordInsertOrder = map[string]int{}
)
var kr *KeywordRegistry
var krLock sync.Mutex

// KeywordRegistry contains a mapping of jsonschema keywords and their
// expected behavior.
type KeywordRegistry struct {
keywordRegistry map[string]KeyMaker
keywordOrder map[string]int
keywordInsertOrder map[string]int
}

func getGlobalKeywordRegistry() (*KeywordRegistry, func()) {
krLock.Lock()
if kr == nil {
kr = &KeywordRegistry{
keywordRegistry: make(map[string]KeyMaker, 0),
keywordOrder: make(map[string]int, 0),
keywordInsertOrder: make(map[string]int, 0),
}
}
return kr, func() { krLock.Unlock() }
}

func copyGlobalKeywordRegistry() *KeywordRegistry {
kr, release := getGlobalKeywordRegistry()
defer release()
return kr.Copy()
}

// Copy creates a new KeywordRegistry populated with the same data.
func (r *KeywordRegistry) Copy() *KeywordRegistry {
dest := &KeywordRegistry{
keywordRegistry: make(map[string]KeyMaker, len(r.keywordRegistry)),
keywordOrder: make(map[string]int, len(r.keywordOrder)),
keywordInsertOrder: make(map[string]int, len(r.keywordInsertOrder)),
}

for k, v := range r.keywordRegistry {
dest.keywordRegistry[k] = v
}

for k, v := range r.keywordOrder {
dest.keywordOrder[k] = v
}

for k, v := range r.keywordInsertOrder {
dest.keywordInsertOrder[k] = v
}

return dest
}

// IsRegisteredKeyword validates if a given prop string is a registered keyword
func IsRegisteredKeyword(prop string) bool {
_, ok := keywordRegistry[prop]
func (r *KeywordRegistry) IsRegisteredKeyword(prop string) bool {
_, ok := r.keywordRegistry[prop]
return ok
}

// GetKeyword returns a new instance of the keyword
func GetKeyword(prop string) Keyword {
if !IsRegisteredKeyword(prop) {
func (r *KeywordRegistry) GetKeyword(prop string) Keyword {
if !r.IsRegisteredKeyword(prop) {
return NewVoid()
}
return keywordRegistry[prop]()
return r.keywordRegistry[prop]()
}

// GetKeywordOrder returns the order index of
// the given keyword or defaults to 1
func GetKeywordOrder(prop string) int {
if order, ok := keywordOrder[prop]; ok {
func (r *KeywordRegistry) GetKeywordOrder(prop string) int {
if order, ok := r.keywordOrder[prop]; ok {
return order
}
return 1
}

// GetKeywordInsertOrder returns the insert index of
// the given keyword
func GetKeywordInsertOrder(prop string) int {
if order, ok := keywordInsertOrder[prop]; ok {
func (r *KeywordRegistry) GetKeywordInsertOrder(prop string) int {
if order, ok := r.keywordInsertOrder[prop]; ok {
return order
}
// TODO(arqu): this is an arbitrary max
return 1000
}

// SetKeywordOrder assignes a given order to a keyword
// SetKeywordOrder assigns a given order to a keyword
func (r *KeywordRegistry) SetKeywordOrder(prop string, order int) {
r.keywordOrder[prop] = order
}

// SetKeywordOrder assigns a given order to a keyword
func SetKeywordOrder(prop string, order int) {
keywordOrder[prop] = order
r, release := getGlobalKeywordRegistry()
defer release()

r.SetKeywordOrder(prop, order)
}

// IsNotSupportedKeyword is a utility function to clarify when
// a given keyword, while expected is not supported
func IsNotSupportedKeyword(prop string) bool {
func (r *KeywordRegistry) IsNotSupportedKeyword(prop string) bool {
_, ok := notSupported[prop]
return ok
}

// IsRegistryLoaded checks if any keywords are present
func IsRegistryLoaded() bool {
return keywordRegistry != nil && len(keywordRegistry) > 0
func (r *KeywordRegistry) IsRegistryLoaded() bool {
return r.keywordRegistry != nil && len(r.keywordRegistry) > 0
}

// RegisterKeyword registers a keyword with the registry
func (r *KeywordRegistry) RegisterKeyword(prop string, maker KeyMaker) {
r.keywordRegistry[prop] = maker
r.keywordInsertOrder[prop] = len(r.keywordInsertOrder)
}

// RegisterKeyword registers a keyword with the registry
func RegisterKeyword(prop string, maker KeyMaker) {
keywordRegistry[prop] = maker
keywordInsertOrder[prop] = len(keywordInsertOrder)
r, release := getGlobalKeywordRegistry()
defer release()

r.RegisterKeyword(prop, maker)
}

// MaxKeywordErrStringLen sets how long a value can be before it's length is truncated
70 changes: 69 additions & 1 deletion keyword_test.go
Original file line number Diff line number Diff line change
@@ -2,10 +2,12 @@ package jsonschema

import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"testing"

"github.com/qri-io/jsonpointer"
jptr "github.com/qri-io/jsonpointer"
)

@@ -63,7 +65,40 @@ func (f *IsFoo) ValidateKeyword(ctx context.Context, currentState *ValidationSta
}
}

// ContentEncoding represents a "custom" Schema property
type ContentEncoding string

// newContentEncoding allocates a new ContentEncoding validator
func newContentEncoding() Keyword {
return new(ContentEncoding)
}

func (c ContentEncoding) Validate(propPath string, data interface{}, errs *[]KeyError) {}

func (c ContentEncoding) ValidateKeyword(ctx context.Context, currentState *ValidationState, data interface{}) {
if obj, ok := data.(string); ok {
switch c {
case "base64":
_, err := base64.StdEncoding.DecodeString(obj)
if err != nil {
currentState.AddError(data, fmt.Sprintf("invalid %s value: %s", c, obj))
}
// Add validation support for other encodings as needed
// See https://json-schema.org/latest/json-schema-validation.html#rfc.section.8.3
default:
currentState.AddError(data, fmt.Sprintf("unsupported or invalid contentEncoding type of %s", c))
}
}
}

func (c ContentEncoding) Register(uri string, registry *SchemaRegistry) {}

func (c ContentEncoding) Resolve(pointer jsonpointer.Pointer, uri string) *Schema {
return nil
}

func ExampleCustomValidator() {

// register a custom validator by supplying a function
// that creates new instances of your Validator.
RegisterKeyword("foo", newIsFoo)
@@ -86,6 +121,39 @@ func ExampleCustomValidator() {
// Output: /: "bar" should be foo. plz make 'bar' == foo. plz
}

func ExampleCustomSchemaValidator() {

// register a custom validator by supplying a function
// that creates new instances of your Validator.
RegisterKeyword("contentEncoding", newContentEncoding)
ctx := context.Background()

schBytes := []byte(`{
"type": "object",
"properties" : {
"file" : {
"type": "string",
"contentEncoding": "base64"
}
},
"required" : ["file"]
}`)

rs := new(Schema)
if err := json.Unmarshal(schBytes, rs); err != nil {
// Real programs handle errors.
panic(err)
}

errs, err := rs.ValidateBytes(ctx, []byte(`{ "file": "abc123" }`))
if err != nil {
panic(err)
}

fmt.Println(errs[0].Error())
// Output: /file: "abc123" invalid base64 value: abc123
}

type FooKeyword uint8

func (f *FooKeyword) Validate(propPath string, data interface{}, errs *[]KeyError) {}
@@ -106,7 +174,7 @@ func TestRegisterFooKeyword(t *testing.T) {

RegisterKeyword("foo", newFoo)

if !IsRegisteredKeyword("foo") {
if !kr.IsRegisteredKeyword("foo") {
t.Errorf("expected %s to be added as a default validator", "foo")
}
}
21 changes: 10 additions & 11 deletions schema.go
Original file line number Diff line number Diff line change
@@ -64,9 +64,9 @@ func (s *Schema) Register(uri string, registry *SchemaRegistry) {
registry.RegisterLocal(s)

// load default keyset if no other is present
if !IsRegistryLoaded() {
LoadDraft2019_09()
}
globalRegistry, release := getGlobalKeywordRegistry()
globalRegistry.DefaultIfEmpty()
release()

address := s.id
if uri != "" && address != "" {
@@ -179,9 +179,8 @@ func (s *Schema) UnmarshalJSON(data []byte) error {
return nil
}

if !IsRegistryLoaded() {
LoadDraft2019_09()
}
keywordRegistry := copyGlobalKeywordRegistry()
keywordRegistry.DefaultIfEmpty()

_s := _schema{}
if err := json.Unmarshal(data, &_s); err != nil {
@@ -200,9 +199,9 @@ func (s *Schema) UnmarshalJSON(data []byte) error {

for prop, rawmsg := range valprops {
var keyword Keyword
if IsRegisteredKeyword(prop) {
keyword = GetKeyword(prop)
} else if IsNotSupportedKeyword(prop) {
if keywordRegistry.IsRegisteredKeyword(prop) {
keyword = keywordRegistry.GetKeyword(prop)
} else if keywordRegistry.IsNotSupportedKeyword(prop) {
schemaDebug(fmt.Sprintf("[Schema] WARN: '%s' is not supported and will be ignored\n", prop))
continue
} else {
@@ -226,13 +225,13 @@ func (s *Schema) UnmarshalJSON(data []byte) error {
for k := range sch.keywords {
keyOrders[i] = _keyOrder{
Key: k,
Order: GetKeywordOrder(k),
Order: keywordRegistry.GetKeywordOrder(k),
}
i++
}
sort.SliceStable(keyOrders, func(i, j int) bool {
if keyOrders[i].Order == keyOrders[j].Order {
return GetKeywordInsertOrder(keyOrders[i].Key) < GetKeywordInsertOrder(keyOrders[j].Key)
return keywordRegistry.GetKeywordInsertOrder(keyOrders[i].Key) < keywordRegistry.GetKeywordInsertOrder(keyOrders[j].Key)
}
return keyOrders[i].Order < keyOrders[j].Order
})
4 changes: 4 additions & 0 deletions schema_test.go
Original file line number Diff line number Diff line change
@@ -546,6 +546,10 @@ func runJSONTests(t *testing.T, testFilepaths []string) {
sc := ts.Schema
for i, c := range ts.Tests {
tests++

// Ensure we can register keywords in go routines
RegisterKeyword(fmt.Sprintf("content-encoding-%d", tests), newContentEncoding)

validationState := sc.Validate(ctx, c.Data)
if validationState.IsValid() != c.Valid {
t.Errorf("%s: %s test case %d: %s. error: %s", base, ts.Description, i, c.Description, *validationState.Errs)
3 changes: 2 additions & 1 deletion validation_state.go
Original file line number Diff line number Diff line change
@@ -17,7 +17,8 @@ type ValidationState struct {
RelativeLocation *jptr.Pointer
BaseRelativeLocation *jptr.Pointer

LocalRegistry *SchemaRegistry
LocalRegistry *SchemaRegistry
LocalKeywordRegistry *KeywordRegistry

EvaluatedPropertyNames *map[string]bool
LocalEvaluatedPropertyNames *map[string]bool