Initial implementation

Bench results:

Lua5.3.3: (Lua5.1/5.2 have roughly the same numbers)

    $ lua bench.lua 10
    testcases loaded:   0.000989637
    run 10 validations: 3.8892e-05
    $ lua bench.lua 100
    testcases loaded:   0.001156522
    run 100 validations:    0.000281029
    $ lua bench.lua 1000
    testcases loaded:   0.001209315
    run 1000 validations:   0.00310095
    $ lua bench.lua 10000
    testcases loaded:   0.003009102
    run 10000 validations:  0.024220092
    $ lua bench.lua 100000
    testcases loaded:   0.001657029
    run 100000 validations: 0.219233237
    $ lua bench.lua 1000000
    testcases loaded:   0.001503246
    run 1000000 validations:    2.234834785
    $ lua bench.lua 10000000
    testcases loaded:   0.001200452
    run 10000000 validations:   21.634801401

LuaJIT 2.0.4:

    $ luajit bench.lua 10
    testcases loaded:   0.00253848
    run 10 validations: 0.000113092
    $ luajit bench.lua 100
    testcases loaded:   0.002582576
    run 100 validations:    0.001147461
    $ luajit bench.lua 1000
    testcases loaded:   0.001086581
    run 1000 validations:   0.003077144
    $ luajit bench.lua 10000
    testcases loaded:   0.001171133
    run 10000 validations:  0.016859904
    $ luajit bench.lua 100000
    testcases loaded:   0.00256524
    run 100000 validations: 0.109411399
    $ luajit bench.lua 1000000
    testcases loaded:   0.001154897
    run 1000000 validations:    1.037945463
    $ luajit bench.lua 10000000
    testcases loaded:   0.000716779
    run 10000000 validations:   10.114214835

Author: jdesgats

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "spec/JSON-Schema-Test-Suite"]
+	path = spec/JSON-Schema-Test-Suite
+	url = https://github.com/json-schema-org/JSON-Schema-Test-Suite.git

bench.lua

@@ -0,0 +1,97 @@
+-- JSON schema validator benchmarking tool
+
+local json = require 'cjson'
+local time = require 'posix.time'
+local jsonschema = require 'jsonschema'
+
+local mrandom = math.random
+local clock_gettime = time.clock_gettime
+local CLOCK_PROCESS_CPUTIME_ID = time.CLOCK_PROCESS_CPUTIME_ID
+local function timer()
+  local start = clock_gettime(CLOCK_PROCESS_CPUTIME_ID)
+  return function()
+    local cur = clock_gettime(CLOCK_PROCESS_CPUTIME_ID)
+    return (cur.tv_sec + cur.tv_nsec/1e9) - (start.tv_sec + start.tv_nsec/1e9)
+  end
+end
+
+local blacklist = {
+  -- edge cases, not supported features
+  ['object type matches objects'] = {
+    ['an array is not an object'] = true, -- empty object/array confusion
+  },
+  ['array type matches arrays'] = {
+    ['an object is not an array'] = true, -- empty object/array confusion
+  },
+  ['regexes are not anchored by default and are case sensitive'] = {
+    ['recognized members are accounted for'] = true, -- uses a unsupported pattern construct
+  },
+  ['minLength validation'] = {
+    ['one supplementary Unicode code point is not long enough'] = true, -- unicode handling
+  },
+  ['maxLength validation'] = {
+    ['two supplementary Unicode code points is long enough'] = true, -- unicode handling
+  },
+}
+
+local supported = {
+  'spec/extra/sanity.json',
+  'spec/extra/empty.json',
+  'spec/JSON-Schema-Test-Suite/tests/draft4/type.json',
+  -- objects
+  'spec/JSON-Schema-Test-Suite/tests/draft4/properties.json',
+  'spec/JSON-Schema-Test-Suite/tests/draft4/required.json',
+  'spec/JSON-Schema-Test-Suite/tests/draft4/additionalProperties.json',
+  'spec/JSON-Schema-Test-Suite/tests/draft4/patternProperties.json',
+  'spec/JSON-Schema-Test-Suite/tests/draft4/minProperties.json',
+  'spec/JSON-Schema-Test-Suite/tests/draft4/maxProperties.json',
+  -- TODO: dependencies
+  -- strings
+  'spec/JSON-Schema-Test-Suite/tests/draft4/minLength.json',
+  'spec/JSON-Schema-Test-Suite/tests/draft4/maxLength.json',
+  'spec/JSON-Schema-Test-Suite/tests/draft4/pattern.json',
+  -- numbers
+  'spec/JSON-Schema-Test-Suite/tests/draft4/multipleOf.json',
+  'spec/JSON-Schema-Test-Suite/tests/draft4/minimum.json',
+  'spec/JSON-Schema-Test-Suite/tests/draft4/maximum.json',
+  -- lists
+  'spec/JSON-Schema-Test-Suite/tests/draft4/items.json',
+  'spec/JSON-Schema-Test-Suite/tests/draft4/additionalItems.json',
+  'spec/JSON-Schema-Test-Suite/tests/draft4/minItems.json',
+  'spec/JSON-Schema-Test-Suite/tests/draft4/maxItems.json',
+  --'spec/JSON-Schema-Test-Suite/tests/draft4/uniqueItems.json',
+  -- misc
+  -- 'spec/JSON-Schema-Test-Suite/tests/draft4/enum.json',
+}
+
+local function decode_descriptor(path)
+  local f = assert(io.open(path))
+  local testsuite = json.decode(assert(f:read('*a')))
+  f:close()
+  return ipairs(testsuite)
+end
+
+local NRUNS = assert(tonumber(arg[1]), 'run count required')
+
+-- read all test cases
+local loadtimer = timer()
+local cases, ncases = {}, 0
+for _, descriptor in ipairs(supported) do
+  for _, suite in decode_descriptor(descriptor) do
+    local skipped = blacklist[suite.description] or {}
+    if skipped ~= true then
+      for _, case in ipairs(suite.tests) do
+        ncases = ncases+1
+        cases[ncases] = { jsonschema.generate_validator(suite.schema), case.data }
+      end
+    end
+  end
+end
+print('testcases loaded:', loadtimer())
+
+local runtimer = timer()
+for i=1, NRUNS do
+  local case = cases[mrandom(ncases)]
+  case[1](case[2])
+end
+print('run ' .. NRUNS .. ' validations:', runtimer())