Skip to content

Commit

Permalink
Add support for passing schema registries along with cases.
Browse files Browse the repository at this point in the history
Only does so at a per-case level. We'll evaluate other options in
#27. It's also possible some global-level request will be useful, which
we can also look at later.

Lets test cases reference arbitrary additional schemas for retrieval.

Also then makes the JSON-Schema-Test-Suite-specific support use this
new registry to pass along remotes/-directory schemas.

This is easier or harder with various implementations it seems, and it
may be I've missed some APIs here for an implementation or two.

Closes: #14.
  • Loading branch information
Julian committed Sep 23, 2022
1 parent 9ac9116 commit 01c7482
Showing 16 changed files with 197 additions and 28 deletions.
24 changes: 19 additions & 5 deletions bowtie/_cli.py
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@
from contextlib import AsyncExitStack
from fnmatch import fnmatch
from pathlib import Path
from urllib.parse import urljoin
import asyncio
import json
import os
@@ -153,7 +154,7 @@ def run(context, input, filter, **kwargs):
Run a sequence of cases provided on standard input.
"""

cases = (TestCase.from_dict(json.loads(line)) for line in input)
cases = (TestCase.from_dict(**json.loads(line)) for line in input)
if filter:
cases = (
case for case in cases
@@ -187,10 +188,12 @@ def suite(context, input, filter, **kwargs):
"""

if input.is_dir():
cases = suite_cases_from(files=input.glob("*.json"))
remotes = input.parent.parent.joinpath("remotes")
cases = suite_cases_from(files=input.glob("*.json"), remotes=remotes)
dialect = DIALECT_SHORTNAMES.get(input.name)
else:
cases = suite_cases_from(files=[input])
remotes = input.parent.parent.parent.joinpath("remotes")
cases = suite_cases_from(files=[input], remotes=remotes)
dialect = DIALECT_SHORTNAMES.get(input.parent.name)
if dialect is None:
raise click.BadParameter(
@@ -277,12 +280,23 @@ def sequenced(cases, reporter):
yield seq, case, reporter.case_started(seq=seq, case=case)


def suite_cases_from(files):
def suite_cases_from(files, remotes):
for file in files:
if file.name == "refRemote.json":
registry = {
urljoin(
"http://localhost:1234",
str(each.relative_to(remotes)).replace("\\", "/"),
): json.loads(each.read_text())
for each in remotes.glob("**/*.json")
}
else:
registry = {}

for case in json.loads(file.read_text()):
for test in case["tests"]:
test["instance"] = test.pop("data")
yield TestCase.from_dict(case)
yield TestCase.from_dict(**case, registry=registry)


def redirect_structlog(file=sys.stderr):
10 changes: 6 additions & 4 deletions bowtie/_commands.py
Original file line number Diff line number Diff line change
@@ -23,11 +23,12 @@ class TestCase:
schema: object
tests: list[Test]
comment: str | None = None
registry: dict | None = None

@classmethod
def from_dict(cls, data):
data["tests"] = [Test(**test) for test in data["tests"]]
return cls(**data)
def from_dict(cls, tests, **kwargs):
kwargs["tests"] = [Test(**test) for test in tests]
return cls(**kwargs)

def without_expected_results(self):
as_dict = {
@@ -40,7 +41,8 @@ def without_expected_results(self):
attrs.asdict(
self,
filter=lambda k, v: k.name != "tests" and (
k.name != "comment" or v is not None
k.name not in {"comment", "registry"}
or v is not None
),
),
)
24 changes: 24 additions & 0 deletions bowtie/schemas/io-schema.json
Original file line number Diff line number Diff line change
@@ -243,6 +243,30 @@
"description": "A valid JSON Schema.",
"$ref": "urn:current-dialect"
},
"registry": {
"description": "A collection of schemas (with URIs) which tests may reference (via $ref) and expect to be retrievable. They should be registered in whatever mechanism is expected by the implementation.",
"type": "array",
"items": {
"type": "object",
"properties": {
"schema": {
"description": "A JSON Schema, in any dialect."
},
"retrievalURIs": {
"description": "An array of retrieval URIs that the implementation should resolve to the schema in this registry entry.",
"type": "array",
"items": { "type": "string", "format": "uri" }
},
"requireDialects": {
"description": "An optional array of dialects which this registry entry (schema) requires the implementation to support. Only needed if the referenced dialect is different from the dialect currently being spoken (as an implementation may not support the referenced dialect).",
"type": "array",
"items": { "type": "string", "format": "uri" }
}
},
"required": ["schema", "retrievalURIs"],
"additionalProperties": false
}
},
"tests": {
"description": "A set of related tests all using the same schema",
"type": "array",
8 changes: 8 additions & 0 deletions implementations/dotnet-json-everything/Program.cs
Original file line number Diff line number Diff line change
@@ -76,8 +76,16 @@

var testCase = root["case"];
var schemaText = testCase["schema"];
var registry = testCase["registry"];

options.SchemaRegistry.Fetch = uri =>
{
return registry[uri.ToString()].Deserialize<JsonSchema>();
};

var schema = schemaText.Deserialize<JsonSchema>();
var tests = testCase["tests"].AsArray();

try {
var results = new JsonArray();

16 changes: 16 additions & 0 deletions implementations/go-jsonschema/bowtie_jsonschema.go
Original file line number Diff line number Diff line change
@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"github.com/santhosh-tekuri/jsonschema/v5"
"io"
"log"
"os"
"strings"
@@ -88,6 +89,21 @@ func main() {
panic("No case!")
}

jsonschema.LoadURL = func(s string) (io.ReadCloser, error) {
refSchema, ok := testCase["registry"].(map[string]interface{})[s]

if !ok {
return nil, fmt.Errorf("%q not found", s)
}

// FIXME: map[string].interface{} -> Schema?
reserializedRef, err := json.Marshal(refSchema)
if err != nil {
panic("This should never happen.")
}
return io.NopCloser(strings.NewReader(string(reserializedRef))), nil
}

// FIXME: map[string].interface{} -> Schema?
reserialized, err := json.Marshal(testCase["schema"])
if err != nil {
6 changes: 6 additions & 0 deletions implementations/js-ajv/bowtie_ajv.js
Original file line number Diff line number Diff line change
@@ -65,7 +65,13 @@ const cmds = {

try {
const testCase = args.case;
const registry = testCase.registry;

ajv.removeSchema(); // Clear the cache.
for (const id in registry) ajv.addSchema(registry[id], id);

const validate = ajv.compile(testCase.schema);

return {
seq: args.seq,
results: testCase.tests.map((test) => ({
6 changes: 6 additions & 0 deletions implementations/js-hyperjump/bowtie_hyperjump.js
Original file line number Diff line number Diff line change
@@ -50,6 +50,12 @@ const cmds = {

const testCase = args.case;

for (const id in testCase.registry) {
try {
JsonSchema.add(testCase.registry[id], id, dialect);
} catch {}
}

const fakeURI = "bowtie.sent.schema." + args.seq.toString() + ".json";
JsonSchema.add(testCase.schema, fakeURI, dialect);
const schema = JsonSchema.get(fakeURI);
8 changes: 7 additions & 1 deletion implementations/lua-jsonschema/bowtie-jsonschema
Original file line number Diff line number Diff line change
@@ -34,7 +34,13 @@ local cmds = {
run = function(request)
assert(STARTED, "Not started!")

local validate = jsonschema.generate_validator(request.case.schema)
local validate = jsonschema.generate_validator(
request.case.schema, {
external_resolver = function(url)
return request.case.registry[url]
end,
}
)
local results = {}
for _, test in ipairs(request.case.tests) do
table.insert(results, { valid = validate(test.instance) })
5 changes: 5 additions & 0 deletions implementations/python-fastjsonschema/bowtie-fastjsonschema
Original file line number Diff line number Diff line change
@@ -54,6 +54,11 @@ class Runner:
assert self._started, "Not started!"
schema = case["schema"]
try:
# The registry parameter isn't used here when building cases.
# Unless I'm missing it, it doesn't seem there's a way to register
# schemas with fastjsonschema. It seems to use RefResolver-like
# objects, but they're not exposed to end-user APIs like compile.

validate = fastjsonschema.compile(schema)

results = []
45 changes: 39 additions & 6 deletions implementations/python-jschon/bowtie-jschon
Original file line number Diff line number Diff line change
@@ -8,6 +8,31 @@ import traceback

import jschon

SUITE_REMOTES_BASE_URI = jschon.URI("http://localhost:1234/")


class InMemorySource(jschon.catalog.Source):
# From what I can tell this seems to be the way to get jschon to look up
# schemas. Just using catalog.add_schema doesn't seem to do it.

def __init__(self, base_url, registry, **kwargs):
# It probably doesn't matter here, but it seems super() isn't the right
# thing to use, as jschon.catalog.Source.__init__ seems to have a
# different signature than its subclasses from the examples I see.
jschon.catalog.Source.__init__(self, **kwargs)
self.base_url = base_url
self.registry = registry

def __call__(self, relative_path):
url = str(jschon.URI(relative_path).resolve(self.base_url))
return self.registry[url]


VOCABULARIES = {
jschon.URI("https://json-schema.org/draft/2020-12/schema"): "2020-12",
jschon.URI("https://json-schema.org/draft/2019-09/schema"): "2019-09",
}


@dataclass
class Runner:
@@ -47,19 +72,27 @@ class Runner:

def cmd_dialect(self, dialect):
assert self._started, "Not started!"
vocabularies = {
"https://json-schema.org/draft/2020-12/schema": "2020-12",
"https://json-schema.org/draft/2019-09/schema": "2019-09",
}
jschon.create_catalog(vocabularies[dialect])
self._metaschema_uri = jschon.URI(dialect)
return dict(ok=True)
return dict(ok=self._metaschema_uri in VOCABULARIES)

def cmd_run(self, case, seq):
assert self._started, "Not started!"
try:
catalog = jschon.create_catalog(
VOCABULARIES[self._metaschema_uri],
name=f"catalog-{seq}",
)
catalog.add_uri_source(
SUITE_REMOTES_BASE_URI,
InMemorySource(
base_url=SUITE_REMOTES_BASE_URI,
registry=case.get("registry", {}),
),
)

schema = jschon.JSONSchema(
case["schema"],
catalog=catalog,
metaschema_uri=self._metaschema_uri,
)

6 changes: 4 additions & 2 deletions implementations/python-jsonschema/bowtie-jsonschema
Original file line number Diff line number Diff line change
@@ -7,7 +7,7 @@ import sys
import traceback

from jsonschema.protocols import Validator
from jsonschema.validators import validator_for
from jsonschema.validators import RefResolver, validator_for


@dataclass
@@ -65,7 +65,9 @@ class Runner:
"No dialect sent and schema is missing $schema."
)

validator = Validator(schema)
registry = case.get("registry", {})
resolver = RefResolver.from_schema(schema, store=registry)
validator = Validator(schema, resolver=resolver)

results = [
{"valid": validator.is_valid(test["instance"])}
19 changes: 11 additions & 8 deletions implementations/ruby-json_schemer/bowtie_json_schemer.rb
Original file line number Diff line number Diff line change
@@ -13,15 +13,15 @@ class NotStarted < StandardError

module BowtieJsonSchemer

started = false
draft = nil
@@started = false
@@draft = nil

ARGF.each_line do |line|
request = JSON.parse(line)
case request["cmd"]
when "start"
raise WrongVersion if request["version"] != 1
started = true
@@started = true
response = {
:ready => true,
:version => 1,
@@ -40,13 +40,16 @@ module BowtieJsonSchemer
}
puts "#{JSON.generate(response)}\n"
when "dialect"
raise NotStarted if not started
draft = JSONSchemer::DRAFT_CLASS_BY_META_SCHEMA[request["dialect"]]
raise NotStarted if not @@started
@@draft = JSONSchemer::DRAFT_CLASS_BY_META_SCHEMA[request["dialect"]]
response = { :ok => true }
puts "#{JSON.generate(response)}\n"
when "run"
raise NotStarted if not started
schemer = draft.new(request["case"]["schema"])
raise NotStarted if not @@started
schemer = @@draft.new(
request["case"]["schema"],
ref_resolver: proc { |uri| request["case"]["registry"][uri.to_s] },
)
response = {
:seq => request["seq"],
:results => request["case"]["tests"].map{ |test|
@@ -55,7 +58,7 @@ module BowtieJsonSchemer
}
puts "#{JSON.generate(response)}\n"
when "stop"
raise NotStarted if not started
raise NotStarted if not @@started
exit(0)
end
end
1 change: 1 addition & 0 deletions implementations/rust-jsonschema/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions implementations/rust-jsonschema/Cargo.toml
Original file line number Diff line number Diff line change
@@ -7,3 +7,4 @@ edition = "2021"
jsonschema = { version = "0.16", features = [ "draft201909", "draft202012" ] }
serde = "1.0"
serde_json = "1.0"
url = "2.2.2"
22 changes: 20 additions & 2 deletions implementations/rust-jsonschema/src/main.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,20 @@
use std::{collections::HashMap, io, process};
use std::{collections::HashMap, io, process, sync::Arc};

use jsonschema::{Draft, JSONSchema};
use jsonschema::{Draft, JSONSchema, SchemaResolver, SchemaResolverError};
use serde_json::{json, Result};
use url::Url;


struct InMemoryResolver {
registry: serde_json::Value,
}

impl SchemaResolver for InMemoryResolver {
fn resolve(&self, _root_schema: &serde_json::Value, url: &Url, _original_reference: &str) -> core::result::Result<Arc<serde_json::Value>, SchemaResolverError> {
return Ok(Arc::new(self.registry[url.to_string()].to_owned()));
}
}


fn main() -> Result<()> {
let dialects = HashMap::from([
@@ -74,6 +87,11 @@ fn main() -> Result<()> {
panic!("Not started!")
};
let case = &request["case"];

let registry = &case["registry"];
let resolver = InMemoryResolver { registry: registry.to_owned() };
compiler = compiler.with_resolver(resolver);

let compiled = compiler.compile(&case["schema"]).expect("Invalid schema!");
let results: Vec<_> = case["tests"]
.as_array()
24 changes: 24 additions & 0 deletions tests/fauxmplementations/lintsonschema/io-schema.json
Original file line number Diff line number Diff line change
@@ -243,6 +243,30 @@
"description": "A valid JSON Schema.",
"$ref": "urn:current-dialect"
},
"registry": {
"description": "A collection of schemas (with URIs) which tests may reference (via $ref) and expect to be retrievable. They should be registered in whatever mechanism is expected by the implementation.",
"type": "array",
"items": {
"type": "object",
"properties": {
"schema": {
"description": "A JSON Schema, in any dialect."
},
"retrievalURIs": {
"description": "An array of retrieval URIs that the implementation should resolve to the schema in this registry entry.",
"type": "array",
"items": { "type": "string", "format": "uri" }
},
"requireDialects": {
"description": "An optional array of dialects which this registry entry (schema) requires the implementation to support. Only needed if the referenced dialect is different from the dialect currently being spoken (as an implementation may not support the referenced dialect).",
"type": "array",
"items": { "type": "string", "format": "uri" }
}
},
"required": ["schema", "retrievalURIs"],
"additionalProperties": false
}
},
"tests": {
"description": "A set of related tests all using the same schema",
"type": "array",

0 comments on commit 01c7482

Please sign in to comment.