Skip to content

Commit 8a3e605

Browse files
fsteegfsteeg
authored
Merge pull request #468 from metafacture/443-jsonValidator
Add JSON schema validator
2 parents e353e4b + 355016c commit 8a3e605

File tree

6 files changed

+393
-0
lines changed

6 files changed

+393
-0
lines changed

‎metafacture-json/build.gradle

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,9 @@ dependencies {
2222
implementation 'com.fasterxml.jackson.core:jackson-core:2.13.0'
2323
implementation 'com.fasterxml.jackson.core:jackson-databind:2.13.0'
2424
implementation 'com.jayway.jsonpath:json-path:2.6.0'
25+
implementation 'com.github.erosb:everit-json-schema:1.14.1'
2526
testImplementation 'junit:junit:4.12'
2627
testImplementation 'org.mockito:mockito-core:2.5.5'
28+
testImplementation 'com.github.tomakehurst:wiremock-jre8:2.33.2'
2729
testRuntimeOnly 'org.slf4j:slf4j-simple:1.7.21'
2830
}

‎metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
/*
2+
* Copyright 2021, 2023 Fabian Steeg, hbz
3+
*
4+
* Licensed under the Apache License, Version 2.0 the "License";
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.metafacture.json;
18+
19+
import org.metafacture.framework.FluxCommand;
20+
import org.metafacture.framework.MetafactureException;
21+
import org.metafacture.framework.ObjectReceiver;
22+
import org.metafacture.framework.annotations.Description;
23+
import org.metafacture.framework.annotations.In;
24+
import org.metafacture.framework.annotations.Out;
25+
import org.metafacture.framework.helpers.DefaultObjectPipe;
26+
27+
import org.everit.json.schema.Schema;
28+
import org.everit.json.schema.ValidationException;
29+
import org.everit.json.schema.loader.SchemaClient;
30+
import org.everit.json.schema.loader.SchemaLoader;
31+
import org.everit.json.schema.loader.SchemaLoader.SchemaLoaderBuilder;
32+
import org.json.JSONException;
33+
import org.json.JSONObject;
34+
import org.json.JSONTokener;
35+
import org.slf4j.Logger;
36+
import org.slf4j.LoggerFactory;
37+
38+
import java.io.FileWriter;
39+
import java.io.IOException;
40+
import java.io.InputStream;
41+
import java.net.URL;
42+
43+
/**
44+
* Validate JSON against a given schema, pass only valid input to the receiver.
45+
*
46+
* @author Fabian Steeg (fsteeg)
47+
*/
48+
@Description("Validate JSON against a given schema, send only valid input to the receiver. Pass the schema location to validate against. " +
49+
"Write valid and/or invalid output to locations specified with `writeValid` and `writeInvalid`. " +
50+
"Set the JSON key for the record ID value with `idKey` (for logging output, defaults to `id`).")
51+
@In(String.class)
52+
@Out(String.class)
53+
@FluxCommand("validate-json")
54+
public final class JsonValidator extends DefaultObjectPipe<String, ObjectReceiver<String>> {
55+
56+
private static final Logger LOG = LoggerFactory.getLogger(JsonValidator.class);
57+
private static final String DEFAULT_ID_KEY = "id";
58+
private Schema schema;
59+
private long fail;
60+
private long success;
61+
private FileWriter writeInvalid;
62+
private FileWriter writeValid;
63+
private String idKey = DEFAULT_ID_KEY;
64+
65+
/**
66+
* @param url The URL of the schema to validate against.
67+
*/
68+
public JsonValidator(final String url) {
69+
initSchema(url);
70+
}
71+
72+
/**
73+
* @param writeValid The location to write valid data to.
74+
*/
75+
public void setWriteValid(final String writeValid) {
76+
this.writeValid = fileWriter(writeValid);
77+
}
78+
79+
/**
80+
* @param writeInvalid The location to write invalid data to.
81+
*/
82+
public void setWriteInvalid(final String writeInvalid) {
83+
this.writeInvalid = fileWriter(writeInvalid);
84+
}
85+
86+
/**
87+
* @param idKey The JSON key for the record ID value.
88+
*/
89+
public void setIdKey(final String idKey) {
90+
this.idKey = idKey;
91+
}
92+
93+
@Override
94+
public void process(final String json) {
95+
try {
96+
validate(json, new JSONObject(json) /* throws JSONException on syntax error */);
97+
}
98+
catch (final JSONException e) {
99+
handleInvalid(json, null, e.getMessage());
100+
}
101+
}
102+
103+
private void validate(final String json, final JSONObject object) {
104+
try {
105+
schema.validate(object); // throws ValidationException if invalid
106+
getReceiver().process(json);
107+
++success;
108+
write(json, writeValid);
109+
}
110+
catch (final ValidationException e) {
111+
handleInvalid(json, object, e.getAllMessages().toString());
112+
}
113+
}
114+
115+
@Override
116+
protected void onCloseStream() {
117+
close(writeInvalid);
118+
close(writeValid);
119+
LOG.debug("Success: {}, Fail: {}", success, fail);
120+
super.onCloseStream();
121+
}
122+
123+
private void initSchema(final String schemaUrl) {
124+
if (schema != null) {
125+
return;
126+
}
127+
SchemaLoaderBuilder schemaLoader = SchemaLoader.builder();
128+
try {
129+
final URL url = new URL(schemaUrl);
130+
schemaLoader = schemaLoader.schemaJson(jsonFrom(url.openStream()))
131+
.resolutionScope(baseFor(url.toString()));
132+
}
133+
catch (final IOException e) {
134+
LOG.debug("Could not read as URL: {}, trying to load from class path", schemaUrl);
135+
schemaLoader = schemaLoader.schemaClient(SchemaClient.classPathAwareClient())
136+
.schemaJson(jsonFrom(getClass().getResourceAsStream(schemaUrl)))
137+
.resolutionScope("classpath://" + baseFor(schemaUrl));
138+
}
139+
schema = schemaLoader.build().load().build();
140+
}
141+
142+
private JSONObject jsonFrom(final InputStream inputStream) {
143+
try {
144+
return new JSONObject(new JSONTokener(inputStream));
145+
}
146+
catch (final JSONException e) {
147+
throw new MetafactureException(e.getMessage(), e);
148+
}
149+
}
150+
151+
private String baseFor(final String path) {
152+
return path.substring(0, path.lastIndexOf('/') + 1);
153+
}
154+
155+
private FileWriter fileWriter(final String fileLocation) {
156+
try {
157+
return new FileWriter(fileLocation);
158+
}
159+
catch (final IOException e) {
160+
throw new MetafactureException(e.getMessage(), e);
161+
}
162+
}
163+
164+
private void handleInvalid(final String json, final JSONObject object,
165+
final String errorMessage) {
166+
LOG.info("Invalid JSON: {} in {}", errorMessage, object != null ? object.opt(idKey) : json);
167+
++fail;
168+
write(json, writeInvalid);
169+
}
170+
171+
private void write(final String json, final FileWriter fileWriter) {
172+
if (fileWriter != null) {
173+
try {
174+
fileWriter.append(json);
175+
fileWriter.append("\n");
176+
}
177+
catch (final IOException e) {
178+
throw new MetafactureException(e.getMessage(), e);
179+
}
180+
}
181+
}
182+
183+
private void close(final FileWriter fileWriter) {
184+
if (fileWriter != null) {
185+
try {
186+
fileWriter.close();
187+
}
188+
catch (final IOException e) {
189+
throw new MetafactureException(e.getMessage(), e);
190+
}
191+
}
192+
}
193+
194+
}

‎metafacture-json/src/main/resources/flux-commands.properties

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,4 @@
1515
#
1616
encode-json org.metafacture.json.JsonEncoder
1717
decode-json org.metafacture.json.JsonDecoder
18+
validate-json org.metafacture.json.JsonValidator

‎metafacture-json/src/test/java/org/metafacture/json/JsonValidatorTest.java

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
/*
2+
* Copyright 2021, 2023 Fabian Steeg, hbz
3+
*
4+
* Licensed under the Apache License, Version 2.0 the "License";
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package org.metafacture.json;
17+
18+
import static org.hamcrest.CoreMatchers.both;
19+
import static org.hamcrest.CoreMatchers.containsString;
20+
import static org.junit.Assert.assertThat;
21+
import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
22+
import static com.github.tomakehurst.wiremock.client.WireMock.request;
23+
24+
import java.io.BufferedReader;
25+
import java.io.IOException;
26+
import java.io.InputStreamReader;
27+
import java.net.MalformedURLException;
28+
import java.net.URL;
29+
import java.nio.charset.StandardCharsets;
30+
import java.util.Arrays;
31+
import java.util.Collection;
32+
import java.util.function.Function;
33+
import java.util.stream.Collectors;
34+
35+
import org.junit.After;
36+
import org.junit.Before;
37+
import org.junit.Rule;
38+
import org.junit.Test;
39+
import org.junit.runner.RunWith;
40+
import org.junit.runners.Parameterized;
41+
import org.metafacture.framework.MetafactureException;
42+
import org.metafacture.framework.ObjectReceiver;
43+
import org.mockito.InOrder;
44+
import org.mockito.Mock;
45+
import org.mockito.Mockito;
46+
import org.mockito.MockitoAnnotations;
47+
48+
import com.github.tomakehurst.wiremock.client.WireMock;
49+
import com.github.tomakehurst.wiremock.core.WireMockConfiguration;
50+
import com.github.tomakehurst.wiremock.junit.WireMockRule;
51+
52+
53+
/**
54+
* Tests for {@link JsonValidator}.
55+
*
56+
* @author Fabian Steeg
57+
*
58+
*/
59+
@RunWith(Parameterized.class)
60+
public final class JsonValidatorTest {
61+
62+
private static final String MAIN_SCHEMA = "/schemas/schema.json";
63+
private static final String ID_SCHEMA = "/schemas/id.json";
64+
private static final String JSON_VALID = "{\"id\":\"http://example.org/\"}";
65+
private static final String JSON_INVALID_MISSING_REQUIRED = "{}";
66+
private static final String JSON_INVALID_URI_FORMAT= "{\"id\":\"example.org/\"}";
67+
private static final String JSON_INVALID_DUPLICATE_KEY = "{\"id\":\"val\",\"id\":\"val\"}";
68+
private static final String JSON_INVALID_SYNTAX_ERROR = "{\"id1\":\"val\",\"id2\":\"val\"";
69+
70+
private JsonValidator validator;
71+
72+
@Mock
73+
private ObjectReceiver<String> receiver;
74+
private InOrder inOrder;
75+
private Function<Object, String> schemaLocationGetter;
76+
77+
@Rule
78+
public WireMockRule wireMockRule = new WireMockRule(WireMockConfiguration.wireMockConfig()
79+
.jettyAcceptors(Runtime.getRuntime().availableProcessors()).dynamicPort());
80+
81+
@Parameterized.Parameters(name = "{index}")
82+
public static Collection<Object[]> siteMaps() {
83+
return Arrays.asList((Object[][]) (new Function[][] { //
84+
// Pass the schema to each test as path on classpath, file url, and http url:
85+
{ (Object rule) -> MAIN_SCHEMA },
86+
{ (Object rule) -> JsonValidatorTest.class.getResource(MAIN_SCHEMA).toString() },
87+
{ (Object rule) -> ((WireMockRule) rule).baseUrl() + MAIN_SCHEMA } }));
88+
}
89+
90+
public JsonValidatorTest(Function<Object, String> schemaLocationGetter) {
91+
this.schemaLocationGetter = schemaLocationGetter;
92+
}
93+
94+
@Before
95+
public void setup() throws IOException {
96+
MockitoAnnotations.initMocks(this);
97+
wireMock(MAIN_SCHEMA, ID_SCHEMA);
98+
String schemaLocation = schemaLocationGetter.apply(wireMockRule);
99+
validator = new JsonValidator(schemaLocation);
100+
validator.setReceiver(receiver);
101+
inOrder = Mockito.inOrder(receiver);
102+
}
103+
104+
private void wireMock(final String... schemaLocations) throws IOException {
105+
for (String schemaLocation : schemaLocations) {
106+
stubFor(request("GET", WireMock.urlEqualTo(schemaLocation)).willReturn(
107+
WireMock.ok().withBody(readToString(getClass().getResource(schemaLocation)))
108+
.withHeader("Content-type", "application/json")));
109+
}
110+
}
111+
112+
private String readToString(final URL url) throws IOException {
113+
return new BufferedReader(new InputStreamReader(url.openStream(), StandardCharsets.UTF_8))
114+
.lines().collect(Collectors.joining("\n"));
115+
}
116+
117+
@Test
118+
public void callWireMockSchema() throws MalformedURLException, IOException {
119+
final String schemaContent = readToString(new URL(wireMockRule.baseUrl() + MAIN_SCHEMA));
120+
assertThat(schemaContent, both(containsString("$schema")).and(containsString("$ref")));
121+
}
122+
123+
@Test
124+
public void testShouldValidate() {
125+
validator.process(JSON_VALID);
126+
inOrder.verify(receiver, Mockito.calls(1)).process(JSON_VALID);
127+
}
128+
129+
@Test
130+
public void testShouldInvalidateMissingRequired() {
131+
validator.process(JSON_INVALID_MISSING_REQUIRED);
132+
inOrder.verifyNoMoreInteractions();
133+
}
134+
135+
@Test
136+
public void testShouldInvalidateUriFormat() {
137+
validator.process(JSON_INVALID_URI_FORMAT);
138+
inOrder.verifyNoMoreInteractions();
139+
}
140+
141+
@Test
142+
public void testShouldInvalidateDuplicateKey() {
143+
validator.process(JSON_INVALID_DUPLICATE_KEY);
144+
inOrder.verifyNoMoreInteractions();
145+
}
146+
147+
@Test
148+
public void testShouldInvalidateSyntaxError() {
149+
validator.process(JSON_INVALID_SYNTAX_ERROR);
150+
inOrder.verifyNoMoreInteractions();
151+
}
152+
153+
@Test(expected = MetafactureException.class)
154+
public void testShouldCatchMissingSchemaFile() {
155+
new JsonValidator("").process("{}");
156+
}
157+
158+
@Test(expected = MetafactureException.class)
159+
public void testShouldCatchMissingValidOutputFile() {
160+
validator.setWriteValid("");
161+
validator.process(JSON_INVALID_MISSING_REQUIRED);
162+
}
163+
164+
@Test(expected = MetafactureException.class)
165+
public void testShouldCatchMissingInvalidOutputFile() {
166+
validator.setWriteInvalid("");
167+
validator.process(JSON_INVALID_MISSING_REQUIRED);
168+
}
169+
170+
@After
171+
public void cleanup() {
172+
validator.closeStream();
173+
}
174+
175+
}

‎metafacture-json/src/test/resources/schemas/id.json

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"$id": "id.json",
3+
"$schema": "http://json-schema.org/draft-07/schema#",
4+
"title": "URL",
5+
"description": "The URL/URI of the resource",
6+
"type": "string",
7+
"format": "uri"
8+
}

0 commit comments

Comments
 (0)