From ad64284c0bcceb54cedb136520922bbd2059fad4 Mon Sep 17 00:00:00 2001 From: Eric Wittmann Date: Fri, 16 Jun 2023 13:49:59 -0400 Subject: [PATCH 1/2] Added a new ReferenceFinder component for artifacts (per-type). Added a proposed change to the maven plugin to auto detect refs --- .../content/refs/AsyncApiReferenceFinder.java | 26 +++ .../content/refs/AvroReferenceFinder.java | 105 ++++++++++ .../content/refs/ExternalReference.java | 150 +++++++++++++ .../refs/JsonPointerExternalReference.java | 54 +++++ .../content/refs/NoOpReferenceFinder.java | 39 ++++ .../content/refs/ReferenceFinder.java | 34 +++ .../refs/JsonSchemaReferenceFinder.java | 65 ++++++ .../AbstractDataModelsReferenceFinder.java | 98 +++++++++ .../content/refs/OpenApiReferenceFinder.java | 26 +++ .../content/refs/ProtobufReferenceFinder.java | 57 +++++ .../provider/ArtifactTypeUtilProvider.java | 3 + .../AsyncApiArtifactTypeUtilProvider.java | 7 + .../AvroArtifactTypeUtilProvider.java | 7 + .../GraphQLArtifactTypeUtilProvider.java | 10 + .../JsonArtifactTypeUtilProvider.java | 7 + .../KConnectArtifactTypeUtilProvider.java | 10 + .../OpenApiArtifactTypeUtilProvider.java | 7 + .../ProtobufArtifactTypeUtilProvider.java | 7 + .../WsdlArtifactTypeUtilProvider.java | 10 + .../provider/XmlArtifactTypeUtilProvider.java | 10 + .../provider/XsdArtifactTypeUtilProvider.java | 10 + .../refs/AsyncApiReferenceFinderTest.java | 47 +++++ .../content/refs/AvroReferenceFinderTest.java | 47 +++++ .../refs/OpenApiReferenceFinderTest.java | 47 +++++ .../refs/ProtobufReferenceFinderTest.java | 48 +++++ .../content/refs/asyncapi-with-refs.json | 33 +++ .../registry/content/refs/avro-with-refs.avsc | 23 ++ .../content/refs/openapi-with-refs.json | 159 ++++++++++++++ .../content/refs/protobuf-with-refs.proto | 22 ++ utils/maven-plugin/pom.xml | 10 +- .../registry/maven/ExistingReference.java | 90 ++++++++ .../registry/maven/RegisterArtifact.java | 18 ++ .../registry/maven/RegisterRegistryMojo.java | 198 ++++++++++++++++-- .../registry/maven/refs/IndexedResource.java | 122 +++++++++++ .../registry/maven/refs/ReferenceIndex.java | 165 +++++++++++++++ 35 files changed, 1739 insertions(+), 32 deletions(-) create mode 100644 schema-util/asyncapi/src/main/java/io/apicurio/registry/content/refs/AsyncApiReferenceFinder.java create mode 100644 schema-util/avro/src/main/java/io/apicurio/registry/content/refs/AvroReferenceFinder.java create mode 100644 schema-util/common/src/main/java/io/apicurio/registry/content/refs/ExternalReference.java create mode 100644 schema-util/common/src/main/java/io/apicurio/registry/content/refs/JsonPointerExternalReference.java create mode 100644 schema-util/common/src/main/java/io/apicurio/registry/content/refs/NoOpReferenceFinder.java create mode 100644 schema-util/common/src/main/java/io/apicurio/registry/content/refs/ReferenceFinder.java create mode 100644 schema-util/json/src/main/java/io/apicurio/registry/content/refs/JsonSchemaReferenceFinder.java create mode 100644 schema-util/openapi/src/main/java/io/apicurio/registry/content/refs/AbstractDataModelsReferenceFinder.java create mode 100644 schema-util/openapi/src/main/java/io/apicurio/registry/content/refs/OpenApiReferenceFinder.java create mode 100644 schema-util/protobuf/src/main/java/io/apicurio/registry/content/refs/ProtobufReferenceFinder.java create mode 100644 schema-util/util-provider/src/test/java/io/apicurio/registry/content/refs/AsyncApiReferenceFinderTest.java create mode 100644 schema-util/util-provider/src/test/java/io/apicurio/registry/content/refs/AvroReferenceFinderTest.java create mode 100644 schema-util/util-provider/src/test/java/io/apicurio/registry/content/refs/OpenApiReferenceFinderTest.java create mode 100644 schema-util/util-provider/src/test/java/io/apicurio/registry/content/refs/ProtobufReferenceFinderTest.java create mode 100644 schema-util/util-provider/src/test/resources/io/apicurio/registry/content/refs/asyncapi-with-refs.json create mode 100644 schema-util/util-provider/src/test/resources/io/apicurio/registry/content/refs/avro-with-refs.avsc create mode 100644 schema-util/util-provider/src/test/resources/io/apicurio/registry/content/refs/openapi-with-refs.json create mode 100644 schema-util/util-provider/src/test/resources/io/apicurio/registry/content/refs/protobuf-with-refs.proto create mode 100644 utils/maven-plugin/src/main/java/io/apicurio/registry/maven/ExistingReference.java create mode 100644 utils/maven-plugin/src/main/java/io/apicurio/registry/maven/refs/IndexedResource.java create mode 100644 utils/maven-plugin/src/main/java/io/apicurio/registry/maven/refs/ReferenceIndex.java diff --git a/schema-util/asyncapi/src/main/java/io/apicurio/registry/content/refs/AsyncApiReferenceFinder.java b/schema-util/asyncapi/src/main/java/io/apicurio/registry/content/refs/AsyncApiReferenceFinder.java new file mode 100644 index 0000000000..dd5b1ae4a7 --- /dev/null +++ b/schema-util/asyncapi/src/main/java/io/apicurio/registry/content/refs/AsyncApiReferenceFinder.java @@ -0,0 +1,26 @@ +/* + * Copyright 2023 Red Hat Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.apicurio.registry.content.refs; + +/** + * OpenAPI implementation of a reference finder. Parses the OpenAPI document, finds all $refs, converts them + * to external references, and returns them. + * @author eric.wittmann@gmail.com + */ +public class AsyncApiReferenceFinder extends AbstractDataModelsReferenceFinder { + +} diff --git a/schema-util/avro/src/main/java/io/apicurio/registry/content/refs/AvroReferenceFinder.java b/schema-util/avro/src/main/java/io/apicurio/registry/content/refs/AvroReferenceFinder.java new file mode 100644 index 0000000000..bed52750f1 --- /dev/null +++ b/schema-util/avro/src/main/java/io/apicurio/registry/content/refs/AvroReferenceFinder.java @@ -0,0 +1,105 @@ +/* + * Copyright 2023 Red Hat Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.apicurio.registry.content.refs; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; + +import io.apicurio.registry.content.ContentHandle; + +/** + * An Apache Avro implementation of a reference finder. + * @author eric.wittmann@gmail.com + */ +public class AvroReferenceFinder implements ReferenceFinder { + + private static final ObjectMapper mapper = new ObjectMapper(); + private static final Logger log = LoggerFactory.getLogger(AvroReferenceFinder.class); + + private static final Set PRIMITIVE_TYPES = Set.of("null", "boolean", "int", "long", "float", "double", "bytes", "string"); + + /** + * @see io.apicurio.registry.content.refs.ReferenceFinder#findExternalReferences(io.apicurio.registry.content.ContentHandle) + */ + @Override + public Set findExternalReferences(ContentHandle content) { + try { + JsonNode tree = mapper.readTree(content.content()); + Set externalTypes = new HashSet<>(); + findExternalTypesIn(tree, externalTypes); + return externalTypes.stream().map(type -> new ExternalReference(type)).collect(Collectors.toSet()); + } catch (Exception e) { + log.error("Error finding external references in an Avro file.", e); + return Collections.emptySet(); + } + } + + private static void findExternalTypesIn(JsonNode schema, Set externalTypes) { + // Null check + if (schema == null || schema.isNull()) { + return; + } + + // Handle primitive/external types + if (schema.isTextual()) { + String type = schema.asText(); + if (!PRIMITIVE_TYPES.contains(type)) { + externalTypes.add(type); + } + } + + // Handle unions + if (schema.isArray()) { + ArrayNode schemas = (ArrayNode) schema; + schemas.forEach(s -> findExternalTypesIn(s, externalTypes)); + } + + // Handle records + if (schema.isObject() && schema.has("type") && !schema.get("type").isNull() && schema.get("type").asText().equals("record")) { + JsonNode fieldsNode = schema.get("fields"); + if (fieldsNode != null && fieldsNode.isArray()) { + ArrayNode fields = (ArrayNode) fieldsNode; + fields.forEach(fieldNode -> { + if (fieldNode.isObject()) { + JsonNode typeNode = fieldNode.get("type"); + findExternalTypesIn(typeNode, externalTypes); + } + }); + } + } + // Handle arrays + if (schema.has("type") && !schema.get("type").isNull() && schema.get("type").asText().equals("array")) { + JsonNode items = schema.get("items"); + findExternalTypesIn(items, externalTypes); + } + // Handle maps + if (schema.has("type") && !schema.get("type").isNull() && schema.get("type").asText().equals("map")) { + JsonNode values = schema.get("values"); + findExternalTypesIn(values, externalTypes); + } + } + +} diff --git a/schema-util/common/src/main/java/io/apicurio/registry/content/refs/ExternalReference.java b/schema-util/common/src/main/java/io/apicurio/registry/content/refs/ExternalReference.java new file mode 100644 index 0000000000..7eb1aae0ed --- /dev/null +++ b/schema-util/common/src/main/java/io/apicurio/registry/content/refs/ExternalReference.java @@ -0,0 +1,150 @@ +/* + * Copyright 2023 Red Hat Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.apicurio.registry.content.refs; + +import java.util.Objects; + +/** + * Models a reference from one artifact to another. This represents the information found in the content + * of an artifact, and is very type-specific. For example, a JSON schema reference might look like this: + * + * 
+ * {
+ *   "$ref" : "types/data-types.json#/$defs/FooType"
+ * }
+ *
+ * + * In this case, the fields of this type will be: + * + *
    + *
  • fullReference: types/data-types.json#/$defs/FooType
    • + *
  • resource: types/data-types.json
    • + *
  • component: #/$defs/FooType
    • + *
+ * + * For an Avro artifact a reference might look like this: + * + * 
+ * {
+ *   "name": "exchange",
+ *   "type": "com.kubetrade.schema.common.Exchange"
+ * }
+ *
+ * + * In this case, the fields of this type will be: + * + *
    + *
  • fullReference: com.kubetrade.schema.common.Exchange
    • + *
  • resource: com.kubetrade.schema.common.Exchange
    • + *
  • component: null
    • + *
+ * + * @author eric.wittmann@gmail.com + */ +public class ExternalReference { + + private String fullReference; + private String resource; + private String component; + + /** + * Constructor. + * @param fullReference + * @param resource + * @param component + */ + public ExternalReference(String fullReference, String resource, String component) { + this.fullReference = fullReference; + this.resource = resource; + this.component = component; + } + + /** + * Constructor. This variant is useful if there is no component part of an external reference. In this + * case the full reference is also the resource (and the component is null). + * @param reference + */ + public ExternalReference(String reference) { + this(reference, reference, null); + } + + /** + * @return the fullReference + */ + public String getFullReference() { + return fullReference; + } + + /** + * @param fullReference the fullReference to set + */ + public void setFullReference(String fullReference) { + this.fullReference = fullReference; + } + + /** + * @return the resource + */ + public String getResource() { + return resource; + } + + /** + * @param resource the resource to set + */ + public void setResource(String resource) { + this.resource = resource; + } + + /** + * @return the component + */ + public String getComponent() { + return component; + } + + /** + * @param component the component to set + */ + public void setComponent(String component) { + this.component = component; + } + + /** + * @see java.lang.Object#hashCode() + */ + @Override + public int hashCode() { + return Objects.hash(fullReference); + } + + /** + * @see java.lang.Object#equals(java.lang.Object) + */ + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + ExternalReference other = (ExternalReference) obj; + return Objects.equals(fullReference, other.fullReference); + } + +} diff --git a/schema-util/common/src/main/java/io/apicurio/registry/content/refs/JsonPointerExternalReference.java b/schema-util/common/src/main/java/io/apicurio/registry/content/refs/JsonPointerExternalReference.java new file mode 100644 index 0000000000..a1afd0d3d7 --- /dev/null +++ b/schema-util/common/src/main/java/io/apicurio/registry/content/refs/JsonPointerExternalReference.java @@ -0,0 +1,54 @@ +/* + * Copyright 2023 Red Hat Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.apicurio.registry.content.refs; + +/** + * @author eric.wittmann@gmail.com + */ +public class JsonPointerExternalReference extends ExternalReference { + + /** + * Constructor. + * @param jsonPointer + */ + public JsonPointerExternalReference(String jsonPointer) { + super(jsonPointer, resourceFrom(jsonPointer), componentFrom(jsonPointer)); + } + + private static String componentFrom(String jsonPointer) { + int idx = jsonPointer.indexOf('#'); + if (idx == 0) { + return jsonPointer; + } else if (idx > 0) { + return jsonPointer.substring(idx); + } else { + return null; + } + } + + private static String resourceFrom(String jsonPointer) { + int idx = jsonPointer.indexOf('#'); + if (idx == 0) { + return null; + } else if (idx > 0) { + return jsonPointer.substring(0, idx); + } else { + return jsonPointer; + } + } + +} diff --git a/schema-util/common/src/main/java/io/apicurio/registry/content/refs/NoOpReferenceFinder.java b/schema-util/common/src/main/java/io/apicurio/registry/content/refs/NoOpReferenceFinder.java new file mode 100644 index 0000000000..9a42418300 --- /dev/null +++ b/schema-util/common/src/main/java/io/apicurio/registry/content/refs/NoOpReferenceFinder.java @@ -0,0 +1,39 @@ +/* + * Copyright 2023 Red Hat Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.apicurio.registry.content.refs; + +import java.util.Collections; +import java.util.Set; + +import io.apicurio.registry.content.ContentHandle; + +/** + * @author eric.wittmann@gmail.com + */ +public class NoOpReferenceFinder implements ReferenceFinder { + + public static final ReferenceFinder INSTANCE = new NoOpReferenceFinder(); + + /** + * @see io.apicurio.registry.content.refs.ReferenceFinder#findExternalReferences(io.apicurio.registry.content.ContentHandle) + */ + @Override + public Set findExternalReferences(ContentHandle content) { + return Collections.emptySet(); + } + +} diff --git a/schema-util/common/src/main/java/io/apicurio/registry/content/refs/ReferenceFinder.java b/schema-util/common/src/main/java/io/apicurio/registry/content/refs/ReferenceFinder.java new file mode 100644 index 0000000000..b662ea280b --- /dev/null +++ b/schema-util/common/src/main/java/io/apicurio/registry/content/refs/ReferenceFinder.java @@ -0,0 +1,34 @@ +/* + * Copyright 2023 Red Hat Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.apicurio.registry.content.refs; + +import java.util.Set; + +import io.apicurio.registry.content.ContentHandle; + +/** + * @author eric.wittmann@gmail.com + */ +public interface ReferenceFinder { + + /** + * Finds the set of external references in a piece of content. + * @param content + */ + public Set findExternalReferences(ContentHandle content); + +} diff --git a/schema-util/json/src/main/java/io/apicurio/registry/content/refs/JsonSchemaReferenceFinder.java b/schema-util/json/src/main/java/io/apicurio/registry/content/refs/JsonSchemaReferenceFinder.java new file mode 100644 index 0000000000..7da956651b --- /dev/null +++ b/schema-util/json/src/main/java/io/apicurio/registry/content/refs/JsonSchemaReferenceFinder.java @@ -0,0 +1,65 @@ +/* + * Copyright 2023 Red Hat Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.apicurio.registry.content.refs; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import io.apicurio.registry.content.ContentHandle; + +/** + * A JSON Schema implementation of a reference finder. + * @author eric.wittmann@gmail.com + */ +public class JsonSchemaReferenceFinder implements ReferenceFinder { + + private static final ObjectMapper mapper = new ObjectMapper(); + private static final Logger log = LoggerFactory.getLogger(JsonSchemaReferenceFinder.class); + + /** + * @see io.apicurio.registry.content.refs.ReferenceFinder#findExternalReferences(io.apicurio.registry.content.ContentHandle) + */ + @Override + public Set findExternalReferences(ContentHandle content) { + try { + JsonNode tree = mapper.readTree(content.content()); + Set externalTypes = new HashSet<>(); + findExternalTypesIn(tree, externalTypes); + + return externalTypes.stream() + .map(type -> new JsonPointerExternalReference(type)) + .filter(ref -> ref.getResource() != null) + .collect(Collectors.toSet()); + } catch (Exception e) { + log.error("Error finding external references in an Avro file.", e); + return Collections.emptySet(); + } + } + + private static void findExternalTypesIn(JsonNode schema, Set externalTypes) { + // TODO implement this!! + } + +} diff --git a/schema-util/openapi/src/main/java/io/apicurio/registry/content/refs/AbstractDataModelsReferenceFinder.java b/schema-util/openapi/src/main/java/io/apicurio/registry/content/refs/AbstractDataModelsReferenceFinder.java new file mode 100644 index 0000000000..0baf5e71db --- /dev/null +++ b/schema-util/openapi/src/main/java/io/apicurio/registry/content/refs/AbstractDataModelsReferenceFinder.java @@ -0,0 +1,98 @@ +/* + * Copyright 2023 Red Hat Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.apicurio.registry.content.refs; + +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; + +import com.fasterxml.jackson.databind.JsonNode; + +import io.apicurio.datamodels.Library; +import io.apicurio.datamodels.TraverserDirection; +import io.apicurio.datamodels.models.Document; +import io.apicurio.datamodels.models.Node; +import io.apicurio.datamodels.models.Referenceable; +import io.apicurio.datamodels.models.asyncapi.AsyncApiMessage; +import io.apicurio.datamodels.models.visitors.AllNodeVisitor; +import io.apicurio.registry.content.ContentHandle; + +/** + * Implementation of a reference finder that uses Apicurio Data Models and so supports any specification + * contained therein. Parses the document, finds all $refs, converts them to external references, and + * returns them. + * + * @author eric.wittmann@gmail.com + */ +public abstract class AbstractDataModelsReferenceFinder implements ReferenceFinder { + + /** + * @see io.apicurio.registry.content.refs.ReferenceFinder#findExternalReferences(io.apicurio.registry.content.ContentHandle) + */ + @Override + public Set findExternalReferences(ContentHandle content) { + Document doc = Library.readDocumentFromJSONString(content.content()); + + // Find all the $refs + RefFinderVisitor visitor = new RefFinderVisitor(); + Library.visitTree(doc, visitor, TraverserDirection.down); + + // Convert to ExternalReference and filter. + return visitor.allReferences.stream() + .map(ref -> new JsonPointerExternalReference(ref)) + .filter(ref -> ref.getResource() != null) + .collect(Collectors.toSet()); + } + + /** + * Visitor that will visit every node looking for "$ref" properties. + * @author eric.wittmann@gmail.com + */ + private static class RefFinderVisitor extends AllNodeVisitor { + + public Set allReferences = new HashSet<>(); + + /** + * @see io.apicurio.datamodels.models.visitors.AllNodeVisitor#visitNode(io.apicurio.datamodels.models.Node) + */ + @Override + protected void visitNode(Node node) { + if (node instanceof Referenceable) { + String ref = ((Referenceable) node).get$ref(); + if (ref != null && !ref.trim().isEmpty()) { + allReferences.add(ref); + } + } + } + + /** + * @see io.apicurio.datamodels.models.visitors.AllNodeVisitor#visitMessage(io.apicurio.datamodels.models.asyncapi.AsyncApiMessage) + */ + @Override + public void visitMessage(AsyncApiMessage node) { + // Note: special handling of message payloads because data-models doesn't fully model the payload yet. + JsonNode payload = node.getPayload(); + if (payload != null && payload.has("$ref") && !payload.get("$ref").isNull()) { + String ref = payload.get("$ref").asText(); + allReferences.add(ref); + } + super.visitMessage(node); + } + + } + +} diff --git a/schema-util/openapi/src/main/java/io/apicurio/registry/content/refs/OpenApiReferenceFinder.java b/schema-util/openapi/src/main/java/io/apicurio/registry/content/refs/OpenApiReferenceFinder.java new file mode 100644 index 0000000000..d56eabbf84 --- /dev/null +++ b/schema-util/openapi/src/main/java/io/apicurio/registry/content/refs/OpenApiReferenceFinder.java @@ -0,0 +1,26 @@ +/* + * Copyright 2023 Red Hat Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.apicurio.registry.content.refs; + +/** + * OpenAPI implementation of a reference finder. Parses the OpenAPI document, finds all $refs, converts them + * to external references, and returns them. + * @author eric.wittmann@gmail.com + */ +public class OpenApiReferenceFinder extends AbstractDataModelsReferenceFinder { + +} diff --git a/schema-util/protobuf/src/main/java/io/apicurio/registry/content/refs/ProtobufReferenceFinder.java b/schema-util/protobuf/src/main/java/io/apicurio/registry/content/refs/ProtobufReferenceFinder.java new file mode 100644 index 0000000000..c01ab830d2 --- /dev/null +++ b/schema-util/protobuf/src/main/java/io/apicurio/registry/content/refs/ProtobufReferenceFinder.java @@ -0,0 +1,57 @@ +/* + * Copyright 2023 Red Hat Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.apicurio.registry.content.refs; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.squareup.wire.schema.internal.parser.ProtoFileElement; + +import io.apicurio.registry.content.ContentHandle; +import io.apicurio.registry.utils.protobuf.schema.ProtobufFile; + +/** + * A Google Protocol Buffer implementation of a reference finder. + * @author eric.wittmann@gmail.com + */ +public class ProtobufReferenceFinder implements ReferenceFinder { + + private static final Logger log = LoggerFactory.getLogger(ProtobufReferenceFinder.class); + + /** + * @see io.apicurio.registry.content.refs.ReferenceFinder#findExternalReferences(io.apicurio.registry.content.ContentHandle) + */ + @Override + public Set findExternalReferences(ContentHandle content) { + try { + ProtoFileElement protoFileElement = ProtobufFile.toProtoFileElement(content.content()); + Set allImports = new HashSet<>(); + allImports.addAll(protoFileElement.getImports()); + allImports.addAll(protoFileElement.getPublicImports()); + return allImports.stream().map(imprt -> new ExternalReference(imprt)).collect(Collectors.toSet()); + } catch (Exception e) { + log.error("Error finding external references in a Protobuf file.", e); + return Collections.emptySet(); + } + } + +} diff --git a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/ArtifactTypeUtilProvider.java b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/ArtifactTypeUtilProvider.java index 8fed5b31c1..1bcb1d2115 100644 --- a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/ArtifactTypeUtilProvider.java +++ b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/ArtifactTypeUtilProvider.java @@ -19,6 +19,7 @@ import io.apicurio.registry.content.canon.ContentCanonicalizer; import io.apicurio.registry.content.dereference.ContentDereferencer; import io.apicurio.registry.content.extract.ContentExtractor; +import io.apicurio.registry.content.refs.ReferenceFinder; import io.apicurio.registry.rules.compatibility.CompatibilityChecker; import io.apicurio.registry.rules.validity.ContentValidator; @@ -44,4 +45,6 @@ public interface ArtifactTypeUtilProvider { ContentExtractor getContentExtractor(); ContentDereferencer getContentDereferencer(); + + ReferenceFinder getReferenceFinder(); } diff --git a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/AsyncApiArtifactTypeUtilProvider.java b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/AsyncApiArtifactTypeUtilProvider.java index bd5bcaed1e..e83795d876 100644 --- a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/AsyncApiArtifactTypeUtilProvider.java +++ b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/AsyncApiArtifactTypeUtilProvider.java @@ -22,6 +22,8 @@ import io.apicurio.registry.content.dereference.JsonSchemaDereferencer; import io.apicurio.registry.content.extract.ContentExtractor; import io.apicurio.registry.content.extract.OpenApiOrAsyncApiContentExtractor; +import io.apicurio.registry.content.refs.AsyncApiReferenceFinder; +import io.apicurio.registry.content.refs.ReferenceFinder; import io.apicurio.registry.rules.compatibility.CompatibilityChecker; import io.apicurio.registry.rules.compatibility.NoopCompatibilityChecker; import io.apicurio.registry.rules.validity.AsyncApiContentValidator; @@ -62,4 +64,9 @@ protected ContentExtractor createContentExtractor() { public ContentDereferencer getContentDereferencer() { return new JsonSchemaDereferencer(); } + + @Override + public ReferenceFinder getReferenceFinder() { + return new AsyncApiReferenceFinder(); + } } diff --git a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/AvroArtifactTypeUtilProvider.java b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/AvroArtifactTypeUtilProvider.java index 98aa80237a..ac1ff3ce3a 100644 --- a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/AvroArtifactTypeUtilProvider.java +++ b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/AvroArtifactTypeUtilProvider.java @@ -22,6 +22,8 @@ import io.apicurio.registry.content.dereference.ContentDereferencer; import io.apicurio.registry.content.extract.AvroContentExtractor; import io.apicurio.registry.content.extract.ContentExtractor; +import io.apicurio.registry.content.refs.JsonSchemaReferenceFinder; +import io.apicurio.registry.content.refs.ReferenceFinder; import io.apicurio.registry.rules.compatibility.AvroCompatibilityChecker; import io.apicurio.registry.rules.compatibility.CompatibilityChecker; import io.apicurio.registry.rules.validity.AvroContentValidator; @@ -62,4 +64,9 @@ protected ContentExtractor createContentExtractor() { public ContentDereferencer getContentDereferencer() { return new AvroDereferencer(); } + + @Override + public ReferenceFinder getReferenceFinder() { + return new JsonSchemaReferenceFinder(); + } } diff --git a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/GraphQLArtifactTypeUtilProvider.java b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/GraphQLArtifactTypeUtilProvider.java index 4ed15b300a..e3c8dc210e 100644 --- a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/GraphQLArtifactTypeUtilProvider.java +++ b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/GraphQLArtifactTypeUtilProvider.java @@ -21,6 +21,8 @@ import io.apicurio.registry.content.dereference.ContentDereferencer; import io.apicurio.registry.content.extract.ContentExtractor; import io.apicurio.registry.content.extract.NoopContentExtractor; +import io.apicurio.registry.content.refs.NoOpReferenceFinder; +import io.apicurio.registry.content.refs.ReferenceFinder; import io.apicurio.registry.rules.compatibility.CompatibilityChecker; import io.apicurio.registry.rules.compatibility.NoopCompatibilityChecker; import io.apicurio.registry.rules.validity.ContentValidator; @@ -60,4 +62,12 @@ protected ContentExtractor createContentExtractor() { public ContentDereferencer getContentDereferencer() { return null; } + + /** + * @see io.apicurio.registry.types.provider.ArtifactTypeUtilProvider#getReferenceFinder() + */ + @Override + public ReferenceFinder getReferenceFinder() { + return NoOpReferenceFinder.INSTANCE; + } } diff --git a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/JsonArtifactTypeUtilProvider.java b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/JsonArtifactTypeUtilProvider.java index 2815819d3e..c6bcf50f99 100644 --- a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/JsonArtifactTypeUtilProvider.java +++ b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/JsonArtifactTypeUtilProvider.java @@ -22,6 +22,8 @@ import io.apicurio.registry.content.dereference.JsonSchemaDereferencer; import io.apicurio.registry.content.extract.ContentExtractor; import io.apicurio.registry.content.extract.JsonContentExtractor; +import io.apicurio.registry.content.refs.JsonSchemaReferenceFinder; +import io.apicurio.registry.content.refs.ReferenceFinder; import io.apicurio.registry.rules.compatibility.CompatibilityChecker; import io.apicurio.registry.rules.compatibility.JsonSchemaCompatibilityChecker; import io.apicurio.registry.rules.validity.ContentValidator; @@ -62,4 +64,9 @@ protected ContentExtractor createContentExtractor() { public ContentDereferencer getContentDereferencer() { return new JsonSchemaDereferencer(); } + + @Override + public ReferenceFinder getReferenceFinder() { + return new JsonSchemaReferenceFinder(); + } } diff --git a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/KConnectArtifactTypeUtilProvider.java b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/KConnectArtifactTypeUtilProvider.java index de3edbe23e..1803c5ee14 100644 --- a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/KConnectArtifactTypeUtilProvider.java +++ b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/KConnectArtifactTypeUtilProvider.java @@ -21,6 +21,8 @@ import io.apicurio.registry.content.dereference.ContentDereferencer; import io.apicurio.registry.content.extract.ContentExtractor; import io.apicurio.registry.content.extract.NoopContentExtractor; +import io.apicurio.registry.content.refs.NoOpReferenceFinder; +import io.apicurio.registry.content.refs.ReferenceFinder; import io.apicurio.registry.rules.compatibility.CompatibilityChecker; import io.apicurio.registry.rules.compatibility.NoopCompatibilityChecker; import io.apicurio.registry.rules.validity.ContentValidator; @@ -60,4 +62,12 @@ protected ContentExtractor createContentExtractor() { public ContentDereferencer getContentDereferencer() { return null; } + + /** + * @see io.apicurio.registry.types.provider.ArtifactTypeUtilProvider#getReferenceFinder() + */ + @Override + public ReferenceFinder getReferenceFinder() { + return NoOpReferenceFinder.INSTANCE; + } } diff --git a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/OpenApiArtifactTypeUtilProvider.java b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/OpenApiArtifactTypeUtilProvider.java index a479b5be2b..68a736fff5 100644 --- a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/OpenApiArtifactTypeUtilProvider.java +++ b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/OpenApiArtifactTypeUtilProvider.java @@ -22,6 +22,8 @@ import io.apicurio.registry.content.dereference.JsonSchemaDereferencer; import io.apicurio.registry.content.extract.ContentExtractor; import io.apicurio.registry.content.extract.OpenApiOrAsyncApiContentExtractor; +import io.apicurio.registry.content.refs.AsyncApiReferenceFinder; +import io.apicurio.registry.content.refs.ReferenceFinder; import io.apicurio.registry.rules.compatibility.CompatibilityChecker; import io.apicurio.registry.rules.compatibility.NoopCompatibilityChecker; import io.apicurio.registry.rules.validity.ContentValidator; @@ -62,4 +64,9 @@ protected ContentExtractor createContentExtractor() { public ContentDereferencer getContentDereferencer() { return new JsonSchemaDereferencer(); } + + @Override + public ReferenceFinder getReferenceFinder() { + return new AsyncApiReferenceFinder(); + } } diff --git a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/ProtobufArtifactTypeUtilProvider.java b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/ProtobufArtifactTypeUtilProvider.java index 856db9a130..d0f0f62456 100644 --- a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/ProtobufArtifactTypeUtilProvider.java +++ b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/ProtobufArtifactTypeUtilProvider.java @@ -22,6 +22,8 @@ import io.apicurio.registry.content.dereference.ProtobufDereferencer; import io.apicurio.registry.content.extract.ContentExtractor; import io.apicurio.registry.content.extract.NoopContentExtractor; +import io.apicurio.registry.content.refs.ProtobufReferenceFinder; +import io.apicurio.registry.content.refs.ReferenceFinder; import io.apicurio.registry.rules.compatibility.CompatibilityChecker; import io.apicurio.registry.rules.compatibility.ProtobufCompatibilityChecker; import io.apicurio.registry.rules.validity.ContentValidator; @@ -61,4 +63,9 @@ protected ContentExtractor createContentExtractor() { public ContentDereferencer getContentDereferencer() { return new ProtobufDereferencer(); } + + @Override + public ReferenceFinder getReferenceFinder() { + return new ProtobufReferenceFinder(); + } } diff --git a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/WsdlArtifactTypeUtilProvider.java b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/WsdlArtifactTypeUtilProvider.java index 07c3cbc47f..86f56a6289 100644 --- a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/WsdlArtifactTypeUtilProvider.java +++ b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/WsdlArtifactTypeUtilProvider.java @@ -21,6 +21,8 @@ import io.apicurio.registry.content.dereference.ContentDereferencer; import io.apicurio.registry.content.extract.ContentExtractor; import io.apicurio.registry.content.extract.WsdlOrXsdContentExtractor; +import io.apicurio.registry.content.refs.NoOpReferenceFinder; +import io.apicurio.registry.content.refs.ReferenceFinder; import io.apicurio.registry.rules.compatibility.CompatibilityChecker; import io.apicurio.registry.rules.compatibility.NoopCompatibilityChecker; import io.apicurio.registry.rules.validity.ContentValidator; @@ -76,4 +78,12 @@ protected ContentExtractor createContentExtractor() { public ContentDereferencer getContentDereferencer() { return null; } + + /** + * @see io.apicurio.registry.types.provider.ArtifactTypeUtilProvider#getReferenceFinder() + */ + @Override + public ReferenceFinder getReferenceFinder() { + return NoOpReferenceFinder.INSTANCE; + } } diff --git a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/XmlArtifactTypeUtilProvider.java b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/XmlArtifactTypeUtilProvider.java index 56b6fb11ff..701e10ecbb 100644 --- a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/XmlArtifactTypeUtilProvider.java +++ b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/XmlArtifactTypeUtilProvider.java @@ -21,6 +21,8 @@ import io.apicurio.registry.content.dereference.ContentDereferencer; import io.apicurio.registry.content.extract.ContentExtractor; import io.apicurio.registry.content.extract.NoopContentExtractor; +import io.apicurio.registry.content.refs.NoOpReferenceFinder; +import io.apicurio.registry.content.refs.ReferenceFinder; import io.apicurio.registry.rules.compatibility.CompatibilityChecker; import io.apicurio.registry.rules.compatibility.NoopCompatibilityChecker; import io.apicurio.registry.rules.validity.ContentValidator; @@ -76,4 +78,12 @@ protected ContentExtractor createContentExtractor() { public ContentDereferencer getContentDereferencer() { return null; } + + /** + * @see io.apicurio.registry.types.provider.ArtifactTypeUtilProvider#getReferenceFinder() + */ + @Override + public ReferenceFinder getReferenceFinder() { + return NoOpReferenceFinder.INSTANCE; + } } diff --git a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/XsdArtifactTypeUtilProvider.java b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/XsdArtifactTypeUtilProvider.java index c887875c08..120609112c 100644 --- a/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/XsdArtifactTypeUtilProvider.java +++ b/schema-util/util-provider/src/main/java/io/apicurio/registry/types/provider/XsdArtifactTypeUtilProvider.java @@ -21,6 +21,8 @@ import io.apicurio.registry.content.dereference.ContentDereferencer; import io.apicurio.registry.content.extract.ContentExtractor; import io.apicurio.registry.content.extract.WsdlOrXsdContentExtractor; +import io.apicurio.registry.content.refs.NoOpReferenceFinder; +import io.apicurio.registry.content.refs.ReferenceFinder; import io.apicurio.registry.rules.compatibility.CompatibilityChecker; import io.apicurio.registry.rules.compatibility.NoopCompatibilityChecker; import io.apicurio.registry.rules.validity.ContentValidator; @@ -76,4 +78,12 @@ protected ContentExtractor createContentExtractor() { public ContentDereferencer getContentDereferencer() { return null; } + + /** + * @see io.apicurio.registry.types.provider.ArtifactTypeUtilProvider#getReferenceFinder() + */ + @Override + public ReferenceFinder getReferenceFinder() { + return NoOpReferenceFinder.INSTANCE; + } } diff --git a/schema-util/util-provider/src/test/java/io/apicurio/registry/content/refs/AsyncApiReferenceFinderTest.java b/schema-util/util-provider/src/test/java/io/apicurio/registry/content/refs/AsyncApiReferenceFinderTest.java new file mode 100644 index 0000000000..013caa9c93 --- /dev/null +++ b/schema-util/util-provider/src/test/java/io/apicurio/registry/content/refs/AsyncApiReferenceFinderTest.java @@ -0,0 +1,47 @@ +/* + * Copyright 2023 Red Hat Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.apicurio.registry.content.refs; + +import java.util.Set; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import io.apicurio.registry.content.ContentHandle; +import io.apicurio.registry.rules.validity.ArtifactUtilProviderTestBase; + +/** + * @author eric.wittmann@gmail.com + */ +public class AsyncApiReferenceFinderTest extends ArtifactUtilProviderTestBase { + + /** + * Test method for {@link io.apicurio.registry.content.refs.AsyncApiReferenceFinder#findExternalReferences(io.apicurio.registry.content.ContentHandle)}. + */ + @Test + public void testFindExternalReferences() { + ContentHandle content = resourceToContentHandle("asyncapi-with-refs.json"); + AsyncApiReferenceFinder finder = new AsyncApiReferenceFinder(); + Set foundReferences = finder.findExternalReferences(content); + Assertions.assertNotNull(foundReferences); + Assertions.assertEquals(2, foundReferences.size()); + Assertions.assertEquals(Set.of( + new JsonPointerExternalReference("./TradeKey.avsc"), + new JsonPointerExternalReference("./common-types.json#/components/schemas/User")), foundReferences); + } + +} diff --git a/schema-util/util-provider/src/test/java/io/apicurio/registry/content/refs/AvroReferenceFinderTest.java b/schema-util/util-provider/src/test/java/io/apicurio/registry/content/refs/AvroReferenceFinderTest.java new file mode 100644 index 0000000000..ac125bb3e0 --- /dev/null +++ b/schema-util/util-provider/src/test/java/io/apicurio/registry/content/refs/AvroReferenceFinderTest.java @@ -0,0 +1,47 @@ +/* + * Copyright 2023 Red Hat Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.apicurio.registry.content.refs; + +import java.util.Set; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import io.apicurio.registry.content.ContentHandle; +import io.apicurio.registry.rules.validity.ArtifactUtilProviderTestBase; + +/** + * @author eric.wittmann@gmail.com + */ +public class AvroReferenceFinderTest extends ArtifactUtilProviderTestBase { + + /** + * Test method for {@link io.apicurio.registry.content.refs.AsyncApiReferenceFinder#findExternalReferences(io.apicurio.registry.content.ContentHandle)}. + */ + @Test + public void testFindExternalReferences() { + ContentHandle content = resourceToContentHandle("avro-with-refs.avsc"); + AvroReferenceFinder finder = new AvroReferenceFinder(); + Set foundReferences = finder.findExternalReferences(content); + Assertions.assertNotNull(foundReferences); + Assertions.assertEquals(2, foundReferences.size()); + Assertions.assertEquals(Set.of( + new ExternalReference("com.kubetrade.schema.trade.TradeKey"), + new ExternalReference("com.kubetrade.schema.trade.TradeValue")), foundReferences); + } + +} diff --git a/schema-util/util-provider/src/test/java/io/apicurio/registry/content/refs/OpenApiReferenceFinderTest.java b/schema-util/util-provider/src/test/java/io/apicurio/registry/content/refs/OpenApiReferenceFinderTest.java new file mode 100644 index 0000000000..276f4be35e --- /dev/null +++ b/schema-util/util-provider/src/test/java/io/apicurio/registry/content/refs/OpenApiReferenceFinderTest.java @@ -0,0 +1,47 @@ +/* + * Copyright 2023 Red Hat Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.apicurio.registry.content.refs; + +import java.util.Set; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import io.apicurio.registry.content.ContentHandle; +import io.apicurio.registry.rules.validity.ArtifactUtilProviderTestBase; + +/** + * @author eric.wittmann@gmail.com + */ +public class OpenApiReferenceFinderTest extends ArtifactUtilProviderTestBase { + + /** + * Test method for {@link io.apicurio.registry.content.refs.AsyncApiReferenceFinder#findExternalReferences(io.apicurio.registry.content.ContentHandle)}. + */ + @Test + public void testFindExternalReferences() { + ContentHandle content = resourceToContentHandle("openapi-with-refs.json"); + OpenApiReferenceFinder finder = new OpenApiReferenceFinder(); + Set foundReferences = finder.findExternalReferences(content); + Assertions.assertNotNull(foundReferences); + Assertions.assertEquals(2, foundReferences.size()); + Assertions.assertEquals(Set.of( + new JsonPointerExternalReference("./types/bar-types.json#/components/schemas/Bar"), + new JsonPointerExternalReference("./types/foo-types.json#/components/schemas/Foo")), foundReferences); + } + +} diff --git a/schema-util/util-provider/src/test/java/io/apicurio/registry/content/refs/ProtobufReferenceFinderTest.java b/schema-util/util-provider/src/test/java/io/apicurio/registry/content/refs/ProtobufReferenceFinderTest.java new file mode 100644 index 0000000000..4729e58ae7 --- /dev/null +++ b/schema-util/util-provider/src/test/java/io/apicurio/registry/content/refs/ProtobufReferenceFinderTest.java @@ -0,0 +1,48 @@ +/* + * Copyright 2023 Red Hat Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.apicurio.registry.content.refs; + +import java.util.Set; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import io.apicurio.registry.content.ContentHandle; +import io.apicurio.registry.rules.validity.ArtifactUtilProviderTestBase; + +/** + * @author eric.wittmann@gmail.com + */ +public class ProtobufReferenceFinderTest extends ArtifactUtilProviderTestBase { + + /** + * Test method for {@link io.apicurio.registry.content.refs.AsyncApiReferenceFinder#findExternalReferences(io.apicurio.registry.content.ContentHandle)}. + */ + @Test + public void testFindExternalReferences() { + ContentHandle content = resourceToContentHandle("protobuf-with-refs.proto"); + ProtobufReferenceFinder finder = new ProtobufReferenceFinder(); + Set foundReferences = finder.findExternalReferences(content); + Assertions.assertNotNull(foundReferences); + Assertions.assertEquals(3, foundReferences.size()); + Assertions.assertEquals(Set.of( + new ExternalReference("google/protobuf/timestamp.proto"), + new ExternalReference("sample/table_info.proto"), + new ExternalReference("sample/table_notification_type.proto")), foundReferences); + } + +} diff --git a/schema-util/util-provider/src/test/resources/io/apicurio/registry/content/refs/asyncapi-with-refs.json b/schema-util/util-provider/src/test/resources/io/apicurio/registry/content/refs/asyncapi-with-refs.json new file mode 100644 index 0000000000..5d3d4c2658 --- /dev/null +++ b/schema-util/util-provider/src/test/resources/io/apicurio/registry/content/refs/asyncapi-with-refs.json @@ -0,0 +1,33 @@ +{ + "asyncapi": "2.6.0", + "info": { + "title": "Account Service", + "version": "1.0.0", + "description": "This service is in charge of processing user signups" + }, + "channels": { + "user/signedup": { + "subscribe": { + "message": { + "$ref": "#/components/messages/UserSignedUp" + } + } + }, + "user/externalAvro": { + "subscribe": { + "message": { + "$ref": "./TradeKey.avsc" + } + } + } + }, + "components": { + "messages": { + "UserSignedUp": { + "payload": { + "$ref": "./common-types.json#/components/schemas/User" + } + } + } + } +} diff --git a/schema-util/util-provider/src/test/resources/io/apicurio/registry/content/refs/avro-with-refs.avsc b/schema-util/util-provider/src/test/resources/io/apicurio/registry/content/refs/avro-with-refs.avsc new file mode 100644 index 0000000000..20ab732e91 --- /dev/null +++ b/schema-util/util-provider/src/test/resources/io/apicurio/registry/content/refs/avro-with-refs.avsc @@ -0,0 +1,23 @@ +{ + "namespace": "com.kubetrade.schema.trade", + "type": "record", + "name": "TradeRaw", + "fields": [ + { + "name": "tradeKey", + "type": "com.kubetrade.schema.trade.TradeKey" + }, + { + "name": "value", + "type": ["string", "com.kubetrade.schema.trade.TradeValue"] + }, + { + "name": "symbol", + "type": "string" + }, + { + "name": "payload", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/schema-util/util-provider/src/test/resources/io/apicurio/registry/content/refs/openapi-with-refs.json b/schema-util/util-provider/src/test/resources/io/apicurio/registry/content/refs/openapi-with-refs.json new file mode 100644 index 0000000000..4ba2c9ba54 --- /dev/null +++ b/schema-util/util-provider/src/test/resources/io/apicurio/registry/content/refs/openapi-with-refs.json @@ -0,0 +1,159 @@ +{ + "openapi": "3.0.2", + "info": { + "title": "Widget API (with refs)", + "version": "1.0.0" + }, + "paths": { + "/widgets": { + "summary": "Path used to manage the list of widgets.", + "description": "The REST endpoint/path used to list and create zero or more `Widget` entities. This path contains a `GET` and `POST` operation to perform the list and create tasks, respectively.", + "get": { + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Widget" + } + } + } + }, + "description": "Successful response - returns an array of `Widget` entities." + } + }, + "operationId": "getWidgets", + "summary": "List All Widgets", + "description": "Gets a list of all `Widget` entities." + }, + "post": { + "requestBody": { + "description": "A new `Widget` to be created.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Widget" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "Successful response." + } + }, + "operationId": "createWidget", + "summary": "Create a Widget", + "description": "Creates a new instance of a `Widget`." + } + }, + "/widgets/{widgetId}": { + "summary": "Path used to manage a single Widget.", + "description": "The REST endpoint/path used to get, update, and delete single instances of an `Widget`. This path contains `GET`, `PUT`, and `DELETE` operations used to perform the get, update, and delete tasks, respectively.", + "get": { + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Widget" + } + } + }, + "description": "Successful response - returns a single `Widget`." + } + }, + "operationId": "getWidget", + "summary": "Get a Widget", + "description": "Gets the details of a single instance of a `Widget`." + }, + "put": { + "requestBody": { + "description": "Updated `Widget` information.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Widget" + } + } + }, + "required": true + }, + "responses": { + "202": { + "description": "Successful response." + } + }, + "operationId": "updateWidget", + "summary": "Update a Widget", + "description": "Updates an existing `Widget`." + }, + "delete": { + "responses": { + "204": { + "description": "Successful response." + } + }, + "operationId": "deleteWidget", + "summary": "Delete a Widget", + "description": "Deletes an existing `Widget`." + }, + "parameters": [ + { + "name": "widgetId", + "description": "A unique identifier for a `Widget`.", + "schema": { + "type": "string" + }, + "in": "path", + "required": true + } + ] + } + }, + "components": { + "schemas": { + "Widget": { + "title": "Root Type for Widget", + "description": "", + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "foo": { + "$ref": "#/components/schemas/Foo", + "properties": { + "prop-1": { + "type": "string" + }, + "prop-2": { + "type": "string" + } + } + } + }, + "example": { + "name": "...", + "description": "...", + "foo": { + "prop-1": "", + "prop-2": "" + } + } + }, + "Bar": { + "$ref": "./types/bar-types.json#/components/schemas/Bar" + }, + "Foo": { + "$ref": "./types/foo-types.json#/components/schemas/Foo" + } + } + } +} \ No newline at end of file diff --git a/schema-util/util-provider/src/test/resources/io/apicurio/registry/content/refs/protobuf-with-refs.proto b/schema-util/util-provider/src/test/resources/io/apicurio/registry/content/refs/protobuf-with-refs.proto new file mode 100644 index 0000000000..81cf94b6f3 --- /dev/null +++ b/schema-util/util-provider/src/test/resources/io/apicurio/registry/content/refs/protobuf-with-refs.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +option java_package = "io.api.sample"; +option java_multiple_files = true; + +import "google/protobuf/timestamp.proto"; +import "sample/table_info.proto"; +import "sample/table_notification_type.proto"; + +message TableNotification { + + google.protobuf.Timestamp timestamp = 1; + + string user = 2; + + string session_id = 3; + + sample.TableNotificationType table_notification_type = 4; + + sample.TableInfo table_info = 5; + +} \ No newline at end of file diff --git a/utils/maven-plugin/pom.xml b/utils/maven-plugin/pom.xml index 2a54753a47..873d02e3b9 100644 --- a/utils/maven-plugin/pom.xml +++ b/utils/maven-plugin/pom.xml @@ -50,17 +50,9 @@ commons-io commons-io - - commons-io - commons-io - - - io.apicurio - apicurio-registry-protobuf-schema-utilities - io.apicurio - apicurio-registry-schema-util-json + apicurio-registry-schema-util-provider diff --git a/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/ExistingReference.java b/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/ExistingReference.java new file mode 100644 index 0000000000..4f7503417a --- /dev/null +++ b/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/ExistingReference.java @@ -0,0 +1,90 @@ +/* + * Copyright 2021 Red Hat + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.apicurio.registry.maven; + +/** + * @author eric.wittmann@gmail.com + */ +public class ExistingReference { + + private String resourceName; + private String groupId; + private String artifactId; + private String version; + + /** + * Constructor. + */ + public ExistingReference() { + } + + /** + * @return the groupId + */ + public String getGroupId() { + return groupId; + } + + /** + * @param groupId the groupId to set + */ + public void setGroupId(String groupId) { + this.groupId = groupId; + } + + /** + * @return the artifactId + */ + public String getArtifactId() { + return artifactId; + } + + /** + * @param artifactId the artifactId to set + */ + public void setArtifactId(String artifactId) { + this.artifactId = artifactId; + } + + /** + * @return the resourceName + */ + public String getResourceName() { + return resourceName; + } + + /** + * @param resourceName the resourceName to set + */ + public void setResourceName(String resourceName) { + this.resourceName = resourceName; + } + + /** + * @return the version + */ + public String getVersion() { + return version; + } + + /** + * @param version the version to set + */ + public void setVersion(String version) { + this.version = version; + } +} diff --git a/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/RegisterArtifact.java b/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/RegisterArtifact.java index 1eedf83b2b..66e3e0fc21 100644 --- a/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/RegisterArtifact.java +++ b/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/RegisterArtifact.java @@ -35,8 +35,10 @@ public class RegisterArtifact { private Boolean canonicalize; private Boolean minify; private Boolean analyzeDirectory; + private Boolean autoRefs; private String contentType; private List references; + private List existingReferences; /** * Constructor. @@ -191,4 +193,20 @@ public Boolean getAnalyzeDirectory() { public void setAnalyzeDirectory(Boolean analyzeDirectory) { this.analyzeDirectory = analyzeDirectory; } + + public Boolean getAutoRefs() { + return autoRefs; + } + + public void setAutoRefs(Boolean autoRefs) { + this.autoRefs = autoRefs; + } + + public List getExistingReferences() { + return existingReferences; + } + + public void setExistingReferences(List existingReferences) { + this.existingReferences = existingReferences; + } } diff --git a/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/RegisterRegistryMojo.java b/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/RegisterRegistryMojo.java index d6c9e4d4d9..340d332266 100644 --- a/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/RegisterRegistryMojo.java +++ b/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/RegisterRegistryMojo.java @@ -17,26 +17,44 @@ package io.apicurio.registry.maven; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.Stack; +import java.util.stream.Collectors; + +import org.apache.avro.Schema; +import org.apache.commons.io.FileUtils; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.protobuf.Descriptors.FileDescriptor; + +import io.apicurio.registry.content.ContentHandle; +import io.apicurio.registry.content.refs.ExternalReference; +import io.apicurio.registry.content.refs.ReferenceFinder; +import io.apicurio.registry.maven.refs.IndexedResource; +import io.apicurio.registry.maven.refs.ReferenceIndex; import io.apicurio.registry.rest.v2.beans.ArtifactMetaData; import io.apicurio.registry.rest.v2.beans.ArtifactReference; import io.apicurio.registry.rest.v2.beans.IfExists; import io.apicurio.registry.types.ArtifactType; import io.apicurio.registry.types.ContentTypes; -import org.apache.avro.Schema; -import org.apache.maven.plugin.MojoExecutionException; -import org.apache.maven.plugins.annotations.Mojo; -import org.apache.maven.plugins.annotations.Parameter; - -import java.io.ByteArrayInputStream; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; +import io.apicurio.registry.types.provider.ArtifactTypeUtilProvider; +import io.apicurio.registry.types.provider.DefaultArtifactTypeUtilProviderImpl; /** * Register artifacts against registry. @@ -46,11 +64,19 @@ @Mojo(name = "register") public class RegisterRegistryMojo extends AbstractRegistryMojo { + /** + * The list of pre-registered artifacts that can be used as references. + */ + @Parameter(required = true) + List existingReferences; + /** * The list of artifacts to register. */ @Parameter(required = true) List artifacts; + + DefaultArtifactTypeUtilProviderImpl utilProviderFactory = new DefaultArtifactTypeUtilProviderImpl(); /** * Validate the configuration. @@ -98,8 +124,15 @@ protected void executeInternal() throws MojoExecutionException { String groupId = artifact.getGroupId(); String artifactId = artifact.getArtifactId(); try { + if (artifact.getAutoRefs() != null && artifact.getAutoRefs()) { + // If we have references, then we'll need to create the local resource index and then process all refs. + ReferenceIndex index = createIndex(artifact.getFile()); + addExistingReferencesToIndex(index, existingReferences); + addExistingReferencesToIndex(index, artifact.getExistingReferences()); + Stack registrationStack = new Stack<>(); - if (artifact.getAnalyzeDirectory() != null && artifact.getAnalyzeDirectory()) { //Auto register selected, we must figure out if the artifact has reference using the directory structure + registerWithAutoRefs(artifact, index, registrationStack); + } else if (artifact.getAnalyzeDirectory() != null && artifact.getAnalyzeDirectory()) { //Auto register selected, we must figure out if the artifact has reference using the directory structure registerDirectory(artifact); } else { @@ -123,6 +156,59 @@ protected void executeInternal() throws MojoExecutionException { } } + private ArtifactMetaData registerWithAutoRefs(RegisterArtifact artifact, ReferenceIndex index, Stack registrationStack) throws IOException { + if (loopDetected(artifact, registrationStack)) { + throw new RuntimeException("Artifact reference loop detected (not supported): " + printLoop(registrationStack)); + } + registrationStack.push(artifact); + + // Read the artifact content. + ContentHandle artifactContent = readContent(artifact.getFile()); + + // Find all references in the content + ArtifactTypeUtilProvider provider = this.utilProviderFactory.getArtifactTypeProvider(artifact.getType()); + ReferenceFinder referenceFinder = provider.getReferenceFinder(); + Set externalReferences = referenceFinder.findExternalReferences(artifactContent); + + // Register all of the references first, then register the artifact. + List registeredReferences = externalReferences.stream().map(externalRef -> { + IndexedResource iresource = index.lookup(externalRef.getResource(), Paths.get(artifact.getFile().toURI())); + + // TODO: need a way to resolve references that are not local (already registered in the registry) + if (iresource == null) { + throw new RuntimeException("Reference could not be resolved. From: " + artifact.getFile().getName() + " To: " + externalRef.getFullReference()); + } + + // If the resource isn't already registered, then register it now. + if (!iresource.isRegistered()) { + // TODO: determine the artifactId better (type-specific logic here?) + String artifactId = externalRef.getResource(); + File localFile = getLocalFile(iresource.getPath()); + RegisterArtifact refArtifact = buildFromRoot(artifact, artifactId); + refArtifact.setType(iresource.getType()); + refArtifact.setVersion(null); + refArtifact.setFile(localFile); + refArtifact.setContentType(getContentTypeByExtension(localFile.getName())); + try { + ArtifactMetaData amd = registerWithAutoRefs(refArtifact, index, registrationStack); + iresource.setRegistration(amd); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + return new ArtifactReference( + iresource.getRegistration().getGroupId(), + iresource.getRegistration().getId(), + iresource.getRegistration().getVersion(), + externalRef.getFullReference()); + + }).sorted((ref1, ref2) -> ref1.getName().compareTo(ref2.getName())).collect(Collectors.toList()); + + registrationStack.pop(); + return registerArtifact(artifact, registeredReferences); + } + private void registerDirectory(RegisterArtifact artifact) throws IOException { switch (artifact.getType()) { case ArtifactType.AVRO: @@ -172,8 +258,7 @@ private ArtifactMetaData registerArtifact(RegisterArtifact artifact, InputStream return amd; } - - private boolean hasReferences(RegisterArtifact artifact) { + private static boolean hasReferences(RegisterArtifact artifact) { return artifact.getReferences() != null && !artifact.getReferences().isEmpty(); } @@ -192,7 +277,11 @@ private boolean hasReferences(RegisterArtifact artifact) { return references; } - private ArtifactReference buildReferenceFromMetadata(ArtifactMetaData amd, String referenceName) { + public void setArtifacts(List artifacts) { + this.artifacts = artifacts; + } + + private static ArtifactReference buildReferenceFromMetadata(ArtifactMetaData amd, String referenceName) { ArtifactReference reference = new ArtifactReference(); reference.setName(referenceName); reference.setArtifactId(amd.getId()); @@ -201,15 +290,78 @@ private ArtifactReference buildReferenceFromMetadata(ArtifactMetaData amd, Strin return reference; } - private String contentType(RegisterArtifact registerArtifact) { - String contentType = registerArtifact.getContentType(); - if (contentType != null) { - return contentType; + /** + * Create a local index relative to the given file location. + * @param file + */ + private static ReferenceIndex createIndex(File file) { + ReferenceIndex index = new ReferenceIndex(file.getParentFile().toPath()); + Collection allFiles = FileUtils.listFiles(file.getParentFile(), null, true); + allFiles.stream().filter(f -> f.isFile()).forEach(f -> { + index.index(f.toPath(), readContent(f)); + }); + return index; + } + + private void addExistingReferencesToIndex(ReferenceIndex index, List existingReferences) { + if (existingReferences != null && !existingReferences.isEmpty()) { + existingReferences.forEach(ref -> { + ArtifactMetaData amd; + if (ref.getVersion() == null || "LATEST".equalsIgnoreCase(ref.getVersion())) { + amd = getClient().getArtifactMetaData(ref.getGroupId(), ref.getArtifactId()); + } else { + amd = new ArtifactMetaData(); + amd.setGroupId(ref.getGroupId()); + amd.setId(ref.getArtifactId()); + amd.setVersion(ref.getVersion()); + } + index.index(ref.getResourceName(), amd); + }); } - return getContentTypeByExtension(registerArtifact.getFile().getName()); } - public void setArtifacts(List artifacts) { - this.artifacts = artifacts; + protected static ContentHandle readContent(File file) { + try { + return ContentHandle.create(Files.readAllBytes(file.toPath())); + } catch (IOException e) { + throw new RuntimeException("Failed to read schema file: " + file, e); + } + } + + protected static RegisterArtifact buildFromRoot(RegisterArtifact rootArtifact, String artifactId) { + RegisterArtifact nestedSchema = new RegisterArtifact(); + nestedSchema.setCanonicalize(rootArtifact.getCanonicalize()); + nestedSchema.setArtifactId(artifactId); + nestedSchema.setGroupId(rootArtifact.getGroupId()); + nestedSchema.setContentType(rootArtifact.getContentType()); + nestedSchema.setType(rootArtifact.getType()); + nestedSchema.setMinify(rootArtifact.getMinify()); + nestedSchema.setContentType(rootArtifact.getContentType()); + nestedSchema.setIfExists(rootArtifact.getIfExists()); + nestedSchema.setAutoRefs(rootArtifact.getAutoRefs()); + return nestedSchema; + } + + private static File getLocalFile(Path path) { + return path.toFile(); + } + + /** + * Detects a loop by looking for the given artifact in the registration stack. + * @param artifact + * @param registrationStack + */ + private static boolean loopDetected(RegisterArtifact artifact, Stack registrationStack) { + for (RegisterArtifact stackArtifact : registrationStack) { + if (artifact.getFile().equals(stackArtifact.getFile())) { + return true; + } + } + return false; + } + + private static String printLoop(Stack registrationStack) { + return registrationStack.stream().map(artifact -> artifact.getFile().getName()).collect(Collectors.joining(" -> ")); } + } diff --git a/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/refs/IndexedResource.java b/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/refs/IndexedResource.java new file mode 100644 index 0000000000..ca272964d4 --- /dev/null +++ b/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/refs/IndexedResource.java @@ -0,0 +1,122 @@ +/* + * Copyright 2023 Red Hat Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.apicurio.registry.maven.refs; + +import java.nio.file.Path; +import java.util.Set; + +import io.apicurio.registry.content.ContentHandle; +import io.apicurio.registry.rest.v2.beans.ArtifactMetaData; +import io.apicurio.registry.types.ArtifactType; + +/** + * @author eric.wittmann@gmail.com + */ +public class IndexedResource { + + private final Path path; + private final String type; + private final String resourceName; + private final ContentHandle content; + private ArtifactMetaData registration; + + /** + * Constructor. + * @param path + * @param type + * @param resourceName + * @param content + */ + public IndexedResource(Path path, String type, String resourceName, ContentHandle content) { + super(); + this.path = path; + this.content = content; + this.type = type; + this.resourceName = resourceName; + } + + /** + * @return the content + */ + public ContentHandle getContent() { + return content; + } + + /** + * @return the type + */ + public String getType() { + return type; + } + + /** + * @return the resourceName + */ + public String getResourceName() { + return resourceName; + } + + /** + * @return the path + */ + public Path getPath() { + return path; + } + + public boolean matches(String resourceName, Path relativeToFile, Set schemaPaths) { + // If this is a pre-registered reference, the match has to happen by resource name. + if (this.path == null) { + return this.resourceName.equals(resourceName); + } + + // For Avro files the match can happen either via path (e.g. when referencing an Avro + // file from an AsyncAPI file) or via resource name (e.g. from Avro to Avro). + if (ArtifactType.AVRO.equals(this.type)) { + if (this.resourceName.equals(resourceName)) { + return true; + } + } + + // The resource name will otherwise be a relative path to the resource. + Path resolvedPath = relativeToFile.getParent().resolve(resourceName); + boolean resolves = this.path.normalize().equals(resolvedPath.normalize()); + + // Protobuf can resolve relative to the "schema paths" (aka --proto-paths in protoc). + if (!resolves && ArtifactType.PROTOBUF.equals(this.type)) { + resolves = schemaPaths.parallelStream().anyMatch(path -> this.path.normalize().equals(path.resolve(resourceName).normalize())); + } + return resolves; + } + + /** + * @return the registration + */ + public ArtifactMetaData getRegistration() { + return registration; + } + + /** + * @param registration the registration to set + */ + public void setRegistration(ArtifactMetaData registration) { + this.registration = registration; + } + + public boolean isRegistered() { + return this.registration != null; + } +} diff --git a/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/refs/ReferenceIndex.java b/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/refs/ReferenceIndex.java new file mode 100644 index 0000000000..b98472ceca --- /dev/null +++ b/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/refs/ReferenceIndex.java @@ -0,0 +1,165 @@ +/* + * Copyright 2023 Red Hat Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.apicurio.registry.maven.refs; + +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Set; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import io.apicurio.datamodels.Library; +import io.apicurio.datamodels.models.Document; +import io.apicurio.datamodels.util.ModelTypeUtil; +import io.apicurio.registry.content.ContentHandle; +import io.apicurio.registry.rest.v2.beans.ArtifactMetaData; +import io.apicurio.registry.types.ArtifactType; +import io.apicurio.registry.utils.protobuf.schema.ProtobufFile; + +/** + * An index of the files available when discovering references in an artifact. This index is + * typically populated by getting a list of all files in a directory, or zip file. + * + * The index maps a resource name (this will vary depending on the artifact type) to the + * content of the resource. For example, Avro schemas will have resource names based on the + * qualified name of the type they define. JSON Schemas will have resources names based on + * the name of the file. + * + * The intent of this index is to resolve an external reference found in an artifact to an + * actual piece of content (e.g. file) in the index. If it cannot be resolved, that would + * typically mean that there is a broken reference in the schema/design. + * + * @author eric.wittmann@gmail.com + */ +public class ReferenceIndex { + + private static final ObjectMapper mapper = new ObjectMapper(); + + private Set index = new HashSet<>(); + private Set schemaPaths = new HashSet<>(); + + /** + * Constructor. + */ + public ReferenceIndex() { + } + + /** + * Constructor. + * @param schemaPath + */ + public ReferenceIndex(Path schemaPath) { + this.schemaPaths.add(schemaPath); + } + + /** + * @param path + */ + public void addSchemaPath(Path path) { + this.schemaPaths.add(path); + } + + /** + * Look up a resource in the index. Returns null if no resource with that + * name is found. + * @param resourceName + * @param relativeToFile + */ + public IndexedResource lookup(String resourceName, Path relativeToFile) { + return index.stream().filter(resource -> resource.matches(resourceName, relativeToFile, schemaPaths)).findFirst().orElse(null); + } + + /** + * Index an existing (remote) reference using a resource name and remote artifact metadata. + * @param resourceName + * @param amd + */ + public void index(String resourceName, ArtifactMetaData amd) { + IndexedResource res = new IndexedResource(null, null, resourceName, null); + res.setRegistration(amd); + this.index.add(res); + } + + /** + * Index the given content. Indexing will parse the content and figure out its resource + * name and type. + * @param path + * @param content + */ + public void index(Path path, ContentHandle content) { + try { + JsonNode tree = mapper.readTree(content.content()); + + // OpenAPI + if (tree.has("openapi") || tree.has("swagger") || tree.has("asyncapi")) { + indexDataModels(path, content); + } + // JSON Schema + if (tree.has("$schema") && tree.get("$schema").asText().contains("json-schema.org") || tree.has("properties")) { + indexJsonSchema(path, content); + } + // Avro + indexAvro(path, content, tree); + } catch (Exception e) { + // Must not be JSON... + } + + try { + indexProto(path, content); + return; + } catch (Exception e) { + // I guess it's not Protobuf. + } + } + + private void indexAvro(Path path, ContentHandle content, JsonNode parsed) { + // TODO: is namespace required for an Avro schema? + String ns = parsed.get("namespace").asText(); + String name = parsed.get("name").asText(); + String resourceName = ns != null ? ns + "." + name : name; + IndexedResource resource = new IndexedResource(path, ArtifactType.AVRO, resourceName, content); + this.index.add(resource); + } + + private void indexProto(Path path, ContentHandle content) { + ProtobufFile.toProtoFileElement(content.content()); + + IndexedResource resource = new IndexedResource(path, ArtifactType.PROTOBUF, null, content); + this.index.add(resource); + } + + private void indexJsonSchema(Path path, ContentHandle content) { + throw new UnsupportedOperationException("Not yet implemented."); + } + + private void indexDataModels(Path path, ContentHandle content) { + Document doc = Library.readDocumentFromJSONString(content.content()); + if (doc == null) { + throw new UnsupportedOperationException("Content is not OpenAPI or AsyncAPI."); + } + + String type = ArtifactType.OPENAPI; + if (ModelTypeUtil.isAsyncApiModel(doc)) { + type = ArtifactType.ASYNCAPI; + } + + IndexedResource resource = new IndexedResource(path, type, null, content); + this.index.add(resource); + } + +} From e1b6667956915e196f2db6302bf938468dc6f379 Mon Sep 17 00:00:00 2001 From: Eric Wittmann Date: Fri, 16 Jun 2023 15:30:39 -0400 Subject: [PATCH 2/2] Added a ref finder for JSON Schema --- .../refs/JsonSchemaReferenceFinder.java | 17 ++++++- .../refs/JsonSchemaReferenceFinderTest.java | 45 +++++++++++++++++++ .../content/refs/json-schema-with-refs.json | 10 +++++ .../registry/maven/refs/ReferenceIndex.java | 13 ++++-- 4 files changed, 80 insertions(+), 5 deletions(-) create mode 100644 schema-util/util-provider/src/test/java/io/apicurio/registry/content/refs/JsonSchemaReferenceFinderTest.java create mode 100644 schema-util/util-provider/src/test/resources/io/apicurio/registry/content/refs/json-schema-with-refs.json diff --git a/schema-util/json/src/main/java/io/apicurio/registry/content/refs/JsonSchemaReferenceFinder.java b/schema-util/json/src/main/java/io/apicurio/registry/content/refs/JsonSchemaReferenceFinder.java index 7da956651b..356e645cab 100644 --- a/schema-util/json/src/main/java/io/apicurio/registry/content/refs/JsonSchemaReferenceFinder.java +++ b/schema-util/json/src/main/java/io/apicurio/registry/content/refs/JsonSchemaReferenceFinder.java @@ -18,6 +18,8 @@ import java.util.Collections; import java.util.HashSet; +import java.util.Iterator; +import java.util.Map.Entry; import java.util.Set; import java.util.stream.Collectors; @@ -59,7 +61,20 @@ public Set findExternalReferences(ContentHandle content) { } private static void findExternalTypesIn(JsonNode schema, Set externalTypes) { - // TODO implement this!! + if (schema.isObject()) { + if (schema.has("$ref")) { + String ref = schema.get("$ref").asText(null); + if (ref != null) { + // TODO: the value of the ref should be resolved against the $id in this schema if it has one + externalTypes.add(ref); + } + } + Iterator> fields = schema.fields(); + while (fields.hasNext()) { + Entry field = fields.next(); + findExternalTypesIn(field.getValue(), externalTypes); + } + } } } diff --git a/schema-util/util-provider/src/test/java/io/apicurio/registry/content/refs/JsonSchemaReferenceFinderTest.java b/schema-util/util-provider/src/test/java/io/apicurio/registry/content/refs/JsonSchemaReferenceFinderTest.java new file mode 100644 index 0000000000..65202d7495 --- /dev/null +++ b/schema-util/util-provider/src/test/java/io/apicurio/registry/content/refs/JsonSchemaReferenceFinderTest.java @@ -0,0 +1,45 @@ +/* + * Copyright 2023 Red Hat Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.apicurio.registry.content.refs; + +import java.util.Set; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import io.apicurio.registry.content.ContentHandle; +import io.apicurio.registry.rules.validity.ArtifactUtilProviderTestBase; + +/** + * @author eric.wittmann@gmail.com + */ +public class JsonSchemaReferenceFinderTest extends ArtifactUtilProviderTestBase { + + /** + * Test method for {@link io.apicurio.registry.content.refs.AsyncApiReferenceFinder#findExternalReferences(io.apicurio.registry.content.ContentHandle)}. + */ + @Test + public void testFindExternalReferences() { + ContentHandle content = resourceToContentHandle("json-schema-with-refs.json"); + JsonSchemaReferenceFinder finder = new JsonSchemaReferenceFinder(); + Set foundReferences = finder.findExternalReferences(content); + Assertions.assertNotNull(foundReferences); + Assertions.assertEquals(2, foundReferences.size()); + Assertions.assertEquals(Set.of(new JsonPointerExternalReference("./address.json"), new JsonPointerExternalReference("./ssn.json")), foundReferences); + } + +} diff --git a/schema-util/util-provider/src/test/resources/io/apicurio/registry/content/refs/json-schema-with-refs.json b/schema-util/util-provider/src/test/resources/io/apicurio/registry/content/refs/json-schema-with-refs.json new file mode 100644 index 0000000000..30969bfc3d --- /dev/null +++ b/schema-util/util-provider/src/test/resources/io/apicurio/registry/content/refs/json-schema-with-refs.json @@ -0,0 +1,10 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "name": { "type": "string" }, + "age": { "type": "integer" }, + "address": { "$ref": "./address.json" }, + "ssn": { "$ref": "./ssn.json" } + } +} diff --git a/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/refs/ReferenceIndex.java b/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/refs/ReferenceIndex.java index b98472ceca..332d12a195 100644 --- a/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/refs/ReferenceIndex.java +++ b/utils/maven-plugin/src/main/java/io/apicurio/registry/maven/refs/ReferenceIndex.java @@ -110,8 +110,8 @@ public void index(Path path, ContentHandle content) { indexDataModels(path, content); } // JSON Schema - if (tree.has("$schema") && tree.get("$schema").asText().contains("json-schema.org") || tree.has("properties")) { - indexJsonSchema(path, content); + if (tree.has("$schema") && !tree.get("$schema").isNull()) { + indexJsonSchema(tree, path, content); } // Avro indexAvro(path, content, tree); @@ -143,8 +143,13 @@ private void indexProto(Path path, ContentHandle content) { this.index.add(resource); } - private void indexJsonSchema(Path path, ContentHandle content) { - throw new UnsupportedOperationException("Not yet implemented."); + private void indexJsonSchema(JsonNode schema, Path path, ContentHandle content) { + String resourceName = null; + if (schema.has("$id")) { + resourceName = schema.get("$id").asText(null); + } + IndexedResource resource = new IndexedResource(path, ArtifactType.JSON, resourceName, content); + this.index.add(resource); } private void indexDataModels(Path path, ContentHandle content) {