Merge pull request #369 from mattpolzin/load-external-references-v4-t…

…ake2

Load external references v4 take2

Package.swift

@@ -5,7 +5,7 @@ import PackageDescription
 let package = Package(
     name: "OpenAPIKit",
     platforms: [
-        .macOS(.v10_13),
+        .macOS(.v10_15),
         .iOS(.v11)
     ],
     products: [

README.md

@@ -220,7 +220,7 @@ You can create an external reference with `JSONReference.external(URL)`. Interna
 
 You can check whether a given `JSONReference` exists in the Components Object with `document.components.contains()`. You can access a referenced object in the Components Object with `document.components[reference]`.
 
-You can create references from the Components Object with `document.components.reference(named:ofType:)`. This method will throw an error if the given component does not exist in the ComponentsObject.
+References can be created from the Components Object with `document.components.reference(named:ofType:)`. This method will throw an error if the given component does not exist in the ComponentsObject.
 
 You can use `document.components.lookup()` or the `Components` type's `subscript` to turn an `Either` containing either a reference or a component into an optional value of that component's type (having either pulled it out of the `Either` or looked it up in the Components Object). The `lookup()` method throws when it can't find an item whereas `subscript` returns `nil`.
 
@@ -284,7 +284,7 @@ let document = OpenAPI.Document(
 ```
 
 #### Specification Extensions
-Many OpenAPIKit types support [Specification Extensions](https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.3.md#specification-extensions). As described in the OpenAPI Specification, these extensions must be objects that are keyed with the prefix "x-". For example, a property named "specialProperty" on the root OpenAPI Object (`OpenAPI.Document`) is invalid but the property "x-specialProperty" is a valid specification extension.
+Many OpenAPIKit types support [Specification Extensions](https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.1.0.md#specification-extensions). As described in the OpenAPI Specification, these extensions must be objects that are keyed with the prefix "x-". For example, a property named "specialProperty" on the root OpenAPI Object (`OpenAPI.Document`) is invalid but the property "x-specialProperty" is a valid specification extension.
 
 You can get or set specification extensions via the `vendorExtensions` property on any object that supports this feature. The keys are `Strings` beginning with the aforementioned "x-" prefix and the values are `AnyCodable`. If you set an extension without using the "x-" prefix, the prefix will be added upon encoding.
 
@@ -323,11 +323,51 @@ try encodeEqual(URL(string: "https://website.com"), AnyCodable(URL(string: "http
 ```
 
 ### Dereferencing & Resolving
+#### External References
+This is currently only available for OAS 3.1 documents (supported by the `OpenAPIKit` module (as opposed to the `OpenAPIKit30` moudle). External dereferencing does not resolve any local (internal) references, it just loads external references into the Document. It does this by storing any loaded externally referenced objects in the Components Object and transforming the reference being resolved from an external reference to an internal one. That way, you can always run internal dereferencing as a second step if you want a fully dereferenced document, but if you simply wanted to load additional referenced files then you can stop after external dereferencing.
+
+OpenAPIKit leaves it to you to decide how to load external files and where to store the results in the Components Object. It does this by requiring that you provide an implementation of the `ExternalLoader` protocol. You provide a `load` function and a `componentKey` function, both of which accept as input the `URL` to load. A simple mock example implementation from the OpenAPIKit tests will go a long way to showing how the `ExternalLoader` can be set up:
+
+```swift
+struct ExampleLoader: ExternalLoader {
+    static func load<T>(_ url: URL) async throws -> T where T : Decodable {
+        // load data from file, perhaps. we will just mock that up for the test:
+        let data = try await mockData(componentKey(type: T.self, at: url))
+
+        // We use the YAML decoder mostly for order-stability in this case but it is
+        // also nice that it will handle both YAML and JSON data.
+        let decoded = try YAMLDecoder().decode(T.self, from: data)
+        let finished: T
+        // while unnecessary, a loader may likely want to attatch some extra info
+        // to keep track of where a reference was loaded from.
+        if var extendable = decoded as? VendorExtendable {
+            extendable.vendorExtensions["x-source-url"] = AnyCodable(url)
+            finished = extendable as! T
+        } else {
+            finished = decoded 
+        }
+        return finished
+    }
+
+    static func componentKey<T>(type: T.Type, at url: URL) throws -> OpenAPIKit.OpenAPI.ComponentKey {
+        // do anything you want here to determine what key the new component should be stored at.
+        // for the example, we will just transform the URL path into a valid components key:
+        let urlString = url.pathComponents
+          .joined(separator: "_")
+          .replacingOccurrences(of: ".", with: "_")
+        return try .forceInit(rawValue: urlString)
+    }
+}
+```
+
+Once you have an `ExternalLoader`, you can call an `OpenAPI.Document`'s `externallyDereference()` method to externally dereference it. You get to choose whether to only load references to a certain depth or to fully resolve references until you run out of them; any given referenced document may itself contain references and these references may point back to things loaded into the Document previously so dereferencing is done recursively up to a given depth (or until fully dereferenced if you use the `.full` depth).
+
+#### Internal References
 In addition to looking something up in the `Components` object, you can entirely derefererence many OpenAPIKit types. A dereferenced type has had all of its references looked up (and all of its properties' references, all the way down).
 
 You use a value's `dereferenced(in:)` method to fully dereference it.
 
-You can even dereference the whole document with the `OpenAPI.Document` `locallyDereferenced()` method. As the name implies, you can only derefence whole documents that are contained within one file (which is another way of saying that all references are "local"). Specifically, all references must be located within the document's Components Object.
+You can even dereference the whole document with the `OpenAPI.Document` `locallyDereferenced()` method. As the name implies, you can only derefence whole documents that are contained within one file (which is another way of saying that all references are "local"). Specifically, all references must be located within the document's Components Object. External dereferencing is done as a separeate step, but you can first dereference externally and then dereference internally if you'd like to perform both.
 
 Unlike what happens when you lookup an individual component using the `lookup()` method on `Components`, dereferencing a whole `OpenAPI.Document` will result in type-level changes that guarantee all references are removed. `OpenAPI.Document`'s `locallyDereferenced()` method returns a `DereferencedDocument` which exposes `DereferencedPathItem`s which have `DereferencedParameter`s and `DereferencedOperation`s and so on.
 

Sources/OpenAPIKit/Callbacks.swift

@@ -37,3 +37,9 @@ extension OpenAPI.CallbackURL: LocallyDereferenceable {
     }
 }
 
+// The following conformance is theoretically unnecessary but the compiler is
+// only able to find the conformance if we explicitly declare it here, though
+// it is apparently able to determine the conformance is already satisfied here
+// at least.
+extension OpenAPI.Callbacks: ExternallyDereferenceable { }
+

Sources/OpenAPIKit/CodableVendorExtendable.swift

@@ -18,7 +18,7 @@ public protocol VendorExtendable {
     /// These should be of the form:
     /// `[ "x-extensionKey": <anything>]`
     /// where the values are anything codable.
-    var vendorExtensions: VendorExtensions { get }
+    var vendorExtensions: VendorExtensions { get set }
 }
 
 public enum VendorExtensionsConfiguration {

Sources/OpenAPIKit/Components Object/Components+Locatable.swift

@@ -6,6 +6,7 @@
 //
 
 import OpenAPIKitCore
+import Foundation
 
 /// Anything conforming to ComponentDictionaryLocatable knows
 /// where to find resources of its type in the Components Dictionary.
@@ -15,57 +16,57 @@ public protocol ComponentDictionaryLocatable: SummaryOverridable {
     /// This can be used to create a JSON path
     /// like `#/name1/name2/name3`
     static var openAPIComponentsKey: String { get }
-    static var openAPIComponentsKeyPath: KeyPath<OpenAPI.Components, OpenAPI.ComponentDictionary<Self>> { get }
+    static var openAPIComponentsKeyPath: WritableKeyPath<OpenAPI.Components, OpenAPI.ComponentDictionary<Self>> { get }
 }
 
 extension JSONSchema: ComponentDictionaryLocatable {
     public static var openAPIComponentsKey: String { "schemas" }
-    public static var openAPIComponentsKeyPath: KeyPath<OpenAPI.Components, OpenAPI.ComponentDictionary<Self>> { \.schemas }
+    public static var openAPIComponentsKeyPath: WritableKeyPath<OpenAPI.Components, OpenAPI.ComponentDictionary<Self>> { \.schemas }
 }
 
 extension OpenAPI.Response: ComponentDictionaryLocatable {
     public static var openAPIComponentsKey: String { "responses" }
-    public static var openAPIComponentsKeyPath: KeyPath<OpenAPI.Components, OpenAPI.ComponentDictionary<Self>> { \.responses }
+    public static var openAPIComponentsKeyPath: WritableKeyPath<OpenAPI.Components, OpenAPI.ComponentDictionary<Self>> { \.responses }
 }
 
 extension OpenAPI.Callbacks: ComponentDictionaryLocatable & SummaryOverridable {
     public static var openAPIComponentsKey: String { "callbacks" }
-    public static var openAPIComponentsKeyPath: KeyPath<OpenAPI.Components, OpenAPI.ComponentDictionary<Self>> { \.callbacks }
+    public static var openAPIComponentsKeyPath: WritableKeyPath<OpenAPI.Components, OpenAPI.ComponentDictionary<Self>> { \.callbacks }
 }
 
 extension OpenAPI.Parameter: ComponentDictionaryLocatable {
     public static var openAPIComponentsKey: String { "parameters" }
-    public static var openAPIComponentsKeyPath: KeyPath<OpenAPI.Components, OpenAPI.ComponentDictionary<Self>> { \.parameters }
+    public static var openAPIComponentsKeyPath: WritableKeyPath<OpenAPI.Components, OpenAPI.ComponentDictionary<Self>> { \.parameters }
 }
 
 extension OpenAPI.Example: ComponentDictionaryLocatable {
     public static var openAPIComponentsKey: String { "examples" }
-    public static var openAPIComponentsKeyPath: KeyPath<OpenAPI.Components, OpenAPI.ComponentDictionary<Self>> { \.examples }
+    public static var openAPIComponentsKeyPath: WritableKeyPath<OpenAPI.Components, OpenAPI.ComponentDictionary<Self>> { \.examples }
 }
 
 extension OpenAPI.Request: ComponentDictionaryLocatable {
     public static var openAPIComponentsKey: String { "requestBodies" }
-    public static var openAPIComponentsKeyPath: KeyPath<OpenAPI.Components, OpenAPI.ComponentDictionary<Self>> { \.requestBodies }
+    public static var openAPIComponentsKeyPath: WritableKeyPath<OpenAPI.Components, OpenAPI.ComponentDictionary<Self>> { \.requestBodies }
 }
 
 extension OpenAPI.Header: ComponentDictionaryLocatable {
     public static var openAPIComponentsKey: String { "headers" }
-    public static var openAPIComponentsKeyPath: KeyPath<OpenAPI.Components, OpenAPI.ComponentDictionary<Self>> { \.headers }
+    public static var openAPIComponentsKeyPath: WritableKeyPath<OpenAPI.Components, OpenAPI.ComponentDictionary<Self>> { \.headers }
 }
 
 extension OpenAPI.SecurityScheme: ComponentDictionaryLocatable {
     public static var openAPIComponentsKey: String { "securitySchemes" }
-    public static var openAPIComponentsKeyPath: KeyPath<OpenAPI.Components, OpenAPI.ComponentDictionary<Self>> { \.securitySchemes }
+    public static var openAPIComponentsKeyPath: WritableKeyPath<OpenAPI.Components, OpenAPI.ComponentDictionary<Self>> { \.securitySchemes }
 }
 
 extension OpenAPI.Link: ComponentDictionaryLocatable {
     public static var openAPIComponentsKey: String { "links" }
-    public static var openAPIComponentsKeyPath: KeyPath<OpenAPI.Components, OpenAPI.ComponentDictionary<Self>> { \.links }
+    public static var openAPIComponentsKeyPath: WritableKeyPath<OpenAPI.Components, OpenAPI.ComponentDictionary<Self>> { \.links }
 }
 
 extension OpenAPI.PathItem: ComponentDictionaryLocatable {
     public static var openAPIComponentsKey: String { "pathItems" }
-    public static var openAPIComponentsKeyPath: KeyPath<OpenAPI.Components, OpenAPI.ComponentDictionary<Self>> { \.pathItems }
+    public static var openAPIComponentsKeyPath: WritableKeyPath<OpenAPI.Components, OpenAPI.ComponentDictionary<Self>> { \.pathItems }
 }
 
 /// A dereferenceable type can be recursively looked up in

Sources/OpenAPIKit/Components Object/Components.swift

@@ -71,6 +71,57 @@ extension OpenAPI {
     }
 }
 
+extension OpenAPI.Components {
+    public struct ComponentCollision: Swift.Error {
+        public let componentType: String
+        public let existingComponent: String
+        public let newComponent: String
+    }
+
+    private func detectCollision<T: Equatable>(type: String) throws -> (_ old: T, _ new: T) throws -> T {
+        return { old, new in
+            // theoretically we can detect collisions here, but we would need to compare
+            // for equality up-to but not including the difference between an external and
+            // internal reference which is not supported yet.
+//            if(old == new) { return old }
+//            throw ComponentCollision(componentType: type, existingComponent: String(describing:old), newComponent: String(describing:new))
+
+            // Given we aren't ensuring there are no collisions, the old version is going to be
+            // the one more likely to have been _further_ dereferenced than the new record, so
+            // we keep that version.
+            return old
+        }
+    }
+
+    public mutating func merge(_ other: OpenAPI.Components) throws {
+        try schemas.merge(other.schemas, uniquingKeysWith: detectCollision(type: "schema"))
+        try responses.merge(other.responses, uniquingKeysWith: detectCollision(type: "responses"))
+        try parameters.merge(other.parameters, uniquingKeysWith: detectCollision(type: "parameters"))
+        try examples.merge(other.examples, uniquingKeysWith: detectCollision(type: "examples"))
+        try requestBodies.merge(other.requestBodies, uniquingKeysWith: detectCollision(type: "requestBodies"))
+        try headers.merge(other.headers, uniquingKeysWith: detectCollision(type: "headers"))
+        try securitySchemes.merge(other.securitySchemes, uniquingKeysWith: detectCollision(type: "securitySchemes"))
+        try links.merge(other.links, uniquingKeysWith: detectCollision(type: "links"))
+        try callbacks.merge(other.callbacks, uniquingKeysWith: detectCollision(type: "callbacks"))
+        try pathItems.merge(other.pathItems, uniquingKeysWith: detectCollision(type: "pathItems"))
+        try vendorExtensions.merge(other.vendorExtensions, uniquingKeysWith: detectCollision(type: "vendorExtensions"))
+    }
+
+    /// Sort the components within each type by the component key.
+    public mutating func sort() {
+        schemas.sortKeys()
+        responses.sortKeys()
+        parameters.sortKeys()
+        examples.sortKeys()
+        requestBodies.sortKeys()
+        headers.sortKeys()
+        securitySchemes.sortKeys()
+        links.sortKeys()
+        callbacks.sortKeys()
+        pathItems.sortKeys()
+    }
+}
+
 extension OpenAPI.Components {
     /// The extension name used to store a Components Object name (the key something is stored under
     /// within the Components Object). This is used by OpenAPIKit to store the previous Component name 
@@ -268,4 +319,83 @@ extension OpenAPI.Components {
     }
 }
 
+extension OpenAPI.Components {
+    internal mutating func externallyDereference<Context: ExternalLoader>(in context: Context.Type, depth: ExternalDereferenceDepth = .iterations(1)) async throws {
+        if case let .iterations(number) = depth,
+           number <= 0 {
+            return
+        }
+
+        let oldSchemas = schemas
+        let oldResponses = responses
+        let oldParameters = parameters
+        let oldExamples = examples
+        let oldRequestBodies = requestBodies
+        let oldHeaders = headers
+        let oldSecuritySchemes = securitySchemes
+        let oldCallbacks = callbacks
+        let oldPathItems = pathItems
+
+        async let (newSchemas, c1) = oldSchemas.externallyDereferenced(with: context)
+        async let (newResponses, c2) = oldResponses.externallyDereferenced(with: context)
+        async let (newParameters, c3) = oldParameters.externallyDereferenced(with: context)
+        async let (newExamples, c4) = oldExamples.externallyDereferenced(with: context)
+        async let (newRequestBodies, c5) = oldRequestBodies.externallyDereferenced(with: context)
+        async let (newHeaders, c6) = oldHeaders.externallyDereferenced(with: context)
+        async let (newSecuritySchemes, c7) = oldSecuritySchemes.externallyDereferenced(with: context)
+        async let (newCallbacks, c8) = oldCallbacks.externallyDereferenced(with: context)
+        async let (newPathItems, c9) = oldPathItems.externallyDereferenced(with: context)
+
+        schemas = try await newSchemas
+        responses = try await newResponses
+        parameters = try await newParameters
+        examples = try await newExamples
+        requestBodies = try await newRequestBodies
+        headers = try await newHeaders
+        securitySchemes = try await newSecuritySchemes
+        callbacks = try await newCallbacks
+        pathItems = try await newPathItems
+
+        let c1Resolved = try await c1
+        let c2Resolved = try await c2
+        let c3Resolved = try await c3
+        let c4Resolved = try await c4
+        let c5Resolved = try await c5
+        let c6Resolved = try await c6
+        let c7Resolved = try await c7
+        let c8Resolved = try await c8
+        let c9Resolved = try await c9
+
+        let noNewComponents =
+            c1Resolved.isEmpty
+            && c2Resolved.isEmpty
+            && c3Resolved.isEmpty
+            && c4Resolved.isEmpty
+            && c5Resolved.isEmpty
+            && c6Resolved.isEmpty
+            && c7Resolved.isEmpty
+            && c8Resolved.isEmpty
+            && c9Resolved.isEmpty
+
+        if noNewComponents { return }
+
+        try merge(c1Resolved)
+        try merge(c2Resolved)
+        try merge(c3Resolved)
+        try merge(c4Resolved)
+        try merge(c5Resolved)
+        try merge(c6Resolved)
+        try merge(c7Resolved)
+        try merge(c8Resolved)
+        try merge(c9Resolved)
+
+        switch depth {
+            case .iterations(let number):
+                try await externallyDereference(in: context, depth: .iterations(number - 1))
+            case .full:
+                try await externallyDereference(in: context, depth: .full)
+        }
+    }
+}
+
 extension OpenAPI.Components: Validatable {}

Sources/OpenAPIKit/Content/DereferencedContent.swift

@@ -75,3 +75,30 @@ extension OpenAPI.Content: LocallyDereferenceable {
         return try DereferencedContent(self, resolvingIn: components, following: references)
     }
 }
+
+extension OpenAPI.Content: ExternallyDereferenceable {
+    public func externallyDereferenced<Context: ExternalLoader>(with loader: Context.Type) async throws -> (Self, OpenAPI.Components) { 
+      let oldSchema = schema
+
+      async let (newSchema, c1) = oldSchema.externallyDereferenced(with: loader)
+
+      var newContent = self
+      var newComponents = try await c1
+
+      newContent.schema = try await newSchema
+
+        if let oldExamples = examples {
+            let (newExamples, c2) = try await oldExamples.externallyDereferenced(with: loader)
+            newContent.examples = newExamples
+            try newComponents.merge(c2)
+        }
+
+        if let oldEncoding = encoding {
+            let (newEncoding, c3) = try await oldEncoding.externallyDereferenced(with: loader)
+            newContent.encoding = newEncoding
+            try newComponents.merge(c3)
+        }
+
+        return (newContent, newComponents)
+    }
+}