Merge pull request #374 from mattpolzin/add-external-loader-message-s…

…ystem

Support arbitrary messaging from external loaders

.github/workflows/codecov.yml

@@ -5,7 +5,7 @@ on: [pull_request]
 jobs:
   codecov:
     container: 
-      image: swift:5.8
+      image: swift:5.10
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3

README.md

@@ -330,23 +330,25 @@ OpenAPIKit leaves it to you to decide how to load external files and where to st
 
 ```swift
 struct ExampleLoader: ExternalLoader {
-    static func load<T>(_ url: URL) async throws -> T where T : Decodable {
+    typealias Message = Void
+
+    static func load<T>(_ url: URL) async throws -> (T, [Message]) where T : Decodable {
         // load data from file, perhaps. we will just mock that up for the test:
         let data = try await mockData(componentKey(type: T.self, at: url))
 
-        // We use the YAML decoder mostly for order-stability in this case but it is
-        // also nice that it will handle both YAML and JSON data.
+        // We use the YAML decoder purely for order-stability.
         let decoded = try YAMLDecoder().decode(T.self, from: data)
         let finished: T
         // while unnecessary, a loader may likely want to attatch some extra info
-        // to keep track of where a reference was loaded from.
+        // to keep track of where a reference was loaded from. This test makes sure
+        // the following strategy of using vendor extensions works.
         if var extendable = decoded as? VendorExtendable {
             extendable.vendorExtensions["x-source-url"] = AnyCodable(url)
             finished = extendable as! T
         } else {
             finished = decoded 
         }
-        return finished
+        return (finished, [])
     }
 
     static func componentKey<T>(type: T.Type, at url: URL) throws -> OpenAPIKit.OpenAPI.ComponentKey {
@@ -362,6 +364,8 @@ struct ExampleLoader: ExternalLoader {
 
 Once you have an `ExternalLoader`, you can call an `OpenAPI.Document`'s `externallyDereference()` method to externally dereference it. You get to choose whether to only load references to a certain depth or to fully resolve references until you run out of them; any given referenced document may itself contain references and these references may point back to things loaded into the Document previously so dereferencing is done recursively up to a given depth (or until fully dereferenced if you use the `.full` depth).
 
+If you have some information that you want to pass back to yourself from the `load()` function, you can specify any type you want as the `Message` type and return any number of messages from each `load()` function execution. These messages could be warnings, additional information about the files that each newly loaded Component came from, etc. If you want to tie some information about file loading to new Components in your messages, you can use the `componentKey()` function to get the key the new Component will be found under once external dereferencing is complete.
+
 #### Internal References
 In addition to looking something up in the `Components` object, you can entirely derefererence many OpenAPIKit types. A dereferenced type has had all of its references looked up (and all of its properties' references, all the way down).
 

Sources/OpenAPIKit/Components Object/Components.swift

@@ -320,31 +320,37 @@ extension OpenAPI.Components {
 }
 
 extension OpenAPI.Components {
-    internal mutating func externallyDereference<Loader: ExternalLoader>(with loader: Loader.Type, depth: ExternalDereferenceDepth = .iterations(1)) async throws {
+    internal mutating func externallyDereference<Loader: ExternalLoader>(with loader: Loader.Type, depth: ExternalDereferenceDepth = .iterations(1), context: [Loader.Message] = []) async throws -> [Loader.Message] {
         if case let .iterations(number) = depth,
            number <= 0 {
-            return
+            return context
         }
 
+        // NOTE: The links and callbacks related code commented out below pushes Swift 5.8 and 5.9
+        //       over the edge and you get exit code 137 crashes in CI.
+        //       Swift 5.10 handles it fine.
+
         let oldSchemas = schemas
         let oldResponses = responses
         let oldParameters = parameters
         let oldExamples = examples
         let oldRequestBodies = requestBodies
         let oldHeaders = headers
         let oldSecuritySchemes = securitySchemes
+        let oldLinks = links
         let oldCallbacks = callbacks
         let oldPathItems = pathItems
 
-        async let (newSchemas, c1) = oldSchemas.externallyDereferenced(with: loader)
-        async let (newResponses, c2) = oldResponses.externallyDereferenced(with: loader)
-        async let (newParameters, c3) = oldParameters.externallyDereferenced(with: loader)
-        async let (newExamples, c4) = oldExamples.externallyDereferenced(with: loader)
-        async let (newRequestBodies, c5) = oldRequestBodies.externallyDereferenced(with: loader)
-        async let (newHeaders, c6) = oldHeaders.externallyDereferenced(with: loader)
-        async let (newSecuritySchemes, c7) = oldSecuritySchemes.externallyDereferenced(with: loader)
-        async let (newCallbacks, c8) = oldCallbacks.externallyDereferenced(with: loader)
-        async let (newPathItems, c9) = oldPathItems.externallyDereferenced(with: loader)
+        async let (newSchemas, c1, m1) = oldSchemas.externallyDereferenced(with: loader)
+        async let (newResponses, c2, m2) = oldResponses.externallyDereferenced(with: loader)
+        async let (newParameters, c3, m3) = oldParameters.externallyDereferenced(with: loader)
+        async let (newExamples, c4, m4) = oldExamples.externallyDereferenced(with: loader)
+        async let (newRequestBodies, c5, m5) = oldRequestBodies.externallyDereferenced(with: loader)
+        async let (newHeaders, c6, m6) = oldHeaders.externallyDereferenced(with: loader)
+        async let (newSecuritySchemes, c7, m7) = oldSecuritySchemes.externallyDereferenced(with: loader)
+//        async let (newLinks, c8, m8) = oldLinks.externallyDereferenced(with: loader)
+//        async let (newCallbacks, c9, m9) = oldCallbacks.externallyDereferenced(with: loader)
+        async let (newPathItems, c10, m10) = oldPathItems.externallyDereferenced(with: loader)
 
         schemas = try await newSchemas
         responses = try await newResponses
@@ -353,7 +359,8 @@ extension OpenAPI.Components {
         requestBodies = try await newRequestBodies
         headers = try await newHeaders
         securitySchemes = try await newSecuritySchemes
-        callbacks = try await newCallbacks
+//        links = try await newLinks
+//        callbacks = try await newCallbacks
         pathItems = try await newPathItems
 
         let c1Resolved = try await c1
@@ -363,8 +370,18 @@ extension OpenAPI.Components {
         let c5Resolved = try await c5
         let c6Resolved = try await c6
         let c7Resolved = try await c7
-        let c8Resolved = try await c8
-        let c9Resolved = try await c9
+//        let c8Resolved = try await c8
+//        let c9Resolved = try await c9
+        let c10Resolved = try await c10
+
+        // For Swift 5.10+ we can delete the following links and callbacks code and uncomment the
+        // preferred code above.
+        let (newLinks, c8, m8) = try await oldLinks.externallyDereferenced(with: loader)
+        links = newLinks
+        let c8Resolved = c8
+        let (newCallbacks, c9, m9) = try await oldCallbacks.externallyDereferenced(with: loader)
+        callbacks = newCallbacks
+        let c9Resolved = c9
 
         let noNewComponents =
             c1Resolved.isEmpty
@@ -376,8 +393,11 @@ extension OpenAPI.Components {
             && c7Resolved.isEmpty
             && c8Resolved.isEmpty
             && c9Resolved.isEmpty
+            && c10Resolved.isEmpty
+
+        let newMessages = try await context + m1 + m2 + m3 + m4 + m5 + m6 + m7 + m8 + m9 + m10 
 
-        if noNewComponents { return }
+        if noNewComponents { return newMessages }
 
         try merge(c1Resolved)
         try merge(c2Resolved)
@@ -388,12 +408,13 @@ extension OpenAPI.Components {
         try merge(c7Resolved)
         try merge(c8Resolved)
         try merge(c9Resolved)
-
+        try merge(c10Resolved)
+
         switch depth {
             case .iterations(let number):
-                try await externallyDereference(with: loader, depth: .iterations(number - 1))
+                return try await externallyDereference(with: loader, depth: .iterations(number - 1), context: newMessages)
             case .full:
-                try await externallyDereference(with: loader, depth: .full)
+                return try await externallyDereference(with: loader, depth: .full, context: newMessages)
         }
     }
 }

Sources/OpenAPIKit/Content/DereferencedContent.swift

@@ -77,28 +77,31 @@ extension OpenAPI.Content: LocallyDereferenceable {
 }
 
 extension OpenAPI.Content: ExternallyDereferenceable {
-    public func externallyDereferenced<Loader: ExternalLoader>(with loader: Loader.Type) async throws -> (Self, OpenAPI.Components) { 
+    public func externallyDereferenced<Loader: ExternalLoader>(with loader: Loader.Type) async throws -> (Self, OpenAPI.Components, [Loader.Message]) { 
       let oldSchema = schema
 
-      async let (newSchema, c1) = oldSchema.externallyDereferenced(with: loader)
+      async let (newSchema, c1, m1) = oldSchema.externallyDereferenced(with: loader)
 
       var newContent = self
       var newComponents = try await c1
+      var newMessages = try await m1
 
       newContent.schema = try await newSchema
 
         if let oldExamples = examples {
-            let (newExamples, c2) = try await oldExamples.externallyDereferenced(with: loader)
+            let (newExamples, c2, m2) = try await oldExamples.externallyDereferenced(with: loader)
             newContent.examples = newExamples
             try newComponents.merge(c2)
+            newMessages += m2
         }
 
         if let oldEncoding = encoding {
-            let (newEncoding, c3) = try await oldEncoding.externallyDereferenced(with: loader)
+            let (newEncoding, c3, m3) = try await oldEncoding.externallyDereferenced(with: loader)
             newContent.encoding = newEncoding
             try newComponents.merge(c3)
+            newMessages += m3
         }
 
-        return (newContent, newComponents)
+        return (newContent, newComponents, newMessages)
     }
 }

Sources/OpenAPIKit/Content/DereferencedContentEncoding.swift

@@ -58,15 +58,17 @@ extension OpenAPI.Content.Encoding: LocallyDereferenceable {
 }
 
 extension OpenAPI.Content.Encoding: ExternallyDereferenceable {
-    public func externallyDereferenced<Loader: ExternalLoader>(with loader: Loader.Type) async throws -> (Self, OpenAPI.Components) { 
+    public func externallyDereferenced<Loader: ExternalLoader>(with loader: Loader.Type) async throws -> (Self, OpenAPI.Components, [Loader.Message]) { 
         let newHeaders: OpenAPI.Header.Map?
         let newComponents: OpenAPI.Components
+        let newMessages: [Loader.Message]
 
         if let oldHeaders = headers {
-          (newHeaders, newComponents) = try await oldHeaders.externallyDereferenced(with: loader)
+          (newHeaders, newComponents, newMessages) = try await oldHeaders.externallyDereferenced(with: loader)
         } else {
           newHeaders = nil
           newComponents = .init()
+          newMessages = []
         }
 
         let newEncoding = OpenAPI.Content.Encoding(
@@ -77,6 +79,6 @@ extension OpenAPI.Content.Encoding: ExternallyDereferenceable {
             allowReserved: allowReserved
         )
 
-        return (newEncoding, newComponents)
+        return (newEncoding, newComponents, newMessages)
     }
 }

Sources/OpenAPIKit/Document/Document.swift

@@ -362,24 +362,44 @@ extension OpenAPI.Document {
         return try DereferencedDocument(self)
     }
 
-    public mutating func externallyDereference<Loader: ExternalLoader>(with loader: Loader.Type, depth: ExternalDereferenceDepth = .iterations(1)) async throws {
+    /// Load all remote references into the document. A remote reference is one
+    /// that points to another file rather than a location within the
+    /// same file.
+    ///
+    /// This function will load remote references into the Components object
+    /// and replace the remote reference with a local reference to that component.
+    /// No local references are modified or resolved by this function. You can
+    /// call `locallyDereferenced()` on the externally dereferenced document if
+    /// you want to also remove local references by inlining all of them.
+    ///
+    /// Externally dereferencing a document requires that you provide both a
+    /// function that produces a `OpenAPI.ComponentKey` for any given remote
+    /// file URI and also a function that loads and decodes the data found in
+    /// that remote file. The latter is less work than it may sound like because
+    /// the function is told what Decodable thing it wants, so you really just
+    /// need to decide what decoder to use and provide the file data to that
+    /// decoder. See `ExternalLoader` documentation for details.
+    @discardableResult
+    public mutating func externallyDereference<Loader: ExternalLoader>(with loader: Loader.Type, depth: ExternalDereferenceDepth = .iterations(1), context: [Loader.Message] = []) async throws -> [Loader.Message] {
         if case let .iterations(number) = depth,
            number <= 0 {
-            return
+            return context
         }
 
         let oldPaths = paths
         let oldWebhooks = webhooks
 
-        async let (newPaths, c1) = oldPaths.externallyDereferenced(with: loader)
-        async let (newWebhooks, c2) = oldWebhooks.externallyDereferenced(with: loader) 
+        async let (newPaths, c1, m1) = oldPaths.externallyDereferenced(with: loader)
+        async let (newWebhooks, c2, m2) = oldWebhooks.externallyDereferenced(with: loader) 
 
         paths = try await newPaths
         webhooks = try await newWebhooks
         try await components.merge(c1)
         try await components.merge(c2)
 
-        try await components.externallyDereference(with: loader, depth: depth)
+        let m3 = try await components.externallyDereference(with: loader, depth: depth, context: context)
+
+        return try await context + m1 + m2 + m3
     }
 }
 

Sources/OpenAPIKit/Either/Either+ExternallyDereferenceable.swift

@@ -10,14 +10,14 @@ import OpenAPIKitCore
 // MARK: - ExternallyDereferenceable
 extension Either: ExternallyDereferenceable where A: ExternallyDereferenceable, B: ExternallyDereferenceable {
 
-    public func externallyDereferenced<Loader: ExternalLoader>(with loader: Loader.Type) async throws -> (Self, OpenAPI.Components) { 
+    public func externallyDereferenced<Loader: ExternalLoader>(with loader: Loader.Type) async throws -> (Self, OpenAPI.Components, [Loader.Message]) { 
         switch self {
         case .a(let a):
-           let (newA, components) = try await a.externallyDereferenced(with: loader)
-           return (.a(newA), components)
+           let (newA, components, messages) = try await a.externallyDereferenced(with: loader)
+           return (.a(newA), components, messages)
         case .b(let b):
-           let (newB, components) = try await b.externallyDereferenced(with: loader)
-           return (.b(newB), components)
+           let (newB, components, messages) = try await b.externallyDereferenced(with: loader)
+           return (.b(newB), components, messages)
         }
     }
 }

Sources/OpenAPIKit/Example.swift

@@ -209,8 +209,8 @@ extension OpenAPI.Example: LocallyDereferenceable {
 }
 
 extension OpenAPI.Example: ExternallyDereferenceable {
-    public func externallyDereferenced<Loader: ExternalLoader>(with loader: Loader.Type) async throws -> (Self, OpenAPI.Components) { 
-        return (self, .init())
+    public func externallyDereferenced<Loader: ExternalLoader>(with loader: Loader.Type) async throws -> (Self, OpenAPI.Components, [Loader.Message]) { 
+        return (self, .init(), [])
     }
 }
 

Sources/OpenAPIKit/ExternalLoader.swift

@@ -12,11 +12,17 @@ import Foundation
 /// without knowing the details of what decoder is being used or how new internal 
 /// references should be named.
 public protocol ExternalLoader {
+    /// This can be anything that an implementor of this protocol wants to pass back from
+    /// the `load()` function and have available after all external loading has been done.
+    ///
+    /// A trivial type if no Messages are needed would be Void.
+    associatedtype Message
+
     /// Load the given URL and decode it as Type `T`. All Types `T` are `Decodable`, so
     /// the only real responsibility of a `load` function is to locate and load the given
     /// `URL` and pass its `Data` or `String` (depending on the decoder) to an appropriate
     /// `Decoder` for the given file type.
-    static func load<T>(_: URL) async throws -> T where T: Decodable
+    static func load<T>(_: URL) async throws -> (T, [Message]) where T: Decodable
 
     /// Determine the next Component Key (where to store something in the 
     /// Components Object) for a new object of the given type that was loaded
@@ -30,5 +36,5 @@ public protocol ExternalLoader {
 }
 
 public protocol ExternallyDereferenceable {
-    func externallyDereferenced<Loader: ExternalLoader>(with loader: Loader.Type) async throws -> (Self, OpenAPI.Components)
+    func externallyDereferenced<Loader: ExternalLoader>(with loader: Loader.Type) async throws -> (Self, OpenAPI.Components, [Loader.Message])
 }

Sources/OpenAPIKit/Header/DereferencedHeader.swift

@@ -84,24 +84,27 @@ extension OpenAPI.Header: LocallyDereferenceable {
 }
 
 extension OpenAPI.Header: ExternallyDereferenceable {
-    public func externallyDereferenced<Loader: ExternalLoader>(with loader: Loader.Type) async throws -> (Self, OpenAPI.Components) { 
+    public func externallyDereferenced<Loader: ExternalLoader>(with loader: Loader.Type) async throws -> (Self, OpenAPI.Components, [Loader.Message]) { 
 
         // if not for a Swift bug, this whole next bit would just be the
         // next line:
 //        let (newSchemaOrContent, components) = try await schemaOrContent.externallyDereferenced(with: loader)
 
         let newSchemaOrContent: Either<SchemaContext, OpenAPI.Content.Map>
         let newComponents: OpenAPI.Components
+        let newMessages: [Loader.Message]
 
         switch schemaOrContent {
         case .a(let schemaContext):
-            let (context, components) = try await schemaContext.externallyDereferenced(with: loader)
+            let (context, components, messages) = try await schemaContext.externallyDereferenced(with: loader)
             newSchemaOrContent = .a(context)
             newComponents = components
+            newMessages = messages
         case .b(let contentMap):
-            let (map, components) = try await contentMap.externallyDereferenced(with: loader)
+            let (map, components, messages) = try await contentMap.externallyDereferenced(with: loader)
             newSchemaOrContent = .b(map)
             newComponents = components
+            newMessages = messages
         }
 
         let newHeader = OpenAPI.Header(
@@ -112,6 +115,6 @@ extension OpenAPI.Header: ExternallyDereferenceable {
             vendorExtensions: vendorExtensions
         )
 
-        return (newHeader, newComponents)
+        return (newHeader, newComponents, newMessages)
     }
 }