wip reconcile

Sources/App/Commands/Reconcile.swift

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+import Dependencies
 import Fluent
 import Vapor
 
@@ -48,16 +49,24 @@ struct ReconcileCommand: AsyncCommand {
 func reconcile(client: Client, database: Database) async throws {
     let start = DispatchTime.now().uptimeNanoseconds
     defer { AppMetrics.reconcileDurationSeconds?.time(since: start) }
-    async let sourcePackageList = try Current.fetchPackageList(client)
-    async let sourcePackageDenyList = try Current.fetchPackageDenyList(client)
-    async let currentList = try fetchCurrentPackageList(database)
 
-    let packageList = processPackageDenyList(packageList: try await sourcePackageList,
-                                             denyList: try await sourcePackageDenyList)
+    do { // reconcile main package list
+        async let sourcePackageList = try Current.fetchPackageList(client)
+        async let sourcePackageDenyList = try Current.fetchPackageDenyList(client)
+        async let currentList = try fetchCurrentPackageList(database)
 
-    try await reconcileLists(db: database,
-                             source: packageList,
-                             target: currentList)
+        let packageList = processPackageDenyList(packageList: try await sourcePackageList,
+                                                 denyList: try await sourcePackageDenyList)
+
+        try await reconcileLists(db: database,
+                                 source: packageList,
+                                 target: currentList)
+    }
+
+    do { // reconcile custom package collections
+        // - fetch custom-package-collections.json
+        // - for each entry: reconcileCustomCollection
+    }
 }
 
 
@@ -115,6 +124,7 @@ func reconcileLists(db: Database, source: [URL], target: [URL]) async throws {
     }
 }
 
+
 func processPackageDenyList(packageList: [URL], denyList: [URL]) -> [URL] {
     // Note: If the implementation of this function ever changes, the `RemoveDenyList`
     // command in the Validator will also need updating to match.
@@ -140,3 +150,38 @@ func processPackageDenyList(packageList: [URL], denyList: [URL]) -> [URL] {
             .subtracting(Set(denyList.map(CaseInsensitiveURL.init)))
     ).map(\.url)
 }
+
+
+func reconcileCustomCollection(client: Client, database: Database, fullPackageList: [URL], collectionURL: URL) async throws {
+    @Dependency(\.packageListRepository) var packageListRepository
+
+    // - fetch list
+    let list = try await packageListRepository.fetchCustomCollection(client: client, url: collectionURL)
+
+    // impose a limit because we're dealing with an input outside our control
+    let urls = list.prefix(50)
+
+    // lookup packages
+    // current assumptions:
+    // - url must exist in the full list
+    // - url must be spelled exactly like it is in the full list
+    let packages = Package.query(on: database).filter(by: urls)
+
+    // - reconcile the list against its custom collection
+    //   - load collection
+    guard let collection = try await CustomCollection.query(on: database)
+        .filter(\.$url == collectionURL.absoluteString)
+        .first() else {
+        throw Abort(.notFound)
+    }
+    try await collection.$packages.load(on: database)
+    let currentURLs = Set(collection.packages.map(\.url))
+    let incomingURLs = Set(urls.map(\.absoluteString))
+    let toAdd = incomingURLs.subtracting(currentURLs)
+    let toDelete = currentURLs.subtracting(incomingURLs)
+
+    //   - add/remove as needed
+    for pkg in toAdd {
+
+    }
+}

Tests/AppTests/ReconcilerTests.swift

@@ -12,10 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+import XCTest
+
 @testable import App
 
+import Dependencies
 import Vapor
-import XCTest
 
 
 class ReconcilerTests: AppTestCase {
@@ -114,4 +116,20 @@ class ReconcilerTests: AppTestCase {
         let packages = try await Package.query(on: app.db).all()
         XCTAssertEqual(packages.map(\.url).sorted(), ["https://example.com/two/two"])
     }
+
+    func test_reconcileCustomCollections() async throws {
+        let fullPackageList = [URL("a"), URL("b"), URL("c")]
+        try await withDependencies {
+            $0.packageListRepository.fetchCustomCollection = { @Sendable _, _ in
+                [URL("b")]
+            }
+        } operation: {
+            // MUT
+//            try await reconcileCustomCollection(client: app.client,
+//                                                database: app.db,
+//                                                fullPackageList: fullPackageList,
+//                                                collectionURL: URL("collection"))
+        }
+    }
+
 }