Merge pull request #2481 from SwiftPackageIndex/macro-search

Macro search

Sources/App/Core/SearchFilter/Filters/ProductTypeSearchFilter.swift

@@ -65,6 +65,7 @@ extension ProductTypeSearchFilter {
     enum ProductType: String, Codable, CaseIterable {
         case executable
         case library
+        case macro
         case plugin
 
         var displayDescription: String {
@@ -73,6 +74,8 @@ extension ProductTypeSearchFilter {
                     return "Executable"
                 case .library:
                     return "Library"
+                case .macro:
+                    return "Macro"
                 case .plugin:
                     return "Plugin"
             }

Sources/App/Migrations/066/UpdateSearchAddMacroProductType.swift

@@ -0,0 +1,104 @@
+// Copyright Dave Verwer, Sven A. Schmidt, and other contributors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Fluent
+import SQLKit
+
+
+struct UpdateSearchAddMacroProductType: AsyncMigration {
+    let dropSQL: SQLQueryString = "DROP MATERIALIZED VIEW search"
+
+    func prepare(on database: Database) async throws {
+        guard let db = database as? SQLDatabase else {
+            fatalError("Database must be an SQLDatabase ('as? SQLDatabase' must succeed)")
+        }
+
+        // Create an index on targets.version_id - this speeds up search view creation
+        // dramatically.
+        try await db.raw("CREATE INDEX idx_targets_version_id ON targets (version_id)")
+            .run()
+
+        // ** IMPORTANT **
+        // When updating the query underlying the materialized view, make sure to also
+        // update the matching performance test in QueryPerformanceTests.test_Search_refresh!
+        try await db.raw(dropSQL).run()
+        try await db.raw("""
+            -- v11
+            CREATE MATERIALIZED VIEW search AS
+            SELECT
+              p.id AS package_id,
+              p.platform_compatibility,
+              p.score,
+              r.keywords,
+              r.last_commit_date,
+              r.license,
+              r.name AS repo_name,
+              r.owner AS repo_owner,
+              r.stars,
+              r.last_activity_at,
+              r.summary,
+              v.package_name,
+              ARRAY_LENGTH(doc_archives, 1) >= 1 AS has_docs,
+              ARRAY(
+                SELECT DISTINCT JSONB_OBJECT_KEYS(type) FROM products WHERE products.version_id = v.id
+                UNION
+                SELECT * FROM (
+                  SELECT DISTINCT JSONB_OBJECT_KEYS(type) AS "type" FROM targets
+                  WHERE targets.version_id = v.id) AS macro_targets
+                WHERE type = 'macro'
+              ) AS product_types,
+              ARRAY(SELECT DISTINCT name FROM products WHERE products.version_id = v.id) AS product_names,
+              TO_TSVECTOR(CONCAT_WS(' ', COALESCE(v.package_name, ''), r.name, COALESCE(r.summary, ''), ARRAY_TO_STRING(r.keywords, ' '))) AS tsvector
+            FROM packages p
+              JOIN repositories r ON r.package_id = p.id
+              JOIN versions v ON v.package_id = p.id
+            WHERE v.reference ->> 'branch' = r.default_branch
+            """).run()
+    }
+
+    func revert(on database: Database) async throws {
+        guard let db = database as? SQLDatabase else {
+            fatalError("Database must be an SQLDatabase ('as? SQLDatabase' must succeed)")
+        }
+
+        try await db.raw(dropSQL).run()
+        try await db.raw("""
+            -- v10
+            CREATE MATERIALIZED VIEW search AS
+            SELECT
+              p.id AS package_id,
+              p.platform_compatibility,
+              p.score,
+              r.keywords,
+              r.last_commit_date,
+              r.license,
+              r.name AS repo_name,
+              r.owner AS repo_owner,
+              r.stars,
+              r.last_activity_at,
+              r.summary,
+              v.package_name,
+              ARRAY_LENGTH(doc_archives, 1) >= 1 AS has_docs,
+              ARRAY(SELECT DISTINCT JSONB_OBJECT_KEYS(type) FROM products WHERE products.version_id = v.id) AS product_types,
+              ARRAY(SELECT DISTINCT name FROM products WHERE products.version_id = v.id) AS product_names,
+              TO_TSVECTOR(CONCAT_WS(' ', COALESCE(v.package_name, ''), r.name, COALESCE(r.summary, ''), ARRAY_TO_STRING(r.keywords, ' '))) AS tsvector
+            FROM packages p
+              JOIN repositories r ON r.package_id = p.id
+              JOIN versions v ON v.package_id = p.id
+            WHERE v.reference ->> 'branch' = r.default_branch
+            """).run()
+
+        try await db.raw("DROP INDEX idx_targets_version_id").run()
+    }
+}

Sources/App/configure.swift

@@ -287,6 +287,9 @@ public func configure(_ app: Application) throws -> String {
     do { // Migration 065 - add linkable_paths_count to doc_uploads
         app.migrations.add(UpdateDocUploadAddLinkablePathsCount())
     }
+    do { // Migration 066 - add virtual macro product type to search view
+        app.migrations.add(UpdateSearchAddMacroProductType())
+    }
 
     app.commands.use(Analyze.Command(), as: "analyze")
     app.commands.use(CreateRestfileCommand(), as: "create-restfile")

Tests/AppTests/QueryPerformanceTests.swift

@@ -124,7 +124,7 @@ class QueryPerformanceTests: XCTestCase {
             return
         }
         let query = db.raw("""
-            -- v10
+            -- v11
             SELECT
               p.id AS package_id,
               p.platform_compatibility,
@@ -139,15 +139,22 @@ class QueryPerformanceTests: XCTestCase {
               r.summary,
               v.package_name,
               array_length(doc_archives, 1) >= 1 AS has_docs,
-              ARRAY(SELECT DISTINCT JSONB_OBJECT_KEYS(type) FROM products WHERE products.version_id = v.id) AS product_types,
+              ARRAY(
+                SELECT DISTINCT JSONB_OBJECT_KEYS(type) FROM products WHERE products.version_id = v.id
+                UNION
+                SELECT * FROM (
+                  SELECT DISTINCT JSONB_OBJECT_KEYS(type) AS "type" FROM targets
+                  WHERE targets.version_id = v.id) AS macro_targets
+                WHERE type = 'macro'
+              ) AS product_types,
               ARRAY(SELECT DISTINCT name FROM products WHERE products.version_id = v.id) AS product_names,
               TO_TSVECTOR(CONCAT_WS(' ', COALESCE(v.package_name, ''), r.name, COALESCE(r.summary, ''), ARRAY_TO_STRING(r.keywords, ' '))) AS tsvector
             FROM packages p
               JOIN repositories r ON r.package_id = p.id
               JOIN versions v ON v.package_id = p.id
             WHERE v.reference ->> 'branch' = r.default_branch
             """)
-        try await assertQueryPerformance(query, expectedCost: 31_300, variation: 500)
+        try await assertQueryPerformance(query, expectedCost: 55_000, variation: 500)
     }
 
 }

Tests/AppTests/SearchFilterTests.swift

@@ -431,11 +431,29 @@ class SearchFilterTests: AppTestCase {
         XCTAssertEqual(binds(filter.rightHandSide), ["{executable}"])
     }
 
+    func test_productTypeFilter_macro() throws {
+        // Test "virtual" macro product filter
+        let filter = try ProductTypeSearchFilter(expression: .init(operator: .is, value: "macro"))
+        XCTAssertEqual(filter.key, .productType)
+        XCTAssertEqual(filter.predicate, .init(operator: .contains,
+                                               bindableValue: .value("macro"),
+                                               displayValue: "Macro"))
+
+        // test view representation
+        XCTAssertEqual(filter.viewModel.description, "Package products contain a Macro")
+
+        // test sql representation
+        XCTAssertEqual(renderSQL(filter.leftHandSide), #""product_types""#)
+        XCTAssertEqual(renderSQL(filter.sqlOperator), "@>")
+        XCTAssertEqual(binds(filter.rightHandSide), ["{macro}"])
+    }
+
     func test_productTypeFilter_spelling() throws {
         let expectedDisplayValues = [
             ProductTypeSearchFilter.ProductType.executable: "Package products contain an Executable",
             ProductTypeSearchFilter.ProductType.plugin: "Package products contain a Plugin",
-            ProductTypeSearchFilter.ProductType.library: "Package products contain a Library"
+            ProductTypeSearchFilter.ProductType.library: "Package products contain a Library",
+            ProductTypeSearchFilter.ProductType.macro: "Package products contain a Macro"
         ]
 
         for type in ProductTypeSearchFilter.ProductType.allCases {

Tests/AppTests/SearchTests.swift

@@ -1427,6 +1427,58 @@ class SearchTests: AppTestCase {
         }
     }
 
+    func test_productTypeFilter_macro() async throws {
+        // setup
+        do {
+            let p1 = Package.init(id: .id0, url: "1".url)
+            try await p1.save(on: app.db)
+            try await Repository(package: p1,
+                                 defaultBranch: "main",
+                                 name: "1",
+                                 owner: "foo",
+                                 stars: 1,
+                                 summary: "test package").save(on: app.db)
+            let v = try Version(package: p1)
+            try await v.save(on: app.db)
+            try await Target(version: v, name: "t1", type: .regular).save(on: app.db)
+        }
+        do {
+            let p2 = Package.init(id: .id1, url: "2".url)
+            try await p2.save(on: app.db)
+            try await Repository(package: p2,
+                                 defaultBranch: "main",
+                                 name: "2",
+                                 owner: "foo",
+                                 summary: "test package").save(on: app.db)
+            let v = try Version(package: p2)
+            try await v.save(on: app.db)
+            try await Target(version: v, name: "t2", type: .macro).save(on: app.db)
+        }
+        try await Search.refresh(on: app.db).get()
+
+        do {
+            // MUT
+            let res = try await Search.fetch(app.db, ["test", "product:macro"], page: 1, pageSize: 20).get()
+
+            // validate
+            XCTAssertEqual(res.results.count, 1)
+            XCTAssertEqual(
+                res.results.compactMap(\.packageResult?.repositoryName), ["2"]
+            )
+        }
+
+        do {
+            // MUT
+            let res = try await Search.fetch(app.db, ["test"], page: 1, pageSize: 20).get()
+
+            // validate
+            XCTAssertEqual(res.results.count, 2)
+            XCTAssertEqual(
+                res.results.compactMap(\.packageResult?.repositoryName), ["1", "2"]
+            )
+        }
+    }
+
     func test_SearchFilter_error() throws {
         // Test error handling in case of an invalid filter
         // Setup