DBP: Support new dataSource property (#2267)

Task/Issue URL: https://app.asana.com/0/1204167627774280/1206602738361283/f **Description**: Add new datasource property support
duckduckgo · Feb 29, 2024 · abe67a7 · abe67a7
1 parent 83eae5d
commit abe67a7
Show file tree

Hide file tree

Showing 59 changed files with 395 additions and 255 deletions.
diff --git a/DuckDuckGo.xcodeproj/project.pbxproj b/DuckDuckGo.xcodeproj/project.pbxproj
@@ -13583,7 +13583,7 @@
 			repositoryURL = "https://github.com/duckduckgo/BrowserServicesKit";
 			requirement = {
 				kind = exactVersion;
-				version = 112.0.1;
+				version = 113.0.0;
 			};
 		};
 		AA06B6B52672AF8100F541C5 /* XCRemoteSwiftPackageReference "Sparkle" */ = {

diff --git a/DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
@@ -23,17 +23,17 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/duckduckgo/BrowserServicesKit",
       "state" : {
-        "revision" : "c513f0f0fedc1de65591d46908ddef530b0dc001",
-        "version" : "112.0.1"
+        "revision" : "f903ffcbc51e85ac262c355b56726e3387957a80",
+        "version" : "113.0.0"
       }
     },
     {
       "identity" : "content-scope-scripts",
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/duckduckgo/content-scope-scripts",
       "state" : {
-        "revision" : "36ddba2cbac52a41b9a9275af06d32fa8a56d2d7",
-        "version" : "4.64.0"
+        "revision" : "a3690b7666a3617693383d948cb492513f6aa569",
+        "version" : "5.0.0"
       }
     },
     {

diff --git a/LocalPackages/DataBrokerProtection/Package.swift b/LocalPackages/DataBrokerProtection/Package.swift
@@ -29,7 +29,7 @@ let package = Package(
             targets: ["DataBrokerProtection"])
     ],
     dependencies: [
-        .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "112.0.1"),
+        .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "113.0.0"),
         .package(path: "../PixelKit"),
         .package(path: "../SwiftUIExtensions"),
         .package(path: "../XPCHelper"),

diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/Action.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/Action.swift
@@ -29,14 +29,19 @@ enum ActionType: String, Codable, Sendable {
     case solveCaptcha
 }
 
+enum DataSource: String, Codable {
+    case userProfile
+    case extractedProfile
+}
+
 protocol Action: Codable, Sendable {
     var id: String { get }
     var actionType: ActionType { get }
     var needsEmail: Bool { get }
+    var dataSource: DataSource { get }
 }
 
 extension Action {
-    var needsEmail: Bool {
-        get { false }
-    }
+    var needsEmail: Bool { false }
+    var dataSource: DataSource { .userProfile }
 }
diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/Actions/Click.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/Actions/Click.swift
@@ -22,4 +22,5 @@ struct ClickAction: Action {
     let id: String
     let actionType: ActionType
     let elements: [PageElement]
+    let dataSource: DataSource?
 }
diff --git a/...s/DataBrokerProtection/Sources/DataBrokerProtection/Model/Actions/EmailConfirmation.swift b/...s/DataBrokerProtection/Sources/DataBrokerProtection/Model/Actions/EmailConfirmation.swift
@@ -22,4 +22,5 @@ struct EmailConfirmationAction: Action {
     let id: String
     let actionType: ActionType
     let pollingTime: TimeInterval
+    let dataSource: DataSource?
 }
diff --git a/...ckages/DataBrokerProtection/Sources/DataBrokerProtection/Model/Actions/Expectaction.swift b/...ckages/DataBrokerProtection/Sources/DataBrokerProtection/Model/Actions/Expectaction.swift
@@ -34,4 +34,5 @@ internal struct ExpectationAction: Action {
     let id: String
     let actionType: ActionType
     let expectations: [Item]
+    let dataSource: DataSource?
 }
diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/Actions/Extract.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/Actions/Extract.swift
@@ -21,4 +21,5 @@ struct ExtractAction: Action {
     let actionType: ActionType
     let selector: String
     let profile: ExtractProfileSelectors
+    let dataSource: DataSource?
 }
diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/Actions/FillForm.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/Actions/FillForm.swift
@@ -23,6 +23,7 @@ struct FillFormAction: Action {
     let actionType: ActionType
     let selector: String
     let elements: [PageElement]
+    let dataSource: DataSource?
 
     var needsEmail: Bool {
         elements.contains { $0.type == "email" }

diff --git a/...ages/DataBrokerProtection/Sources/DataBrokerProtection/Model/Actions/GetCaptchaInfo.swift b/...ages/DataBrokerProtection/Sources/DataBrokerProtection/Model/Actions/GetCaptchaInfo.swift
@@ -22,4 +22,5 @@ struct GetCaptchaInfoAction: Action {
     let id: String
     let actionType: ActionType
     let selector: String
+    let dataSource: DataSource?
 }
diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/Actions/Navigate.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/Actions/Navigate.swift
@@ -21,4 +21,5 @@ struct NavigateAction: Action {
     let actionType: ActionType
     let url: String
     let ageRange: [String]?
+    let dataSource: DataSource?
 }
diff --git a/...ckages/DataBrokerProtection/Sources/DataBrokerProtection/Model/Actions/SolveCaptcha.swift b/...ckages/DataBrokerProtection/Sources/DataBrokerProtection/Model/Actions/SolveCaptcha.swift
@@ -22,4 +22,5 @@ struct SolveCaptchaAction: Action {
     let id: String
     let actionType: ActionType
     let selector: String
+    let dataSource: DataSource?
 }
diff --git a/...ckages/DataBrokerProtection/Sources/DataBrokerProtection/Model/CCFRequestParameters.swift b/...ckages/DataBrokerProtection/Sources/DataBrokerProtection/Model/CCFRequestParameters.swift
@@ -19,9 +19,8 @@
 import Foundation
 
 enum CCFRequestData: Encodable {
-    case profile(ProfileQuery)
-    case extractedProfile(ExtractedProfile)
     case solveCaptcha(CaptchaToken)
+    case userData(ProfileQuery, ExtractedProfile?)
 }
 
 struct CaptchaToken: Encodable, Sendable {
@@ -33,6 +32,11 @@ struct InitParams: Encodable {
     let dataBrokerData: DataBroker
 }
 
+private enum UserDataCodingKeys: String, CodingKey {
+    case userProfile
+    case extractedProfile
+}
+
 struct ActionRequest: Encodable {
     let action: Action
     let data: CCFRequestData?
@@ -47,12 +51,14 @@ struct ActionRequest: Encodable {
         var container = encoder.container(keyedBy: CodingKeys.self)
 
         switch data {
-        case .profile(let profileQuery):
-            try container.encode(profileQuery, forKey: .data)
         case .solveCaptcha(let captchaToken):
             try container.encode(captchaToken, forKey: .data)
-        case .extractedProfile(let extractedProfile):
-            try container.encode(extractedProfile, forKey: .data)
+        case .userData(let profileQuery, let extractedProfile):
+            var userDataContainer = container.nestedContainer(keyedBy: UserDataCodingKeys.self, forKey: .data)
+            try userDataContainer.encode(profileQuery, forKey: .userProfile)
+            if let extractedProfile = extractedProfile {
+                try userDataContainer.encode(extractedProfile, forKey: .extractedProfile)
+            }
         default:
             assertionFailure("Data not found. Please add the mission data to the encoding list.")
         }

diff --git a/...kages/DataBrokerProtection/Sources/DataBrokerProtection/Model/CCFResponseParameters.swift b/...kages/DataBrokerProtection/Sources/DataBrokerProtection/Model/CCFResponseParameters.swift
@@ -74,17 +74,20 @@ struct CCFSuccessResponse: Decodable {
     let actionID: String
     let actionType: ActionType
     let response: CCFSuccessData?
+    let meta: [String: Any]?
 
     enum CodingKeys: CodingKey {
         case actionID
         case actionType
         case response
+        case meta
     }
 
     init(from decoder: Decoder) throws {
         let container = try decoder.container(keyedBy: CodingKeys.self)
         self.actionID = try container.decode(String.self, forKey: .actionID)
         self.actionType = try container.decode(ActionType.self, forKey: .actionType)
+        self.meta = try container.decodeIfPresent([String: Any].self, forKey: .meta)
 
         switch actionType {
         case .navigate:

diff --git a/...es/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift b/...es/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift
@@ -111,11 +111,7 @@ extension DataBrokerOperation {
             stageCalculator?.setStage(.captchaParse)
         }
 
-        if let extractedProfile = self.extractedProfile {
-            await webViewHandler?.execute(action: action, data: .extractedProfile(extractedProfile))
-        } else {
-            await webViewHandler?.execute(action: action, data: .profile(query.profileQuery))
-        }
+        await webViewHandler?.execute(action: action, data: .userData(query.profileQuery, self.extractedProfile))
     }
 
     private func runEmailConfirmationAction(action: EmailConfirmationAction) async throws {

diff --git a/...rProtection/Sources/DataBrokerProtection/Resources/JSON/advancedbackgroundchecks.com.json b/...rProtection/Sources/DataBrokerProtection/Resources/JSON/advancedbackgroundchecks.com.json
@@ -1,7 +1,7 @@
 {
   "name": "AdvancedBackgroundChecks",
   "url": "advancedbackgroundchecks.com",
-  "version": "0.1.5",
+  "version": "0.1.6",
   "parent": "peoplefinders.com",
   "addedDatetime": 1678060800000,
   "steps": [
@@ -45,7 +45,9 @@
               "separator": ","
             },
             "profileUrl": {
-              "selector": ".link-to-details"
+              "selector": ".link-to-details",
+              "identifierType": "path",
+              "identifier": "https://www.advancedbackgroundchecks.com/${id}"
             }
           }
         }

diff --git a/...DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/backgroundcheck.run.json b/...DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/backgroundcheck.run.json
@@ -1,7 +1,7 @@
 {
   "name": "backgroundcheck.run",
   "url": "backgroundcheck.run",
-  "version": "0.1.4",
+  "version": "0.1.5",
   "parent": "verecor.com",
   "addedDatetime": 1677736800000,
   "steps": [
@@ -11,12 +11,12 @@
       "actions": [
         {
           "actionType": "navigate",
-          "id": "5f90e39f-cb94-4b8d-94ed-48ba0060dc08",
+          "id": "64967cc5-ebf6-4c27-b12c-8f15dceb2678",
           "url": "https://backgroundcheck.run/profile/search?fname=${firstName}&lname=${lastName}&state=${state}&city=${city}"
         },
         {
           "actionType": "extract",
-          "id": "3225fa15-4e00-4e6a-bfc7-a85dfb504c86",
+          "id": "57dfbc89-94c8-49f0-9952-c58c92883ffd",
           "selector": ".b-pfl-list",
           "profile": {
             "name": {
@@ -39,7 +39,9 @@
               "findElements": true
             },
             "profileUrl": {
-              "selector": "a"
+              "selector": "a",
+              "identifierType": "path",
+              "identifier": "https://backgroundcheck.run/person/${id}"
             }
           }
         }
@@ -56,4 +58,4 @@
     "confirmOptOutScan": 72,
     "maintenanceScan": 240
   }
-}
+}
diff --git a/...ackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/centeda.com.json b/...ackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/centeda.com.json
@@ -1,17 +1,17 @@
 {
   "name": "Centeda",
   "url": "centeda.com",
-  "version": "0.1.4",
+  "version": "0.1.5",
   "parent": "verecor.com",
-  "addedDatetime": 1677715200000,
+  "addedDatetime": 1677736800000,
   "steps": [
     {
       "stepType": "scan",
       "scanType": "templatedUrl",
       "actions": [
         {
           "actionType": "navigate",
-          "id": "af9c9f03-e778-4c29-85fc-e5cbbfec563c",
+          "id": "ce7caa09-bcf4-4bf9-bc7a-024c2c50bd1f",
           "url": "https://centeda.com/profile/search?fname=${firstName}&lname=${lastName}&state=${state}&city=${city}&fage=${age|ageRange}",
           "ageRange": [
             "18-30",
@@ -25,7 +25,7 @@
         },
         {
           "actionType": "extract",
-          "id": "79fa2a1c-65b4-417a-a8ac-2ca6d729ffc1",
+          "id": "e779b827-702d-402d-ae79-f3681ad50008",
           "selector": ".search-result > a",
           "profile": {
             "name": {
@@ -47,7 +47,9 @@
               "selector": ".//div[@class='col-sm-24 col-md-8 related-to']//li"
             },
             "profileUrl": {
-              "selector": "a"
+              "selector": "a",
+              "identifierType": "param",
+              "identifier": "pid"
             }
           }
         }

diff --git a/...ackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/clubset.com.json b/...ackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/clubset.com.json
@@ -1,7 +1,7 @@
 {
   "name": "Clubset",
   "url": "clubset.com",
-  "version": "0.1.4",
+  "version": "0.1.5",
   "parent": "verecor.com",
   "addedDatetime": 1702965600000,
   "steps": [
@@ -11,7 +11,7 @@
       "actions": [
         {
           "actionType": "navigate",
-          "id": "5c559c67-c13c-4055-a318-6ba35d62a2cf",
+          "id": "9b8c7984-a54a-40d1-8e2e-56cd1910ec61",
           "url": "https://clubset.com/profile/search?fname=${firstName}&lname=${lastName}&state=${state|upcase}&city=${city|capitalize}&fage=${age|ageRange}",
           "ageRange": [
             "18-30",
@@ -25,7 +25,7 @@
         },
         {
           "actionType": "extract",
-          "id": "866bdfc5-069e-4734-9ce0-a19976fa796b",
+          "id": "dee8a12e-b8bd-4a37-b77d-8330d8eda4cb",
           "selector": ".card",
           "profile": {
             "name": {
@@ -51,7 +51,9 @@
               "findElements": true
             },
             "profileUrl": {
-              "selector": "a"
+              "selector": "a",
+              "identifierType": "path",
+              "identifier": "https://clubset.com/profile/${id}"
             }
           }
         }
@@ -68,4 +70,4 @@
     "confirmOptOutScan": 72,
     "maintenanceScan": 240
   }
-}
+}
diff --git a/...ages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/clustrmaps.com.json b/...ages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/clustrmaps.com.json
@@ -1,7 +1,7 @@
 {
   "name": "ClustrMaps",
   "url": "clustrmaps.com",
-  "version": "0.1.4",
+  "version": "0.1.5",
   "parent": "neighbor.report",
   "addedDatetime": 1692594000000,
   "steps": [
@@ -11,12 +11,12 @@
       "actions": [
         {
           "actionType": "navigate",
-          "id": "e6929e37-4764-450a-be2a-73479f11842a",
+          "id": "c4d4ce8d-0b33-46cb-8e73-a544b362460d",
           "url": "https://clustrmaps.com/persons/${firstName}-${lastName}/${state|stateFull|capitalize}/${city|hyphenated}"
         },
         {
           "actionType": "extract",
-          "id": "06f39df7-89c2-40da-b288-cdf3ed0e4bfd",
+          "id": "f0e4f08c-c999-451c-b195-93f481c0a4de",
           "selector": ".//div[@itemprop='Person']",
           "profile": {
             "name": {
@@ -39,7 +39,9 @@
               "afterText": "Associated persons:"
             },
             "profileUrl": {
-              "selector": ".persons"
+              "selector": ".persons",
+              "identifierType": "path",
+              "identifier": "https://clustrmaps.com/person/${id}"
             }
           }
         }
@@ -56,4 +58,4 @@
     "confirmOptOutScan": 72,
     "maintenanceScan": 240
   }
-}
+}