Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DBP: Support new dataSource property #2267

Merged
merged 15 commits into from
Feb 29, 2024
Merged

DBP: Support new dataSource property #2267

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
f6f36ca
WIP: new datasource type
Bunn Feb 26, 2024
33284b3
Rename userData to userProfile
Bunn Feb 26, 2024
58dc407
Improve formatting
Bunn Feb 26, 2024
7cb38ce
Update actions
Bunn Feb 26, 2024
f4f85b7
Add meta object to CCF response
jotaemepereira Feb 27, 2024
c5474bb
Update BSK to point to C-S-S 5.0
Bunn Feb 28, 2024
6af9d65
Update JSON
Bunn Feb 28, 2024
769d59f
Merge branch 'main' into bunn/dbp/datasource
Bunn Feb 28, 2024
10ed728
Change reference to bsk
Bunn Feb 28, 2024
d8ac97a
Fix tests
Bunn Feb 28, 2024
14f0fa0
Bump advancedbackgroundchecks.com.json
brianhall Feb 28, 2024
ea16eab
Point to new BSK version
Bunn Feb 29, 2024
eec785b
Merge branch 'main' into bunn/dbp/datasource
Bunn Feb 29, 2024
dc0708c
Update BSK to 113.0.0
Bunn Feb 29, 2024
f125094
Fix merge issue
Bunn Feb 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion DuckDuckGo.xcodeproj/project.pbxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13583,7 +13583,7 @@
repositoryURL = "https://github.com/duckduckgo/BrowserServicesKit";
requirement = {
kind = exactVersion;
version = 112.0.1;
version = 113.0.0;
};
};
AA06B6B52672AF8100F541C5 /* XCRemoteSwiftPackageReference "Sparkle" */ = {
Expand Down
8 changes: 4 additions & 4 deletions DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,17 @@
"kind" : "remoteSourceControl",
"location" : "https://github.com/duckduckgo/BrowserServicesKit",
"state" : {
"revision" : "c513f0f0fedc1de65591d46908ddef530b0dc001",
"version" : "112.0.1"
"revision" : "f903ffcbc51e85ac262c355b56726e3387957a80",
"version" : "113.0.0"
}
},
{
"identity" : "content-scope-scripts",
"kind" : "remoteSourceControl",
"location" : "https://github.com/duckduckgo/content-scope-scripts",
"state" : {
"revision" : "36ddba2cbac52a41b9a9275af06d32fa8a56d2d7",
"version" : "4.64.0"
"revision" : "a3690b7666a3617693383d948cb492513f6aa569",
"version" : "5.0.0"
}
},
{
Expand Down
2 changes: 1 addition & 1 deletion LocalPackages/DataBrokerProtection/Package.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ let package = Package(
targets: ["DataBrokerProtection"])
],
dependencies: [
.package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "112.0.1"),
.package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "113.0.0"),
.package(path: "../PixelKit"),
.package(path: "../SwiftUIExtensions"),
.package(path: "../XPCHelper"),
Expand Down
11 changes: 8 additions & 3 deletions LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/Action.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,19 @@ enum ActionType: String, Codable, Sendable {
case solveCaptcha
}

enum DataSource: String, Codable {
case userProfile
case extractedProfile
}

protocol Action: Codable, Sendable {
var id: String { get }
var actionType: ActionType { get }
var needsEmail: Bool { get }
var dataSource: DataSource { get }
}

extension Action {
var needsEmail: Bool {
get { false }
}
var needsEmail: Bool { false }
var dataSource: DataSource { .userProfile }
}
1 change: 1 addition & 0 deletions LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/Actions/Click.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,5 @@ struct ClickAction: Action {
let id: String
let actionType: ActionType
let elements: [PageElement]
let dataSource: DataSource?
}
1 change: 1 addition & 0 deletions ...s/DataBrokerProtection/Sources/DataBrokerProtection/Model/Actions/EmailConfirmation.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,5 @@ struct EmailConfirmationAction: Action {
let id: String
let actionType: ActionType
let pollingTime: TimeInterval
let dataSource: DataSource?
}
1 change: 1 addition & 0 deletions ...ckages/DataBrokerProtection/Sources/DataBrokerProtection/Model/Actions/Expectaction.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,4 +34,5 @@ internal struct ExpectationAction: Action {
let id: String
let actionType: ActionType
let expectations: [Item]
let dataSource: DataSource?
}
1 change: 1 addition & 0 deletions LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/Actions/Extract.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,4 +21,5 @@ struct ExtractAction: Action {
let actionType: ActionType
let selector: String
let profile: ExtractProfileSelectors
let dataSource: DataSource?
}
1 change: 1 addition & 0 deletions LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/Actions/FillForm.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ struct FillFormAction: Action {
let actionType: ActionType
let selector: String
let elements: [PageElement]
let dataSource: DataSource?

var needsEmail: Bool {
elements.contains { $0.type == "email" }
Expand Down
1 change: 1 addition & 0 deletions ...ages/DataBrokerProtection/Sources/DataBrokerProtection/Model/Actions/GetCaptchaInfo.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,5 @@ struct GetCaptchaInfoAction: Action {
let id: String
let actionType: ActionType
let selector: String
let dataSource: DataSource?
}
1 change: 1 addition & 0 deletions LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/Actions/Navigate.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,4 +21,5 @@ struct NavigateAction: Action {
let actionType: ActionType
let url: String
let ageRange: [String]?
let dataSource: DataSource?
}
1 change: 1 addition & 0 deletions ...ckages/DataBrokerProtection/Sources/DataBrokerProtection/Model/Actions/SolveCaptcha.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,5 @@ struct SolveCaptchaAction: Action {
let id: String
let actionType: ActionType
let selector: String
let dataSource: DataSource?
}
18 changes: 12 additions & 6 deletions ...ckages/DataBrokerProtection/Sources/DataBrokerProtection/Model/CCFRequestParameters.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,8 @@
import Foundation

enum CCFRequestData: Encodable {
case profile(ProfileQuery)
case extractedProfile(ExtractedProfile)
case solveCaptcha(CaptchaToken)
case userData(ProfileQuery, ExtractedProfile?)
}

struct CaptchaToken: Encodable, Sendable {
Expand All @@ -33,6 +32,11 @@ struct InitParams: Encodable {
let dataBrokerData: DataBroker
}

private enum UserDataCodingKeys: String, CodingKey {
case userProfile
case extractedProfile
}

struct ActionRequest: Encodable {
let action: Action
let data: CCFRequestData?
Expand All @@ -47,12 +51,14 @@ struct ActionRequest: Encodable {
var container = encoder.container(keyedBy: CodingKeys.self)

switch data {
case .profile(let profileQuery):
try container.encode(profileQuery, forKey: .data)
case .solveCaptcha(let captchaToken):
try container.encode(captchaToken, forKey: .data)
case .extractedProfile(let extractedProfile):
try container.encode(extractedProfile, forKey: .data)
case .userData(let profileQuery, let extractedProfile):
var userDataContainer = container.nestedContainer(keyedBy: UserDataCodingKeys.self, forKey: .data)
try userDataContainer.encode(profileQuery, forKey: .userProfile)
if let extractedProfile = extractedProfile {
try userDataContainer.encode(extractedProfile, forKey: .extractedProfile)
}
default:
assertionFailure("Data not found. Please add the mission data to the encoding list.")
}
Expand Down
3 changes: 3 additions & 0 deletions ...kages/DataBrokerProtection/Sources/DataBrokerProtection/Model/CCFResponseParameters.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,17 +74,20 @@ struct CCFSuccessResponse: Decodable {
let actionID: String
let actionType: ActionType
let response: CCFSuccessData?
let meta: [String: Any]?

enum CodingKeys: CodingKey {
case actionID
case actionType
case response
case meta
}

init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
self.actionID = try container.decode(String.self, forKey: .actionID)
self.actionType = try container.decode(ActionType.self, forKey: .actionType)
self.meta = try container.decodeIfPresent([String: Any].self, forKey: .meta)

switch actionType {
case .navigate:
Expand Down
6 changes: 1 addition & 5 deletions ...es/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,11 +111,7 @@ extension DataBrokerOperation {
stageCalculator?.setStage(.captchaParse)
}

if let extractedProfile = self.extractedProfile {
await webViewHandler?.execute(action: action, data: .extractedProfile(extractedProfile))
} else {
await webViewHandler?.execute(action: action, data: .profile(query.profileQuery))
}
await webViewHandler?.execute(action: action, data: .userData(query.profileQuery, self.extractedProfile))
}

private func runEmailConfirmationAction(action: EmailConfirmationAction) async throws {
Expand Down
6 changes: 4 additions & 2 deletions ...rProtection/Sources/DataBrokerProtection/Resources/JSON/advancedbackgroundchecks.com.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "AdvancedBackgroundChecks",
"url": "advancedbackgroundchecks.com",
"version": "0.1.5",
"version": "0.1.6",
"parent": "peoplefinders.com",
"addedDatetime": 1678060800000,
"steps": [
Expand Down Expand Up @@ -45,7 +45,9 @@
"separator": ","
},
"profileUrl": {
"selector": ".link-to-details"
"selector": ".link-to-details",
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Here, we added the identifier, but we didn’t bump the version on this file. What will happen to users who find this? Or we didn’t release the last change for this?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're right. I think this needs to be bumped to 0.1.6.
@brianhall, wdyt?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This whole versioning thing was inherited and needs to be defined a little better, I'll work on it. 😄 Technically the identifier isn't visible to any client, so it won't matter if they see it, but happy for this to get bumped to 0.1.6, I'll add a commit with all changes based on the feedback in a minute.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@brianhall Let me change the wording. What I meant with “users who find this”, I meant that if a user has this extract JSON broker version on their computers, they will not get this identifier change, because for them it will be the same version.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@brianhall, I believe it matters because we only parse/save the new JSON if there's a version change. The JSON is not always read from the file; we store it serialized in the DB. Therefore, if we don't bump the version here, we won't send the identifier to C-S-S.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, thanks both! That's useful context, understood.

Copy link
Collaborator Author

@Bunn Bunn Feb 28, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll add a commit with all changes based on the feedback in a minute.

Just to confirm, are you doing a commit on this branch or in the json repo for me to fetch the changes here?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oops, sorry. Just bumped a commit here and will reflect back in the broker json repo.

"identifierType": "path",
"identifier": "https://www.advancedbackgroundchecks.com/${id}"
}
}
}
Expand Down
12 changes: 7 additions & 5 deletions ...DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/backgroundcheck.run.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "backgroundcheck.run",
"url": "backgroundcheck.run",
"version": "0.1.4",
"version": "0.1.5",
"parent": "verecor.com",
"addedDatetime": 1677736800000,
"steps": [
Expand All @@ -11,12 +11,12 @@
"actions": [
{
"actionType": "navigate",
"id": "5f90e39f-cb94-4b8d-94ed-48ba0060dc08",
"id": "64967cc5-ebf6-4c27-b12c-8f15dceb2678",
"url": "https://backgroundcheck.run/profile/search?fname=${firstName}&lname=${lastName}&state=${state}&city=${city}"
},
{
"actionType": "extract",
"id": "3225fa15-4e00-4e6a-bfc7-a85dfb504c86",
"id": "57dfbc89-94c8-49f0-9952-c58c92883ffd",
"selector": ".b-pfl-list",
"profile": {
"name": {
Expand All @@ -39,7 +39,9 @@
"findElements": true
},
"profileUrl": {
"selector": "a"
"selector": "a",
"identifierType": "path",
"identifier": "https://backgroundcheck.run/person/${id}"
}
}
}
Expand All @@ -56,4 +58,4 @@
"confirmOptOutScan": 72,
"maintenanceScan": 240
}
}
}
12 changes: 7 additions & 5 deletions ...ackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/centeda.com.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
{
"name": "Centeda",
"url": "centeda.com",
"version": "0.1.4",
"version": "0.1.5",
"parent": "verecor.com",
"addedDatetime": 1677715200000,
"addedDatetime": 1677736800000,
"steps": [
{
"stepType": "scan",
"scanType": "templatedUrl",
"actions": [
{
"actionType": "navigate",
"id": "af9c9f03-e778-4c29-85fc-e5cbbfec563c",
"id": "ce7caa09-bcf4-4bf9-bc7a-024c2c50bd1f",
"url": "https://centeda.com/profile/search?fname=${firstName}&lname=${lastName}&state=${state}&city=${city}&fage=${age|ageRange}",
"ageRange": [
"18-30",
Expand All @@ -25,7 +25,7 @@
},
{
"actionType": "extract",
"id": "79fa2a1c-65b4-417a-a8ac-2ca6d729ffc1",
"id": "e779b827-702d-402d-ae79-f3681ad50008",
"selector": ".search-result > a",
"profile": {
"name": {
Expand All @@ -47,7 +47,9 @@
"selector": ".//div[@class='col-sm-24 col-md-8 related-to']//li"
},
"profileUrl": {
"selector": "a"
"selector": "a",
"identifierType": "param",
"identifier": "pid"
}
}
}
Expand Down
12 changes: 7 additions & 5 deletions ...ackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/clubset.com.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "Clubset",
"url": "clubset.com",
"version": "0.1.4",
"version": "0.1.5",
"parent": "verecor.com",
"addedDatetime": 1702965600000,
"steps": [
Expand All @@ -11,7 +11,7 @@
"actions": [
{
"actionType": "navigate",
"id": "5c559c67-c13c-4055-a318-6ba35d62a2cf",
"id": "9b8c7984-a54a-40d1-8e2e-56cd1910ec61",
"url": "https://clubset.com/profile/search?fname=${firstName}&lname=${lastName}&state=${state|upcase}&city=${city|capitalize}&fage=${age|ageRange}",
"ageRange": [
"18-30",
Expand All @@ -25,7 +25,7 @@
},
{
"actionType": "extract",
"id": "866bdfc5-069e-4734-9ce0-a19976fa796b",
"id": "dee8a12e-b8bd-4a37-b77d-8330d8eda4cb",
"selector": ".card",
"profile": {
"name": {
Expand All @@ -51,7 +51,9 @@
"findElements": true
},
"profileUrl": {
"selector": "a"
"selector": "a",
"identifierType": "path",
"identifier": "https://clubset.com/profile/${id}"
}
}
}
Expand All @@ -68,4 +70,4 @@
"confirmOptOutScan": 72,
"maintenanceScan": 240
}
}
}
12 changes: 7 additions & 5 deletions ...ages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/clustrmaps.com.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "ClustrMaps",
"url": "clustrmaps.com",
"version": "0.1.4",
"version": "0.1.5",
"parent": "neighbor.report",
"addedDatetime": 1692594000000,
"steps": [
Expand All @@ -11,12 +11,12 @@
"actions": [
{
"actionType": "navigate",
"id": "e6929e37-4764-450a-be2a-73479f11842a",
"id": "c4d4ce8d-0b33-46cb-8e73-a544b362460d",
"url": "https://clustrmaps.com/persons/${firstName}-${lastName}/${state|stateFull|capitalize}/${city|hyphenated}"
},
{
"actionType": "extract",
"id": "06f39df7-89c2-40da-b288-cdf3ed0e4bfd",
"id": "f0e4f08c-c999-451c-b195-93f481c0a4de",
"selector": ".//div[@itemprop='Person']",
"profile": {
"name": {
Expand All @@ -39,7 +39,9 @@
"afterText": "Associated persons:"
},
"profileUrl": {
"selector": ".persons"
"selector": ".persons",
"identifierType": "path",
"identifier": "https://clustrmaps.com/person/${id}"
}
}
}
Expand All @@ -56,4 +58,4 @@
"confirmOptOutScan": 72,
"maintenanceScan": 240
}
}
}
Loading
Loading