diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/CCF/DataBrokerProtectionFeature.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/CCF/DataBrokerProtectionFeature.swift index c596488888..be97096c4e 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/CCF/DataBrokerProtectionFeature.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/CCF/DataBrokerProtectionFeature.swift @@ -24,7 +24,7 @@ import Common protocol CCFCommunicationDelegate: AnyObject { func loadURL(url: URL) async - func extractedProfiles(profiles: [ExtractedProfile]) async + func extractedProfiles(profiles: [ExtractedProfile], meta: [String: Any]?) async func captchaInformation(captchaInfo: GetCaptchaInfoResponse) async func solveCaptcha(with response: SolveCaptchaResponse) async func success(actionId: String, actionType: ActionType) async @@ -101,7 +101,7 @@ struct DataBrokerProtectionFeature: Subfeature { await delegate?.onError(error: DataBrokerProtectionError.malformedURL) } case .extract(let profiles): - await delegate?.extractedProfiles(profiles: profiles) + await delegate?.extractedProfiles(profiles: profiles, meta: success.meta) case .getCaptchaInfo(let captchaInfo): await delegate?.captchaInformation(captchaInfo: captchaInfo) case .solveCaptcha(let response): diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/CCF/WebViewHandler.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/CCF/WebViewHandler.swift index c89644eec4..ab98fa98b9 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/CCF/WebViewHandler.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/CCF/WebViewHandler.swift @@ -25,6 +25,8 @@ import Common protocol WebViewHandler: NSObject { func initializeWebView(showWebView: Bool) async func load(url: URL) async throws + func takeSnaphost(path: String, fileName: String) async throws + func saveHTML(path: String, fileName: String) async throws func waitForWebViewLoad(timeoutInSeconds: Int) async throws func finish() async func execute(action: Action, data: CCFRequestData) async @@ -122,6 +124,75 @@ final class DataBrokerProtectionWebViewHandler: NSObject, WebViewHandler { func evaluateJavaScript(_ javaScript: String) async throws { _ = webView?.evaluateJavaScript(javaScript, in: nil, in: WKContentWorld.page) } + + func takeSnaphost(path: String, fileName: String) async throws { + let script = "document.body.scrollHeight" + + let result = try await webView?.evaluateJavaScript(script) + + if let height = result as? CGFloat { + webView?.frame = CGRect(origin: .zero, size: CGSize(width: 1024, height: height)) + let configuration = WKSnapshotConfiguration() + configuration.rect = CGRect(x: 0, y: 0, width: webView?.frame.size.width ?? 0.0, height: height) + if let image = try await webView?.takeSnapshot(configuration: configuration) { + saveToDisk(image: image, path: path, fileName: fileName) + } + } + } + + func saveHTML(path: String, fileName: String) async throws { + let result = try await webView?.evaluateJavaScript("document.documentElement.outerHTML") + let fileManager = FileManager.default + + if let htmlString = result as? String { + do { + if !fileManager.fileExists(atPath: path) { + try fileManager.createDirectory(atPath: path, withIntermediateDirectories: true, attributes: nil) + } + + let fileURL = URL(fileURLWithPath: "\(path)/\(fileName)") + try htmlString.write(to: fileURL, atomically: true, encoding: .utf8) + print("HTML content saved to file: \(fileURL)") + } catch { + print("Error writing HTML content to file: \(error)") + } + } + } + + private func saveToDisk(image: NSImage, path: String, fileName: String) { + guard let tiffData = image.tiffRepresentation else { + // Handle the case where tiff representation is not available + return + } + + // Create a bitmap representation from the tiff data + guard let bitmapImageRep = NSBitmapImageRep(data: tiffData) else { + // Handle the case where bitmap representation cannot be created + return + } + + let fileManager = FileManager.default + + if !fileManager.fileExists(atPath: path) { + do { + try fileManager.createDirectory(atPath: path, withIntermediateDirectories: true, attributes: nil) + } catch { + print("Error creating folder: \(error)") + } + } + + if let pngData = bitmapImageRep.representation(using: .png, properties: [:]) { + // Save the PNG data to a file + do { + let fileURL = URL(fileURLWithPath: "\(path)/\(fileName)") + try pngData.write(to: fileURL) + } catch { + print("Error writing PNG: \(error)") + } + } else { + print("Error png data was not respresented") + } + } } extension DataBrokerProtectionWebViewHandler: WKNavigationDelegate { diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/DebugUI/DataBrokerRunCustomJSONView.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/DebugUI/DataBrokerRunCustomJSONView.swift index 33671421d5..22b8c6b90a 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/DebugUI/DataBrokerRunCustomJSONView.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/DebugUI/DataBrokerRunCustomJSONView.swift @@ -28,26 +28,42 @@ struct DataBrokerRunCustomJSONView: View { if viewModel.results.isEmpty { VStack(alignment: .leading) { Text("macOS App version: \(viewModel.appVersion())") - Text("C-S-S version: \(viewModel.contentScopeScriptsVersion())") Divider() - HStack { - TextField("First name", text: $viewModel.firstName) - .padding() - TextField("Last name", text: $viewModel.lastName) - .padding() - TextField("Middle", text: $viewModel.middle) - .padding() + ForEach(viewModel.names.indices, id: \.self) { index in + HStack { + TextField("First name", text: $viewModel.names[index].first) + .padding() + TextField("Middle", text: $viewModel.names[index].middle) + .padding() + TextField("Last name", text: $viewModel.names[index].last) + .padding() + } + } + + Button("Add other name") { + viewModel.names.append(.empty()) } Divider() - HStack { - TextField("City", text: $viewModel.city) - .padding() - TextField("State", text: $viewModel.state) - .padding() + ForEach(viewModel.addresses.indices, id: \.self) { index in + HStack { + TextField("City", text: $viewModel.addresses[index].city) + .padding() + TextField("State (two characters format)", text: $viewModel.addresses[index].state) + .onChange(of: viewModel.addresses[index].state) { newValue in + if newValue.count > 2 { + viewModel.addresses[index].state = String(newValue.prefix(2)) + } + } + .padding() + } + } + + Button("Add other address") { + viewModel.addresses.append(.empty()) } Divider() @@ -76,6 +92,14 @@ struct DataBrokerRunCustomJSONView: View { Button("Run") { viewModel.runJSON(jsonString: jsonText) } + + if viewModel.isRunningOnAllBrokers { + ProgressView("Scanning...") + } else { + Button("Run all brokers") { + viewModel.runAllBrokers() + } + } } .padding() .frame(minWidth: 600, minHeight: 800) @@ -88,19 +112,19 @@ struct DataBrokerRunCustomJSONView: View { } else { VStack { VStack { - List(viewModel.results, id: \.name) { extractedProfile in + List(viewModel.results, id: \.id) { scanResult in HStack { - Text(extractedProfile.name ?? "No name") + Text(scanResult.extractedProfile.name ?? "No name") .padding(.horizontal, 10) Divider() - Text(extractedProfile.addresses?.first?.fullAddress ?? "No address") + Text(scanResult.extractedProfile.addresses?.first?.fullAddress ?? "No address") .padding(.horizontal, 10) Divider() - Text(extractedProfile.relatives?.joined(separator: ",") ?? "No relatives") + Text(scanResult.extractedProfile.relatives?.joined(separator: ",") ?? "No relatives") .padding(.horizontal, 10) Divider() Button("Opt-out") { - viewModel.runOptOut(extractedProfile: extractedProfile) + viewModel.runOptOut(scanResult: scanResult) } } }.navigationTitle("Results") diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/DebugUI/DataBrokerRunCustomJSONViewModel.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/DebugUI/DataBrokerRunCustomJSONViewModel.swift index e0241d5542..0cb75a8f1d 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/DebugUI/DataBrokerRunCustomJSONViewModel.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/DebugUI/DataBrokerRunCustomJSONViewModel.swift @@ -22,6 +22,49 @@ import Common import ContentScopeScripts import Combine +struct ExtractedAddress: Codable { + let state: String + let city: String +} + +struct UserData: Codable { + let firstName: String + let lastName: String + let middleName: String? + let state: String + let email: String? + let city: String + let age: Int + let addresses: [ExtractedAddress] +} + +struct ProfileUrl: Codable { + let profileUrl: String + let identifier: String +} + +struct ScrapedData: Codable { + let name: String? + let alternativeNamesList: [String]? + let age: String? + let addressCityState: String? + let addressCityStateList: [ExtractedAddress]? + let relativesList: [String]? + let profileUrl: ProfileUrl? +} + +struct ExtractResult: Codable { + let scrapedData: ScrapedData + let result: Bool + let score: Int + let matchedFields: [String] +} + +struct Metadata: Codable { + let userData: UserData + let extractResults: [ExtractResult] +} + struct AlertUI { var title: String = "" var description: String = "" @@ -30,28 +73,80 @@ struct AlertUI { AlertUI(title: "No results", description: "No results were found.") } + static func finishedScanningAllBrokers() -> AlertUI { + AlertUI(title: "Finished!", description: "We finished scanning all brokers. You should find the data inside ~/Desktop/PIR-Debug/") + } + static func from(error: DataBrokerProtectionError) -> AlertUI { AlertUI(title: error.title, description: error.description) } } -final class DataBrokerRunCustomJSONViewModel: ObservableObject { +final class NameUI: ObservableObject { + let id = UUID() + @Published var first: String + @Published var middle: String + @Published var last: String + + init(first: String, middle: String = "", last: String) { + self.first = first + self.middle = middle + self.last = last + } - @Published var firstName: String = "" - @Published var lastName: String = "" - @Published var middle: String = "" - @Published var city: String = "" - @Published var state: String = "" + static func empty() -> NameUI { + .init(first: "", middle: "", last: "") + } + + func toModel() -> DataBrokerProtectionProfile.Name { + .init(firstName: first, lastName: last, middleName: middle.isEmpty ? nil : middle) + } +} + +final class AddressUI: ObservableObject { + let id = UUID() + @Published var city: String + @Published var state: String + + init(city: String, state: String) { + self.city = city + self.state = state + } + + static func empty() -> AddressUI { + .init(city: "", state: "") + } + + func toModel() -> DataBrokerProtectionProfile.Address { + .init(city: city, state: state) + } +} + +struct ScanResult { + let id = UUID() + let dataBroker: DataBroker + let profileQuery: ProfileQuery + let extractedProfile: ExtractedProfile +} + +final class DataBrokerRunCustomJSONViewModel: ObservableObject { @Published var birthYear: String = "" - @Published var results = [ExtractedProfile]() + @Published var results = [ScanResult]() @Published var showAlert = false @Published var showNoResults = false + @Published var isRunningOnAllBrokers = false + @Published var names = [NameUI.empty()] + @Published var addresses = [AddressUI.empty()] + var alert: AlertUI? var selectedDataBroker: DataBroker? let brokers: [DataBroker] private let runnerProvider: OperationRunnerProvider + private let privacyConfigManager: PrivacyConfigurationManaging + private let contentScopeProperties: ContentScopeProperties + private let csvColumns = ["name_input", "age_input", "city_input", "state_input", "name_scraped", "age_scraped", "address_scraped", "relatives_scraped", "url", "broker name", "screenshot_id", "error", "matched_fields", "result_match", "expected_match"] init() { let privacyConfigurationManager = PrivacyConfigurationManagingMock() @@ -69,45 +164,202 @@ final class DataBrokerRunCustomJSONViewModel: ObservableObject { let contentScopeProperties = ContentScopeProperties(gpcEnabled: false, sessionKey: sessionKey, featureToggles: features) + self.runnerProvider = DataBrokerOperationRunnerProvider( privacyConfigManager: privacyConfigurationManager, contentScopeProperties: contentScopeProperties, emailService: EmailService(), captchaService: CaptchaService()) + self.privacyConfigManager = privacyConfigurationManager + self.contentScopeProperties = contentScopeProperties let fileResources = FileResources() self.brokers = fileResources.fetchBrokerFromResourceFiles() ?? [DataBroker]() } - func runJSON(jsonString: String) { - if firstName.isEmpty || lastName.isEmpty || city.isEmpty || state.isEmpty || birthYear.isEmpty { + func runAllBrokers() { + isRunningOnAllBrokers = true + + let brokerProfileQueryData = createBrokerProfileQueryData() + + Task.detached { + var scanResults = [DebugScanReturnValue]() + let semaphore = DispatchSemaphore(value: 10) + try await withThrowingTaskGroup(of: DebugScanReturnValue.self) { group in + for queryData in brokerProfileQueryData { + semaphore.wait() + let debugScanOperation = DebugScanOperation(privacyConfig: self.privacyConfigManager, prefs: self.contentScopeProperties, query: queryData) { + true + } + + group.addTask { + defer { + semaphore.signal() + } + do { + return try await debugScanOperation.run(inputValue: (), stageCalculator: FakeStageDurationCalculator(), showWebView: false) + } catch { + return DebugScanReturnValue(brokerURL: "ERROR - with broker: \(queryData.dataBroker.name)", extractedProfiles: [ExtractedProfile](), brokerProfileQueryData: queryData) + } + } + } + + for try await result in group { + scanResults.append(result) + } + + self.formCSV(with: scanResults) + + self.finishLoading() + } + } + } + + private func finishLoading() { + DispatchQueue.main.async { + self.alert = AlertUI.finishedScanningAllBrokers() self.showAlert = true - self.alert = AlertUI(title: "Error", description: "Some required fields were not entered.") - return + self.isRunningOnAllBrokers = false + } + } + + private func formCSV(with scanResults: [DebugScanReturnValue]) { + var csvText = csvColumns.map { $0 }.joined(separator: ",") + csvText.append("\n") + + for result in scanResults { + if let error = result.error { + csvText.append(append(error: error, for: result)) + } else { + csvText.append(append(result)) + } + } + + save(csv: csvText) + } + + private func append(error: Error, for result: DebugScanReturnValue) -> String { + if let dbpError = error as? DataBrokerProtectionError { + if dbpError.is404 { + return createRowFor(matched: false, result: result, error: "404 - No results") + } else { + return createRowFor(matched: false, result: result, error: "\(dbpError.title)-\(dbpError.description)") + } + } else { + return createRowFor(matched: false, result: result, error: error.localizedDescription) + } + } + + private func append(_ result: DebugScanReturnValue) -> String { + var resultsText = "" + + if let meta = result.meta{ + do { + let jsonData = try JSONSerialization.data(withJSONObject: meta, options: []) + let decoder = JSONDecoder() + let decodedMeta = try decoder.decode(Metadata.self, from: jsonData) + + for extractedResult in decodedMeta.extractResults { + resultsText.append(createRowFor(matched: extractedResult.result, result: result, extractedResult: extractedResult)) + } + } catch { + print("Error decoding JSON: \(error)") + } + } else { + print("No meta object") } + return resultsText + } + + private func createRowFor(matched: Bool, + result: DebugScanReturnValue, + error: String? = nil, + extractedResult: ExtractResult? = nil) -> String { + let matchedString = matched ? "TRUE" : "FALSE" + let profileQuery = result.brokerProfileQueryData.profileQuery + + var csvRow = "" + + csvRow.append("\(profileQuery.fullName),") // Name (input) + csvRow.append("\(profileQuery.age),") // Age (input) + csvRow.append("\(profileQuery.city),") // City (input) + csvRow.append("\(profileQuery.state),") // State (input) + + if let extractedResult = extractedResult { + csvRow.append("\(extractedResult.scrapedData.nameCSV),") // Name (scraped) + csvRow.append("\(extractedResult.scrapedData.ageCSV),") // Age (scraped) + csvRow.append("\(extractedResult.scrapedData.addressesCSV),") // Address (scraped) + csvRow.append("\(extractedResult.scrapedData.relativesCSV),") // Relatives (matched) + } else { + csvRow.append(",") // Name (scraped) + csvRow.append(",") // Age (scraped) + csvRow.append(",") // Address (scraped) + csvRow.append(",") // Relatives (scraped) + } + + csvRow.append("\(result.brokerURL),") // Broker URL + csvRow.append("\(result.brokerProfileQueryData.dataBroker.name),") // Broker Name + csvRow.append("\(profileQuery.id ?? 0)_\(result.brokerProfileQueryData.dataBroker.name),") // Screenshot name + + if let error = error { + csvRow.append("\(error),") // Error + } else { + csvRow.append(",") // Error empty + } + + if let extractedResult = extractedResult { + csvRow.append("\(extractedResult.matchedFields.joined(separator: "-")),") // matched_fields + } else { + csvRow.append(",") // matched_fields + } + + csvRow.append("\(matchedString),") // result_match + csvRow.append(",") // expected_match + csvRow.append("\n") + + return csvRow + } + + private func save(csv: String) { + do { + if let desktopPath = FileManager.default.urls(for: .desktopDirectory, in: .userDomainMask).first?.relativePath { + let path = desktopPath + "/PIR-Debug" + let fileName = "output.csv" + let fileURL = URL(fileURLWithPath: "\(path)/\(fileName)") + try csv.write(to: fileURL, atomically: true, encoding: .utf8) + } else { + os_log("Error getting path") + } + } catch { + os_log("Error writing to file: \(error)") + } + } + + func runJSON(jsonString: String) { if let data = jsonString.data(using: .utf8) { do { let decoder = JSONDecoder() let dataBroker = try decoder.decode(DataBroker.self, from: data) self.selectedDataBroker = dataBroker let brokerProfileQueryData = createBrokerProfileQueryData(for: dataBroker) - let runner = runnerProvider.getOperationRunner() - Task { - do { - let extractedProfiles = try await runner.scan(brokerProfileQueryData, stageCalculator: FakeStageDurationCalculator(), showWebView: true) { true } - - DispatchQueue.main.async { - if extractedProfiles.isEmpty { - self.showNoResultsAlert() - } else { - self.results = extractedProfiles + for query in brokerProfileQueryData { + Task { + do { + let extractedProfiles = try await runner.scan(query, stageCalculator: FakeStageDurationCalculator(), showWebView: true) { true } + + DispatchQueue.main.async { + for extractedProfile in extractedProfiles { + self.results.append(ScanResult(dataBroker: query.dataBroker, + profileQuery: query.profileQuery, + extractedProfile: extractedProfile)) + } } + } catch { + print("Error when scanning: \(error)") } - } catch { - showAlert(for: error) } } } catch { @@ -116,18 +368,16 @@ final class DataBrokerRunCustomJSONViewModel: ObservableObject { } } - func runOptOut(extractedProfile: ExtractedProfile) { + func runOptOut(scanResult: ScanResult) { let runner = runnerProvider.getOperationRunner() - guard let dataBroker = self.selectedDataBroker else { - print("No broker selected") - return - } - - let brokerProfileQueryData = createBrokerProfileQueryData(for: dataBroker) - + let brokerProfileQueryData = BrokerProfileQueryData( + dataBroker: scanResult.dataBroker, + profileQuery: scanResult.profileQuery, + scanOperationData: ScanOperationData(brokerId: 1, profileQueryId: 1, historyEvents: [HistoryEvent]()) + ) Task { do { - try await runner.optOut(profileQuery: brokerProfileQueryData, extractedProfile: extractedProfile, stageCalculator: FakeStageDurationCalculator(), showWebView: true) { + try await runner.optOut(profileQuery: brokerProfileQueryData, extractedProfile: scanResult.extractedProfile, stageCalculator: FakeStageDurationCalculator(), showWebView: true) { true } @@ -142,10 +392,54 @@ final class DataBrokerRunCustomJSONViewModel: ObservableObject { } } - private func createBrokerProfileQueryData(for dataBroker: DataBroker) -> BrokerProfileQueryData { - let profile = createProfile() - let fakeScanOperationData = ScanOperationData(brokerId: 0, profileQueryId: 0, historyEvents: [HistoryEvent]()) - return BrokerProfileQueryData(dataBroker: dataBroker, profileQuery: profile.profileQueries.first!, scanOperationData: fakeScanOperationData) + private func createBrokerProfileQueryData(for broker: DataBroker) -> [BrokerProfileQueryData] { + let profile: DataBrokerProtectionProfile = + .init( + names: names.map { $0.toModel() }, + addresses: addresses.map { $0.toModel() }, + phones: [String](), + birthYear: Int(birthYear) ?? 1990 + ) + let profileQueries = profile.profileQueries + var brokerProfileQueryData = [BrokerProfileQueryData]() + + var profileQueryIndex: Int64 = 1 + for profileQuery in profileQueries { + let fakeScanOperationData = ScanOperationData(brokerId: 0, profileQueryId: profileQueryIndex, historyEvents: [HistoryEvent]()) + brokerProfileQueryData.append( + .init(dataBroker: broker, profileQuery: profileQuery.with(id: profileQueryIndex), scanOperationData: fakeScanOperationData) + ) + + profileQueryIndex += 1 + } + + return brokerProfileQueryData + } + + private func createBrokerProfileQueryData() -> [BrokerProfileQueryData] { + let profile: DataBrokerProtectionProfile = + .init( + names: names.map { $0.toModel() }, + addresses: addresses.map { $0.toModel() }, + phones: [String](), + birthYear: Int(birthYear) ?? 1990 + ) + let profileQueries = profile.profileQueries + var brokerProfileQueryData = [BrokerProfileQueryData]() + + var profileQueryIndex: Int64 = 1 + for profileQuery in profileQueries { + let fakeScanOperationData = ScanOperationData(brokerId: 0, profileQueryId: profileQueryIndex, historyEvents: [HistoryEvent]()) + for broker in brokers { + brokerProfileQueryData.append( + .init(dataBroker: broker, profileQuery: profileQuery.with(id: profileQueryIndex), scanOperationData: fakeScanOperationData) + ) + } + + profileQueryIndex += 1 + } + + return brokerProfileQueryData } private func showNoResultsAlert() { @@ -166,21 +460,9 @@ final class DataBrokerRunCustomJSONViewModel: ObservableObject { } } - private func createProfile() -> DataBrokerProtectionProfile { - let names = DataBrokerProtectionProfile.Name(firstName: firstName, lastName: lastName, middleName: middle) - let addresses = DataBrokerProtectionProfile.Address(city: city, state: state) - - return DataBrokerProtectionProfile(names: [names], addresses: [addresses], phones: [String](), birthYear: Int(birthYear) ?? 1990) - } - func appVersion() -> String { AppVersion.shared.versionNumber } - - func contentScopeScriptsVersion() -> String { - // How can I return this? - return "4.59.2" - } } final class FakeStageDurationCalculator: StageDurationCalculator { @@ -367,4 +649,51 @@ extension DataBrokerProtectionError { default: return name } } + + var is404: Bool { + switch self { + case .httpError(let code): + return code == 404 + default: return false + } + } +} + +extension ScrapedData { + + var nameCSV: String { + if let name = self.name { + return name.replacingOccurrences(of: ",", with: "-") + } else if let alternativeNamesList = self.alternativeNamesList { + return alternativeNamesList.joined(separator: "/").replacingOccurrences(of: ",", with: "-") + } else { + return "" + } + } + + var ageCSV: String { + if let age = self.age { + return age + } else { + return "" + } + } + + var addressesCSV: String { + if let address = self.addressCityState { + return address + } else if let addressFull = self.addressCityStateList { + return addressFull.map { "\($0.city)-\($0.state)" }.joined(separator: "/") + } else { + return "" + } + } + + var relativesCSV: String { + if let relatives = self.relativesList { + return relatives.joined(separator: "-").replacingOccurrences(of: ",", with: "-") + } else { + return "" + } + } } diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/DebugUI/DebugScanOperation.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/DebugUI/DebugScanOperation.swift new file mode 100644 index 0000000000..94ee1b481c --- /dev/null +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/DebugUI/DebugScanOperation.swift @@ -0,0 +1,181 @@ +// +// DebugScanOperation.swift +// +// Copyright © 2023 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import Foundation +import WebKit +import BrowserServicesKit +import UserScript +import Common + +struct DebugScanReturnValue { + let brokerURL: String + let extractedProfiles: [ExtractedProfile] + let error: Error? + let brokerProfileQueryData: BrokerProfileQueryData + let meta: [String: Any]? + + init(brokerURL: String, + extractedProfiles: [ExtractedProfile] = [ExtractedProfile](), + error: Error? = nil, + brokerProfileQueryData: BrokerProfileQueryData, + meta: [String: Any]? = nil) { + self.brokerURL = brokerURL + self.extractedProfiles = extractedProfiles + self.error = error + self.brokerProfileQueryData = brokerProfileQueryData + self.meta = meta + } +} + +final class DebugScanOperation: DataBrokerOperation { + typealias ReturnValue = DebugScanReturnValue + typealias InputValue = Void + + let privacyConfig: PrivacyConfigurationManaging + let prefs: ContentScopeProperties + let query: BrokerProfileQueryData + let emailService: EmailServiceProtocol + let captchaService: CaptchaServiceProtocol + var webViewHandler: WebViewHandler? + var actionsHandler: ActionsHandler? + var continuation: CheckedContinuation? + var extractedProfile: ExtractedProfile? + var stageCalculator: StageDurationCalculator? + private let operationAwaitTime: TimeInterval + let shouldRunNextStep: () -> Bool + var retriesCountOnError: Int = 0 + var scanURL: String? + + private let fileManager = FileManager.default + private let debugScanContentPath: String? + + init(privacyConfig: PrivacyConfigurationManaging, + prefs: ContentScopeProperties, + query: BrokerProfileQueryData, + emailService: EmailServiceProtocol = EmailService(), + captchaService: CaptchaServiceProtocol = CaptchaService(), + operationAwaitTime: TimeInterval = 3, + shouldRunNextStep: @escaping () -> Bool + ) { + self.privacyConfig = privacyConfig + self.prefs = prefs + self.query = query + self.emailService = emailService + self.captchaService = captchaService + self.operationAwaitTime = operationAwaitTime + self.shouldRunNextStep = shouldRunNextStep + if let desktopPath = fileManager.urls(for: .desktopDirectory, in: .userDomainMask).first?.relativePath { + self.debugScanContentPath = desktopPath + "/PIR-Debug" + } else { + self.debugScanContentPath = nil + } + } + + func run(inputValue: Void, + webViewHandler: WebViewHandler? = nil, + actionsHandler: ActionsHandler? = nil, + stageCalculator: StageDurationCalculator, // We do not need it for scans - for now. + showWebView: Bool) async throws -> DebugScanReturnValue { + try await withCheckedThrowingContinuation { continuation in + self.continuation = continuation + Task { + await initialize(handler: webViewHandler, isFakeBroker: query.dataBroker.isFakeBroker, showWebView: showWebView) + + do { + let scanStep = try query.dataBroker.scanStep() + if let actionsHandler = actionsHandler { + self.actionsHandler = actionsHandler + } else { + self.actionsHandler = ActionsHandler(step: scanStep) + } + if self.shouldRunNextStep() { + await executeNextStep() + } else { + failed(with: DataBrokerProtectionError.cancelled) + } + } catch { + failed(with: DataBrokerProtectionError.unknown(error.localizedDescription)) + } + } + } + } + + func runNextAction(_ action: Action) async { + if action as? ExtractAction != nil { + do { + if let path = self.debugScanContentPath { + let fileName = "\(query.profileQuery.id ?? 0)_\(query.dataBroker.name)" + try await webViewHandler?.takeSnaphost(path: path + "/screenshots/", fileName: "\(fileName).png") + try await webViewHandler?.saveHTML(path: path + "/html/", fileName: "\(fileName).html") + } + } catch { + print("Error: \(error)") + } + } + + await webViewHandler?.execute(action: action, data: .userData(query.profileQuery, self.extractedProfile)) + } + + func extractedProfiles(profiles: [ExtractedProfile], meta: [String: Any]?) async { + if let scanURL = self.scanURL { + let debugScanReturnValue = DebugScanReturnValue( + brokerURL: scanURL, + extractedProfiles: profiles, + brokerProfileQueryData: query, + meta: meta + ) + complete(debugScanReturnValue) + } + + await executeNextStep() + } + + func completeWith(error: Error) async { + if let scanURL = self.scanURL { + let debugScanReturnValue = DebugScanReturnValue(brokerURL: scanURL, error: error, brokerProfileQueryData: query) + complete(debugScanReturnValue) + } + + await executeNextStep() + } + + func executeNextStep() async { + retriesCountOnError = 0 // We reset the retries on error when it is successful + os_log("SCAN Waiting %{public}f seconds...", log: .action, operationAwaitTime) + + try? await Task.sleep(nanoseconds: UInt64(operationAwaitTime) * 1_000_000_000) + + if let action = actionsHandler?.nextAction() { + os_log("Next action: %{public}@", log: .action, String(describing: action.actionType.rawValue)) + await runNextAction(action) + } else { + os_log("Releasing the web view", log: .action) + await webViewHandler?.finish() // If we executed all steps we release the web view + } + } + + func loadURL(url: URL) async { + do { + self.scanURL = url.absoluteString + try await webViewHandler?.load(url: url) + await executeNextStep() + } catch { + await completeWith(error: error) + } + } +} diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBrokerProtectionProfile.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBrokerProtectionProfile.swift index a77770d5f9..c76645a804 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBrokerProtectionProfile.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBrokerProtectionProfile.swift @@ -69,7 +69,7 @@ public struct DataBrokerProtectionProfile: Codable { } } -internal extension DataBrokerProtectionProfile { +extension DataBrokerProtectionProfile { var profileQueries: [ProfileQuery] { return addresses.flatMap { address in names.map { name in diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/ProfileQuery.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/ProfileQuery.swift index 0850e3027c..c2cbde5fa5 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/ProfileQuery.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/ProfileQuery.swift @@ -117,6 +117,21 @@ extension ProfileQuery { birthYear: birthYear, deprecated: deprecated) } + + func with(id: Int64) -> ProfileQuery { + return ProfileQuery(id: id, + firstName: firstName, + lastName: lastName, + middleName: middleName, + suffix: suffix, + city: city, + state: state, + street: street, + zipCode: zip, + phone: phone, + birthYear: birthYear, + deprecated: deprecated) + } } extension ProfileQuery: Hashable { diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OptOutOperation.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OptOutOperation.swift index ba59229aeb..7f12d1112f 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OptOutOperation.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OptOutOperation.swift @@ -99,7 +99,7 @@ final class OptOutOperation: DataBrokerOperation { } } - func extractedProfiles(profiles: [ExtractedProfile]) async { + func extractedProfiles(profiles: [ExtractedProfile], meta: [String: Any]?) async { // No - op } diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/ScanOperation.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/ScanOperation.swift index ac26510a64..06dfddd03d 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/ScanOperation.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/ScanOperation.swift @@ -87,7 +87,7 @@ final class ScanOperation: DataBrokerOperation { } } - func extractedProfiles(profiles: [ExtractedProfile]) async { + func extractedProfiles(profiles: [ExtractedProfile], meta: [String: Any]?) async { complete(profiles) await executeNextStep() } diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProtectionFeatureTests.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProtectionFeatureTests.swift index 4fe2f44458..280b30050d 100644 --- a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProtectionFeatureTests.swift +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProtectionFeatureTests.swift @@ -118,7 +118,7 @@ final class MockCSSCommunicationDelegate: CCFCommunicationDelegate { self.url = url } - func extractedProfiles(profiles: [ExtractedProfile]) { + func extractedProfiles(profiles: [ExtractedProfile], meta: [String: Any]?) async { self.profiles = profiles } diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/Mocks.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/Mocks.swift index 24a55ee92d..cc36c1e9de 100644 --- a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/Mocks.swift +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/Mocks.swift @@ -214,6 +214,14 @@ final class WebViewHandlerMock: NSObject, WebViewHandler { wasExecuteJavascriptCalled = true } + func takeSnaphost(path: String, fileName: String) async throws { + + } + + func saveHTML(path: String, fileName: String) async throws { + + } + func reset() { wasInitializeWebViewCalled = false wasLoadCalledWithURL = nil