From a64d9ed89ab4278ef74f63b7073ce4efa333a976 Mon Sep 17 00:00:00 2001 From: Juan Manuel Pereira Date: Mon, 26 Feb 2024 19:05:09 -0300 Subject: [PATCH] WIP --- .../CCF/WebViewHandler.swift | 41 ++++- .../DebugUI/DataBrokerRunCustomJSONView.swift | 8 + .../DataBrokerRunCustomJSONViewModel.swift | 95 ++++++++++ .../DebugUI/DebugScanOperation.swift | 173 ++++++++++++++++++ .../Operations/DataBrokerOperation.swift | 1 - 5 files changed, 308 insertions(+), 10 deletions(-) create mode 100644 LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/DebugUI/DebugScanOperation.swift diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/CCF/WebViewHandler.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/CCF/WebViewHandler.swift index 5c315db12ef..4af0f5773f3 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/CCF/WebViewHandler.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/CCF/WebViewHandler.swift @@ -25,7 +25,8 @@ import Common protocol WebViewHandler: NSObject { func initializeWebView(showWebView: Bool) async func load(url: URL) async throws - func takeSnaphost() async throws + func takeSnaphost(path: String, fileName: String) async throws + func saveHTML(path: String, fileName: String) async throws func waitForWebViewLoad(timeoutInSeconds: Int) async throws func finish() async func execute(action: Action, data: CCFRequestData) async @@ -124,7 +125,7 @@ final class DataBrokerProtectionWebViewHandler: NSObject, WebViewHandler { _ = webView?.evaluateJavaScript(javaScript, in: nil, in: WKContentWorld.page) } - func takeSnaphost() async throws { + func takeSnaphost(path: String, fileName: String) async throws { let script = "document.body.scrollHeight" let result = try await webView?.evaluateJavaScript(script) @@ -134,12 +135,26 @@ final class DataBrokerProtectionWebViewHandler: NSObject, WebViewHandler { let configuration = WKSnapshotConfiguration() configuration.rect = CGRect(x: 0, y: 0, width: webView?.frame.size.width ?? 0.0, height: height) if let image = try await webView?.takeSnapshot(configuration: configuration) { - saveToDisk(image: image) + saveToDisk(image: image, path: path, fileName: fileName) } } } - private func saveToDisk(image: NSImage) { + func saveHTML(path: String, fileName: String) async throws { + let result = try await webView?.evaluateJavaScript("document.documentElement.outerHTML") + + if let htmlString = result as? String { + do { + let fileURL = URL(fileURLWithPath: "\(path)/\(fileName)") + try htmlString.write(to: fileURL, atomically: true, encoding: .utf8) + print("HTML content saved to file: \(fileURL)") + } catch { + print("Error writing HTML content to file: \(error)") + } + } + } + + private func saveToDisk(image: NSImage, path: String, fileName: String) { guard let tiffData = image.tiffRepresentation else { // Handle the case where tiff representation is not available return @@ -151,18 +166,26 @@ final class DataBrokerProtectionWebViewHandler: NSObject, WebViewHandler { return } - // Convert the bitmap representation to JPEG or PNG data + let fileManager = FileManager.default + + if !fileManager.fileExists(atPath: path) { + do { + try fileManager.createDirectory(atPath: path, withIntermediateDirectories: true, attributes: nil) + } catch { + print("Error creating folder: \(error)") + } + } + if let pngData = bitmapImageRep.representation(using: .png, properties: [:]) { // Save the PNG data to a file do { - let fileURL = URL(fileURLWithPath: "/Users/juanpereira/Desktop/test.png") + let fileURL = URL(fileURLWithPath: "\(path)/\(fileName)") try pngData.write(to: fileURL) - // Image saved successfully } catch { - // Handle the error + print("Error writing PNG: \(error)") } } else { - // Handle the case where PNG data cannot be created + print("Error png data was not respresented") } } } diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/DebugUI/DataBrokerRunCustomJSONView.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/DebugUI/DataBrokerRunCustomJSONView.swift index 33671421d54..fac4b3c2351 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/DebugUI/DataBrokerRunCustomJSONView.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/DebugUI/DataBrokerRunCustomJSONView.swift @@ -76,6 +76,14 @@ struct DataBrokerRunCustomJSONView: View { Button("Run") { viewModel.runJSON(jsonString: jsonText) } + + if viewModel.isRunningOnAllBrokers { + ProgressView("Scanning...") + } else { + Button("Run all brokers") { + viewModel.runAllBrokers() + } + } } .padding() .frame(minWidth: 600, minHeight: 800) diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/DebugUI/DataBrokerRunCustomJSONViewModel.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/DebugUI/DataBrokerRunCustomJSONViewModel.swift index 9109f0baed7..998c54bdefb 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/DebugUI/DataBrokerRunCustomJSONViewModel.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/DebugUI/DataBrokerRunCustomJSONViewModel.swift @@ -30,6 +30,10 @@ struct AlertUI { AlertUI(title: "No results", description: "No results were found.") } + static func finishedScanningAllBrokers() -> AlertUI { + AlertUI(title: "Finished!", description: "We finished scanning all brokers. You should find the data inside ~/Desktop/PIR-Debug/ folder") + } + static func from(error: DataBrokerProtectionError) -> AlertUI { AlertUI(title: error.title, description: error.description) } @@ -46,6 +50,8 @@ final class DataBrokerRunCustomJSONViewModel: ObservableObject { @Published var results = [ExtractedProfile]() @Published var showAlert = false @Published var showNoResults = false + @Published var isRunningOnAllBrokers = false + var alert: AlertUI? var selectedDataBroker: DataBroker? @@ -79,6 +85,87 @@ final class DataBrokerRunCustomJSONViewModel: ObservableObject { self.brokers = fileResources.fetchBrokerFromResourceFiles() ?? [DataBroker]() } + func runAllBrokers() { + let privacyConfigurationManager = PrivacyConfigurationManagingMock() + let features = ContentScopeFeatureToggles(emailProtection: false, + emailProtectionIncontextSignup: false, + credentialsAutofill: false, + identitiesAutofill: false, + creditCardsAutofill: false, + credentialsSaving: false, + passwordGeneration: false, + inlineIconCredentials: false, + thirdPartyCredentialsProvider: false) + + let sessionKey = UUID().uuidString + let contentScopeProperties = ContentScopeProperties(gpcEnabled: false, + sessionKey: sessionKey, + featureToggles: features) + + isRunningOnAllBrokers = true + + Task.detached { + var scanResults = [DebugScanReturnValue]() + try await withThrowingTaskGroup(of: DebugScanReturnValue.self) { group in + for broker in self.brokers { + let brokerProfileQueryData = self.createBrokerProfileQueryData(for: broker) + let debugScanOperation = DebugScanOperation(privacyConfig: privacyConfigurationManager, prefs: contentScopeProperties, query: brokerProfileQueryData) { + true + } + + group.addTask { + do { + return try await debugScanOperation.run(inputValue: (), stageCalculator: FakeStageDurationCalculator(), showWebView: false) + } catch { + return DebugScanReturnValue(brokerURL: "ERROR - with broker: \(broker.name)", extractedProfiles: [ExtractedProfile]()) + } + } + } + + for try await result in group { + scanResults.append(result) + } + + var csvText = "Broker-URL,Number-Of-Matches-Found,Error\n" + + for result in scanResults { + + if let error = result.error { + if let dbpError = error as? DataBrokerProtectionError { + if dbpError.is404 { + let rowText = "\(result.brokerURL),\(result.extractedProfiles.count)" + csvText.append(rowText + "\n") + } else { + let rowText = "\(result.brokerURL),\(result.extractedProfiles.count),\(dbpError.title)-\(dbpError.description)" + csvText.append(rowText + "\n") + } + } else { + let rowText = "\(result.brokerURL),\(result.extractedProfiles.count),\(error.localizedDescription)" + csvText.append(rowText + "\n") + } + } else { + let rowText = "\(result.brokerURL),\(result.extractedProfiles.count)" + csvText.append(rowText + "\n") + } + } + + do { + let fileURL = URL(fileURLWithPath: "/Users/juanpereira/Desktop/PIR-Debug/output.csv") + try csvText.write(to: fileURL, atomically: true, encoding: .utf8) + os_log("File created successfully") + } catch { + os_log("Error writing to file: \(error)") + } + + DispatchQueue.main.async { + self.alert = AlertUI.finishedScanningAllBrokers() + self.showAlert = true + self.isRunningOnAllBrokers = false + } + } + } + } + func runJSON(jsonString: String) { if firstName.isEmpty || lastName.isEmpty || city.isEmpty || state.isEmpty || birthYear.isEmpty { self.showAlert = true @@ -350,4 +437,12 @@ extension DataBrokerProtectionError { default: return name } } + + var is404: Bool { + switch self { + case .httpError(let code): + return code == 404 + default: return false + } + } } diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/DebugUI/DebugScanOperation.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/DebugUI/DebugScanOperation.swift new file mode 100644 index 00000000000..e13fa0e3849 --- /dev/null +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/DebugUI/DebugScanOperation.swift @@ -0,0 +1,173 @@ +// +// DebugScanOperation.swift +// +// Copyright © 2023 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import Foundation +import WebKit +import BrowserServicesKit +import UserScript +import Common + +struct DebugScanReturnValue { + let brokerURL: String + let extractedProfiles: [ExtractedProfile] + let error: Error? + + init(brokerURL: String, + extractedProfiles: [ExtractedProfile] = [ExtractedProfile](), + error: Error? = nil) { + self.brokerURL = brokerURL + self.extractedProfiles = extractedProfiles + self.error = error + } +} + +final class DebugScanOperation: DataBrokerOperation { + typealias ReturnValue = DebugScanReturnValue + typealias InputValue = Void + + let privacyConfig: PrivacyConfigurationManaging + let prefs: ContentScopeProperties + let query: BrokerProfileQueryData + let emailService: EmailServiceProtocol + let captchaService: CaptchaServiceProtocol + var webViewHandler: WebViewHandler? + var actionsHandler: ActionsHandler? + var continuation: CheckedContinuation? + var extractedProfile: ExtractedProfile? + var stageCalculator: StageDurationCalculator? + private let operationAwaitTime: TimeInterval + let shouldRunNextStep: () -> Bool + var retriesCountOnError: Int = 0 + var scanURL: String? + + private let fileManager = FileManager.default + private let debugScanContentPath: String? + + init(privacyConfig: PrivacyConfigurationManaging, + prefs: ContentScopeProperties, + query: BrokerProfileQueryData, + emailService: EmailServiceProtocol = EmailService(), + captchaService: CaptchaServiceProtocol = CaptchaService(), + operationAwaitTime: TimeInterval = 3, + shouldRunNextStep: @escaping () -> Bool + ) { + self.privacyConfig = privacyConfig + self.prefs = prefs + self.query = query + self.emailService = emailService + self.captchaService = captchaService + self.operationAwaitTime = operationAwaitTime + self.shouldRunNextStep = shouldRunNextStep + if let desktopPath = fileManager.urls(for: .desktopDirectory, in: .userDomainMask).first?.relativePath { + self.debugScanContentPath = desktopPath + "/PIR-Debug" + } else { + self.debugScanContentPath = nil + } + } + + func run(inputValue: Void, + webViewHandler: WebViewHandler? = nil, + actionsHandler: ActionsHandler? = nil, + stageCalculator: StageDurationCalculator, // We do not need it for scans - for now. + showWebView: Bool) async throws -> DebugScanReturnValue { + try await withCheckedThrowingContinuation { continuation in + self.continuation = continuation + Task { + await initialize(handler: webViewHandler, isFakeBroker: query.dataBroker.isFakeBroker, showWebView: showWebView) + + do { + let scanStep = try query.dataBroker.scanStep() + if let actionsHandler = actionsHandler { + self.actionsHandler = actionsHandler + } else { + self.actionsHandler = ActionsHandler(step: scanStep) + } + if self.shouldRunNextStep() { + await executeNextStep() + } else { + failed(with: DataBrokerProtectionError.cancelled) + } + } catch { + failed(with: DataBrokerProtectionError.unknown(error.localizedDescription)) + } + } + } + } + + func runNextAction(_ action: Action) async { + if action as? ExtractAction != nil { + do { + if let path = self.debugScanContentPath { + try await webViewHandler?.takeSnaphost(path: path, fileName: "\(query.dataBroker.name).png") + try await webViewHandler?.saveHTML(path: path, fileName: "\(query.dataBroker.name).html") + } + } catch { + print("Error: \(error)") + } + } + + if let extractedProfile = self.extractedProfile { + await webViewHandler?.execute(action: action, data: .extractedProfile(extractedProfile)) + } else { + await webViewHandler?.execute(action: action, data: .profile(query.profileQuery)) + } + } + + func extractedProfiles(profiles: [ExtractedProfile]) async { + if let scanURL = self.scanURL { + let debugScanReturnValue = DebugScanReturnValue(brokerURL: scanURL, extractedProfiles: profiles) + complete(debugScanReturnValue) + } + + await executeNextStep() + } + + func completeWith(error: Error) async { + if let scanURL = self.scanURL { + let debugScanReturnValue = DebugScanReturnValue(brokerURL: scanURL, error: error) + complete(debugScanReturnValue) + } + + await executeNextStep() + } + + func executeNextStep() async { + retriesCountOnError = 0 // We reset the retries on error when it is successful + os_log("SCAN Waiting %{public}f seconds...", log: .action, operationAwaitTime) + + try? await Task.sleep(nanoseconds: UInt64(operationAwaitTime) * 1_000_000_000) + + if let action = actionsHandler?.nextAction() { + os_log("Next action: %{public}@", log: .action, String(describing: action.actionType.rawValue)) + await runNextAction(action) + } else { + os_log("Releasing the web view", log: .action) + await webViewHandler?.finish() // If we executed all steps we release the web view + } + } + + func loadURL(url: URL) async { + do { + self.scanURL = url.absoluteString + try await webViewHandler?.load(url: url) + await executeNextStep() + } catch { + await completeWith(error: error) + } + } +} diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift index 1bc5e8c2515..37cd385e4e9 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift @@ -169,7 +169,6 @@ extension DataBrokerOperation { func loadURL(url: URL) async { do { try await webViewHandler?.load(url: url) - try await webViewHandler?.takeSnaphost() await executeNextStep() } catch { await onError(error: error)