Skip to content

Commit

Permalink
WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
jotaemepereira committed Feb 27, 2024
1 parent f41b365 commit a64d9ed
Show file tree
Hide file tree
Showing 5 changed files with 308 additions and 10 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ import Common
protocol WebViewHandler: NSObject {
func initializeWebView(showWebView: Bool) async
func load(url: URL) async throws
func takeSnaphost() async throws
func takeSnaphost(path: String, fileName: String) async throws
func saveHTML(path: String, fileName: String) async throws
func waitForWebViewLoad(timeoutInSeconds: Int) async throws
func finish() async
func execute(action: Action, data: CCFRequestData) async
Expand Down Expand Up @@ -124,7 +125,7 @@ final class DataBrokerProtectionWebViewHandler: NSObject, WebViewHandler {
_ = webView?.evaluateJavaScript(javaScript, in: nil, in: WKContentWorld.page)
}

func takeSnaphost() async throws {
func takeSnaphost(path: String, fileName: String) async throws {
let script = "document.body.scrollHeight"

let result = try await webView?.evaluateJavaScript(script)
Expand All @@ -134,12 +135,26 @@ final class DataBrokerProtectionWebViewHandler: NSObject, WebViewHandler {
let configuration = WKSnapshotConfiguration()
configuration.rect = CGRect(x: 0, y: 0, width: webView?.frame.size.width ?? 0.0, height: height)
if let image = try await webView?.takeSnapshot(configuration: configuration) {
saveToDisk(image: image)
saveToDisk(image: image, path: path, fileName: fileName)
}
}
}

private func saveToDisk(image: NSImage) {
func saveHTML(path: String, fileName: String) async throws {
let result = try await webView?.evaluateJavaScript("document.documentElement.outerHTML")

if let htmlString = result as? String {
do {
let fileURL = URL(fileURLWithPath: "\(path)/\(fileName)")
try htmlString.write(to: fileURL, atomically: true, encoding: .utf8)
print("HTML content saved to file: \(fileURL)")
} catch {
print("Error writing HTML content to file: \(error)")
}
}
}

private func saveToDisk(image: NSImage, path: String, fileName: String) {
guard let tiffData = image.tiffRepresentation else {
// Handle the case where tiff representation is not available
return
Expand All @@ -151,18 +166,26 @@ final class DataBrokerProtectionWebViewHandler: NSObject, WebViewHandler {
return
}

// Convert the bitmap representation to JPEG or PNG data
let fileManager = FileManager.default

if !fileManager.fileExists(atPath: path) {
do {
try fileManager.createDirectory(atPath: path, withIntermediateDirectories: true, attributes: nil)
} catch {
print("Error creating folder: \(error)")
}
}

if let pngData = bitmapImageRep.representation(using: .png, properties: [:]) {
// Save the PNG data to a file
do {
let fileURL = URL(fileURLWithPath: "/Users/juanpereira/Desktop/test.png")
let fileURL = URL(fileURLWithPath: "\(path)/\(fileName)")
try pngData.write(to: fileURL)
// Image saved successfully
} catch {
// Handle the error
print("Error writing PNG: \(error)")
}
} else {
// Handle the case where PNG data cannot be created
print("Error png data was not respresented")
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,14 @@ struct DataBrokerRunCustomJSONView: View {
Button("Run") {
viewModel.runJSON(jsonString: jsonText)
}

if viewModel.isRunningOnAllBrokers {
ProgressView("Scanning...")
} else {
Button("Run all brokers") {
viewModel.runAllBrokers()
}
}
}
.padding()
.frame(minWidth: 600, minHeight: 800)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@ struct AlertUI {
AlertUI(title: "No results", description: "No results were found.")
}

static func finishedScanningAllBrokers() -> AlertUI {
AlertUI(title: "Finished!", description: "We finished scanning all brokers. You should find the data inside ~/Desktop/PIR-Debug/ folder")
}

static func from(error: DataBrokerProtectionError) -> AlertUI {
AlertUI(title: error.title, description: error.description)
}
Expand All @@ -46,6 +50,8 @@ final class DataBrokerRunCustomJSONViewModel: ObservableObject {
@Published var results = [ExtractedProfile]()
@Published var showAlert = false
@Published var showNoResults = false
@Published var isRunningOnAllBrokers = false

var alert: AlertUI?
var selectedDataBroker: DataBroker?

Expand Down Expand Up @@ -79,6 +85,87 @@ final class DataBrokerRunCustomJSONViewModel: ObservableObject {
self.brokers = fileResources.fetchBrokerFromResourceFiles() ?? [DataBroker]()
}

func runAllBrokers() {
let privacyConfigurationManager = PrivacyConfigurationManagingMock()
let features = ContentScopeFeatureToggles(emailProtection: false,
emailProtectionIncontextSignup: false,
credentialsAutofill: false,
identitiesAutofill: false,
creditCardsAutofill: false,
credentialsSaving: false,
passwordGeneration: false,
inlineIconCredentials: false,
thirdPartyCredentialsProvider: false)

let sessionKey = UUID().uuidString
let contentScopeProperties = ContentScopeProperties(gpcEnabled: false,
sessionKey: sessionKey,
featureToggles: features)

isRunningOnAllBrokers = true

Task.detached {
var scanResults = [DebugScanReturnValue]()
try await withThrowingTaskGroup(of: DebugScanReturnValue.self) { group in
for broker in self.brokers {
let brokerProfileQueryData = self.createBrokerProfileQueryData(for: broker)
let debugScanOperation = DebugScanOperation(privacyConfig: privacyConfigurationManager, prefs: contentScopeProperties, query: brokerProfileQueryData) {
true
}

group.addTask {
do {
return try await debugScanOperation.run(inputValue: (), stageCalculator: FakeStageDurationCalculator(), showWebView: false)
} catch {
return DebugScanReturnValue(brokerURL: "ERROR - with broker: \(broker.name)", extractedProfiles: [ExtractedProfile]())
}
}
}

for try await result in group {
scanResults.append(result)
}

var csvText = "Broker-URL,Number-Of-Matches-Found,Error\n"

for result in scanResults {

if let error = result.error {
if let dbpError = error as? DataBrokerProtectionError {
if dbpError.is404 {
let rowText = "\(result.brokerURL),\(result.extractedProfiles.count)"
csvText.append(rowText + "\n")
} else {
let rowText = "\(result.brokerURL),\(result.extractedProfiles.count),\(dbpError.title)-\(dbpError.description)"
csvText.append(rowText + "\n")
}
} else {
let rowText = "\(result.brokerURL),\(result.extractedProfiles.count),\(error.localizedDescription)"
csvText.append(rowText + "\n")
}
} else {
let rowText = "\(result.brokerURL),\(result.extractedProfiles.count)"
csvText.append(rowText + "\n")
}
}

do {
let fileURL = URL(fileURLWithPath: "/Users/juanpereira/Desktop/PIR-Debug/output.csv")
try csvText.write(to: fileURL, atomically: true, encoding: .utf8)
os_log("File created successfully")
} catch {
os_log("Error writing to file: \(error)")
}

DispatchQueue.main.async {
self.alert = AlertUI.finishedScanningAllBrokers()
self.showAlert = true
self.isRunningOnAllBrokers = false
}
}
}
}

func runJSON(jsonString: String) {
if firstName.isEmpty || lastName.isEmpty || city.isEmpty || state.isEmpty || birthYear.isEmpty {
self.showAlert = true
Expand Down Expand Up @@ -350,4 +437,12 @@ extension DataBrokerProtectionError {
default: return name
}
}

var is404: Bool {
switch self {
case .httpError(let code):
return code == 404
default: return false
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
//
// DebugScanOperation.swift
//
// Copyright © 2023 DuckDuckGo. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

import Foundation
import WebKit
import BrowserServicesKit
import UserScript
import Common

struct DebugScanReturnValue {
let brokerURL: String
let extractedProfiles: [ExtractedProfile]
let error: Error?

init(brokerURL: String,
extractedProfiles: [ExtractedProfile] = [ExtractedProfile](),
error: Error? = nil) {
self.brokerURL = brokerURL
self.extractedProfiles = extractedProfiles
self.error = error
}
}

final class DebugScanOperation: DataBrokerOperation {
typealias ReturnValue = DebugScanReturnValue
typealias InputValue = Void

let privacyConfig: PrivacyConfigurationManaging
let prefs: ContentScopeProperties
let query: BrokerProfileQueryData
let emailService: EmailServiceProtocol
let captchaService: CaptchaServiceProtocol
var webViewHandler: WebViewHandler?
var actionsHandler: ActionsHandler?
var continuation: CheckedContinuation<DebugScanReturnValue, Error>?
var extractedProfile: ExtractedProfile?
var stageCalculator: StageDurationCalculator?
private let operationAwaitTime: TimeInterval
let shouldRunNextStep: () -> Bool
var retriesCountOnError: Int = 0
var scanURL: String?

private let fileManager = FileManager.default
private let debugScanContentPath: String?

init(privacyConfig: PrivacyConfigurationManaging,
prefs: ContentScopeProperties,
query: BrokerProfileQueryData,
emailService: EmailServiceProtocol = EmailService(),
captchaService: CaptchaServiceProtocol = CaptchaService(),
operationAwaitTime: TimeInterval = 3,
shouldRunNextStep: @escaping () -> Bool
) {
self.privacyConfig = privacyConfig
self.prefs = prefs
self.query = query
self.emailService = emailService
self.captchaService = captchaService
self.operationAwaitTime = operationAwaitTime
self.shouldRunNextStep = shouldRunNextStep
if let desktopPath = fileManager.urls(for: .desktopDirectory, in: .userDomainMask).first?.relativePath {
self.debugScanContentPath = desktopPath + "/PIR-Debug"
} else {
self.debugScanContentPath = nil
}
}

func run(inputValue: Void,
webViewHandler: WebViewHandler? = nil,
actionsHandler: ActionsHandler? = nil,
stageCalculator: StageDurationCalculator, // We do not need it for scans - for now.
showWebView: Bool) async throws -> DebugScanReturnValue {
try await withCheckedThrowingContinuation { continuation in
self.continuation = continuation
Task {
await initialize(handler: webViewHandler, isFakeBroker: query.dataBroker.isFakeBroker, showWebView: showWebView)

do {
let scanStep = try query.dataBroker.scanStep()
if let actionsHandler = actionsHandler {
self.actionsHandler = actionsHandler
} else {
self.actionsHandler = ActionsHandler(step: scanStep)
}
if self.shouldRunNextStep() {
await executeNextStep()
} else {
failed(with: DataBrokerProtectionError.cancelled)
}
} catch {
failed(with: DataBrokerProtectionError.unknown(error.localizedDescription))
}
}
}
}

func runNextAction(_ action: Action) async {
if action as? ExtractAction != nil {
do {
if let path = self.debugScanContentPath {
try await webViewHandler?.takeSnaphost(path: path, fileName: "\(query.dataBroker.name).png")
try await webViewHandler?.saveHTML(path: path, fileName: "\(query.dataBroker.name).html")
}
} catch {
print("Error: \(error)")
}
}

if let extractedProfile = self.extractedProfile {
await webViewHandler?.execute(action: action, data: .extractedProfile(extractedProfile))
} else {
await webViewHandler?.execute(action: action, data: .profile(query.profileQuery))
}
}

func extractedProfiles(profiles: [ExtractedProfile]) async {
if let scanURL = self.scanURL {
let debugScanReturnValue = DebugScanReturnValue(brokerURL: scanURL, extractedProfiles: profiles)
complete(debugScanReturnValue)
}

await executeNextStep()
}

func completeWith(error: Error) async {
if let scanURL = self.scanURL {
let debugScanReturnValue = DebugScanReturnValue(brokerURL: scanURL, error: error)
complete(debugScanReturnValue)
}

await executeNextStep()
}

func executeNextStep() async {
retriesCountOnError = 0 // We reset the retries on error when it is successful
os_log("SCAN Waiting %{public}f seconds...", log: .action, operationAwaitTime)

try? await Task.sleep(nanoseconds: UInt64(operationAwaitTime) * 1_000_000_000)

if let action = actionsHandler?.nextAction() {
os_log("Next action: %{public}@", log: .action, String(describing: action.actionType.rawValue))
await runNextAction(action)
} else {
os_log("Releasing the web view", log: .action)
await webViewHandler?.finish() // If we executed all steps we release the web view
}
}

func loadURL(url: URL) async {
do {
self.scanURL = url.absoluteString
try await webViewHandler?.load(url: url)
await executeNextStep()
} catch {
await completeWith(error: error)
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,6 @@ extension DataBrokerOperation {
func loadURL(url: URL) async {
do {
try await webViewHandler?.load(url: url)
try await webViewHandler?.takeSnaphost()
await executeNextStep()
} catch {
await onError(error: error)
Expand Down

0 comments on commit a64d9ed

Please sign in to comment.