Allow automated fetching of synced bookmarks' favicons

Core/PixelEvent.swift

@@ -498,6 +498,8 @@ extension Pixel {
         case bookmarksCleanupFailed
         case bookmarksCleanupAttemptedWhileSyncWasEnabled
         case favoritesCleanupFailed
+        case bookmarksFaviconsFetcherStateStoreInitializationFailed
+        case bookmarksFaviconsFetcherFailed
 
         case credentialsDatabaseCleanupFailed
         case credentialsCleanupAttemptedWhileSyncWasEnabled
@@ -990,6 +992,8 @@ extension Pixel.Event {
         case .bookmarksCleanupFailed: return "m_d_bookmarks_cleanup_failed"
         case .bookmarksCleanupAttemptedWhileSyncWasEnabled: return "m_d_bookmarks_cleanup_attempted_while_sync_was_enabled"
         case .favoritesCleanupFailed: return "m_d_favorites_cleanup_failed"
+        case .bookmarksFaviconsFetcherStateStoreInitializationFailed: return "m_d_bookmarks_favicons_fetcher_state_store_initialization_failed"
+        case .bookmarksFaviconsFetcherFailed: return "m_d_bookmarks_favicons_fetcher_failed"
 
         case .credentialsDatabaseCleanupFailed: return "m_d_credentials_database_cleanup_failed_2"
         case .credentialsCleanupAttemptedWhileSyncWasEnabled: return "m_d_credentials_cleanup_attempted_while_sync_was_enabled"

Core/SyncBookmarksAdapter.swift

@@ -30,14 +30,27 @@ public protocol FavoritesDisplayModeStoring: AnyObject {
     var favoritesDisplayMode: FavoritesDisplayMode { get set }
 }
 
+public class BookmarksFaviconsFetcherErrorHandler: EventMapping<BookmarksFaviconsFetcherError> {
+
+    public init() {
+        super.init { event, _, _, _ in
+            Pixel.fire(pixel: .bookmarksFaviconsFetcherFailed, error: event.underlyingError)
+        }
+    }
+
+    override init(mapping: @escaping EventMapping<BookmarksFaviconsFetcherError>.Mapping) {
+        fatalError("Use init()")
+    }
+}
+
 public final class SyncBookmarksAdapter {
 
+    public static let syncBookmarksPausedStateChanged = Notification.Name("com.duckduckgo.app.SyncPausedStateChanged")
+    public static let bookmarksSyncLimitReached = Notification.Name("com.duckduckgo.app.SyncBookmarksLimitReached")
+
     public private(set) var provider: BookmarksProvider?
     public let databaseCleaner: BookmarkDatabaseCleaner
     public let syncDidCompletePublisher: AnyPublisher<Void, Never>
-    public let widgetRefreshCancellable: AnyCancellable
-    public static let syncBookmarksPausedStateChanged = Notification.Name("com.duckduckgo.app.SyncPausedStateChanged")
-    public static let bookmarksSyncLimitReached = Notification.Name("com.duckduckgo.app.SyncBookmarksLimitReached")
 
     public var shouldResetBookmarksSyncTimestamp: Bool = false {
         willSet {
@@ -55,6 +68,22 @@ public final class SyncBookmarksAdapter {
     @UserDefaultsWrapper(key: .syncBookmarksPausedErrorDisplayed, defaultValue: false)
     static private var didShowBookmarksSyncPausedError: Bool
 
+    @Published
+    public var isFaviconsFetchingEnabled: Bool = UserDefaultsWrapper(key: .syncAutomaticallyFetchFavicons, defaultValue: false).wrappedValue {
+        didSet {
+            var udWrapper = UserDefaultsWrapper(key: .syncAutomaticallyFetchFavicons, defaultValue: false)
+            udWrapper.wrappedValue = isFaviconsFetchingEnabled
+            if isFaviconsFetchingEnabled {
+                faviconsFetcher?.initializeFetcherState()
+            } else {
+                faviconsFetcher?.cancelOngoingFetchingIfNeeded()
+            }
+        }
+    }
+
+    @UserDefaultsWrapper(key: .syncIsEligibleForFaviconsFetcherOnboarding, defaultValue: false)
+    public var isEligibleForFaviconsFetcherOnboarding: Bool
+
     public init(database: CoreDataDatabase, favoritesDisplayModeStorage: FavoritesDisplayModeStoring) {
         self.database = database
         self.favoritesDisplayModeStorage = favoritesDisplayModeStorage
@@ -74,6 +103,7 @@ public final class SyncBookmarksAdapter {
         if shouldEnable {
             databaseCleaner.scheduleRegularCleaning()
             handleFavoritesAfterDisablingSync()
+            isFaviconsFetchingEnabled = false
         } else {
             databaseCleaner.cancelCleaningSchedule()
         }
@@ -84,19 +114,60 @@ public final class SyncBookmarksAdapter {
             return
         }
 
+        let faviconsFetcher = setUpFaviconsFetcher()
+
         let provider = BookmarksProvider(
             database: database,
             metadataStore: metadataStore,
-            syncDidUpdateData: { [syncDidCompleteSubject] in
-                syncDidCompleteSubject.send()
+            syncDidUpdateData: { [weak self] in
+                self?.syncDidCompleteSubject.send()
                 Self.isSyncBookmarksPaused = false
                 Self.didShowBookmarksSyncPausedError = false
+            },
+            syncDidFinish: { [weak self] faviconsFetcherInput in
+                if let faviconsFetcher, self?.isFaviconsFetchingEnabled == true {
+                    if let faviconsFetcherInput {
+                        faviconsFetcher.updateBookmarkIDs(
+                            modified: faviconsFetcherInput.modifiedBookmarksUUIDs,
+                            deleted: faviconsFetcherInput.deletedBookmarksUUIDs
+                        )
+                    }
+                    faviconsFetcher.startFetching()
+                }
             }
         )
         if shouldResetBookmarksSyncTimestamp {
             provider.lastSyncTimestamp = nil
         }
 
+        bindSyncErrorPublisher(provider)
+
+        self.provider = provider
+        self.faviconsFetcher = faviconsFetcher
+    }
+
+    private func setUpFaviconsFetcher() -> BookmarksFaviconsFetcher? {
+        let stateStore: BookmarksFaviconsFetcherStateStore
+        do {
+            let url = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first!
+            stateStore = try BookmarksFaviconsFetcherStateStore(applicationSupportURL: url)
+        } catch {
+            Pixel.fire(pixel: .bookmarksFaviconsFetcherStateStoreInitializationFailed, error: error)
+            os_log(.error, log: .syncLog, "Failed to initialize BookmarksFaviconsFetcherStateStore: %{public}s", String(reflecting: error))
+            return nil
+        }
+
+        return BookmarksFaviconsFetcher(
+            database: database,
+            stateStore: stateStore,
+            fetcher: FaviconFetcher(),
+            faviconStore: Favicons.shared,
+            errorEvents: BookmarksFaviconsFetcherErrorHandler(),
+            log: .syncLog
+        )
+    }
+
+    private func bindSyncErrorPublisher(_ provider: BookmarksProvider) {
         syncErrorCancellable = provider.syncErrorPublisher
             .sink { error in
                 switch error {
@@ -126,15 +197,31 @@ public final class SyncBookmarksAdapter {
                 }
                 os_log(.error, log: OSLog.syncLog, "Bookmarks Sync error: %{public}s", String(reflecting: error))
             }
-
-        self.provider = provider
     }
 
-    static private func notifyBookmarksSyncLimitReached() {
-        if !Self.didShowBookmarksSyncPausedError {
-            NotificationCenter.default.post(name: Self.bookmarksSyncLimitReached, object: nil)
-            Self.didShowBookmarksSyncPausedError = true
+    public func cancelFaviconsFetching(_ application: UIApplication) {
+        guard let faviconsFetcher else {
+            return
         }
+        if faviconsFetcher.isFetchingInProgress == true {
+            os_log(.debug, log: .syncLog, "Favicons Fetching is in progress. Starting background task to allow it to gracefully complete.")
+
+            var taskID: UIBackgroundTaskIdentifier!
+            taskID = application.beginBackgroundTask(withName: "Cancelled Favicons Fetching Completion Task") {
+                os_log(.debug, log: .syncLog, "Forcing background task completion")
+                application.endBackgroundTask(taskID)
+            }
+            faviconsFetchingDidFinishCancellable?.cancel()
+            faviconsFetchingDidFinishCancellable = faviconsFetcher.$isFetchingInProgress.dropFirst().filter { !$0 }
+                .prefix(1)
+                .receive(on: DispatchQueue.main)
+                .sink { _ in
+                    os_log(.debug, log: .syncLog, "Ending background task")
+                    application.endBackgroundTask(taskID)
+                }
+        }
+
+        faviconsFetcher.cancelOngoingFetchingIfNeeded()
     }
 
     private func handleFavoritesAfterDisablingSync() {
@@ -158,8 +245,19 @@ public final class SyncBookmarksAdapter {
         }
     }
 
+    static private func notifyBookmarksSyncLimitReached() {
+        if !Self.didShowBookmarksSyncPausedError {
+            NotificationCenter.default.post(name: Self.bookmarksSyncLimitReached, object: nil)
+            Self.didShowBookmarksSyncPausedError = true
+        }
+    }
+
     private var syncDidCompleteSubject = PassthroughSubject<Void, Never>()
     private var syncErrorCancellable: AnyCancellable?
+
     private let database: CoreDataDatabase
     private let favoritesDisplayModeStorage: FavoritesDisplayModeStoring
+    private var faviconsFetcher: BookmarksFaviconsFetcher?
+    private var faviconsFetchingDidFinishCancellable: AnyCancellable?
+    private var widgetRefreshCancellable: AnyCancellable?
 }

Core/SyncCredentialsAdapter.swift

@@ -39,6 +39,7 @@ public final class SyncCredentialsAdapter {
             NotificationCenter.default.post(name: syncCredentialsPausedStateChanged, object: nil)
         }
     }
+
     @UserDefaultsWrapper(key: .syncCredentialsPausedErrorDisplayed, defaultValue: false)
     static private var didShowCredentialsSyncPausedError: Bool
 

Core/UserDefaultsPropertyWrapper.swift

@@ -101,6 +101,10 @@ public struct UserDefaultsWrapper<T> {
         case syncCredentialsPaused = "com.duckduckgo.ios.sync-credentialsPaused"
         case syncBookmarksPausedErrorDisplayed = "com.duckduckgo.ios.sync-bookmarksPausedErrorDisplayed"
         case syncCredentialsPausedErrorDisplayed = "com.duckduckgo.ios.sync-credentialsPausedErrorDisplayed"
+        case syncAutomaticallyFetchFavicons = "com.duckduckgo.ios.sync-automatically-fetch-favicons"
+        case syncIsFaviconsFetcherEnabled = "com.duckduckgo.ios.sync-is-favicons-fetcher-enabled"
+        case syncIsEligibleForFaviconsFetcherOnboarding = "com.duckduckgo.ios.sync-is-eligible-for-favicons-fetcher-onboarding"
+        case syncDidPresentFaviconsFetcherOnboarding = "com.duckduckgo.ios.sync-did-present-favicons-fetcher-onboarding"
 
         case networkProtectionDebugOptionAlwaysOnDisabled = "com.duckduckgo.network-protection.always-on.disabled"
 

DuckDuckGo.xcodeproj/project.pbxproj

@@ -253,6 +253,7 @@
 		3760DFED299315EF0045A446 /* Waitlist in Frameworks */ = {isa = PBXBuildFile; productRef = 3760DFEC299315EF0045A446 /* Waitlist */; };
 		377D80222AB48554002AF251 /* FavoritesDisplayModeSyncHandler.swift in Sources */ = {isa = PBXBuildFile; fileRef = 377D80212AB48554002AF251 /* FavoritesDisplayModeSyncHandler.swift */; };
 		379E877429E97C8D001C8BB0 /* BookmarksCleanupErrorHandling.swift in Sources */ = {isa = PBXBuildFile; fileRef = 379E877329E97C8D001C8BB0 /* BookmarksCleanupErrorHandling.swift */; };
+		37A6A8FE2AFD0208008580A3 /* FaviconsFetcherOnboarding.swift in Sources */ = {isa = PBXBuildFile; fileRef = 37A6A8FD2AFD0208008580A3 /* FaviconsFetcherOnboarding.swift */; };
 		37CBCA9E2A8A659C0050218F /* SyncSettingsAdapter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 37CBCA9D2A8A659C0050218F /* SyncSettingsAdapter.swift */; };
 		37CEFCAC2A673B90001EF741 /* CredentialsCleanupErrorHandling.swift in Sources */ = {isa = PBXBuildFile; fileRef = 37CEFCAB2A673B90001EF741 /* CredentialsCleanupErrorHandling.swift */; };
 		37DF000A29F9C416002B7D3E /* SyncMetadataDatabase.swift in Sources */ = {isa = PBXBuildFile; fileRef = 37DF000929F9C416002B7D3E /* SyncMetadataDatabase.swift */; };
@@ -1278,6 +1279,7 @@
 		37445F962A155F7C0029F789 /* SyncDataProviders.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SyncDataProviders.swift; sourceTree = "<group>"; };
 		377D80212AB48554002AF251 /* FavoritesDisplayModeSyncHandler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FavoritesDisplayModeSyncHandler.swift; sourceTree = "<group>"; };
 		379E877329E97C8D001C8BB0 /* BookmarksCleanupErrorHandling.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BookmarksCleanupErrorHandling.swift; sourceTree = "<group>"; };
+		37A6A8FD2AFD0208008580A3 /* FaviconsFetcherOnboarding.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FaviconsFetcherOnboarding.swift; sourceTree = "<group>"; };
 		37CBCA9D2A8A659C0050218F /* SyncSettingsAdapter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SyncSettingsAdapter.swift; sourceTree = "<group>"; };
 		37CEFCAB2A673B90001EF741 /* CredentialsCleanupErrorHandling.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CredentialsCleanupErrorHandling.swift; sourceTree = "<group>"; };
 		37DF000929F9C416002B7D3E /* SyncMetadataDatabase.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SyncMetadataDatabase.swift; sourceTree = "<group>"; };
@@ -4037,6 +4039,7 @@
 		85F98F8C296F0ED100742F4A /* Sync */ = {
 			isa = PBXGroup;
 			children = (
+				37A6A8FD2AFD0208008580A3 /* FaviconsFetcherOnboarding.swift */,
 				377D80202AB4853A002AF251 /* SettingSyncHandlers */,
 				85F98F97296F4CB100742F4A /* SyncAssets.xcassets */,
 				85F0E97229952D7A003D5181 /* DuckDuckGo Recovery Document.pdf */,
@@ -6412,6 +6415,7 @@
 				85DB12ED2A1FED0C000A4A72 /* AppDelegate+AppDeepLinks.swift in Sources */,
 				98DA6ECA2181E41F00E65433 /* ThemeManager.swift in Sources */,
 				C159DF072A430B60007834BB /* EmailSignupViewController.swift in Sources */,
+				37A6A8FE2AFD0208008580A3 /* FaviconsFetcherOnboarding.swift in Sources */,
 				1E016AB6294A5EB100F21625 /* CustomDaxDialog.swift in Sources */,
 				02341FA42A437999008A1531 /* OnboardingStepView.swift in Sources */,
 				F1CA3C3B1F045B65005FADB3 /* Authenticator.swift in Sources */,
@@ -9142,8 +9146,8 @@
 			isa = XCRemoteSwiftPackageReference;
 			repositoryURL = "https://github.com/DuckDuckGo/BrowserServicesKit";
 			requirement = {
-				kind = exactVersion;
-				version = 84.2.0;
+				branch = "dominik/sync-favicons-fetching";
+				kind = branch;
 			};
 		};
 		C14882EB27F211A000D59F0C /* XCRemoteSwiftPackageReference "SwiftSoup" */ = {

DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved

@@ -14,9 +14,9 @@
         "package": "BrowserServicesKit",
         "repositoryURL": "https://github.com/DuckDuckGo/BrowserServicesKit",
         "state": {
-          "branch": null,
-          "revision": "af63158c03de7a47fb9e2cbcd97fbd28a64ab1fd",
-          "version": "84.2.0"
+          "branch": "dominik/sync-favicons-fetching",
+          "revision": "2d1d73cee34c5e2b897209b78522f27d6a991ea3",
+          "version": null
         }
       },
       {

DuckDuckGo/AppDelegate.swift

@@ -541,6 +541,7 @@ class AppDelegate: UIResponder, UIApplicationDelegate {
         lastBackgroundDate = Date()
         AppDependencyProvider.shared.autofillLoginSession.endSession()
         suspendSync()
+        syncDataProviders.bookmarksAdapter.cancelFaviconsFetching(application)
     }
 
     private func suspendSync() {

DuckDuckGo/BookmarksDataSource.swift

@@ -24,6 +24,7 @@ import Core
 class BookmarksDataSource: NSObject, UITableViewDataSource {
 
     let viewModel: BookmarkListInteracting
+    var onFaviconMissing: ((String) -> Void)?
 
     var isEmpty: Bool {
         viewModel.bookmarks.isEmpty
@@ -50,7 +51,11 @@ class BookmarksDataSource: NSObject, UITableViewDataSource {
             return cell
         } else {
             let cell = BookmarksViewControllerCellFactory.makeBookmarkCell(tableView, forIndexPath: indexPath)
-            cell.faviconImageView.loadFavicon(forDomain: bookmark.urlObject?.host, usingCache: .fireproof)
+            cell.faviconImageView.loadFavicon(forDomain: bookmark.urlObject?.host, usingCache: .fireproof) { [weak self] _, isFake in
+                if isFake, let host = bookmark.urlObject?.host {
+                    self?.onFaviconMissing?(host)
+                }
+            }
             cell.titleLabel.text = bookmark.title
             cell.favoriteImageViewContainer.isHidden = !bookmark.isFavorite(on: viewModel.favoritesDisplayMode.displayedFolder)
             return cell

DuckDuckGo/BookmarksViewController.swift

@@ -96,7 +96,14 @@ class BookmarksViewController: UIViewController, UITableViewDelegate {
     fileprivate let viewModel: BookmarkListInteracting
 
     fileprivate lazy var dataSource: BookmarksDataSource = {
-        return BookmarksDataSource(viewModel: viewModel)
+        let dataSource = BookmarksDataSource(viewModel: viewModel)
+        dataSource.onFaviconMissing = { [weak self] _ in
+            guard let self else {
+                return
+            }
+            self.faviconsFetcherOnboarding.presentOnboardingIfNeeded(from: self)
+        }
+        return dataSource
     }()
 
     var searchDataSource: SearchBookmarksDataSource
@@ -833,6 +840,8 @@ class BookmarksViewController: UIViewController, UITableViewDelegate {
         }
     }
 
+    private(set) lazy var faviconsFetcherOnboarding: FaviconsFetcherOnboarding =
+        .init(syncService: syncService, syncBookmarksAdapter: syncDataProviders.bookmarksAdapter)
 }
 
 extension BookmarksViewController: UISearchBarDelegate {

DuckDuckGo/Favicons.swift

@@ -17,6 +17,7 @@
 //  limitations under the License.
 //
 
+import Bookmarks
 import Common
 import Kingfisher
 import UIKit
@@ -481,4 +482,35 @@ public class Favicons {
     }
 
 }
+
+extension Favicons: Bookmarks.FaviconStoring {
+
+    public func hasFavicon(for domain: String) -> Bool {
+        guard let targetCache = Favicons.Constants.caches[.fireproof],
+              let resource = defaultResource(forDomain: domain)
+        else {
+            return false
+        }
+
+        return targetCache.isCached(forKey: resource.cacheKey)
+    }
+
+    public func storeFavicon(_ imageData: Data, with url: URL?, for documentURL: URL) async throws {
+
+        guard let domain = documentURL.host,
+              let options = kfOptions(forDomain: domain, withURL: documentURL, usingCache: .fireproof),
+              let resource = defaultResource(forDomain: domain),
+              let targetCache = Favicons.Constants.caches[.fireproof],
+              let image = UIImage(data: imageData)
+        else {
+            return
+        }
+
+        Task {
+            let image = self.scaleDownIfNeeded(image: image, toFit: Constants.maxFaviconSize)
+            targetCache.store(image, forKey: resource.cacheKey, options: .init(options))
+            WidgetCenter.shared.reloadAllTimelines()
+        }
+    }
+}
 // swiftlint:enable type_body_length file_length