Sensitive content analysis

CHANGELOG.md

@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Release Notes
 - Fix typo in minimum age warning
 - Fix crash when tapping Post button on macOS. [#1687](https://github.com/planetary-social/nos/issues/1687)
+- Added sensitive content analysis for beta testing.
 
 ### Internal Changes
 

Nos/Assets/Localization/Localizable.xcstrings

@@ -3361,6 +3361,17 @@
         }
       }
     },
+    "contentWarning" : {
+      "extractionState" : "manual",
+      "localizations" : {
+        "en" : {
+          "stringUnit" : {
+            "state" : "translated",
+            "value" : "Content Warning"
+          }
+        }
+      }
+    },
     "contentWarningExplanation" : {
       "extractionState" : "manual",
       "localizations" : {
@@ -16919,6 +16930,17 @@
         }
       }
     },
+    "sensitiveContentUploadWarning" : {
+      "extractionState" : "manual",
+      "localizations" : {
+        "en" : {
+          "stringUnit" : {
+            "state" : "translated",
+            "value" : "The media you've selected may contain nudity. Nos prefers to foster a healthy and safe online community. Please confirm that you’re comfortable with this content being posted online."
+          }
+        }
+      }
+    },
     "settings" : {
       "extractionState" : "manual",
       "localizations" : {
@@ -18918,6 +18940,17 @@
         }
       }
     },
+    "upload" : {
+      "extractionState" : "manual",
+      "localizations" : {
+        "en" : {
+          "stringUnit" : {
+            "state" : "translated",
+            "value" : "Upload"
+          }
+        }
+      }
+    },
     "url" : {
       "extractionState" : "manual",
       "localizations" : {

Nos/Controller/SensitiveContentController.swift

@@ -0,0 +1,147 @@
+import Combine
+import Dependencies
+import Foundation
+import SensitiveContentAnalysis
+
+extension SCSensitivityAnalysisPolicy {
+    var description: String {
+        // TODO: Localize
+        switch self {
+        case .disabled:
+            "Sensitive Content Analysis is currently disabled. To enable, go to Settings app -> Privacy & Security -> \"Sensitive Content Warning\"." // swiftlint:disable:this line_length
+        default:
+            "Sensitive Content Analysis is currently enabled. To disable, go to Settings app -> Privacy & Security -> \"Sensitive Content Warning\"." // swiftlint:disable:this line_length
+        }
+    }
+}
+
+/// An object that analyzes images for nudity and manages and publishes related state.
+///
+/// > Note: This object is a Swift actor because many images may be analyzed simultaneously, and the analysis state
+///         cache needs to be read from and written to with isolated access.
+actor SensitiveContentController: FileDownloading {
+
+    @Dependency(\.featureFlags) private var featureFlags
+
+    /// The state of analysis of content at a given file URL.
+    enum AnalysisState: Equatable {
+        /// The content is currently being analyzed by the system.
+        case analyzing
+        /// The content has been analyzed. The associated Bool indicates whether the content has been deemed sensitive.
+        case analyzed(Bool) // true == sensitive
+        /// The content has been explicitly allowed by the user.
+        case allowed
+
+        var shouldObfuscate: Bool {
+            switch self {
+            case .analyzing: true
+            case .analyzed(let isSensitive):
+                isSensitive
+            case .allowed: false
+            }
+        }
+    }
+
+    static let shared = SensitiveContentController()
+
+    private var cache = [String: AnalysisState]()
+
+    private var publishers = [String: CurrentValueSubject<AnalysisState, Never>]()
+
+    private let analyzer = SCSensitivityAnalyzer()
+
+    /// Indicates whether sensitivity analysis can be performed.
+    nonisolated var isSensitivityAnalysisEnabled: Bool {
+        analyzer.analysisPolicy != .disabled
+    }
+
+    /// Analyzes content at the provided URL for nudity.
+    /// - Parameter url: The URL to get the content from.
+    func analyzeContent(atURL url: URL) async {
+        guard isSensitivityAnalysisEnabled && url.isImage else {
+            return  // the content cannot be analyzed
+        }
+
+        if cache[url.absoluteString] != nil {
+            return  // the content is already being analyzed
+        }
+
+        do {
+            #if DEBUG
+            let shouldOverrideAnalyzer = featureFlags.isEnabled(.sensitiveContentFlagAllAsSensitive)
+            if shouldOverrideAnalyzer {
+                try await Task.sleep(nanoseconds: 1 * 1_000_000_000)    // simulate time to analyze
+                updateState(.analyzed(true), for: url)
+                return
+            }
+            #endif
+
+            let fileURLToAnalyze = url.isFileURL ? url : try await file(byDownloadingFrom: url)
+            let result = try await analyzer.analyzeImage(at: fileURLToAnalyze)
+            updateState(.analyzed(result.isSensitive), for: url)
+        } catch {
+            print("⚠️ SensitiveContentController: Failed to analyze content at \(url): \(error)")
+        }
+    }
+
+    /// Marks content at a provided URL as allowed.
+    /// - Parameter url: The URL to mark as allowed.
+    func allowContent(at url: URL) {
+        updateState(.allowed, for: url)
+    }
+
+    /// Analyzes content the user wants to upload for nudity.
+    /// - Parameter fileURL: The file URL to analyze.
+    /// - Returns: True if the content is sensitive.
+    func shouldWarnUserUploadingFile(at fileURL: URL) async -> Bool {
+        guard isSensitivityAnalysisEnabled else {
+            return false
+        }
+
+        #if DEBUG
+        let shouldOverrideAnalyzer = featureFlags.isEnabled(.sensitiveContentFlagAllAsSensitive)
+        if shouldOverrideAnalyzer {
+            try? await Task.sleep(nanoseconds: 250_000_000) // simulate time to analyze
+            updateState(.analyzed(true), for: fileURL)
+            return true
+        }
+        #endif
+
+        do {
+            let result = try await analyzer.analyzeImage(at: fileURL)
+            return result.isSensitive
+        } catch {
+            return false
+        }
+    }
+
+    /// A publisher for a listener to monitor for analysis state changes for a given URL.
+    /// - Parameter url: The URL to monitor state on.
+    /// - Returns: The requested publisher.
+    func analysisStatePublisher(for url: URL) -> AnyPublisher<AnalysisState, Never> {
+        if let publisher = publishers[url.absoluteString] {
+            return publisher.eraseToAnyPublisher()
+        } else {
+            let publisher = CurrentValueSubject<AnalysisState, Never>(.analyzing)
+            publishers[url.absoluteString] = publisher
+            cache[url.absoluteString] = .analyzing
+            return publisher.eraseToAnyPublisher()
+        }
+    }
+
+    /// Updates the analysis state for a provided URL.
+    /// - Parameters:
+    ///   - url: The URL for which to update state.
+    ///   - newState: The state to update to.
+    ///
+    /// The state is cached locally and published to listeners.
+    private func updateState(_ state: AnalysisState, for url: URL) {
+        cache[url.absoluteString] = state
+        if let publisher = publishers[url.absoluteString] {
+            publisher.send(state)
+        } else {
+            let publisher = CurrentValueSubject<AnalysisState, Never>(state)
+            publishers[url.absoluteString] = publisher
+        }
+    }
+}

Nos/Nos.entitlements

@@ -4,6 +4,10 @@
 <dict>
 	<key>aps-environment</key>
 	<string>development</string>
+	<key>com.apple.developer.sensitivecontentanalysis.client</key>
+	<array>
+		<string>analysis</string>
+	</array>
 	<key>com.apple.security.app-sandbox</key>
 	<true/>
 	<key>com.apple.security.files.user-selected.read-only</key>

Nos/NosDev.entitlements

@@ -4,6 +4,10 @@
 <dict>
 	<key>aps-environment</key>
 	<string>development</string>
+	<key>com.apple.developer.sensitivecontentanalysis.client</key>
+	<array>
+		<string>analysis</string>
+	</array>
 	<key>com.apple.security.app-sandbox</key>
 	<true/>
 	<key>com.apple.security.files.user-selected.read-only</key>

Nos/NosStaging.entitlements

@@ -4,6 +4,10 @@
 <dict>
 	<key>aps-environment</key>
 	<string>development</string>
+	<key>com.apple.developer.sensitivecontentanalysis.client</key>
+	<array>
+		<string>analysis</string>
+	</array>
 	<key>com.apple.security.app-sandbox</key>
 	<true/>
 	<key>com.apple.security.files.user-selected.read-only</key>

Nos/Service/FeatureFlags.swift

@@ -8,6 +8,9 @@ enum FeatureFlag {
     /// - Note: See [Figma](https://www.figma.com/design/6MeujQUXzC1AuviHEHCs0J/Nos---In-Progress?node-id=9221-8504)
     ///         for the new flow.
     case newOnboardingFlow
+
+    case sensitiveContentAnalysisEnabled
+    case sensitiveContentFlagAllAsSensitive
 }
 
 /// The set of feature flags used by the app.
@@ -31,7 +34,10 @@ protocol FeatureFlags {
 
     /// Feature flags and their values.
     private var featureFlags: [FeatureFlag: Bool] = [
-        .newOnboardingFlow: true
+        .newOnboardingFlow: true,
+
+        .sensitiveContentAnalysisEnabled: true,
+        .sensitiveContentFlagAllAsSensitive: false
     ]
 
     /// Returns true if the feature is enabled.

Nos/Service/FileDownloading.swift

@@ -0,0 +1,17 @@
+import Foundation
+
+/// Adds file downloading capability to any type.
+protocol FileDownloading {}
+extension FileDownloading {
+
+    /// Downloads a file asynchronously to a temporary directory.
+    /// - Parameter url: The URL to download content from.
+    /// - Returns: A file URL pointing to the downloaded content.
+    func file(byDownloadingFrom url: URL) async throws -> URL {
+        let temporaryDirectory = FileManager.default.temporaryDirectory
+        let fileURL = temporaryDirectory.appendingPathComponent(url.lastPathComponent)
+        let (data, _) = try await URLSession.shared.data(from: url)
+        try data.write(to: fileURL)
+        return fileURL
+    }
+}

Nos/Views/Components/Media/GalleryView.swift

@@ -9,6 +9,9 @@ struct GalleryView: View {
 
     /// Inline metadata describing the data in ``urls``.
     let metadata: InlineMetadataCollection?
+
+    /// The ``Author`` that posted the content.
+    let author: Author?
 
     /// The currently-selected tab in the tab view.
     @State private var selectedTab = 0
@@ -19,6 +22,12 @@ struct GalleryView: View {
     /// The media service that loads content from URLs and determines the orientation for this gallery.
     @Dependency(\.mediaService) private var mediaService
 
+    init(urls: [URL], metadata: InlineMetadataCollection?, author: Author? = nil) {
+        self.urls = urls
+        self.metadata = metadata
+        self.author = author
+    }
+
     /// The orientation determined by the `metadata`, if any.
     private var metadataOrientation: MediaOrientation? {
         metadata?[urls.first?.absoluteString]?.orientation
@@ -50,6 +59,9 @@ struct GalleryView: View {
                         LinkView(url: urls[index])
                             .tag(index)
                     }
+                    .overlay {
+                        SensitiveContentOverlayView(url: urls[index], author: author)
+                    }
                 }
             }
             .tabViewStyle(.page(indexDisplayMode: .never))
@@ -81,6 +93,9 @@ struct GalleryView: View {
         AspectRatioContainer(orientation: orientation) {
             LinkView(url: url)
         }
+        .overlay {
+            SensitiveContentOverlayView(url: url, author: author)
+        }
     }
 
     /// A loading view that determines the orientation for the gallery. When possible, the aspect ratio of the