Merge branch 'develop' into jacek/compilation-pixel

.github/workflows/alpha.yml

@@ -0,0 +1,113 @@
+name: Make TestFlight Alpha Build
+
+on:
+  workflow_dispatch:
+    inputs:
+      destination:
+        description: "TestFlight Group"
+        required: true
+        default: "Latest Alpha Group"
+        type: string
+  workflow_call:
+    inputs:
+      destination:
+        description: "TestFlight Group"
+        required: true
+        default: "Latest Alpha Group"
+        type: string
+    secrets:
+      SSH_PRIVATE_KEY_FASTLANE_MATCH:
+        required: true
+      APPLE_API_KEY_BASE64:
+        required: true
+      APPLE_API_KEY_ID:
+        required: true
+      APPLE_API_KEY_ISSUER:
+        required: true
+      MATCH_PASSWORD:
+        required: true
+      ASANA_ACCESS_TOKEN:
+        required: true
+
+jobs:
+  make-alpha:
+    runs-on: macos-13
+    name: Make TestFlight Alpha Build
+
+    env:
+      destination: ${{ github.event.inputs.destination || inputs.destination }}
+
+    steps:
+
+    - name: Assert develop branch
+      run: |
+        case "${{ github.ref }}" in
+          *develop) ;;
+          *) echo "👎 Not develop branch"; exit 1 ;;
+        esac
+
+    - name: Register SSH keys for access to certificates
+      uses: webfactory/[email protected]
+      with:
+        ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY_FASTLANE_MATCH }}
+
+    - name: Check out the code
+      uses: actions/checkout@v3
+      with:
+        submodules: recursive
+
+    - name: Set cache key hash
+      run: |
+         has_only_tags=$(jq '[ .object.pins[].state | has("version") ] | all' DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved)
+         if [[ "$has_only_tags" == "true" ]]; then
+           echo "cache_key_hash=${{ hashFiles('DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }}" >> $GITHUB_ENV
+         else
+           echo "Package.resolved contains dependencies specified by branch or commit, skipping cache."
+         fi
+
+    - name: Cache SPM
+      if: env.cache_key_hash
+      uses: actions/cache@v3
+      with:
+        path: DerivedData/SourcePackages
+        key: ${{ runner.os }}-spm-${{ env.cache_key_hash }}
+        restore-keys: |
+          ${{ runner.os }}-spm-
+
+    - name: Select Xcode
+      run: sudo xcode-select -s /Applications/Xcode_$(<.xcode-version).app/Contents/Developer
+
+    - name: Prepare fastlane
+      run: bundle install
+
+    - name: Archive and upload the app
+      env:
+        APPLE_API_KEY_BASE64: ${{ secrets.APPLE_API_KEY_BASE64 }}
+        APPLE_API_KEY_ID: ${{ secrets.APPLE_API_KEY_ID }}
+        APPLE_API_KEY_ISSUER: ${{ secrets.APPLE_API_KEY_ISSUER }}
+        MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
+      run: |
+        app_version="$(cut -d ' ' -f 3 < Configuration/Version.xcconfig)"
+        bundle exec fastlane increment_build_number_for_version version:$app_version app_identifier:"com.duckduckgo.mobile.ios.alpha"
+        bundle exec fastlane release_alpha groups:"${{ env.destination }}"
+        build_version="$(xcodebuild -configuration Alpha -showBuildSettings | grep CURRENT_PROJECT_VERSION | tr -d 'CURRENT_PROJECT_VERSION =')"
+        echo "dsyms_path=${{ github.workspace }}/DuckDuckGo-Alpha.app.dSYM.zip" >> $GITHUB_ENV
+        echo "app_version=${app_version}" >> $GITHUB_ENV
+        echo "build_version=${build_version}" >> $GITHUB_ENV
+
+    - name: Upload dSYMs artifact
+      uses: actions/upload-artifact@v3
+      with:
+        name: DuckDuckGo-Alpha-dSYM-${{ env.app_version }}
+        path: ${{ env.dsyms_path }}
+
+    - name: Upload debug symbols to Asana
+      env:
+        ASANA_ACCESS_TOKEN: ${{ secrets.ASANA_ACCESS_TOKEN }}
+      run: |
+        asana_dsyms_path="${{ github.workspace }}/DuckDuckGo-Alpha-${{ env.app_version }}(${{ env.build_version }})-dSYM.zip"
+        mv -f "${{ env.dsyms_path }}" "$asana_dsyms_path"
+
+        curl -s "https://app.asana.com/api/1.0/tasks/1205344386326139/attachments" \
+          -H "Authorization: Bearer ${{ secrets.ASANA_ACCESS_TOKEN }}" \
+          --form "file=@${asana_dsyms_path};type=application/zip"

.github/workflows/nightly.yml

@@ -1,4 +1,4 @@
-name: Nightly Integration Tests
+name: Nightly Test and Deploy
 
 on:
   schedule:
@@ -94,3 +94,15 @@ jobs:
       with:
         report_paths: unittests.xml
 
+  deploy-alpha:
+    name: Deploy Nightly Alpha Build
+    uses: ./.github/workflows/alpha.yml
+    with:
+      destination: "Nightly Alpha Group"
+    secrets:
+      APPLE_API_KEY_BASE64: ${{ secrets.APPLE_API_KEY_BASE64 }}
+      APPLE_API_KEY_ID: ${{ secrets.APPLE_API_KEY_ID }}
+      APPLE_API_KEY_ISSUER: ${{ secrets.APPLE_API_KEY_ISSUER }}
+      MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
+      SSH_PRIVATE_KEY_FASTLANE_MATCH: ${{ secrets.SSH_PRIVATE_KEY_FASTLANE_MATCH }}
+      ASANA_ACCESS_TOKEN: ${{ secrets.ASANA_ACCESS_TOKEN }}

.github/workflows/pr.yml

@@ -103,7 +103,7 @@ jobs:
       if: always() # always run even if the previous step fails
       env:
         ASANA_ACCESS_TOKEN: ${{ secrets.ASANA_ACCESS_TOKEN }}
-        WORKFLOW_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        WORKFLOW_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}
       run: |
         # Extract failed tests from the junit report
         # Only keep failures unique by classname and name (column 1 and 2 of the yq output)

.github/workflows/release.yml

@@ -73,20 +73,21 @@ jobs:
         APPLE_API_KEY_ISSUER: ${{ secrets.APPLE_API_KEY_ISSUER }}
         MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
       run: |
-        bundle exec fastlane release_${{ steps.destination.outputs.destination }}
         app_version="$(cut -d ' ' -f 3 < Configuration/Version.xcconfig)"
         echo "dsyms_path=${{ github.workspace }}/DuckDuckGo.app.dSYM.zip" >> $GITHUB_ENV
         echo "app_version=${app_version}" >> $GITHUB_ENV
+        bundle exec fastlane release_${{ steps.destination.outputs.destination }}
 
     - name: Upload dSYMs artifact
+      if: always()
       uses: actions/upload-artifact@v3
       with:
         name: DuckDuckGo-${{ steps.destination.outputs.destination }}-dSYM-${{ env.app_version }}
         path: ${{ env.dsyms_path }}
 
     - name: Get Asana Task ID
       id: get-task-id
-      if: github.event.inputs.asana-task-url
+      if: ${{ always() && github.event.inputs.asana-task-url }}
       run: |
         task_url_regex='^https://app.asana.com/[0-9]/[0-9]*/([0-9]*)/f$'
         if [[ "${{ github.event.inputs.asana-task-url }}" =~ ${task_url_regex} ]]; then
@@ -96,16 +97,18 @@ jobs:
         fi
 
     - name: Upload debug symbols to Asana
-      if: github.event.inputs.asana-task-url
+      if: ${{ always() && github.event.inputs.asana-task-url }}
       env:
         ASANA_ACCESS_TOKEN: ${{ secrets.ASANA_ACCESS_TOKEN }}
       run: |
-        asana_dsyms_path="${{ github.workspace }}/DuckDuckGo-${{ env.app_version }}-dSYM.zip"
-        mv -f "${{ env.dsyms_path }}" "$asana_dsyms_path"
+        if [[ -f ${{ env.dsyms_path }} ]]; then
+          asana_dsyms_path="${{ github.workspace }}/DuckDuckGo-${{ env.app_version }}-dSYM.zip"
+          mv -f "${{ env.dsyms_path }}" "$asana_dsyms_path"
 
-        curl -s "https://app.asana.com/api/1.0/tasks/${{ steps.get-task-id.outputs.task_id }}/attachments" \
-          -H "Authorization: Bearer ${{ secrets.ASANA_ACCESS_TOKEN }}" \
-          --form "file=@${asana_dsyms_path};type=application/zip"
+          curl -s "https://app.asana.com/api/1.0/tasks/${{ steps.get-task-id.outputs.task_id }}/attachments" \
+            -H "Authorization: Bearer ${{ secrets.ASANA_ACCESS_TOKEN }}" \
+            --form "file=@${asana_dsyms_path};type=application/zip"
+        fi
 
     - name: Send Mattermost message
       env:

Configuration/Version.xcconfig

@@ -1 +1 @@
-MARKETING_VERSION = 7.88.0
+MARKETING_VERSION = 7.90.0

Core/AppPrivacyConfigurationDataProvider.swift

@@ -23,8 +23,8 @@ import BrowserServicesKit
 final public class AppPrivacyConfigurationDataProvider: EmbeddedDataProvider {
 
     public struct Constants {
-        public static let embeddedDataETag = "\"b3acf772994c82ed870835c27a94de36\""
-        public static let embeddedDataSHA = "955ebdcac92bf5589e34e174e9cb8bcbe5170c9bbb5f6d0b4fa32290368a22b2"
+        public static let embeddedDataETag = "\"18cf17e7f3383d2f9d1f0f6643c90c04\""
+        public static let embeddedDataSHA = "3f37f6c999e2d358a343c9150b6f1ac3120931c027a173aa99dae571c2398f37"
     }
 
     public var embeddedDataEtag: String {

Core/DailyPixel.swift

@@ -44,7 +44,11 @@ public final class DailyPixel {
     }
 
     private static let storage: UserDefaults = UserDefaults(suiteName: Constant.dailyPixelStorageIdentifier)!
-
+
+    /// Sends a given Pixel once per day.
+    /// This means a pixel will get sent twice the first time it is called per-day.
+    /// This is useful in situations where pixels receive spikes in volume, as the daily pixel can be used to determine how many users are actually affected.
+    /// Does not append any suffix unlike the alternative function below
     public static func fire(pixel: Pixel.Event,
                             withAdditionalParameters params: [String: String] = [:],
                             onComplete: @escaping (Swift.Error?) -> Void = { _ in }) {
@@ -56,6 +60,27 @@ public final class DailyPixel {
             onComplete(Error.alreadyFired)
         }
     }
+
+    /// Sends a given Pixel once per day with a `_d` suffix, in addition to every time it is called with a `_c` suffix.
+    /// This means a pixel will get sent twice the first time it is called per-day, and subsequent calls that day will only send the `_c` variant.
+    /// This is useful in situations where pixels receive spikes in volume, as the daily pixel can be used to determine how many users are actually affected.
+    public static func fireDailyAndCount(pixel: Pixel.Event,
+                                         error: Swift.Error? = nil,
+                                         withAdditionalParameters params: [String: String] = [:],
+                                         onDailyComplete: @escaping (Swift.Error?) -> Void = { _ in },
+                                         onCountComplete: @escaping (Swift.Error?) -> Void = { _ in }) {
+        if !pixel.hasBeenFiredToday(dailyPixelStorage: storage) {
+            Pixel.fire(pixelNamed: pixel.name + "_d", withAdditionalParameters: params, onComplete: onDailyComplete)
+        } else {
+            onDailyComplete(Error.alreadyFired)
+        }
+        updatePixelLastFireDate(pixel: pixel)
+        var newParams = params
+        if let error {
+            newParams.appendErrorPixelParams(error: error)
+        }
+        Pixel.fire(pixelNamed: pixel.name + "_c", withAdditionalParameters: newParams, onComplete: onCountComplete)
+    }
 
     private static func updatePixelLastFireDate(pixel: Pixel.Event) {
         storage.set(Date(), forKey: pixel.name)

Core/DefaultVariantManager.swift

@@ -46,21 +46,32 @@ public struct VariantIOS: Variant {
             return false
         }
     }
-
+
+    /// This variant is used for returning users to separate them from really new users.
+    static let returningUser = VariantIOS(name: "ru", weight: doNotAllocate, isIncluded: When.always, features: [])
+
     static let doNotAllocate = 0
-
-    // Note: Variants with `doNotAllocate` weight, should always be included so that previous installations are unaffected
+
+    /// The list of cohorts in active ATB experiments.
+    ///
+    /// Variants set to `doNotAllocate` are active, but not adding users to a new cohort, do not change them unless you're sure the experiment is finished.
     public static let defaultVariants: [Variant] = [
-        // SERP testing
         VariantIOS(name: "sc", weight: doNotAllocate, isIncluded: When.always, features: []),
         VariantIOS(name: "sd", weight: doNotAllocate, isIncluded: When.always, features: []),
-        VariantIOS(name: "se", weight: doNotAllocate, isIncluded: When.always, features: [])
-
+        VariantIOS(name: "se", weight: doNotAllocate, isIncluded: When.always, features: []),
+        returningUser
     ]
-
+
+    /// The name of the variant.  Shuld be a two character string like `ma` or `mb`
     public var name: String
+
+    /// The relative weight of this variant, e.g. if two variants have the same weight they will get 50% of the cohorts each.
     public var weight: Int
+
+    /// Function to determine inclusion, e.g. if you want to only run an experiment on English users use `When.inEnglish`
     public var isIncluded: () -> Bool
+
+    /// The experimental feature(s) being tested.
     public var features: [FeatureName]
 
 }
@@ -81,13 +92,26 @@ public class DefaultVariantManager: VariantManager {
     private let variants: [Variant]
     private let storage: StatisticsStore
     private let rng: VariantRNG
+    private let returningUserMeasurement: ReturnUserMeasurement
 
-    public init(variants: [Variant] = VariantIOS.defaultVariants,
-                storage: StatisticsStore = StatisticsUserDefaults(),
-                rng: VariantRNG = Arc4RandomUniformVariantRNG()) {
+    init(variants: [Variant],
+         storage: StatisticsStore,
+         rng: VariantRNG,
+         returningUserMeasurement: ReturnUserMeasurement) {
+
         self.variants = variants
         self.storage = storage
         self.rng = rng
+        self.returningUserMeasurement = returningUserMeasurement
+    }
+
+    public convenience init() {
+        self.init(
+            variants: VariantIOS.defaultVariants,
+            storage: StatisticsUserDefaults(),
+            rng: Arc4RandomUniformVariantRNG(),
+            returningUserMeasurement: KeychainReturnUserMeasurement()
+        )
     }
 
     public func isSupported(feature: FeatureName) -> Bool {
@@ -118,6 +142,10 @@ public class DefaultVariantManager: VariantManager {
     }
 
     private func selectVariant() -> Variant? {
+        if returningUserMeasurement.isReturningUser {
+            return VariantIOS.returningUser
+        }
+
         let totalWeight = variants.reduce(0, { $0 + $1.weight })
         let randomPercent = rng.nextInt(upperBound: totalWeight)
 

Core/FileLoader.swift

@@ -32,7 +32,12 @@ class FileLoader {
         let baseName = fileUrl.deletingPathExtension().path
         let ext = fileUrl.pathExtension
 
-        guard let path = bundle.path(forResource: baseName, ofType: ext) else { throw  FileError.unknownFile }
+        var path = bundle.path(forResource: baseName, ofType: ext)
+        if path == nil {
+            let fileName = fileName.dropping(suffix: ext).dropping(suffix: ".")
+            path = bundle.path(forResource: fileName, ofType: ext)
+        }
+        guard let path else { throw  FileError.unknownFile }
         let url = URL(fileURLWithPath: path)
         guard let data = try? Data(contentsOf: url, options: [.mappedIfSafe]) else { throw  FileError.invalidFileContents }
         return data