diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8201c51..06bf967 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -99,7 +99,7 @@ jobs: ${{ runner.os }}-spm- - name: Select Xcode - run: sudo xcode-select -s /Applications/Xcode_14.3.app/Contents/Developer + run: sudo xcode-select -s /Applications/Xcode_15.0.app/Contents/Developer - name: Install xcbeautify continue-on-error: true @@ -113,7 +113,7 @@ jobs: uses: mikepenz/action-junit-report@v3 if: always() with: - check_name: Test Report + check_name: BSK Test Report report_paths: ./BrowserServicesKit/tests.xml require_tests: true diff --git a/broken-site-reporting/README.md b/broken-site-reporting/README.md index bed9fce..1428175 100644 --- a/broken-site-reporting/README.md +++ b/broken-site-reporting/README.md @@ -27,10 +27,16 @@ Test suite specific fields: - `gpcEnabled` - boolean - if GPC is enabled or not (native apps only) - GPC can be disabled by user or by remote config - `expectReportURLPrefix` - string - resulting report URL should be prefixed with this string - `expectReportURLParams` - Array of `{name: '', value: ''}` objects - resulting report URL should have the following set of URL parameters with matching values + - `value` is an optional check for an exact value match. + - `match` is an optional check for a regex match. + - `present` is an optional check for the presence of a parameter. + - `matchesCurrentDay` is an optional check that the provided value is the current day when the test is ran (e.g., `2023-11-01`). - `remoteConfigEtag` - string - string representation of remote configuration etag - `remoteConfigVersion` - string - string representation of remote configuration version (note, this is the numeric version found in the remote config (e.g, `1680178584671`, not `v1` or `v2`)) - `providedDescription` - string - user-provided breakage description +All of these custom fields are supported by the `reports` array objects within the multiple_report_tests.json file. + ## Pseudo-code implementation ``` @@ -59,7 +65,57 @@ for $testSet in test.json if $test.expectReportURLParams for $param in $test.expectReportURLParams - expect($url.matchesRegex(/[?&] + $param.name + '=' + $param.value + [&$]/)) + if $param.value + expect($url.matchesRegex(/[?&] + $param.name + '=' + $param.value + [&$]/)) + if $param.match + $value = findParamValue($url, $param.name) + expect($value.matchesRegex($param.match)) + if $param.present + expect($url.matchesRegex(/[?&] + $param.name + '=[^&]*[&$]/)) + if $param.matchesCurrentDay + $value = findParamValue($url, $param.name) + expect($value = getCurrentDay()) +``` + +For multiple_report_tests.json: + +``` +for $testSet in test.json + + for $test in $testSet + if $test.exceptPlatforms includes 'current-platform' + skip + + for $report in $testSet.reports + $url = getReportURL( + siteURL=$report.siteURL, + wasUpgraded=$report.wasUpgraded, + reportCategory=$report.category, + blockedTrackers=$report.blockedTrackers, + surrogates=$report.surrogates, + atb=$report.atb, + blocklistVersion=$report.blocklistVersion, + manufacturer=$report.manufacturer, + model=$report.model, + os=$report.os, + gpcEnabled=$report.gpcEnabled + ) + + if $report.expectReportURLPrefix + expect($url.startsWith($report.expectReportURLPrefix)) + + if $report.expectReportURLParams + for $param in $report.expectReportURLParams + if $param.value + expect($url.matchesRegex(/[?&] + $param.name + '=' + $param.value + [&$]/)) + if $param.match + $value = findParamValue($url, $param.name) + expect($value.matchesRegex($param.match)) + if $param.present + expect($url.matchesRegex(/[?&] + $param.name + '=[^&]*[&$]/)) + if $param.matchesCurrentDay + $value = findParamValue($url, $param.name) + expect($value = getCurrentDay()) ``` ## Platform exceptions diff --git a/broken-site-reporting/multiple_report_tests.json b/broken-site-reporting/multiple_report_tests.json new file mode 100644 index 0000000..5f249d3 --- /dev/null +++ b/broken-site-reporting/multiple_report_tests.json @@ -0,0 +1,126 @@ +{ + "reportURL": { + "name": "Broken site report testing", + "tests": [ + { + "name": "Simple test with a common set of fields (without config version)", + "reports": [ + { + "siteURL": "https://example.test/", + "wasUpgraded": true, + "category": "content", + "blockedTrackers": ["bad.tracker.test", "tracking.test"], + "surrogates": ["surrogate.domain.test", "domain2.test"], + "atb": "v123-456g", + "blocklistVersion": "abc123", + "remoteConfigEtag": "abd142", + "remoteConfigVersion": "1234", + "expectReportURLPrefix": "https://improving.duckduckgo.com/t/epbf", + "expectReportURLParams": [ + {"name": "category", "value": "content"}, + {"name": "siteUrl", "value": "https%3A%2F%2Fexample.test%2F"}, + {"name": "upgradedHttps", "value": "true"}, + {"name": "tds", "value": "abc123"}, + {"name": "blockedTrackers", "value": "bad.tracker.test,tracking.test"}, + {"name": "surrogates", "value": "surrogate.domain.test,domain2.test"}, + {"name": "lastSentDay", "present": false} + ] + }, + { + "siteURL": "https://example.test/", + "wasUpgraded": true, + "category": "content", + "blockedTrackers": ["bad.tracker.test", "tracking.test"], + "surrogates": ["surrogate.domain.test", "domain2.test"], + "atb": "v123-456g", + "blocklistVersion": "abc123", + "remoteConfigEtag": "abd142", + "remoteConfigVersion": "1234", + "expectReportURLPrefix": "https://improving.duckduckgo.com/t/epbf", + "expectReportURLParams": [ + {"name": "category", "value": "content"}, + {"name": "siteUrl", "value": "https%3A%2F%2Fexample.test%2F"}, + {"name": "upgradedHttps", "value": "true"}, + {"name": "tds", "value": "abc123"}, + {"name": "remoteConfigEtag", "value": "abd142"}, + {"name": "remoteConfigVersion", "value": "1234"}, + {"name": "blockedTrackers", "value": "bad.tracker.test,tracking.test"}, + {"name": "surrogates", "value": "surrogate.domain.test,domain2.test"}, + {"name": "lastSentDay", "present": true, "matchesCurrentDay": true, "matches": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"} + ] + }, + { + "siteURL": "https://sub.example.test/", + "wasUpgraded": true, + "category": "content", + "blockedTrackers": ["bad.tracker.test", "tracking.test"], + "surrogates": ["surrogate.domain.test", "domain2.test"], + "atb": "v123-456g", + "blocklistVersion": "abc123", + "remoteConfigEtag": "abd142", + "remoteConfigVersion": "1234", + "expectReportURLPrefix": "https://improving.duckduckgo.com/t/epbf", + "expectReportURLParams": [ + {"name": "category", "value": "content"}, + {"name": "siteUrl", "value": "https%3A%2F%2Fsub.example.test%2F"}, + {"name": "upgradedHttps", "value": "true"}, + {"name": "tds", "value": "abc123"}, + {"name": "remoteConfigEtag", "value": "abd142"}, + {"name": "remoteConfigVersion", "value": "1234"}, + {"name": "blockedTrackers", "value": "bad.tracker.test,tracking.test"}, + {"name": "surrogates", "value": "surrogate.domain.test,domain2.test"}, + {"name": "lastSentDay", "present": false} + ] + }, + { + "siteURL": "https://sub.example.test/", + "wasUpgraded": true, + "category": "content", + "blockedTrackers": ["bad.tracker.test", "tracking.test"], + "surrogates": ["surrogate.domain.test", "domain2.test"], + "atb": "v123-456g", + "blocklistVersion": "abc123", + "remoteConfigEtag": "abd142", + "remoteConfigVersion": "1234", + "expectReportURLPrefix": "https://improving.duckduckgo.com/t/epbf", + "expectReportURLParams": [ + {"name": "category", "value": "content"}, + {"name": "siteUrl", "value": "https%3A%2F%2Fsub.example.test%2F"}, + {"name": "upgradedHttps", "value": "true"}, + {"name": "tds", "value": "abc123"}, + {"name": "remoteConfigEtag", "value": "abd142"}, + {"name": "remoteConfigVersion", "value": "1234"}, + {"name": "blockedTrackers", "value": "bad.tracker.test,tracking.test"}, + {"name": "surrogates", "value": "surrogate.domain.test,domain2.test"}, + {"name": "lastSentDay", "present": true, "matchesCurrentDay": true, "matches": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"} + ] + }, + { + "siteURL": "https://sub.other.test/", + "wasUpgraded": true, + "category": "content", + "blockedTrackers": ["bad.tracker.test", "tracking.test"], + "surrogates": ["surrogate.domain.test", "domain2.test"], + "atb": "v123-456g", + "blocklistVersion": "abc123", + "remoteConfigEtag": "abd142", + "remoteConfigVersion": "1234", + "expectReportURLPrefix": "https://improving.duckduckgo.com/t/epbf", + "expectReportURLParams": [ + {"name": "category", "value": "content"}, + {"name": "siteUrl", "value": "https%3A%2F%2Fsub.other.test%2F"}, + {"name": "upgradedHttps", "value": "true"}, + {"name": "tds", "value": "abc123"}, + {"name": "remoteConfigEtag", "value": "abd142"}, + {"name": "remoteConfigVersion", "value": "1234"}, + {"name": "blockedTrackers", "value": "bad.tracker.test,tracking.test"}, + {"name": "surrogates", "value": "surrogate.domain.test,domain2.test"}, + {"name": "lastSentDay", "present": false} + ] + } + ], + "exceptPlatforms": [] + } + ] + } +}