From 4164f19f0376fda7c61502cbfee2f8c7037508c5 Mon Sep 17 00:00:00 2001 From: Anna Scholtz Date: Thu, 9 Oct 2025 12:42:14 -0700 Subject: [PATCH 1/3] [DENG-9376] Publish DOH adoption and usage rates --- .../telemetry/doh_adoption_rate/view.sql | 7 + .../doh_adoption_rate_v1/metadata.yaml | 22 ++ .../doh_adoption_rate_v1/query.sql | 262 ++++++++++++++++++ .../doh_adoption_rate_v1/schema.yaml | 0 4 files changed, 291 insertions(+) create mode 100644 sql/moz-fx-data-shared-prod/telemetry/doh_adoption_rate/view.sql create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/doh_adoption_rate_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/doh_adoption_rate_v1/query.sql create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/doh_adoption_rate_v1/schema.yaml diff --git a/sql/moz-fx-data-shared-prod/telemetry/doh_adoption_rate/view.sql b/sql/moz-fx-data-shared-prod/telemetry/doh_adoption_rate/view.sql new file mode 100644 index 00000000000..8ea94b432c3 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry/doh_adoption_rate/view.sql @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.telemetry.doh_adoption_rate` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.telemetry_derived.doh_adoption_rate_v1` diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/doh_adoption_rate_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/doh_adoption_rate_v1/metadata.yaml new file mode 100644 index 00000000000..20b15608f26 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/doh_adoption_rate_v1/metadata.yaml @@ -0,0 +1,22 @@ +friendly_name: DOH Adoption Rate +description: |- + A distinct client count of a combination of metrics to measure + the adoption of DOH (DNS over HTTPS) in Firefox. +owners: + - ascholtz@mozilla.com +labels: + incremental: true + owner1: ascholtz + public_bigquery: true + public_json: true + review_bugs: + - '1982897' +scheduling: + dag_name: bqetl_ech_adoption_rate +bigquery: + time_partitioning: + type: day + field: 'submission_date' + require_partition_filter: true + expiration_days: null + range_partitioning: null diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/doh_adoption_rate_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/doh_adoption_rate_v1/query.sql new file mode 100644 index 00000000000..40185d495dd --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/doh_adoption_rate_v1/query.sql @@ -0,0 +1,262 @@ +WITH unioned AS ( + /* ---------- doh.evaluate_v2_heuristics ---------- */ + SELECT + DATE(submission_timestamp) AS submission_date, + 'doh.evaluate_v2_heuristics' AS metric, + "value" AS key, + COALESCE(extra.value, '') AS value, + CASE + WHEN COUNT(DISTINCT client_info.client_id) >= 5000 + THEN normalized_country_code + ELSE 'OTHER' + END AS country_code, + COUNT(DISTINCT client_info.client_id) AS client_count + FROM + `moz-fx-data-shared-prod.firefox_desktop.events` AS e, + UNNEST(e.events) AS event, + UNNEST(event.extra) AS extra + WHERE + DATE(submission_timestamp) = @submission_date + AND client_info.client_id IS NOT NULL + AND client_info.app_channel = 'release' + AND event.category = 'doh' + AND event.name = 'evaluate_v2_heuristics' + AND extra.key = 'value' + GROUP BY + submission_date, + key, + value, + normalized_country_code + UNION ALL + /* ---------- doh.state_disabled ---------- */ + SELECT + DATE(submission_timestamp) AS submission_date, + 'doh.state_disabled' AS metric, + '' AS key, + '' AS value, + CASE + WHEN COUNT(DISTINCT client_info.client_id) >= 5000 + THEN normalized_country_code + ELSE 'OTHER' + END AS country_code, + COUNT(DISTINCT client_info.client_id) AS client_count + FROM + `moz-fx-data-shared-prod.firefox_desktop.events` AS e, + UNNEST(e.events) AS event + WHERE + DATE(submission_timestamp) = @submission_date + AND client_info.client_id IS NOT NULL + AND client_info.app_channel = 'release' + AND event.category = 'doh' + AND event.name = 'state_disabled' + GROUP BY + submission_date, + key, + value, + normalized_country_code + UNION ALL + /* ---------- doh.state_enabled ---------- */ + SELECT + DATE(submission_timestamp) AS submission_date, + 'doh.state_enabled' AS metric, + '' AS key, + '' AS value, + CASE + WHEN COUNT(DISTINCT client_info.client_id) >= 5000 + THEN normalized_country_code + ELSE 'OTHER' + END AS country_code, + COUNT(DISTINCT client_info.client_id) AS client_count + FROM + `moz-fx-data-shared-prod.firefox_desktop.events` AS e, + UNNEST(e.events) AS event + WHERE + DATE(submission_timestamp) = @submission_date + AND client_info.client_id IS NOT NULL + AND client_info.app_channel = 'release' + AND event.category = 'doh' + AND event.name = 'state_enabled' + GROUP BY + submission_date, + key, + value, + normalized_country_code + UNION ALL + /* ---------- doh.state_manually_disabled ---------- */ + SELECT + DATE(submission_timestamp) AS submission_date, + 'doh.state_manually_disabled' AS metric, + '' AS key, + '' AS value, + CASE + WHEN COUNT(DISTINCT client_info.client_id) >= 5000 + THEN normalized_country_code + ELSE 'OTHER' + END AS country_code, + COUNT(DISTINCT client_info.client_id) AS client_count + FROM + `moz-fx-data-shared-prod.firefox_desktop.events` AS e, + UNNEST(e.events) AS event + WHERE + DATE(submission_timestamp) = @submission_date + AND client_info.client_id IS NOT NULL + AND client_info.app_channel = 'release' + AND event.category = 'doh' + AND event.name = 'state_manually_disabled' + GROUP BY + submission_date, + key, + value, + normalized_country_code + UNION ALL + /* ---------- doh.state_policy_disabled ---------- */ + SELECT + DATE(submission_timestamp) AS submission_date, + 'doh.state_policy_disabled' AS metric, + '' AS key, + '' AS value, + CASE + WHEN COUNT(DISTINCT client_info.client_id) >= 5000 + THEN normalized_country_code + ELSE 'OTHER' + END AS country_code, + COUNT(DISTINCT client_info.client_id) AS client_count + FROM + `moz-fx-data-shared-prod.firefox_desktop.events` AS e, + UNNEST(e.events) AS event + WHERE + DATE(submission_timestamp) = @submission_date + AND client_info.client_id IS NOT NULL + AND client_info.app_channel = 'release' + AND event.category = 'doh' + AND event.name = 'state_policy_disabled' + GROUP BY + submission_date, + key, + value, + normalized_country_code + UNION ALL + /* ---------- networking.doh_heuristics_attempts ---------- */ + SELECT + DATE(submission_timestamp) AS submission_date, + 'networking.doh_heuristics_attempts' AS metric, + 'networking.doh_heuristics_attempts' AS key, + CAST(metrics.counter.networking_doh_heuristics_attempts AS STRING) AS value, + CASE + WHEN COUNT(DISTINCT client_info.client_id) >= 5000 + THEN metadata.geo.country + ELSE 'OTHER' + END AS country_code, + COUNT(DISTINCT client_info.client_id) AS client_count + FROM + `moz-fx-data-shared-prod.firefox_desktop.metrics` + WHERE + DATE(submission_timestamp) = @submission_date + AND client_info.client_id IS NOT NULL + AND client_info.app_channel = 'release' + AND metrics.counter.networking_doh_heuristics_attempts IS NOT NULL + GROUP BY + submission_date, + key, + value, + metadata.geo.country + UNION ALL + /* ---------- networking.doh_heuristics_pass_count ---------- */ + SELECT + DATE(submission_timestamp) AS submission_date, + 'networking.doh_heuristics_pass_count' AS metric, + 'networking.doh_heuristics_attempts' AS key, + CAST(metrics.counter.networking_doh_heuristics_pass_count AS STRING) AS value, + CASE + WHEN COUNT(DISTINCT client_info.client_id) >= 5000 + THEN metadata.geo.country + ELSE 'OTHER' + END AS country_code, + COUNT(DISTINCT client_info.client_id) AS client_count + FROM + `moz-fx-data-shared-prod.firefox_desktop.metrics` + WHERE + DATE(submission_timestamp) = @submission_date + AND client_info.client_id IS NOT NULL + AND client_info.app_channel = 'release' + AND metrics.counter.networking_doh_heuristics_pass_count IS NOT NULL + GROUP BY + submission_date, + key, + value, + metadata.geo.country + UNION ALL + /* ---------- networking.doh_heuristics_result ---------- */ + SELECT + DATE(submission_timestamp) AS submission_date, + 'networking.doh_heuristics_result' AS metric, + 'networking.doh_heuristics_result' AS key, + CAST(metrics.quantity.networking_doh_heuristics_result AS STRING) AS value, + CASE + WHEN COUNT(DISTINCT client_info.client_id) >= 5000 + THEN metadata.geo.country + ELSE 'OTHER' + END AS country_code, + COUNT(DISTINCT client_info.client_id) AS client_count + FROM + `moz-fx-data-shared-prod.firefox_desktop.metrics` + WHERE + DATE(submission_timestamp) = @submission_date + AND client_info.client_id IS NOT NULL + AND client_info.app_channel = 'release' + AND metrics.quantity.networking_doh_heuristics_result IS NOT NULL + GROUP BY + submission_date, + key, + value, + metadata.geo.country + UNION ALL + /* ---------- security.doh.settings.provider_choice_value ---------- */ + SELECT + DATE(submission_timestamp) AS submission_date, + 'security.doh.settings.provider_choice_value' AS metric, + "value" AS key, + COALESCE(extra.value, '') AS value, + CASE + WHEN COUNT(DISTINCT client_info.client_id) >= 5000 + THEN normalized_country_code + ELSE 'OTHER' + END AS country_code, + COUNT(DISTINCT client_info.client_id) AS client_count + FROM + `moz-fx-data-shared-prod.firefox_desktop.events` AS e, + UNNEST(e.events) AS event, + UNNEST(event.extra) AS extra + WHERE + DATE(submission_timestamp) = @submission_date + AND client_info.client_id IS NOT NULL + AND client_info.app_channel = 'release' + AND event.category = 'security.doh.settings' + AND event.name = 'provider_choice_value' + AND extra.key = 'value' + GROUP BY + submission_date, + key, + value, + normalized_country_code +) +SELECT + submission_date, + metric key, + value, + country_code, + SUM(client_count) AS total_client_count +FROM + unioned +GROUP BY + submission_date, + metric, + key, + value, + country_code +ORDER BY + submission_date DESC, + metric, + country_code, + key, + value diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/doh_adoption_rate_v1/schema.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/doh_adoption_rate_v1/schema.yaml new file mode 100644 index 00000000000..e69de29bb2d From 12406eb1f9798ef9b5e044039b927487d24af9aa Mon Sep 17 00:00:00 2001 From: Anna Scholtz Date: Thu, 16 Oct 2025 13:40:11 -0700 Subject: [PATCH 2/3] Add schema and use events_stream table --- .../doh_adoption_rate_v1/query.sql | 86 ++++++++----------- .../doh_adoption_rate_v1/schema.yaml | 39 +++++++++ 2 files changed, 77 insertions(+), 48 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/doh_adoption_rate_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/doh_adoption_rate_v1/query.sql index 40185d495dd..945baf71680 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/doh_adoption_rate_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/doh_adoption_rate_v1/query.sql @@ -4,24 +4,21 @@ WITH unioned AS ( DATE(submission_timestamp) AS submission_date, 'doh.evaluate_v2_heuristics' AS metric, "value" AS key, - COALESCE(extra.value, '') AS value, + COALESCE(STRING(event_extra.value), '') AS value, CASE - WHEN COUNT(DISTINCT client_info.client_id) >= 5000 + WHEN COUNT(DISTINCT client_id) >= 5000 THEN normalized_country_code ELSE 'OTHER' END AS country_code, - COUNT(DISTINCT client_info.client_id) AS client_count + COUNT(DISTINCT client_id) AS client_count FROM - `moz-fx-data-shared-prod.firefox_desktop.events` AS e, - UNNEST(e.events) AS event, - UNNEST(event.extra) AS extra + `moz-fx-data-shared-prod.firefox_desktop.events_stream` WHERE DATE(submission_timestamp) = @submission_date - AND client_info.client_id IS NOT NULL + AND client_id IS NOT NULL AND client_info.app_channel = 'release' - AND event.category = 'doh' - AND event.name = 'evaluate_v2_heuristics' - AND extra.key = 'value' + AND event_category = 'doh' + AND event_name = 'evaluate_v2_heuristics' GROUP BY submission_date, key, @@ -35,20 +32,19 @@ WITH unioned AS ( '' AS key, '' AS value, CASE - WHEN COUNT(DISTINCT client_info.client_id) >= 5000 + WHEN COUNT(DISTINCT client_id) >= 5000 THEN normalized_country_code ELSE 'OTHER' END AS country_code, - COUNT(DISTINCT client_info.client_id) AS client_count + COUNT(DISTINCT client_id) AS client_count FROM - `moz-fx-data-shared-prod.firefox_desktop.events` AS e, - UNNEST(e.events) AS event + `moz-fx-data-shared-prod.firefox_desktop.events_stream` WHERE DATE(submission_timestamp) = @submission_date - AND client_info.client_id IS NOT NULL + AND client_id IS NOT NULL AND client_info.app_channel = 'release' - AND event.category = 'doh' - AND event.name = 'state_disabled' + AND event_category = 'doh' + AND event_name = 'state_disabled' GROUP BY submission_date, key, @@ -62,20 +58,19 @@ WITH unioned AS ( '' AS key, '' AS value, CASE - WHEN COUNT(DISTINCT client_info.client_id) >= 5000 + WHEN COUNT(DISTINCT client_id) >= 5000 THEN normalized_country_code ELSE 'OTHER' END AS country_code, - COUNT(DISTINCT client_info.client_id) AS client_count + COUNT(DISTINCT client_id) AS client_count FROM - `moz-fx-data-shared-prod.firefox_desktop.events` AS e, - UNNEST(e.events) AS event + `moz-fx-data-shared-prod.firefox_desktop.events_stream` WHERE DATE(submission_timestamp) = @submission_date - AND client_info.client_id IS NOT NULL + AND client_id IS NOT NULL AND client_info.app_channel = 'release' - AND event.category = 'doh' - AND event.name = 'state_enabled' + AND event_category = 'doh' + AND event_name = 'state_enabled' GROUP BY submission_date, key, @@ -89,20 +84,19 @@ WITH unioned AS ( '' AS key, '' AS value, CASE - WHEN COUNT(DISTINCT client_info.client_id) >= 5000 + WHEN COUNT(DISTINCT client_id) >= 5000 THEN normalized_country_code ELSE 'OTHER' END AS country_code, - COUNT(DISTINCT client_info.client_id) AS client_count + COUNT(DISTINCT client_id) AS client_count FROM - `moz-fx-data-shared-prod.firefox_desktop.events` AS e, - UNNEST(e.events) AS event + `moz-fx-data-shared-prod.firefox_desktop.events_stream` WHERE DATE(submission_timestamp) = @submission_date - AND client_info.client_id IS NOT NULL + AND client_id IS NOT NULL AND client_info.app_channel = 'release' - AND event.category = 'doh' - AND event.name = 'state_manually_disabled' + AND event_category = 'doh' + AND event_name = 'state_manually_disabled' GROUP BY submission_date, key, @@ -116,20 +110,19 @@ WITH unioned AS ( '' AS key, '' AS value, CASE - WHEN COUNT(DISTINCT client_info.client_id) >= 5000 + WHEN COUNT(DISTINCT client_id) >= 5000 THEN normalized_country_code ELSE 'OTHER' END AS country_code, - COUNT(DISTINCT client_info.client_id) AS client_count + COUNT(DISTINCT client_id) AS client_count FROM - `moz-fx-data-shared-prod.firefox_desktop.events` AS e, - UNNEST(e.events) AS event + `moz-fx-data-shared-prod.firefox_desktop.events_stream` WHERE DATE(submission_timestamp) = @submission_date - AND client_info.client_id IS NOT NULL + AND client_id IS NOT NULL AND client_info.app_channel = 'release' - AND event.category = 'doh' - AND event.name = 'state_policy_disabled' + AND event_category = 'doh' + AND event_name = 'state_policy_disabled' GROUP BY submission_date, key, @@ -216,24 +209,21 @@ WITH unioned AS ( DATE(submission_timestamp) AS submission_date, 'security.doh.settings.provider_choice_value' AS metric, "value" AS key, - COALESCE(extra.value, '') AS value, + COALESCE(STRING(event_extra.value), '') AS value, CASE - WHEN COUNT(DISTINCT client_info.client_id) >= 5000 + WHEN COUNT(DISTINCT client_id) >= 5000 THEN normalized_country_code ELSE 'OTHER' END AS country_code, - COUNT(DISTINCT client_info.client_id) AS client_count + COUNT(DISTINCT client_id) AS client_count FROM - `moz-fx-data-shared-prod.firefox_desktop.events` AS e, - UNNEST(e.events) AS event, - UNNEST(event.extra) AS extra + `moz-fx-data-shared-prod.firefox_desktop.events_stream` WHERE DATE(submission_timestamp) = @submission_date - AND client_info.client_id IS NOT NULL + AND client_id IS NOT NULL AND client_info.app_channel = 'release' - AND event.category = 'security.doh.settings' - AND event.name = 'provider_choice_value' - AND extra.key = 'value' + AND event_category = 'security.doh.settings' + AND event_name = 'provider_choice_value' GROUP BY submission_date, key, diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/doh_adoption_rate_v1/schema.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/doh_adoption_rate_v1/schema.yaml index e69de29bb2d..5e185808ea3 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/doh_adoption_rate_v1/schema.yaml +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/doh_adoption_rate_v1/schema.yaml @@ -0,0 +1,39 @@ +fields: +- mode: NULLABLE + name: submission_date + type: DATE + description: The date when the telemetry data was submitted to the server. +- mode: NULLABLE + name: metric + type: STRING + description: The specific DOH (DNS-over-HTTPS) metric being measured. Can be one + of doh.evaluate_v2_heuristics (DOH heuristics evaluation), doh.state_disabled + (DOH disabled state), doh.state_enabled (DOH enabled state), doh.state_manually_disabled + (DOH manually disabled by user), doh.state_policy_disabled (DOH disabled by policy), + networking.doh_heuristics_attempts (count of heuristics attempts), networking.doh_heuristics_pass_count + (count of heuristics that passed), networking.doh_heuristics_result (result code + of heuristics evaluation), or security.doh.settings.provider_choice_value (selected + DOH provider). +- mode: NULLABLE + name: key + type: STRING + description: The key or dimension associated with the metric value. For event-based + metrics with extra fields, this represents the field name (e.g., 'value'). For + counter/quantity metrics, this is typically the metric name itself. +- mode: NULLABLE + name: value + type: STRING + description: The value associated with the metric. For event extra fields, this + contains the extra field value. For counters and quantities, this contains the + stringified numeric value. May be empty for events without extra fields. +- mode: NULLABLE + name: country_code + type: STRING + description: The ISO country code where the client is located, normalized by geolocation. + Countries with fewer than 5000 distinct clients are aggregated into 'OTHER' for + privacy reasons. +- mode: NULLABLE + name: total_client_count + type: INTEGER + description: The total count of distinct clients (by client_id) that reported this + metric, key, value, and country combination on the submission date. From 98610607c04475d981ec611a265726d0177ac6a8 Mon Sep 17 00:00:00 2001 From: Anna Scholtz Date: Thu, 16 Oct 2025 14:12:27 -0700 Subject: [PATCH 3/3] Update sql/moz-fx-data-shared-prod/telemetry_derived/doh_adoption_rate_v1/query.sql Co-authored-by: Ben Wu <12437227+BenWu@users.noreply.github.com> --- .../telemetry_derived/doh_adoption_rate_v1/query.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/doh_adoption_rate_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/doh_adoption_rate_v1/query.sql index 945baf71680..2f1d4555f5b 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/doh_adoption_rate_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/doh_adoption_rate_v1/query.sql @@ -232,7 +232,8 @@ WITH unioned AS ( ) SELECT submission_date, - metric key, + metric, + key, value, country_code, SUM(client_count) AS total_client_count