From 111361f2b15af927d9b3be40e9b9f37115dd680f Mon Sep 17 00:00:00 2001 From: Rafa Audibert Date: Fri, 7 Mar 2025 22:52:37 -0300 Subject: [PATCH 1/3] feat: Use `currencyConvert` function when calculating revenue Let's now actually use the `currencyConvert` function when computing Revenue on the Web Analytics panel. We make a lot of effort to only actually try converting it if there's something to convert. --- .../hogql/database/schema/exchange_rate.py | 155 +++++++++++++ posthog/hogql_queries/utils/revenue.py | 95 -------- .../web_analytics/revenue_example_events.py | 45 +++- .../test_revenue_example_events.ambr | 211 ++++++++++++++++++ .../test/__snapshots__/test_web_overview.ambr | 54 ++++- .../test/test_revenue_example_events.py | 92 +++++++- .../web_analytics/test/test_web_overview.py | 53 ++++- .../web_analytics_query_runner.py | 19 +- 8 files changed, 608 insertions(+), 116 deletions(-) delete mode 100644 posthog/hogql_queries/utils/revenue.py create mode 100644 posthog/hogql_queries/web_analytics/test/__snapshots__/test_revenue_example_events.ambr diff --git a/posthog/hogql/database/schema/exchange_rate.py b/posthog/hogql/database/schema/exchange_rate.py index b10a0f666a664..a465503d054f5 100644 --- a/posthog/hogql/database/schema/exchange_rate.py +++ b/posthog/hogql/database/schema/exchange_rate.py @@ -1,3 +1,7 @@ +from typing import Union + +from posthog.hogql import ast +from posthog.schema import CurrencyCode, RevenueTrackingConfig, RevenueTrackingEventItem from posthog.hogql.database.models import ( StringDatabaseField, DateDatabaseField, @@ -19,3 +23,154 @@ def to_printed_clickhouse(self, context): def to_printed_hogql(self): return "exchange_rate" + + +def convert_currency_call( + amount: ast.Expr, currency_from: ast.Expr, currency_to: ast.Expr, timestamp: ast.Expr | None = None +) -> ast.Expr: + args = [currency_from, currency_to, amount] + if timestamp: + args.append(timestamp) + + return ast.Call(name="convertCurrency", args=args) + + +def revenue_currency_expression(config: RevenueTrackingConfig) -> ast.Expr: + exprs = [] + for event in config.events: + exprs.extend( + [ + ast.CompareOperation( + left=ast.Field(chain=["event"]), + op=ast.CompareOperationOp.Eq, + right=ast.Constant(value=event.eventName), + ), + ast.Field(chain=["events", "properties", event.revenueCurrencyProperty]) + if event.revenueCurrencyProperty + else ast.Constant(value=None), + ] + ) + + if len(exprs) == 0: + return ast.Constant(value=None) + + # Else clause, make sure there's a None at the end + exprs.append(ast.Constant(value=None)) + + return ast.Call(name="multiIf", args=exprs) + + +def revenue_comparison_and_value_exprs( + event: RevenueTrackingEventItem, + config: RevenueTrackingConfig, + do_currency_conversion: bool = False, +) -> tuple[ast.Expr, ast.Expr]: + # Check whether the event is the one we're looking for + comparison_expr = ast.CompareOperation( + left=ast.Field(chain=["event"]), + op=ast.CompareOperationOp.Eq, + right=ast.Constant(value=event.eventName), + ) + + # If there's a revenueCurrencyProperty, convert the revenue to the base currency from that property + # Otherwise, assume we're already in the base currency + # Also, assume that `base_currency` is USD by default, it'll be empty for most customers + if event.revenueCurrencyProperty and do_currency_conversion: + value_expr = ast.Call( + name="if", + args=[ + ast.Call( + name="isNull", args=[ast.Field(chain=["events", "properties", event.revenueCurrencyProperty])] + ), + ast.Call( + name="toDecimal", + args=[ + ast.Field(chain=["events", "properties", event.revenueProperty]), + ast.Constant(value=10), + ], + ), + convert_currency_call( + ast.Field(chain=["events", "properties", event.revenueProperty]), + ast.Field(chain=["events", "properties", event.revenueCurrencyProperty]), + ast.Constant(value=(config.baseCurrency or CurrencyCode.USD).value), + ast.Call(name="DATE", args=[ast.Field(chain=["events", "timestamp"])]), + ), + ], + ) + else: + value_expr = ast.Call( + name="toDecimal", + args=[ast.Field(chain=["events", "properties", event.revenueProperty]), ast.Constant(value=10)], + ) + + return (comparison_expr, value_expr) + + +def revenue_expression( + config: Union[RevenueTrackingConfig, dict, None], + do_currency_conversion: bool = False, +) -> ast.Expr: + if isinstance(config, dict): + config = RevenueTrackingConfig.model_validate(config) + + if not config or not config.events: + return ast.Constant(value=None) + + exprs: list[ast.Expr] = [] + for event in config.events: + comparison_expr, value_expr = revenue_comparison_and_value_exprs(event, config, do_currency_conversion) + exprs.extend([comparison_expr, value_expr]) + + # Else clause, make sure there's a None at the end + exprs.append(ast.Constant(value=None)) + + return ast.Call(name="multiIf", args=exprs) + + +def revenue_sum_expression( + config: Union[RevenueTrackingConfig, dict, None], + do_currency_conversion: bool = False, +) -> ast.Expr: + if isinstance(config, dict): + config = RevenueTrackingConfig.model_validate(config) + + if not config or not config.events: + return ast.Constant(value=None) + + exprs: list[ast.Expr] = [] + for event in config.events: + comparison_expr, value_expr = revenue_comparison_and_value_exprs(event, config, do_currency_conversion) + + exprs.append( + ast.Call( + name="sumIf", + args=[ + ast.Call(name="ifNull", args=[value_expr, ast.Constant(value=0)]), + comparison_expr, + ], + ) + ) + + if len(exprs) == 1: + return exprs[0] + + return ast.Call(name="plus", args=exprs) + + +def revenue_events_where_expr(config: Union[RevenueTrackingConfig, dict, None]) -> ast.Expr: + if isinstance(config, dict): + config = RevenueTrackingConfig.model_validate(config) + + if not config or not config.events: + return ast.Constant(value=False) + + exprs: list[ast.Expr] = [] + for event in config.events: + # NOTE: Dont care about conversion, only care about comparison which is independent of conversion + comparison_expr, _value_expr = revenue_comparison_and_value_exprs(event, config, do_currency_conversion=False) + exprs.append(comparison_expr) + + if len(exprs) == 1: + return exprs[0] + + return ast.Or(exprs=exprs) diff --git a/posthog/hogql_queries/utils/revenue.py b/posthog/hogql_queries/utils/revenue.py deleted file mode 100644 index 8109afe6d41cd..0000000000000 --- a/posthog/hogql_queries/utils/revenue.py +++ /dev/null @@ -1,95 +0,0 @@ -from typing import Union - -from posthog.hogql import ast -from posthog.schema import RevenueTrackingConfig - - -def revenue_expression(config: Union[RevenueTrackingConfig, dict, None]) -> ast.Expr: - if isinstance(config, dict): - config = RevenueTrackingConfig.model_validate(config) - - if not config or not config.events: - return ast.Constant(value=None) - - exprs: list[ast.Expr] = [] - for event in config.events: - exprs.append( - ast.CompareOperation( - left=ast.Field(chain=["event"]), - op=ast.CompareOperationOp.Eq, - right=ast.Constant(value=event.eventName), - ) - ) - exprs.append( - ast.Call( - name="toFloat", - args=[ast.Field(chain=["events", "properties", event.revenueProperty])], - ) - ) - exprs.append(ast.Constant(value=None)) - - return ast.Call(name="multiIf", args=exprs) - - -def revenue_sum_expression(config: Union[RevenueTrackingConfig, dict, None]) -> ast.Expr: - if isinstance(config, dict): - config = RevenueTrackingConfig.model_validate(config) - - if not config or not config.events: - return ast.Constant(value=None) - - exprs: list[ast.Expr] = [] - for event in config.events: - exprs.append( - ast.Call( - name="sumIf", - args=[ - ast.Call( - name="ifNull", - args=[ - ast.Call( - name="toFloat", - args=[ast.Field(chain=["events", "properties", event.revenueProperty])], - ), - ast.Constant(value=0), - ], - ), - ast.CompareOperation( - left=ast.Field(chain=["event"]), - op=ast.CompareOperationOp.Eq, - right=ast.Constant(value=event.eventName), - ), - ], - ) - ) - if len(exprs) == 1: - return exprs[0] - return ast.Call(name="plus", args=exprs) - - -def revenue_events_exprs(config: Union[RevenueTrackingConfig, dict, None]) -> list[ast.Expr]: - if isinstance(config, dict): - config = RevenueTrackingConfig.model_validate(config) - - if not config or not config.events: - return [] - - exprs: list[ast.Expr] = [] - for event in config.events: - exprs.append( - ast.CompareOperation( - left=ast.Field(chain=["event"]), - op=ast.CompareOperationOp.Eq, - right=ast.Constant(value=event.eventName), - ) - ) - return exprs - - -def revenue_events_expr(config: Union[RevenueTrackingConfig, dict, None]) -> ast.Expr: - exprs = revenue_events_exprs(config) - if not exprs: - return ast.Constant(value=False) - if len(exprs) == 1: - return exprs[0] - return ast.Or(exprs=exprs) diff --git a/posthog/hogql_queries/web_analytics/revenue_example_events.py b/posthog/hogql_queries/web_analytics/revenue_example_events.py index 657f31afb944c..67bcedb0b8ef2 100644 --- a/posthog/hogql_queries/web_analytics/revenue_example_events.py +++ b/posthog/hogql_queries/web_analytics/revenue_example_events.py @@ -1,12 +1,18 @@ import json +import posthoganalytics from posthog.hogql import ast from posthog.hogql.ast import CompareOperationOp from posthog.hogql.constants import LimitContext from posthog.hogql_queries.insights.paginators import HogQLHasMorePaginator from posthog.hogql_queries.query_runner import QueryRunner -from posthog.hogql_queries.utils.revenue import revenue_expression, revenue_events_expr +from posthog.hogql.database.schema.exchange_rate import ( + revenue_expression, + revenue_events_where_expr, + revenue_currency_expression, +) from posthog.schema import ( + CurrencyCode, RevenueExampleEventsQuery, RevenueExampleEventsQueryResponse, CachedRevenueExampleEventsQueryResponse, @@ -18,6 +24,7 @@ class RevenueExampleEventsQueryRunner(QueryRunner): response: RevenueExampleEventsQueryResponse cached_response: CachedRevenueExampleEventsQueryResponse paginator: HogQLHasMorePaginator + do_currency_conversion: bool = False def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -25,6 +32,13 @@ def __init__(self, *args, **kwargs): limit_context=LimitContext.QUERY, limit=self.query.limit if self.query.limit else None ) + self.do_currency_conversion = posthoganalytics.feature_enabled( + "web-analytics-revenue-tracking-conversion", + str(self.team.organization_id), + groups={"organization": str(self.team.organization_id)}, + group_properties={"organization": {"id": str(self.team.organization_id)}}, + ) + def to_query(self) -> ast.SelectQuery: tracking_config = self.query.revenueTrackingConfig @@ -40,7 +54,14 @@ def to_query(self) -> ast.SelectQuery: ], ), ast.Field(chain=["event"]), - ast.Alias(alias="revenue", expr=revenue_expression(tracking_config)), + ast.Alias( + alias="original_revenue", expr=revenue_expression(tracking_config, do_currency_conversion=False) + ), + ast.Alias(alias="revenue", expr=revenue_expression(tracking_config, self.do_currency_conversion)), + ast.Alias(alias="original_currency", expr=revenue_currency_expression(tracking_config)), + ast.Alias( + alias="currency", expr=ast.Constant(value=(tracking_config.baseCurrency or CurrencyCode.USD).value) + ), ast.Call( name="tuple", args=[ @@ -56,7 +77,7 @@ def to_query(self) -> ast.SelectQuery: select_from=ast.JoinExpr(table=ast.Field(chain=["events"])), where=ast.And( exprs=[ - revenue_events_expr(tracking_config), + revenue_events_where_expr(tracking_config), ast.CompareOperation( op=CompareOperationOp.NotEq, left=ast.Field(chain=["revenue"]), # refers to the Alias above @@ -88,14 +109,17 @@ def calculate(self): }, row[1], row[2], - { - "id": row[3][0], - "created_at": row[3][1], - "distinct_id": row[3][2], - "properties": json.loads(row[3][3]), - }, + row[3], row[4], row[5], + { + "id": row[6][0], + "created_at": row[6][1], + "distinct_id": row[6][2], + "properties": json.loads(row[6][3]), + }, + row[7], + row[8], ) for row in response.results ] @@ -104,7 +128,10 @@ def calculate(self): columns=[ "*", "event", + "original_revenue", "revenue", + "original_revenue_currency", + "revenue_currency", "person", "session_id", "timestamp", diff --git a/posthog/hogql_queries/web_analytics/test/__snapshots__/test_revenue_example_events.ambr b/posthog/hogql_queries/web_analytics/test/__snapshots__/test_revenue_example_events.ambr new file mode 100644 index 0000000000000..9aba2b017fc23 --- /dev/null +++ b/posthog/hogql_queries/web_analytics/test/__snapshots__/test_revenue_example_events.ambr @@ -0,0 +1,211 @@ +# serializer version: 1 +# name: TestRevenueExampleEventsQueryRunner.test_multiple_events + ''' + SELECT tuple(events.uuid, events.event, events.distinct_id, events.properties), + events.event AS event, + multiIf(equals(events.event, 'purchase_a'), toDecimal64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'revenue_a'), ''), 'null'), '^"|"$', ''), 10), equals(events.event, 'purchase_b'), toDecimal64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'revenue_b'), ''), 'null'), '^"|"$', ''), 10), NULL) AS original_revenue, + multiIf(equals(events.event, 'purchase_a'), toDecimal64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'revenue_a'), ''), 'null'), '^"|"$', ''), 10), equals(events.event, 'purchase_b'), toDecimal64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'revenue_b'), ''), 'null'), '^"|"$', ''), 10), NULL) AS revenue, + multiIf(equals(events.event, 'purchase_a'), NULL, equals(events.event, 'purchase_b'), NULL, NULL) AS original_currency, + 'USD' AS currency, + tuple(events__person.id, events__person.created_at, events.distinct_id, events__person.properties), + nullIf(nullIf(events.`$session_id`, ''), 'null') AS session_id, + toTimeZone(events.timestamp, 'UTC') AS timestamp + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, + toTimeZone(person.created_at, 'UTC') AS created_at, + person.properties AS properties + FROM person + WHERE and(equals(person.team_id, 99999), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 99999) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS events__person ON equals(if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id), events__person.id) + WHERE and(equals(events.team_id, 99999), or(equals(events.event, 'purchase_a'), equals(events.event, 'purchase_b')), isNotNull(revenue)) + ORDER BY toTimeZone(events.timestamp, 'UTC') DESC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestRevenueExampleEventsQueryRunner.test_no_crash_when_no_data + ''' + SELECT tuple(events.uuid, events.event, events.distinct_id, events.properties), + events.event AS event, + NULL AS original_revenue, + NULL AS revenue, + NULL AS original_currency, + 'USD' AS currency, + tuple(events__person.id, events__person.created_at, events.distinct_id, events__person.properties), + nullIf(nullIf(events.`$session_id`, ''), 'null') AS session_id, + toTimeZone(events.timestamp, 'UTC') AS timestamp + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, + toTimeZone(person.created_at, 'UTC') AS created_at, + person.properties AS properties + FROM person + WHERE and(equals(person.team_id, 99999), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 99999) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS events__person ON equals(if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id), events__person.id) + WHERE and(equals(events.team_id, 99999), 0, isNotNull(revenue)) + ORDER BY toTimeZone(events.timestamp, 'UTC') DESC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestRevenueExampleEventsQueryRunner.test_revenue_currency_property + ''' + SELECT tuple(events.uuid, events.event, events.distinct_id, events.properties), + events.event AS event, + multiIf(equals(events.event, 'purchase_a'), toDecimal64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'revenue_a'), ''), 'null'), '^"|"$', ''), 10), equals(events.event, 'purchase_b'), toDecimal64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'revenue_b'), ''), 'null'), '^"|"$', ''), 10), NULL) AS original_revenue, + multiIf(equals(events.event, 'purchase_a'), if(isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'currency_a'), ''), 'null'), '^"|"$', '')), toDecimal64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'revenue_a'), ''), 'null'), '^"|"$', ''), 10), if(dictGetOrDefault(`posthog_test`.`exchange_rate_dict`, 'rate', replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'currency_a'), ''), 'null'), '^"|"$', ''), toDate(toTimeZone(events.timestamp, 'UTC')), toDecimal64(0, 10)) = 0, toDecimal64(0, 10), multiplyDecimal(divideDecimal(toDecimal64(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'revenue_a'), ''), 'null'), '^"|"$', ''), 10), dictGetOrDefault(`posthog_test`.`exchange_rate_dict`, 'rate', replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'currency_a'), ''), 'null'), '^"|"$', ''), toDate(toTimeZone(events.timestamp, 'UTC')), toDecimal64(0, 10))), dictGetOrDefault(`posthog_test`.`exchange_rate_dict`, 'rate', 'EUR', toDate(toTimeZone(events.timestamp, 'UTC')), toDecimal64(0, 10))))), equals(events.event, 'purchase_b'), if(isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'currency_b'), ''), 'null'), '^"|"$', '')), toDecimal64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'revenue_b'), ''), 'null'), '^"|"$', ''), 10), if(dictGetOrDefault(`posthog_test`.`exchange_rate_dict`, 'rate', replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'currency_b'), ''), 'null'), '^"|"$', ''), toDate(toTimeZone(events.timestamp, 'UTC')), toDecimal64(0, 10)) = 0, toDecimal64(0, 10), multiplyDecimal(divideDecimal(toDecimal64(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'revenue_b'), ''), 'null'), '^"|"$', ''), 10), dictGetOrDefault(`posthog_test`.`exchange_rate_dict`, 'rate', replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'currency_b'), ''), 'null'), '^"|"$', ''), toDate(toTimeZone(events.timestamp, 'UTC')), toDecimal64(0, 10))), dictGetOrDefault(`posthog_test`.`exchange_rate_dict`, 'rate', 'EUR', toDate(toTimeZone(events.timestamp, 'UTC')), toDecimal64(0, 10))))), NULL) AS revenue, + multiIf(equals(events.event, 'purchase_a'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'currency_a'), ''), 'null'), '^"|"$', ''), equals(events.event, 'purchase_b'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'currency_b'), ''), 'null'), '^"|"$', ''), NULL) AS original_currency, + 'EUR' AS currency, + tuple(events__person.id, events__person.created_at, events.distinct_id, events__person.properties), + nullIf(nullIf(events.`$session_id`, ''), 'null') AS session_id, + toTimeZone(events.timestamp, 'UTC') AS timestamp + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + INNER JOIN + (SELECT person.id AS id, + toTimeZone(person.created_at, 'UTC') AS created_at, + person.properties AS properties + FROM person + WHERE and(equals(person.team_id, 99999), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 99999) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS events__person ON equals(if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id), events__person.id) + WHERE and(equals(events.team_id, 99999), or(equals(events.event, 'purchase_a'), equals(events.event, 'purchase_b')), isNotNull(revenue)) + ORDER BY toTimeZone(events.timestamp, 'UTC') DESC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestRevenueExampleEventsQueryRunner.test_revenue_currency_property_without_feature_flag + ''' + SELECT tuple(events.uuid, events.event, events.distinct_id, events.properties), + events.event AS event, + multiIf(equals(events.event, 'purchase_a'), toDecimal64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'revenue_a'), ''), 'null'), '^"|"$', ''), 10), equals(events.event, 'purchase_b'), toDecimal64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'revenue_b'), ''), 'null'), '^"|"$', ''), 10), NULL) AS original_revenue, + multiIf(equals(events.event, 'purchase_a'), toDecimal64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'revenue_a'), ''), 'null'), '^"|"$', ''), 10), equals(events.event, 'purchase_b'), toDecimal64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'revenue_b'), ''), 'null'), '^"|"$', ''), 10), NULL) AS revenue, + multiIf(equals(events.event, 'purchase_a'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'currency_a'), ''), 'null'), '^"|"$', ''), equals(events.event, 'purchase_b'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'currency_b'), ''), 'null'), '^"|"$', ''), NULL) AS original_currency, + 'EUR' AS currency, + tuple(events__person.id, events__person.created_at, events.distinct_id, events__person.properties), + nullIf(nullIf(events.`$session_id`, ''), 'null') AS session_id, + toTimeZone(events.timestamp, 'UTC') AS timestamp + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, + toTimeZone(person.created_at, 'UTC') AS created_at, + person.properties AS properties + FROM person + WHERE and(equals(person.team_id, 99999), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 99999) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS events__person ON equals(if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id), events__person.id) + WHERE and(equals(events.team_id, 99999), or(equals(events.event, 'purchase_a'), equals(events.event, 'purchase_b')), isNotNull(revenue)) + ORDER BY toTimeZone(events.timestamp, 'UTC') DESC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestRevenueExampleEventsQueryRunner.test_single_event + ''' + SELECT tuple(events.uuid, events.event, events.distinct_id, events.properties), + events.event AS event, + multiIf(equals(events.event, 'purchase'), toDecimal64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'revenue'), ''), 'null'), '^"|"$', ''), 10), NULL) AS original_revenue, + multiIf(equals(events.event, 'purchase'), toDecimal64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'revenue'), ''), 'null'), '^"|"$', ''), 10), NULL) AS revenue, + multiIf(equals(events.event, 'purchase'), NULL, NULL) AS original_currency, + 'USD' AS currency, + tuple(events__person.id, events__person.created_at, events.distinct_id, events__person.properties), + nullIf(nullIf(events.`$session_id`, ''), 'null') AS session_id, + toTimeZone(events.timestamp, 'UTC') AS timestamp + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, + toTimeZone(person.created_at, 'UTC') AS created_at, + person.properties AS properties + FROM person + WHERE and(equals(person.team_id, 99999), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 99999) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS events__person ON equals(if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id), events__person.id) + WHERE and(equals(events.team_id, 99999), equals(events.event, 'purchase'), isNotNull(revenue)) + ORDER BY toTimeZone(events.timestamp, 'UTC') DESC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- diff --git a/posthog/hogql_queries/web_analytics/test/__snapshots__/test_web_overview.ambr b/posthog/hogql_queries/web_analytics/test/__snapshots__/test_web_overview.ambr index a1d7b1a4fb157..998658dc326db 100644 --- a/posthog/hogql_queries/web_analytics/test/__snapshots__/test_web_overview.ambr +++ b/posthog/hogql_queries/web_analytics/test/__snapshots__/test_web_overview.ambr @@ -993,7 +993,7 @@ any(events__session.`$session_duration`) AS session_duration, countIf(or(equals(events.event, '$pageview'), equals(events.event, '$screen'))) AS filtered_pageview_count, any(events__session.`$is_bounce`) AS is_bounce, - sumIf(ifNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'revenue'), ''), 'null'), '^"|"$', ''), 'Float64'), 0), equals(events.event, 'purchase')) AS session_revenue + sumIf(ifNull(if(isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'currency'), ''), 'null'), '^"|"$', '')), toDecimal64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'revenue'), ''), 'null'), '^"|"$', ''), 10), if(dictGetOrDefault(`posthog_test`.`exchange_rate_dict`, 'rate', replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'currency'), ''), 'null'), '^"|"$', ''), toDate(toTimeZone(events.timestamp, 'UTC')), toDecimal64(0, 10)) = 0, toDecimal64(0, 10), multiplyDecimal(divideDecimal(toDecimal64(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'revenue'), ''), 'null'), '^"|"$', ''), 10), dictGetOrDefault(`posthog_test`.`exchange_rate_dict`, 'rate', replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'currency'), ''), 'null'), '^"|"$', ''), toDate(toTimeZone(events.timestamp, 'UTC')), toDecimal64(0, 10))), dictGetOrDefault(`posthog_test`.`exchange_rate_dict`, 'rate', 'GBP', toDate(toTimeZone(events.timestamp, 'UTC')), toDecimal64(0, 10))))), 0), equals(events.event, 'purchase')) AS session_revenue FROM events LEFT JOIN (SELECT toString(reinterpretAsUUID(bitOr(bitShiftLeft(raw_sessions.session_id_v7, 64), bitShiftRight(raw_sessions.session_id_v7, 64)))) AS session_id, @@ -1269,3 +1269,55 @@ max_bytes_before_external_group_by=0 ''' # --- +# name: TestWebOverviewQueryRunner.test_revenue_without_feature_flag + ''' + SELECT uniqIf(session_person_id, and(ifNull(greaterOrEquals(start_timestamp, assumeNotNull(toDateTime('2023-12-01 00:00:00', 'UTC'))), 0), ifNull(less(start_timestamp, assumeNotNull(toDateTime('2023-12-03 23:59:59', 'UTC'))), 0))) AS unique_users, + uniqIf(session_person_id, and(ifNull(greaterOrEquals(start_timestamp, assumeNotNull(toDateTime('2023-11-28 00:00:00', 'UTC'))), 0), ifNull(less(start_timestamp, assumeNotNull(toDateTime('2023-11-30 23:59:59', 'UTC'))), 0))) AS previous_unique_users, + sumIf(filtered_pageview_count, and(ifNull(greaterOrEquals(start_timestamp, assumeNotNull(toDateTime('2023-12-01 00:00:00', 'UTC'))), 0), ifNull(less(start_timestamp, assumeNotNull(toDateTime('2023-12-03 23:59:59', 'UTC'))), 0))) AS total_filtered_pageview_count, + sumIf(filtered_pageview_count, and(ifNull(greaterOrEquals(start_timestamp, assumeNotNull(toDateTime('2023-11-28 00:00:00', 'UTC'))), 0), ifNull(less(start_timestamp, assumeNotNull(toDateTime('2023-11-30 23:59:59', 'UTC'))), 0))) AS previous_filtered_pageview_count, + uniqIf(session_id, and(ifNull(greaterOrEquals(start_timestamp, assumeNotNull(toDateTime('2023-12-01 00:00:00', 'UTC'))), 0), ifNull(less(start_timestamp, assumeNotNull(toDateTime('2023-12-03 23:59:59', 'UTC'))), 0))) AS unique_sessions, + uniqIf(session_id, and(ifNull(greaterOrEquals(start_timestamp, assumeNotNull(toDateTime('2023-11-28 00:00:00', 'UTC'))), 0), ifNull(less(start_timestamp, assumeNotNull(toDateTime('2023-11-30 23:59:59', 'UTC'))), 0))) AS previous_unique_sessions, + avgIf(session_duration, and(ifNull(greaterOrEquals(start_timestamp, assumeNotNull(toDateTime('2023-12-01 00:00:00', 'UTC'))), 0), ifNull(less(start_timestamp, assumeNotNull(toDateTime('2023-12-03 23:59:59', 'UTC'))), 0))) AS avg_duration_s, + avgIf(session_duration, and(ifNull(greaterOrEquals(start_timestamp, assumeNotNull(toDateTime('2023-11-28 00:00:00', 'UTC'))), 0), ifNull(less(start_timestamp, assumeNotNull(toDateTime('2023-11-30 23:59:59', 'UTC'))), 0))) AS prev_avg_duration_s, + avgIf(is_bounce, and(ifNull(greaterOrEquals(start_timestamp, assumeNotNull(toDateTime('2023-12-01 00:00:00', 'UTC'))), 0), ifNull(less(start_timestamp, assumeNotNull(toDateTime('2023-12-03 23:59:59', 'UTC'))), 0))) AS bounce_rate, + avgIf(is_bounce, and(ifNull(greaterOrEquals(start_timestamp, assumeNotNull(toDateTime('2023-11-28 00:00:00', 'UTC'))), 0), ifNull(less(start_timestamp, assumeNotNull(toDateTime('2023-11-30 23:59:59', 'UTC'))), 0))) AS prev_bounce_rate, + sumIf(session_revenue, and(ifNull(greaterOrEquals(start_timestamp, assumeNotNull(toDateTime('2023-12-01 00:00:00', 'UTC'))), 0), ifNull(less(start_timestamp, assumeNotNull(toDateTime('2023-12-03 23:59:59', 'UTC'))), 0))) AS revenue, + sumIf(session_revenue, and(ifNull(greaterOrEquals(start_timestamp, assumeNotNull(toDateTime('2023-11-28 00:00:00', 'UTC'))), 0), ifNull(less(start_timestamp, assumeNotNull(toDateTime('2023-11-30 23:59:59', 'UTC'))), 0))) AS previous_revenue + FROM + (SELECT any(if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id)) AS session_person_id, + events__session.session_id AS session_id, + min(events__session.`$start_timestamp`) AS start_timestamp, + any(events__session.`$session_duration`) AS session_duration, + countIf(or(equals(events.event, '$pageview'), equals(events.event, '$screen'))) AS filtered_pageview_count, + any(events__session.`$is_bounce`) AS is_bounce, + sumIf(ifNull(toDecimal64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'revenue'), ''), 'null'), '^"|"$', ''), 10), 0), equals(events.event, 'purchase')) AS session_revenue + FROM events + LEFT JOIN + (SELECT toString(reinterpretAsUUID(bitOr(bitShiftLeft(raw_sessions.session_id_v7, 64), bitShiftRight(raw_sessions.session_id_v7, 64)))) AS session_id, + min(toTimeZone(raw_sessions.min_timestamp, 'UTC')) AS `$start_timestamp`, + dateDiff('second', min(toTimeZone(raw_sessions.min_timestamp, 'UTC')), max(toTimeZone(raw_sessions.max_timestamp, 'UTC'))) AS `$session_duration`, + if(ifNull(equals(uniqMerge(raw_sessions.pageview_uniq), 0), 0), NULL, not(or(ifNull(greater(uniqMerge(raw_sessions.pageview_uniq), 1), 0), ifNull(greater(uniqMerge(raw_sessions.autocapture_uniq), 0), 0), ifNull(greaterOrEquals(dateDiff('second', min(toTimeZone(raw_sessions.min_timestamp, 'UTC')), max(toTimeZone(raw_sessions.max_timestamp, 'UTC'))), 10), 0)))) AS `$is_bounce`, + raw_sessions.session_id_v7 AS session_id_v7 + FROM raw_sessions + WHERE and(equals(raw_sessions.team_id, 99999), or(and(ifNull(greaterOrEquals(plus(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toIntervalDay(3)), assumeNotNull(toDateTime('2023-12-01 00:00:00', 'UTC'))), 0), ifNull(lessOrEquals(minus(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toIntervalDay(3)), assumeNotNull(toDateTime('2023-12-03 23:59:59', 'UTC'))), 0)), and(ifNull(greaterOrEquals(plus(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toIntervalDay(3)), assumeNotNull(toDateTime('2023-11-28 00:00:00', 'UTC'))), 0), ifNull(lessOrEquals(minus(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toIntervalDay(3)), assumeNotNull(toDateTime('2023-11-30 23:59:59', 'UTC'))), 0)))) + GROUP BY raw_sessions.session_id_v7, + raw_sessions.session_id_v7) AS events__session ON equals(toUInt128(accurateCastOrNull(events.`$session_id`, 'UUID')), events__session.session_id_v7) + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(isNotNull(events.`$session_id`), or(equals(events.event, '$pageview'), equals(events.event, '$screen'), equals(events.event, 'purchase')), or(and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(toDateTime('2023-12-01 00:00:00', 'UTC'))), less(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(toDateTime('2023-12-03 23:59:59', 'UTC')))), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(toDateTime('2023-11-28 00:00:00', 'UTC'))), less(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(toDateTime('2023-11-30 23:59:59', 'UTC'))))), 1)) + GROUP BY session_id + HAVING or(and(ifNull(greaterOrEquals(start_timestamp, assumeNotNull(toDateTime('2023-12-01 00:00:00', 'UTC'))), 0), ifNull(less(start_timestamp, assumeNotNull(toDateTime('2023-12-03 23:59:59', 'UTC'))), 0)), and(ifNull(greaterOrEquals(start_timestamp, assumeNotNull(toDateTime('2023-11-28 00:00:00', 'UTC'))), 0), ifNull(less(start_timestamp, assumeNotNull(toDateTime('2023-11-30 23:59:59', 'UTC'))), 0)))) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- diff --git a/posthog/hogql_queries/web_analytics/test/test_revenue_example_events.py b/posthog/hogql_queries/web_analytics/test/test_revenue_example_events.py index 31de1d5f30784..306fa224419a8 100644 --- a/posthog/hogql_queries/web_analytics/test/test_revenue_example_events.py +++ b/posthog/hogql_queries/web_analytics/test/test_revenue_example_events.py @@ -1,11 +1,14 @@ +from decimal import Decimal from typing import Optional from freezegun import freeze_time +from unittest.mock import patch from posthog.hogql.constants import LimitContext from posthog.hogql_queries.web_analytics.revenue_example_events import RevenueExampleEventsQueryRunner from posthog.models.utils import uuid7 from posthog.schema import ( + CurrencyCode, RevenueExampleEventsQuery, RevenueTrackingConfig, RevenueExampleEventsQueryResponse, @@ -14,6 +17,7 @@ from posthog.test.base import ( APIBaseTest, ClickhouseTestMixin, + snapshot_clickhouse_queries, _create_event, _create_person, ) @@ -31,7 +35,24 @@ ] ) +REVENUE_TRACKING_CONFIG_WITH_REVENUE_CURRENCY_PROPERTY = RevenueTrackingConfig( + events=[ + RevenueTrackingEventItem( + eventName="purchase_a", + revenueProperty="revenue_a", + revenueCurrencyProperty="currency_a", + ), + RevenueTrackingEventItem( + eventName="purchase_b", + revenueProperty="revenue_b", + revenueCurrencyProperty="currency_b", + ), + ], + baseCurrency=CurrencyCode.EUR, +) + +@snapshot_clickhouse_queries class TestRevenueExampleEventsQueryRunner(ClickhouseTestMixin, APIBaseTest): QUERY_TIMESTAMP = "2025-01-29" @@ -53,8 +74,9 @@ def _create_events(self, data, event="$pageview"): elements = None lcp_score = None revenue = None + currency = None revenue_property = "revenue" - + currency_property = "currency" if event == "$pageview": url = extra[0] if extra else None elif event == "$autocapture": @@ -62,10 +84,11 @@ def _create_events(self, data, event="$pageview"): elif event == "$web_vitals": lcp_score = extra[0] if extra else None elif event.startswith("purchase"): - # purchase_a -> revenue_a, purchase_b -> revenue_b, etc + # purchase_a -> revenue_a/currency_a, purchase_b -> revenue_b/currency_b, etc revenue_property += event[8:] + currency_property += event[8:] revenue = extra[0] if extra else None - properties = extra[1] if extra and len(extra) > 1 else {} + currency = extra[1] if extra and len(extra) > 1 else None event_ids.append( _create_event( @@ -78,7 +101,7 @@ def _create_events(self, data, event="$pageview"): "$current_url": url, "$web_vitals_LCP_value": lcp_score, revenue_property: revenue, - **properties, + currency_property: currency, }, elements=elements, ) @@ -142,3 +165,64 @@ def test_multiple_events(self): assert results[0][2] == 43 assert results[1][1] == "purchase_a" assert results[1][2] == 42 + + @patch("posthoganalytics.feature_enabled", return_value=True) + def test_revenue_currency_property(self, feature_enabled_mock): + s1 = str(uuid7("2023-12-02")) + self._create_events( + [ + ("p1", [("2023-12-02", s1, 42, "USD")]), + ], + event="purchase_a", + ) + s2 = str(uuid7("2023-12-03")) + self._create_events( + [ + ("p2", [("2023-12-03", s2, 43, "BRL")]), + ], + event="purchase_b", + ) + + results = self._run_revenue_example_events_query(REVENUE_TRACKING_CONFIG_WITH_REVENUE_CURRENCY_PROPERTY).results + + assert len(results) == 2 + + purchase_b, purchase_a = results + + assert purchase_a[1] == "purchase_a" + assert purchase_a[2] == Decimal("42") + assert purchase_a[3] == Decimal("38.619") # 42 USD -> 38.61 EUR + assert purchase_a[4] == CurrencyCode.USD.value + assert purchase_a[5] == CurrencyCode.EUR.value + + assert purchase_b[1] == "purchase_b" + assert purchase_b[2] == Decimal("43") + assert purchase_b[3] == Decimal("8.0388947625") # 43 BRL -> 8.03 EUR + assert purchase_b[4] == CurrencyCode.BRL.value + assert purchase_b[5] == CurrencyCode.EUR.value + + @patch("posthoganalytics.feature_enabled", return_value=False) + def test_revenue_currency_property_without_feature_flag(self, feature_enabled_mock): + s1 = str(uuid7("2023-12-02")) + self._create_events( + [ + ("p1", [("2023-12-02", s1, 42, "USD")]), + ], + event="purchase_a", + ) + s2 = str(uuid7("2023-12-03")) + self._create_events( + [ + ("p2", [("2023-12-03", s2, 43, "BRL")]), + ], + event="purchase_b", + ) + + results = self._run_revenue_example_events_query(REVENUE_TRACKING_CONFIG_WITH_REVENUE_CURRENCY_PROPERTY).results + + # Keep in the original revenue values + assert len(results) == 2 + assert results[0][1] == "purchase_b" + assert results[0][2] == 43 + assert results[1][1] == "purchase_a" + assert results[1][2] == 42 diff --git a/posthog/hogql_queries/web_analytics/test/test_web_overview.py b/posthog/hogql_queries/web_analytics/test/test_web_overview.py index a2e6bf7af4f5d..73bc99760fe89 100644 --- a/posthog/hogql_queries/web_analytics/test/test_web_overview.py +++ b/posthog/hogql_queries/web_analytics/test/test_web_overview.py @@ -10,6 +10,7 @@ from posthog.models.utils import uuid7 from posthog.schema import ( CompareFilter, + CurrencyCode, WebOverviewQuery, DateRange, SessionTableVersion, @@ -52,6 +53,7 @@ def _create_events(self, data, event="$pageview"): elements = None lcp_score = None revenue = None + currency = None if event == "$pageview": url = extra[0] if extra else None elif event == "$autocapture": @@ -60,7 +62,8 @@ def _create_events(self, data, event="$pageview"): lcp_score = extra[0] if extra else None elif event.startswith("purchase"): revenue = extra[0] if extra else None - properties = extra[1] if extra and len(extra) > 1 else {} + currency = extra[1] if extra and len(extra) > 1 and extra[1] else None + properties = extra[1] if extra and len(extra) > 1 and isinstance(extra[1], dict) else {} _create_event( team=self.team, @@ -72,6 +75,7 @@ def _create_events(self, data, event="$pageview"): "$current_url": url, "$web_vitals_LCP_value": lcp_score, "revenue": revenue, + "currency": currency, **properties, }, elements=elements, @@ -643,15 +647,56 @@ def test_conversion_rate(self): conversion_rate = results[3] self.assertAlmostEqual(conversion_rate.value, 100 * 2 / 3) - def test_revenue(self): + @patch("posthoganalytics.feature_enabled", return_value=True) + def test_revenue(self, feature_enabled_mock): s1 = str(uuid7("2023-12-02")) - self.team.revenue_tracking_config = {"events": [{"eventName": "purchase", "revenueProperty": "revenue"}]} + self.team.revenue_tracking_config = { + "events": [{"eventName": "purchase", "revenueProperty": "revenue", "revenueCurrencyProperty": "currency"}], + "baseCurrency": CurrencyCode.GBP.value, + } self.team.save() self._create_events( [ - ("p1", [("2023-12-02", s1, 100)]), + ("p1", [("2023-12-02", s1, 100, "BRL")]), + ], + event="purchase", + ) + results = self._run_web_overview_query("2023-12-01", "2023-12-03", include_revenue=True).results + + visitors = results[0] + assert visitors.value == 1 + + views = results[1] + assert views.value == 0 + + sessions = results[2] + assert sessions.value == 1 + + duration = results[3] + assert duration.value == 0 + + bounce = results[4] + assert bounce.value is None + + revenue = results[5] + assert revenue.kind == "currency" + assert revenue.value == 16.0763662979 + + @patch("posthoganalytics.feature_enabled", return_value=False) + def test_revenue_without_feature_flag(self, feature_enabled_mock): + s1 = str(uuid7("2023-12-02")) + + self.team.revenue_tracking_config = { + "events": [{"eventName": "purchase", "revenueProperty": "revenue", "revenueCurrencyProperty": "currency"}], + "baseCurrency": CurrencyCode.GBP.value, + } + self.team.save() + + self._create_events( + [ + ("p1", [("2023-12-02", s1, 100, "BRL")]), ], event="purchase", ) diff --git a/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py b/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py index 9dae3f98be185..b600ef258baf1 100644 --- a/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py +++ b/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py @@ -3,6 +3,7 @@ from datetime import timedelta from math import ceil from typing import Optional, Union +import posthoganalytics from django.conf import settings from django.core.cache import cache @@ -17,7 +18,7 @@ from posthog.hogql_queries.utils.query_date_range import QueryDateRange from posthog.hogql_queries.utils.query_compare_to_date_range import QueryCompareToDateRange from posthog.hogql_queries.utils.query_previous_period_date_range import QueryPreviousPeriodDateRange -from posthog.hogql_queries.utils.revenue import revenue_sum_expression, revenue_events_exprs +from posthog.hogql.database.schema.exchange_rate import revenue_sum_expression, revenue_events_where_expr from posthog.models import Action from posthog.models.filters.mixins.utils import cached_property @@ -45,6 +46,16 @@ class WebAnalyticsQueryRunner(QueryRunner, ABC): query: WebQueryNode query_type: type[WebQueryNode] + do_currency_conversion: bool = False + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.do_currency_conversion = posthoganalytics.feature_enabled( + "web-analytics-revenue-tracking-conversion", + str(self.team.organization_id), + groups={"organization": str(self.team.organization_id)}, + group_properties={"organization": {"id": str(self.team.organization_id)}}, + ) @cached_property def query_date_range(self): @@ -183,8 +194,10 @@ def conversion_revenue_expr(self) -> ast.Expr: if self.team.revenue_tracking_config else None ) + if not config: return ast.Constant(value=None) + if isinstance(self.query.conversionGoal, CustomEventConversionGoal): event_name = self.query.conversionGoal.customEventName revenue_property = next( @@ -219,7 +232,7 @@ def conversion_revenue_expr(self) -> ast.Expr: @cached_property def revenue_sum_expression(self) -> ast.Expr: - return revenue_sum_expression(self.team.revenue_tracking_config) + return revenue_sum_expression(self.team.revenue_tracking_config, self.do_currency_conversion) @cached_property def event_type_expr(self) -> ast.Expr: @@ -237,7 +250,7 @@ def event_type_expr(self) -> ast.Expr: elif self.query.includeRevenue: # Use elif here, we don't need to include revenue events if we already included conversion events, because # if there is a conversion goal set then we only show revenue from conversion events. - exprs.extend(revenue_events_exprs(self.team.revenue_tracking_config)) + exprs.append(revenue_events_where_expr(self.team.revenue_tracking_config)) return ast.Or(exprs=exprs) From 0a09f262ccc2428c90152d89485559143b02b07f Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 10 Mar 2025 15:49:19 +0000 Subject: [PATCH 2/3] Update query snapshots --- .../web_analytics/test/__snapshots__/test_web_overview.ambr | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/posthog/hogql_queries/web_analytics/test/__snapshots__/test_web_overview.ambr b/posthog/hogql_queries/web_analytics/test/__snapshots__/test_web_overview.ambr index 998658dc326db..e0abb4633d0ba 100644 --- a/posthog/hogql_queries/web_analytics/test/__snapshots__/test_web_overview.ambr +++ b/posthog/hogql_queries/web_analytics/test/__snapshots__/test_web_overview.ambr @@ -1186,7 +1186,7 @@ any(events__session.`$session_duration`) AS session_duration, countIf(or(equals(events.event, '$pageview'), equals(events.event, '$screen'))) AS filtered_pageview_count, any(events__session.`$is_bounce`) AS is_bounce, - plus(sumIf(ifNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'revenue'), ''), 'null'), '^"|"$', ''), 'Float64'), 0), equals(events.event, 'purchase1')), sumIf(ifNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'revenue'), ''), 'null'), '^"|"$', ''), 'Float64'), 0), equals(events.event, 'purchase2'))) AS session_revenue + plus(sumIf(ifNull(toDecimal64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'revenue'), ''), 'null'), '^"|"$', ''), 10), 0), equals(events.event, 'purchase1')), sumIf(ifNull(toDecimal64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'revenue'), ''), 'null'), '^"|"$', ''), 10), 0), equals(events.event, 'purchase2'))) AS session_revenue FROM events LEFT JOIN (SELECT toString(reinterpretAsUUID(bitOr(bitShiftLeft(raw_sessions.session_id_v7, 64), bitShiftRight(raw_sessions.session_id_v7, 64)))) AS session_id, @@ -1205,7 +1205,7 @@ WHERE equals(person_distinct_id_overrides.team_id, 99999) GROUP BY person_distinct_id_overrides.distinct_id HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(isNotNull(events.`$session_id`), or(equals(events.event, '$pageview'), equals(events.event, '$screen'), equals(events.event, 'purchase1'), equals(events.event, 'purchase2')), or(and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(toDateTime('2023-12-01 00:00:00', 'UTC'))), less(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(toDateTime('2023-12-03 23:59:59', 'UTC')))), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(toDateTime('2023-11-28 00:00:00', 'UTC'))), less(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(toDateTime('2023-11-30 23:59:59', 'UTC'))))), 1)) + WHERE and(equals(events.team_id, 99999), and(isNotNull(events.`$session_id`), or(equals(events.event, '$pageview'), equals(events.event, '$screen'), or(equals(events.event, 'purchase1'), equals(events.event, 'purchase2'))), or(and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(toDateTime('2023-12-01 00:00:00', 'UTC'))), less(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(toDateTime('2023-12-03 23:59:59', 'UTC')))), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(toDateTime('2023-11-28 00:00:00', 'UTC'))), less(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(toDateTime('2023-11-30 23:59:59', 'UTC'))))), 1)) GROUP BY session_id HAVING or(and(ifNull(greaterOrEquals(start_timestamp, assumeNotNull(toDateTime('2023-12-01 00:00:00', 'UTC'))), 0), ifNull(less(start_timestamp, assumeNotNull(toDateTime('2023-12-03 23:59:59', 'UTC'))), 0)), and(ifNull(greaterOrEquals(start_timestamp, assumeNotNull(toDateTime('2023-11-28 00:00:00', 'UTC'))), 0), ifNull(less(start_timestamp, assumeNotNull(toDateTime('2023-11-30 23:59:59', 'UTC'))), 0)))) LIMIT 100 SETTINGS readonly=2, @@ -1257,7 +1257,7 @@ WHERE equals(person_distinct_id_overrides.team_id, 99999) GROUP BY person_distinct_id_overrides.distinct_id HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(isNotNull(events.`$session_id`), or(equals(events.event, '$pageview'), equals(events.event, '$screen')), or(and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(toDateTime('2023-12-01 00:00:00', 'UTC'))), less(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(toDateTime('2023-12-03 23:59:59', 'UTC')))), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(toDateTime('2023-11-28 00:00:00', 'UTC'))), less(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(toDateTime('2023-11-30 23:59:59', 'UTC'))))), 1)) + WHERE and(equals(events.team_id, 99999), and(isNotNull(events.`$session_id`), or(equals(events.event, '$pageview'), equals(events.event, '$screen'), 0), or(and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(toDateTime('2023-12-01 00:00:00', 'UTC'))), less(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(toDateTime('2023-12-03 23:59:59', 'UTC')))), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(toDateTime('2023-11-28 00:00:00', 'UTC'))), less(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(toDateTime('2023-11-30 23:59:59', 'UTC'))))), 1)) GROUP BY session_id HAVING or(and(ifNull(greaterOrEquals(start_timestamp, assumeNotNull(toDateTime('2023-12-01 00:00:00', 'UTC'))), 0), ifNull(less(start_timestamp, assumeNotNull(toDateTime('2023-12-03 23:59:59', 'UTC'))), 0)), and(ifNull(greaterOrEquals(start_timestamp, assumeNotNull(toDateTime('2023-11-28 00:00:00', 'UTC'))), 0), ifNull(less(start_timestamp, assumeNotNull(toDateTime('2023-11-30 23:59:59', 'UTC'))), 0)))) LIMIT 100 SETTINGS readonly=2, From adcbea6dbe78be3c281247469f9267832cc09dea Mon Sep 17 00:00:00 2001 From: Rafa Audibert Date: Wed, 12 Mar 2025 14:04:04 -0300 Subject: [PATCH 3/3] refactor: Extract decimal precision to separate constant This makes the hogql generation much more understandable --- posthog/hogql/database/schema/exchange_rate.py | 8 ++++++-- posthog/models/exchange_rate/sql.py | 18 ++++++++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/posthog/hogql/database/schema/exchange_rate.py b/posthog/hogql/database/schema/exchange_rate.py index a465503d054f5..943e15f12dd90 100644 --- a/posthog/hogql/database/schema/exchange_rate.py +++ b/posthog/hogql/database/schema/exchange_rate.py @@ -2,6 +2,7 @@ from posthog.hogql import ast from posthog.schema import CurrencyCode, RevenueTrackingConfig, RevenueTrackingEventItem +from posthog.models.exchange_rate.sql import EXCHANGE_RATE_DECIMAL_PRECISION from posthog.hogql.database.models import ( StringDatabaseField, DateDatabaseField, @@ -86,7 +87,7 @@ def revenue_comparison_and_value_exprs( name="toDecimal", args=[ ast.Field(chain=["events", "properties", event.revenueProperty]), - ast.Constant(value=10), + ast.Constant(value=EXCHANGE_RATE_DECIMAL_PRECISION), ], ), convert_currency_call( @@ -100,7 +101,10 @@ def revenue_comparison_and_value_exprs( else: value_expr = ast.Call( name="toDecimal", - args=[ast.Field(chain=["events", "properties", event.revenueProperty]), ast.Constant(value=10)], + args=[ + ast.Field(chain=["events", "properties", event.revenueProperty]), + ast.Constant(value=EXCHANGE_RATE_DECIMAL_PRECISION), + ], ) return (comparison_expr, value_expr) diff --git a/posthog/models/exchange_rate/sql.py b/posthog/models/exchange_rate/sql.py index 8673872fc009c..063f8f6ddcc19 100644 --- a/posthog/models/exchange_rate/sql.py +++ b/posthog/models/exchange_rate/sql.py @@ -97,6 +97,14 @@ def HISTORICAL_EXCHANGE_RATE_TUPLES(): EXCHANGE_RATE_TABLE_NAME = "exchange_rate" EXCHANGE_RATE_DICTIONARY_NAME = "exchange_rate_dict" +# Storing 10 decimal places is more than enough +# Ideally we should have gone with 4 because that's all we need for most currencies +# but Bitcoin messes this up because it's so valuable compared to the Dollar (our base currency) +# +# If Bitcoin ever moons it even further, we can increase this to 12 or 14 +# but for now 10 is more than enough +EXCHANGE_RATE_DECIMAL_PRECISION = 10 + # `version` is used to ensure the latest version is kept, see https://clickhouse.com/docs/engines/table-engines/mergetree-family/replacingmergetree def EXCHANGE_RATE_TABLE_SQL(on_cluster=True): @@ -104,14 +112,15 @@ def EXCHANGE_RATE_TABLE_SQL(on_cluster=True): CREATE TABLE IF NOT EXISTS {table_name} {on_cluster_clause} ( currency String, date Date, - rate Decimal64(10), + rate Decimal64({decimal_precision}), version UInt32 DEFAULT toUnixTimestamp(now()) ) ENGINE = {engine} ORDER BY (date, currency); """.format( table_name=f"`{CLICKHOUSE_DATABASE}`.`{EXCHANGE_RATE_TABLE_NAME}`", - engine=ReplacingMergeTree("exchange_rate", ver="version"), on_cluster_clause=ON_CLUSTER_CLAUSE(on_cluster), + decimal_precision=EXCHANGE_RATE_DECIMAL_PRECISION, + engine=ReplacingMergeTree("exchange_rate", ver="version"), ) @@ -204,7 +213,7 @@ def EXCHANGE_RATE_DICTIONARY_SQL(on_cluster=True): currency String, start_date Date, end_date Nullable(Date), - rate Decimal64(10) + rate Decimal64({decimal_precision}) ) PRIMARY KEY currency SOURCE(CLICKHOUSE(QUERY '{query}' PASSWORD '{clickhouse_password}')) @@ -212,9 +221,10 @@ def EXCHANGE_RATE_DICTIONARY_SQL(on_cluster=True): LAYOUT(RANGE_HASHED(range_lookup_strategy 'max')) RANGE(MIN start_date MAX end_date)""".format( exchange_rate_dictionary_name=f"`{CLICKHOUSE_DATABASE}`.`{EXCHANGE_RATE_DICTIONARY_NAME}`", + on_cluster_clause=ON_CLUSTER_CLAUSE(on_cluster), + decimal_precision=EXCHANGE_RATE_DECIMAL_PRECISION, query=EXCHANGE_RATE_DICTIONARY_QUERY, clickhouse_password=CLICKHOUSE_PASSWORD, - on_cluster_clause=ON_CLUSTER_CLAUSE(on_cluster), )