Skip to content

Commit

Permalink
Dedupe filter specs in join to time spine queries (#1578)
Browse files Browse the repository at this point in the history 
There was a duplicate spec being passed into one of the
`FilterElementsNodes`, which caused duplicates to show up in the printed
dataflow plan. This fixes that.
  • Loading branch information
@courtneyholcomb
courtneyholcomb authored Dec 21, 2024
1 parent 102169a commit f6f63d7
Show file tree
Hide file tree
Showing 188 changed files with 197 additions and 228 deletions.
4 changes: 3 additions & 1 deletion metricflow/dataflow/builder/dataflow_plan_builder.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1648,7 +1648,9 @@ def _build_aggregated_measure_from_measure_source_node(
join_on_time_dimension_spec = self._determine_time_spine_join_spec(
measure_properties=measure_properties, required_time_spine_specs=base_queried_agg_time_dimension_specs
)
required_time_spine_specs = (join_on_time_dimension_spec,) + base_queried_agg_time_dimension_specs
required_time_spine_specs = base_queried_agg_time_dimension_specs
if join_on_time_dimension_spec not in required_time_spine_specs:
required_time_spine_specs = (join_on_time_dimension_spec,) + required_time_spine_specs
time_spine_node = self._build_time_spine_node(required_time_spine_specs)
unaggregated_measure_node = JoinToTimeSpineNode.create(
metric_source_node=unaggregated_measure_node,
Expand Down
2 changes: 1 addition & 1 deletion ...tom_granularity.py/SqlPlan/BigQuery/test_offset_metric_with_custom_granularity__plan0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ FROM (
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
, subq_5.martian_day AS booking__ds__martian_day
FROM (
-- Pass Only Elements: ['booking__ds__day', 'booking__ds__day']
-- Pass Only Elements: ['booking__ds__day',]
SELECT
subq_3.booking__ds__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...lan/BigQuery/test_offset_metric_with_custom_granularity_filter_not_in_group_by__plan0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ FROM (
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
, subq_5.martian_day AS metric_time__martian_day
FROM (
-- Pass Only Elements: ['metric_time__day', 'metric_time__day']
-- Pass Only Elements: ['metric_time__day',]
SELECT
subq_3.metric_time__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...m_granularity.py/SqlPlan/Databricks/test_offset_metric_with_custom_granularity__plan0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ FROM (
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
, subq_5.martian_day AS booking__ds__martian_day
FROM (
-- Pass Only Elements: ['booking__ds__day', 'booking__ds__day']
-- Pass Only Elements: ['booking__ds__day',]
SELECT
subq_3.booking__ds__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...n/Databricks/test_offset_metric_with_custom_granularity_filter_not_in_group_by__plan0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ FROM (
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
, subq_5.martian_day AS metric_time__martian_day
FROM (
-- Pass Only Elements: ['metric_time__day', 'metric_time__day']
-- Pass Only Elements: ['metric_time__day',]
SELECT
subq_3.metric_time__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...ustom_granularity.py/SqlPlan/DuckDB/test_offset_metric_with_custom_granularity__plan0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ FROM (
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
, subq_5.martian_day AS booking__ds__martian_day
FROM (
-- Pass Only Elements: ['booking__ds__day', 'booking__ds__day']
-- Pass Only Elements: ['booking__ds__day',]
SELECT
subq_3.booking__ds__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...lPlan/DuckDB/test_offset_metric_with_custom_granularity_filter_not_in_group_by__plan0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ FROM (
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
, subq_5.martian_day AS metric_time__martian_day
FROM (
-- Pass Only Elements: ['metric_time__day', 'metric_time__day']
-- Pass Only Elements: ['metric_time__day',]
SELECT
subq_3.metric_time__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...tom_granularity.py/SqlPlan/Postgres/test_offset_metric_with_custom_granularity__plan0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ FROM (
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
, subq_5.martian_day AS booking__ds__martian_day
FROM (
-- Pass Only Elements: ['booking__ds__day', 'booking__ds__day']
-- Pass Only Elements: ['booking__ds__day',]
SELECT
subq_3.booking__ds__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...lan/Postgres/test_offset_metric_with_custom_granularity_filter_not_in_group_by__plan0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ FROM (
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
, subq_5.martian_day AS metric_time__martian_day
FROM (
-- Pass Only Elements: ['metric_time__day', 'metric_time__day']
-- Pass Only Elements: ['metric_time__day',]
SELECT
subq_3.metric_time__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...tom_granularity.py/SqlPlan/Redshift/test_offset_metric_with_custom_granularity__plan0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ FROM (
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
, subq_5.martian_day AS booking__ds__martian_day
FROM (
-- Pass Only Elements: ['booking__ds__day', 'booking__ds__day']
-- Pass Only Elements: ['booking__ds__day',]
SELECT
subq_3.booking__ds__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...lan/Redshift/test_offset_metric_with_custom_granularity_filter_not_in_group_by__plan0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ FROM (
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
, subq_5.martian_day AS metric_time__martian_day
FROM (
-- Pass Only Elements: ['metric_time__day', 'metric_time__day']
-- Pass Only Elements: ['metric_time__day',]
SELECT
subq_3.metric_time__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...om_granularity.py/SqlPlan/Snowflake/test_offset_metric_with_custom_granularity__plan0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ FROM (
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
, subq_5.martian_day AS booking__ds__martian_day
FROM (
-- Pass Only Elements: ['booking__ds__day', 'booking__ds__day']
-- Pass Only Elements: ['booking__ds__day',]
SELECT
subq_3.booking__ds__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...an/Snowflake/test_offset_metric_with_custom_granularity_filter_not_in_group_by__plan0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ FROM (
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
, subq_5.martian_day AS metric_time__martian_day
FROM (
-- Pass Only Elements: ['metric_time__day', 'metric_time__day']
-- Pass Only Elements: ['metric_time__day',]
SELECT
subq_3.metric_time__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...custom_granularity.py/SqlPlan/Trino/test_offset_metric_with_custom_granularity__plan0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ FROM (
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
, subq_5.martian_day AS booking__ds__martian_day
FROM (
-- Pass Only Elements: ['booking__ds__day', 'booking__ds__day']
-- Pass Only Elements: ['booking__ds__day',]
SELECT
subq_3.booking__ds__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...qlPlan/Trino/test_offset_metric_with_custom_granularity_filter_not_in_group_by__plan0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ FROM (
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
, subq_5.martian_day AS metric_time__martian_day
FROM (
-- Pass Only Elements: ['metric_time__day', 'metric_time__day']
-- Pass Only Elements: ['metric_time__day',]
SELECT
subq_3.metric_time__day
FROM (
Expand Down
7 changes: 1 addition & 6 deletions ...test_dataflow_plan_builder.py/DataflowPlan/test_derived_metric_offset_to_grain__dfp_0.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,18 +96,13 @@ docstring:
</ReadSqlSourceNode>
</MetricTimeDimensionTransformNode>
<FilterElementsNode>
<!-- description = "Pass Only Elements: ['metric_time__day', 'metric_time__day']" -->
<!-- description = "Pass Only Elements: ['metric_time__day',]" -->
<!-- node_id = NodeId(id_str='pfe_1') -->
<!-- include_spec = -->
<!-- TimeDimensionSpec( -->
<!-- element_name='metric_time', -->
<!-- time_granularity=ExpandedTimeGranularity(name='day', base_granularity=DAY), -->
<!-- ) -->
<!-- include_spec = -->
<!-- TimeDimensionSpec( -->
<!-- element_name='metric_time', -->
<!-- time_granularity=ExpandedTimeGranularity(name='day', base_granularity=DAY), -->
<!-- ) -->
<!-- distinct = False -->
<AliasSpecsNode>
<!-- description = 'Change Column Aliases' -->
Expand Down
7 changes: 1 addition & 6 deletions ...s/test_dataflow_plan_builder.py/DataflowPlan/test_derived_metric_offset_window__dfp_0.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,18 +62,13 @@ docstring:
</ReadSqlSourceNode>
</MetricTimeDimensionTransformNode>
<FilterElementsNode>
<!-- description = "Pass Only Elements: ['metric_time__day', 'metric_time__day']" -->
<!-- description = "Pass Only Elements: ['metric_time__day',]" -->
<!-- node_id = NodeId(id_str='pfe_0') -->
<!-- include_spec = -->
<!-- TimeDimensionSpec( -->
<!-- element_name='metric_time', -->
<!-- time_granularity=ExpandedTimeGranularity(name='day', base_granularity=DAY), -->
<!-- ) -->
<!-- include_spec = -->
<!-- TimeDimensionSpec( -->
<!-- element_name='metric_time', -->
<!-- time_granularity=ExpandedTimeGranularity(name='day', base_granularity=DAY), -->
<!-- ) -->
<!-- distinct = False -->
<AliasSpecsNode>
<!-- description = 'Change Column Aliases' -->
Expand Down
7 changes: 1 addition & 6 deletions ...st_dataflow_plan_builder.py/DataflowPlan/test_derived_offset_cumulative_metric__dfp_0.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,18 +73,13 @@ test_filename: test_dataflow_plan_builder.py
</MetricTimeDimensionTransformNode>
</JoinOverTimeRangeNode>
<FilterElementsNode>
<!-- description = "Pass Only Elements: ['metric_time__day', 'metric_time__day']" -->
<!-- description = "Pass Only Elements: ['metric_time__day',]" -->
<!-- node_id = NodeId(id_str='pfe_0') -->
<!-- include_spec = -->
<!-- TimeDimensionSpec( -->
<!-- element_name='metric_time', -->
<!-- time_granularity=ExpandedTimeGranularity(name='day', base_granularity=DAY), -->
<!-- ) -->
<!-- include_spec = -->
<!-- TimeDimensionSpec( -->
<!-- element_name='metric_time', -->
<!-- time_granularity=ExpandedTimeGranularity(name='day', base_granularity=DAY), -->
<!-- ) -->
<!-- distinct = False -->
<AliasSpecsNode>
<!-- description = 'Change Column Aliases' -->
Expand Down
11 changes: 1 addition & 10 deletions ...t_dataflow_plan_builder.py/DataflowPlan/test_join_to_time_spine_derived_metric__dfp_0.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,8 +169,7 @@ test_filename: test_dataflow_plan_builder.py
</ReadSqlSourceNode>
</MetricTimeDimensionTransformNode>
<FilterElementsNode>
<!-- description = -->
<!-- "Pass Only Elements: ['metric_time__day', 'metric_time__day']" -->
<!-- description = "Pass Only Elements: ['metric_time__day',]" -->
<!-- node_id = NodeId(id_str='pfe_2') -->
<!-- include_spec = -->
<!-- TimeDimensionSpec( -->
Expand All @@ -180,14 +179,6 @@ test_filename: test_dataflow_plan_builder.py
<!-- base_granularity=DAY, -->
<!-- ), -->
<!-- ) -->
<!-- include_spec = -->
<!-- TimeDimensionSpec( -->
<!-- element_name='metric_time', -->
<!-- time_granularity=ExpandedTimeGranularity( -->
<!-- name='day', -->
<!-- base_granularity=DAY, -->
<!-- ), -->
<!-- ) -->
<!-- distinct = False -->
<AliasSpecsNode>
<!-- description = 'Change Column Aliases' -->
Expand Down
11 changes: 1 addition & 10 deletions ...flow_plan_builder.py/DataflowPlan/test_nested_derived_metric_with_outer_offset__dfp_0.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,7 @@ test_filename: test_dataflow_plan_builder.py
</ReadSqlSourceNode>
</MetricTimeDimensionTransformNode>
<FilterElementsNode>
<!-- description = -->
<!-- "Pass Only Elements: ['metric_time__day', 'metric_time__day']" -->
<!-- description = "Pass Only Elements: ['metric_time__day',]" -->
<!-- node_id = NodeId(id_str='pfe_0') -->
<!-- include_spec = -->
<!-- TimeDimensionSpec( -->
Expand All @@ -96,14 +95,6 @@ test_filename: test_dataflow_plan_builder.py
<!-- base_granularity=DAY, -->
<!-- ), -->
<!-- ) -->
<!-- include_spec = -->
<!-- TimeDimensionSpec( -->
<!-- element_name='metric_time', -->
<!-- time_granularity=ExpandedTimeGranularity( -->
<!-- name='day', -->
<!-- base_granularity=DAY, -->
<!-- ), -->
<!-- ) -->
<!-- distinct = False -->
<AliasSpecsNode>
<!-- description = 'Change Column Aliases' -->
Expand Down
2 changes: 1 addition & 1 deletion ...ng.py/SqlPlan/BigQuery/test_cumulative_time_offset_metric_with_time_constraint__plan0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,7 @@ FROM (
, subq_4.approximate_continuous_booking_value_p99 AS approximate_continuous_booking_value_p99
, subq_4.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
FROM (
-- Pass Only Elements: ['metric_time__day', 'metric_time__day']
-- Pass Only Elements: ['metric_time__day',]
SELECT
subq_6.metric_time__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...py/SqlPlan/BigQuery/test_derived_metric_with_month_dimension_and_offset_window__plan0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ FROM (
, subq_1.booking_monthly__listing AS booking_monthly__listing
, subq_1.bookings_monthly AS bookings_monthly
FROM (
-- Pass Only Elements: ['metric_time__month', 'metric_time__month']
-- Pass Only Elements: ['metric_time__month',]
SELECT
subq_3.metric_time__month
FROM (
Expand Down
2 changes: 1 addition & 1 deletion .../BigQuery/test_derived_metric_with_month_dimension_and_offset_window__plan0_optimized.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ FROM (
FROM (
-- Read From Time Spine 'mf_time_spine'
-- Change Column Aliases
-- Pass Only Elements: ['metric_time__month', 'metric_time__month']
-- Pass Only Elements: ['metric_time__month',]
SELECT
DATETIME_TRUNC(ds, month) AS metric_time__month
FROM ***************************.mf_time_spine time_spine_src_16006
Expand Down
2 changes: 1 addition & 1 deletion ..._metric_rendering.py/SqlPlan/BigQuery/test_derived_metric_with_offset_to_grain__plan0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -344,7 +344,7 @@ FROM (
, subq_6.approximate_continuous_booking_value_p99 AS approximate_continuous_booking_value_p99
, subq_6.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
FROM (
-- Pass Only Elements: ['metric_time__day', 'metric_time__day']
-- Pass Only Elements: ['metric_time__day',]
SELECT
subq_8.metric_time__day
FROM (
Expand Down
2 changes: 1 addition & 1 deletion ...ed_metric_rendering.py/SqlPlan/BigQuery/test_derived_metric_with_offset_window__plan0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -344,7 +344,7 @@ FROM (
, subq_6.approximate_continuous_booking_value_p99 AS approximate_continuous_booking_value_p99
, subq_6.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
FROM (
-- Pass Only Elements: ['metric_time__day', 'metric_time__day']
-- Pass Only Elements: ['metric_time__day',]
SELECT
subq_8.metric_time__day
FROM (
Expand Down
4 changes: 2 additions & 2 deletions ...py/SqlPlan/BigQuery/test_derived_metric_with_offset_window_and_offset_to_grain__plan0.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ FROM (
, subq_1.approximate_continuous_booking_value_p99 AS approximate_continuous_booking_value_p99
, subq_1.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
FROM (
-- Pass Only Elements: ['metric_time__day', 'metric_time__day']
-- Pass Only Elements: ['metric_time__day',]
SELECT
subq_3.metric_time__day
FROM (
Expand Down Expand Up @@ -486,7 +486,7 @@ FROM (
, subq_10.approximate_continuous_booking_value_p99 AS approximate_continuous_booking_value_p99
, subq_10.approximate_discrete_booking_value_p99 AS approximate_discrete_booking_value_p99
FROM (
-- Pass Only Elements: ['metric_time__day', 'metric_time__day']
-- Pass Only Elements: ['metric_time__day',]
SELECT
subq_12.metric_time__day
FROM (
Expand Down
Loading

0 comments on commit f6f63d7

Please sign in to comment.