Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

(WIP) Refactor blocking rule for clarity #1699

Closed
wants to merge 2 commits into from
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion splink/analyse_blocking.py
Original file line number Diff line number Diff line change
@@ -118,7 +118,7 @@ def cumulative_comparisons_generated_by_blocking_rules(
for row, br in zip(br_count, brs_as_objs):
out_dict = {
"row_count": row,
"rule": br.blocking_rule,
"rule": br.blocking_rule_sql,
}
if output_chart:
cumulative_sum += row
154 changes: 109 additions & 45 deletions splink/blocking.py
Original file line number Diff line number Diff line change
@@ -3,7 +3,7 @@
from sqlglot import parse_one
from sqlglot.expressions import Join, Column
from sqlglot.optimizer.eliminate_joins import join_condition
from typing import TYPE_CHECKING, Union
from typing import TYPE_CHECKING, List
import logging

from .misc import ensure_is_list
@@ -37,6 +37,27 @@ def blocking_rule_to_obj(br):
return br


class SaltedBlockingRuleSegment:
"""If a BlockingRule is salted, it will generate a list of
SaltedBlockingRuleSegments. If it's not salted, it will
produce just one
"""

def __init__(
self,
parent_blocking_rule: BlockingRule,
blocking_rule_sql: str,
salt: int = None,
):
self.parent_blocking_rule = parent_blocking_rule
self.blocking_rule_sql = blocking_rule_sql
self.salt = salt

@property
def is_salted(self):
return self.parent_blocking_rule.is_salted


class BlockingRule:
def __init__(
self,
@@ -47,10 +68,10 @@ def __init__(
if sqlglot_dialect:
self._sql_dialect = sqlglot_dialect

self.blocking_rule = blocking_rule
self.blocking_rule_sql = blocking_rule
self.preceding_rules = []
self.sqlglot_dialect = sqlglot_dialect
self.salting_partitions = salting_partitions
self.salting_partitions: int = salting_partitions

@property
def sql_dialect(self):
@@ -61,39 +82,94 @@ def match_key(self):
return len(self.preceding_rules)

@property
def sql(self):
# Wrapper to reveal the underlying SQL
return self.blocking_rule
def is_salted(self):
# Salting does not work in some backends due to the operation of random
# number generators https://github.com/duckdb/duckdb/issues/3974
# Currently we've only tested it's working corerctly in spark

if self.salting_partitions > 1 and self.sql_dialect == "spark":
return True

elif self.salting_partitions > 1:
logger.warning(
"WARNING: Salting is not currently supported by this linker backend and"
" will not be implemented for this run."
)

return False

def add_preceding_rules(self, rules):
rules = ensure_is_list(rules)
self.preceding_rules = rules

@property
def and_not_preceding_rules_sql(self):
if not self.preceding_rules:
return ""
def exclude_pairs_generated_by_this_rule_sql(self, linker: Linker):
"""A SQL string specifying how to exclude the results
of THIS blocking rule from subseqent blocking statements,
so that subsequent statements do not produce duplicate pairs
"""

# Note the coalesce function is important here - otherwise
# you filter out any records with nulls in the previous rules
# meaning these comparisons get lost
return f"coalesce(({self.blocking_rule_sql}),false)"

def exclude_pairs_generated_by_all_preceding_rules_sql(self, linker: Linker):
"""A SQL string that excludes the results of ALL previous blocking rules from
the pairwise comparisons generated.
"""
if not self.preceding_rules:
return ""
or_clauses = [
f"coalesce(({r.blocking_rule}), false)" for r in self.preceding_rules
br.exclude_pairs_generated_by_this_rule_sql(linker)
for br in self.preceding_rules
]
previous_rules = " OR ".join(or_clauses)
return f"AND NOT ({previous_rules})"

def create_pairwise_comparisons_sql(
self,
linker: Linker,
sql_select_expr: str,
salted_br: SaltedBlockingRuleSegment,
probability: str,
where_condition: str,
):
sql = f"""
select
{sql_select_expr}
, '{self.match_key}' as match_key
{probability}
from {linker._input_tablename_l} as l
inner join {linker._input_tablename_r} as r
on
({salted_br.blocking_rule_sql})
{where_condition}
{self.exclude_pairs_generated_by_all_preceding_rules_sql(linker)}
"""
return sql

@property
def salted_blocking_rules(self):
if self.salting_partitions == 1:
yield self.blocking_rule
else:
def salted_blocking_rule_segments(self) -> List[SaltedBlockingRuleSegment]:
if self.is_salted:
for n in range(self.salting_partitions):
yield f"{self.blocking_rule} and ceiling(l.__splink_salt * {self.salting_partitions}) = {n+1}" # noqa: E501
rule_sql = (
f"{self.blocking_rule_sql} and "
f"ceiling(l.__splink_salt * {self.salting_partitions}) "
f"= {n+1}"
)

br_seg = SaltedBlockingRuleSegment(self, rule_sql, n)

yield br_seg

else:
rule_sql = self.blocking_rule_sql
br_seg = SaltedBlockingRuleSegment(self, rule_sql)
yield br_seg

@property
def _parsed_join_condition(self):
br = self.blocking_rule
br = self.blocking_rule_sql
return parse_one("INNER JOIN r", into=Join).on(
br, dialect=self.sqlglot_dialect
) # using sqlglot==11.4.1
@@ -147,17 +223,17 @@ def as_dict(self):
"The minimal representation of the blocking rule"
output = {}

output["blocking_rule"] = self.blocking_rule
output["blocking_rule"] = self.blocking_rule_sql
output["sql_dialect"] = self.sql_dialect

if self.salting_partitions > 1 and self.sql_dialect == "spark":
if self.is_salted:
output["salting_partitions"] = self.salting_partitions

return output

def _as_completed_dict(self):
if not self.salting_partitions > 1 and self.sql_dialect == "spark":
return self.blocking_rule
if not self.is_salted:
return self.blocking_rule_sql
else:
return self.as_dict()

@@ -166,7 +242,7 @@ def descr(self):
return "Custom" if not hasattr(self, "_description") else self._description

def _abbreviated_sql(self, cutoff=75):
sql = self.blocking_rule
sql = self.blocking_rule_sql
return (sql[:cutoff] + "...") if len(sql) > cutoff else sql

def __repr__(self):
@@ -306,29 +382,17 @@ def block_using_rules_sqls(linker: Linker):
probability = ""

br_sqls = []
for br in blocking_rules:
# Apply our salted rules to resolve skew issues. If no salt was
# selected to be added, then apply the initial blocking rule.
if apply_salt:
salted_blocking_rules = br.salted_blocking_rules
else:
salted_blocking_rules = [br.blocking_rule]

for salted_br in salted_blocking_rules:
sql = f"""
select
{sql_select_expr}
, '{br.match_key}' as match_key
{probability}
from {linker._input_tablename_l} as l
inner join {linker._input_tablename_r} as r
on
({salted_br})
{br.and_not_preceding_rules_sql}
{where_condition}
"""

br_sqls.append(sql)
salted_blocking_rules = (
salted_br
for br in blocking_rules
for salted_br in br.salted_blocking_rule_segments
)
for salted_br in salted_blocking_rules:
parent_br = salted_br.parent_blocking_rule
sql = parent_br.create_pairwise_comparisons_sql(
linker, sql_select_expr, salted_br, probability, where_condition
)
br_sqls.append(sql)

sql = "union all".join(br_sqls)

6 changes: 3 additions & 3 deletions splink/blocking_rule_composition.py
Original file line number Diff line number Diff line change
@@ -295,7 +295,7 @@ def not_(*brls: BlockingRule | dict | str, salting_partitions: int = 1) -> Block

brls, sql_dialect, salt = _parse_blocking_rules(*brls)
br = brls[0]
blocking_rule = f"NOT ({br.blocking_rule})"
blocking_rule = f"NOT ({br.blocking_rule_sql})"

return BlockingRule(
blocking_rule,
@@ -314,9 +314,9 @@ def _br_merge(

brs, sql_dialect, salt = _parse_blocking_rules(*brls)
if len(brs) > 1:
conditions = (f"({br.blocking_rule})" for br in brs)
conditions = (f"({br.blocking_rule_sql})" for br in brs)
else:
conditions = (br.blocking_rule for br in brs)
conditions = (br.blocking_rule_sql for br in brs)

blocking_rule = f" {clause} ".join(conditions)

12 changes: 6 additions & 6 deletions splink/em_training_session.py
Original file line number Diff line number Diff line change
@@ -135,7 +135,7 @@ def _training_log_message(self):
else:
mu = "m and u probabilities"

blocking_rule = self._blocking_rule_for_training.blocking_rule
blocking_rule = self._blocking_rule_for_training.blocking_rule_sql

logger.info(
f"Estimating the {mu} of the model by blocking on:\n"
@@ -176,7 +176,7 @@ def _train(self):
# check that the blocking rule actually generates _some_ record pairs,
# if not give the user a helpful message
if not cvv.as_record_dict(limit=1):
br_sql = f"`{self._blocking_rule_for_training.blocking_rule}`"
br_sql = f"`{self._blocking_rule_for_training.blocking_rule_sql}`"
raise EMTrainingException(
f"Training rule {br_sql} resulted in no record pairs. "
"This means that in the supplied data set "
@@ -195,7 +195,7 @@ def _train(self):
# in the original (main) setting object
expectation_maximisation(self, cvv)

rule = self._blocking_rule_for_training.blocking_rule
rule = self._blocking_rule_for_training.blocking_rule_sql
training_desc = f"EM, blocked on: {rule}"

# Add m and u values to original settings
@@ -254,7 +254,7 @@ def _blocking_adjusted_probability_two_random_records_match(self):
comp_levels = self._comparison_levels_to_reverse_blocking_rule
if not comp_levels:
comp_levels = self._original_settings_obj._get_comparison_levels_corresponding_to_training_blocking_rule( # noqa
self._blocking_rule_for_training.blocking_rule
self._blocking_rule_for_training.blocking_rule_sql
)

for cl in comp_levels:
@@ -271,7 +271,7 @@ def _blocking_adjusted_probability_two_random_records_match(self):
logger.log(
15,
f"\nProb two random records match adjusted for blocking on "
f"{self._blocking_rule_for_training.blocking_rule}: "
f"{self._blocking_rule_for_training.blocking_rule_sql}: "
f"{adjusted_prop_m:.3f}",
)
return adjusted_prop_m
@@ -411,7 +411,7 @@ def __repr__(self):
for cc in self._comparisons_that_cannot_be_estimated
]
)
blocking_rule = self._blocking_rule_for_training.blocking_rule
blocking_rule = self._blocking_rule_for_training.blocking_rule_sql
return (
f"<EMTrainingSession, blocking on {blocking_rule}, "
f"deactivating comparisons {deactivated_cols}>"
2 changes: 1 addition & 1 deletion splink/linker.py
Original file line number Diff line number Diff line change
@@ -1630,7 +1630,7 @@ def estimate_parameters_using_expectation_maximisation(
self._initialise_df_concat_with_tf()

# Extract the blocking rule
blocking_rule = blocking_rule_to_obj(blocking_rule).blocking_rule
blocking_rule = blocking_rule_to_obj(blocking_rule).blocking_rule_sql

if comparisons_to_deactivate:
# If user provided a string, convert to Comparison object
2 changes: 1 addition & 1 deletion splink/settings.py
Original file line number Diff line number Diff line change
@@ -125,7 +125,7 @@ def _get_additional_columns_to_retain(self):
used_by_brs = []
for br in self._blocking_rules_to_generate_predictions:
used_by_brs.extend(
get_columns_used_from_sql(br.blocking_rule, br.sql_dialect)
get_columns_used_from_sql(br.blocking_rule_sql, br.sql_dialect)
)

used_by_brs = [InputColumn(c) for c in used_by_brs]
2 changes: 1 addition & 1 deletion splink/settings_validation/settings_validator.py
Original file line number Diff line number Diff line change
@@ -51,7 +51,7 @@ def uid(self):
@property
def blocking_rules(self):
brs = self.settings_obj._blocking_rules_to_generate_predictions
return [br.blocking_rule for br in brs]
return [br.blocking_rule_sql for br in brs]

@property
def comparisons(self):
2 changes: 1 addition & 1 deletion tests/test_blocking.py
Original file line number Diff line number Diff line change
@@ -40,7 +40,7 @@ def test_binary_composition_internals_OR(test_helpers, dialect):
brl.exact_match_rule("help4"),
]
brs_as_objs = settings_tester._brs_as_objs(brs_as_strings)
brs_as_txt = [blocking_rule_to_obj(br).blocking_rule for br in brs_as_strings]
brs_as_txt = [blocking_rule_to_obj(br).blocking_rule_sql for br in brs_as_strings]

assert brs_as_objs[0].preceding_rules == []