Skip to content

Commit

Permalink
Address PR comments
Browse files Browse the repository at this point in the history
  • Loading branch information
leo-mazzone committed Dec 18, 2024
1 parent 83d5d0c commit 1febf53
Show file tree
Hide file tree
Showing 2 changed files with 18 additions and 19 deletions.
34 changes: 17 additions & 17 deletions src/matchbox/common/hash.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,15 +113,15 @@ def columns_to_value_ordered_hash(data: DataFrame, columns: list[str]) -> Series

class IntMap:
"""
A data structure taking unordered sets of integers, and mapping them a to a key that
1) is a negative integer; 2) does not collide with other keys generated by other
A data structure taking unordered sets of integers, and mapping them a to an ID that
1) is a negative integer; 2) does not collide with other IDs generated by other
instances of this class, as long as they are initialised with a different salt.
The fact that keys are always negative means that it's possible to build a hierarchy
where keys are themselves parts of keyed sets, and it's easy to distinguish integers
mapped to raw data points (which will be non-negative), to integers that are keys to
sets (which will be negative). The salt allows to work with a parallel execution
model, where each worker maintains their separate key space, as long as each worker
The fact that IDs are always negative means that it's possible to build a hierarchy
where IDs are themselves parts of other sets, and it's easy to distinguish integers
mapped to raw data points (which will be non-negative), to integers that are IDs
(which will be negative). The salt allows to work with a parallel execution
model, where each worker maintains their separate ID space, as long as each worker
operates on disjoint subsets of positive integers.
"""

Expand All @@ -131,28 +131,28 @@ def __init__(self, salt: int):
raise ValueError("The salt must be a positive integer")
self.salt: int = salt

def _salt_key(self, k: int) -> int:
def _salt_id(self, i: int) -> int:
"""
Use Cantor pairing function on the salt and a negative int key, and minus it.
Use Cantor pairing function on the salt and a negative int ID, and minus it.
"""
if k >= 0:
raise ValueError("Key must be a negative integer")
return -int(0.5 * (self.salt - k) * (self.salt - k + 1) - k)
if i >= 0:
raise ValueError("ID must be a negative integer")
return -int(0.5 * (self.salt - i) * (self.salt - i + 1) - i)

def index(self, *values: int) -> int:
"""
Args:
values: the integers in the set you want to index
Returns:
The old or new key corresponding to the set
The old or new ID corresponding to the set
"""
value_set = frozenset(values)
if value_set in self.mapping:
return self.mapping[value_set]

new_key: int = -len(self.mapping) - 1
salted_key = self._salt_key(new_key)
self.mapping[value_set] = salted_key
new_id: int = -len(self.mapping) - 1
salted_id = self._salt_id(new_id)
self.mapping[value_set] = salted_id

return salted_key
return salted_id
3 changes: 1 addition & 2 deletions src/matchbox/common/results.py
Original file line number Diff line number Diff line change
Expand Up @@ -559,14 +559,13 @@ def component_to_hierarchy(
hierarchy.extend([(parent, old_comp.pop(), threshold)])

parents, children, probs = zip(*hierarchy, strict=True)
hierarchy_results = pa.table(
return pa.table(
{
"parent": pa.array(parents, type=dtype()),
"child": pa.array(children, type=dtype()),
"probability": pa.array(probs, type=pa.uint8()),
}
)
return hierarchy_results


def to_hierarchical_clusters(
Expand Down

0 comments on commit 1febf53

Please sign in to comment.