Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

optimize modulo calculation #214

Open
wants to merge 8 commits into
base: master
Choose a base branch
from
Open

optimize modulo calculation #214

Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
45f85f6
optimize modulo calculation
chris-ha458 Aug 2, 2023
14e54ba
ATTENTION-this causes minor errors
chris-ha458 Aug 7, 2023
b73e841
Revert "ATTENTION-this causes minor errors"
chris-ha458 Aug 7, 2023
baf82af
change dtypes
chris-ha458 Aug 7, 2023
82e1a54
change initial value
chris-ha458 Aug 7, 2023
8db4dd7
Update minhash.py
chris-ha458 Aug 7, 2023
4de2662
mersenne trick step1 formatting and documentation
chris-ha458 Aug 8, 2023
a402a5b
mersenne trick step 2. bitwise and and right shift
chris-ha458 Aug 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 16 additions & 6 deletions datasketch/minhash.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@

# http://en.wikipedia.org/wiki/Mersenne_prime
_mersenne_prime = np.uint64((1 << 61) - 1)
_max_hash = np.uint64((1 << 32) - 1)
_mersenne_exponent = np.uint8(61)
_max_hash = np.uint32((1 << 32) - 1)
_hash_range = (1 << 32)

class MinHash(object):
Expand Down Expand Up @@ -92,7 +93,7 @@ def __init__(self, num_perm=128, seed=1,
raise ValueError("Numbers of hash values and permutations mismatch")

def _init_hashvalues(self, num_perm):
return np.ones(num_perm, dtype=np.uint64)*_max_hash
return np.full(num_perm, _max_hash,dtype=np.uint32)

def _init_permutations(self, num_perm):
# Create parameters for a random bijective permutation function
Expand All @@ -104,7 +105,7 @@ def _init_permutations(self, num_perm):
], dtype=np.uint64).T

def _parse_hashvalues(self, hashvalues):
return np.array(hashvalues, dtype=np.uint64)
return np.array(hashvalues, dtype=np.uint32)

def update(self, b):
'''Update this MinHash with a new value.
Expand Down Expand Up @@ -135,7 +136,16 @@ def _hash_32(b):
'''
hv = self.hashfunc(b)
a, b = self.permutations
phv = np.bitwise_and((a * hv + b) % _mersenne_prime, _max_hash)
_dividend = np.uint64(a * hv + b)

phv = np.bitwise_and(
# we can apply mersenne's modulo trick here
np.add(
np.bitwise_and(_dividend, _mersenne_prime),
np.right_shift(_dividend, _mersenne_exponent)
),
_max_hash
).astype(np.uint32)
self.hashvalues = np.minimum(phv, self.hashvalues)

def update_batch(self, b):
Expand All @@ -155,9 +165,9 @@ def update_batch(self, b):
minhash = Minhash()
minhash.update_batch([s.encode('utf-8') for s in ["token1", "token2"]])
'''
hv = np.array([self.hashfunc(_b) for _b in b], dtype=np.uint64)
hv = np.array([self.hashfunc(_b) for _b in b], dtype=np.uint32)
a, b = self.permutations
phv = np.bitwise_and(((hv * np.tile(a, (len(hv), 1)).T).T + b) % _mersenne_prime, _max_hash)
phv = np.bitwise_and(np.mod(((hv * np.tile(a, (len(hv), 1)).T).T + b), _mersenne_prime), _max_hash).astype(np.uint32)
self.hashvalues = np.vstack([phv, self.hashvalues]).min(axis=0)

def jaccard(self, other):
Expand Down