Skip to content

Commit

Permalink
Let apriori always use low_memory processing
Browse files Browse the repository at this point in the history 
Thanks to previous optimizations, processing with low_memory=True is
now as efficient as with low_memory=False, and allows to process much
larger datasets.

Removing processing with low_memory=False makes code simpler.

The downside is that we do not know in advance the number of itemsets to
process, thus it is displayed afterwards.  We now display the number of
itemsets after prune step.
Note that commit 2f928cb introduced a bug, the number of processing
combinations was multiplied by itemset's length.

Since vectorized operations are no more performed on frequent itemsets,
they are stored as list of tuples.
  • Loading branch information
@dbarbier
dbarbier committed Jan 3, 2020
1 parent f8131a7 commit 053a9cb
Show file tree
Hide file tree
Showing 2 changed files with 55 additions and 155 deletions.
208 changes: 54 additions & 154 deletions mlxtend/frequent_patterns/apriori.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,18 +52,17 @@ def generate_new_combinations(old_combinations):
Generator of all combinations based on the last state of Apriori algorithm
Parameters
-----------
old_combinations: np.array
old_combinations: list of tuples
All combinations with enough support in the last step
Combinations are represented by a matrix.
Number of columns is equal to the combination size
Combinations are represented by a list of tuples.
All tuples have the same length, which is equal to the combination size
of the previous step.
Each row represents one combination
Each tuple represents one combination
and contains item type ids in the ascending order
```
0 1
0 15 20
1 15 22
2 17 19
15 20
15 22
17 19
```
Returns
Expand All @@ -89,7 +88,7 @@ def generate_new_combinations(old_combinations):
if head_i != head_j:
break
# Prune old_combination+(item,) if any subset is not frequent
candidate = tuple(old_combination) + (tail_j,)
candidate = old_combination + (tail_j,)
# No need to check the last two values, because test_candidate
# is then old_combinations[i] and old_combinations[j]
for idx in range(len(candidate) - 2):
Expand All @@ -99,90 +98,10 @@ def generate_new_combinations(old_combinations):
# early exit from for-loop skips else clause just below
break
else:
yield from candidate
yield candidate
j = j + 1


def generate_new_combinations_low_memory(old_combinations, X, min_support,
is_sparse):
"""
Generator of all combinations based on the last state of Apriori algorithm
Parameters
-----------
old_combinations: np.array
All combinations with enough support in the last step
Combinations are represented by a matrix.
Number of columns is equal to the combination size
of the previous step.
Each row represents one combination
and contains item type ids in the ascending order
```
0 1
0 15 20
1 15 22
2 17 19
```
X: np.array or scipy sparse matrix
The allowed values are either 0/1 or True/False.
For example,
```
0 True False True True False True
1 True False True False False True
2 True False True False False False
3 True True False False False False
4 False False True True True True
5 False False True False True True
6 False False True False True False
7 True True False False False False
```
min_support : float (default: 0.5)
A float between 0 and 1 for minumum support of the itemsets returned.
The support is computed as the fraction
`transactions_where_item(s)_occur / total_transactions`.
is_sparse : bool True if X is sparse
Returns
-----------
Generator of all combinations from the last step x items
from the previous step. Every combination contains the
number of transactions where this item occurs, followed
by item type ids in the ascending order.
No combination other than generated
do not have a chance to get enough support
Examples
-----------
For usage examples, please see
http://rasbt.github.io/mlxtend/user_guide/frequent_patterns/generate_new_combinations/
"""

items_types_in_previous_step = np.unique(old_combinations.flatten())
rows_count = X.shape[0]
threshold = min_support * rows_count
for old_combination in old_combinations:
max_combination = old_combination[-1]
mask = items_types_in_previous_step > max_combination
valid_items = items_types_in_previous_step[mask]
old_tuple = tuple(old_combination)
if is_sparse:
mask_rows = X[:, old_tuple].toarray().all(axis=1)
X_cols = X[:, valid_items].toarray()
supports = X_cols[mask_rows].sum(axis=0)
else:
mask_rows = X[:, old_tuple].all(axis=1)
supports = X[mask_rows][:, valid_items].sum(axis=0)
valid_indices = (supports >= threshold).nonzero()[0]
for index in valid_indices:
yield supports[index]
yield from old_tuple
yield valid_items[index]


def apriori(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0,
low_memory=False):
"""Get frequent itemsets from a one-hot DataFrame
Expand Down Expand Up @@ -220,16 +139,7 @@ def apriori(df, min_support=0.5, use_colnames=False, max_len=None, verbose=0,
possible itemsets lengths (under the apriori condition) are evaluated.
verbose : int (default: 0)
Shows the number of iterations if >= 1 and `low_memory` is `True`. If
>=1 and `low_memory` is `False`, shows the number of combinations.
low_memory : bool (default: False)
If `True`, uses an iterator to search for combinations above
`min_support`.
Note that while `low_memory=True` should only be used for large dataset
if memory resources are limited, because this implementation is approx.
3-6x slower than the default.
Shows the number of combinations if >= 1.
Returns
-----------
Expand Down Expand Up @@ -292,80 +202,70 @@ def _support(_x, _n_rows, _is_sparse):
X = df.values
else:
X = df.to_coo().tocsc()
# See comment below
X.eliminate_zeros()
is_sparse = True
elif hasattr(df, "sparse"):
# DataFrame with SparseArray (pandas >= 0.24)
if df.size == 0:
X = df.values
else:
X = df.sparse.to_coo().tocsc()
# See comment below
X.eliminate_zeros()
is_sparse = True
else:
# dense DataFrame
X = df.values
is_sparse = False
support = _support(X, X.shape[0], is_sparse)
ary_col_idx = np.arange(X.shape[1])
support_dict = {1: support[support >= min_support]}
itemset_dict = {1: ary_col_idx[support >= min_support].reshape(-1, 1)}
itemset_dict = {1: [(idx,) for idx in np.where(support >= min_support)[0]]}
max_itemset = 1
rows_count = float(X.shape[0])

all_ones = np.ones((int(rows_count), 1))

while max_itemset and max_itemset < (max_len or float('inf')):
next_max_itemset = max_itemset + 1

# With exceptionally large datasets, the matrix operations can use a
# substantial amount of memory. For low memory applications or large
# datasets, set `low_memory=True` to use a slower but more memory-
# efficient implementation.
if low_memory:
combin = generate_new_combinations_low_memory(
itemset_dict[max_itemset], X, min_support, is_sparse)
# slightly faster than creating an array from a list of tuples
combin = np.fromiter(combin, dtype=int)
combin = combin.reshape(-1, next_max_itemset + 1)

if combin.size == 0:
break
if verbose:
print(
'\rProcessing %d combinations | Sampling itemset size %d' %
(combin.size, next_max_itemset), end="")

itemset_dict[next_max_itemset] = combin[:, 1:]
support_dict[next_max_itemset] = combin[:, 0].astype(float) \
/ rows_count
max_itemset = next_max_itemset
combin = generate_new_combinations(itemset_dict[max_itemset])
# count supports
frequent_itemsets = []
frequent_supports = []
processed = 0
if is_sparse:
count = np.empty(X.shape[0], dtype=int)
for itemset in combin:
processed += 1
count[:] = 0
for item in itemset:
# Count nonnull entries via direct access to X indices;
# this requires X to be stored in CSC format, and to call
# X.eliminate_zeros() to remove null entries from X.
count[X.indices[X.indptr[item]:X.indptr[item+1]]] += 1
support = np.count_nonzero(count == len(itemset)) / X.shape[0]
if support >= min_support:
frequent_itemsets.append(itemset)
frequent_supports.append(support)
else:
combin = generate_new_combinations(itemset_dict[max_itemset])
combin = np.fromiter(combin, dtype=int)
combin = combin.reshape(-1, next_max_itemset)

if combin.size == 0:
break
if verbose:
print(
'\rProcessing %d combinations | Sampling itemset size %d' %
(combin.size, next_max_itemset), end="")

if is_sparse:
_bools = X[:, combin[:, 0]] == all_ones
for n in range(1, combin.shape[1]):
_bools = _bools & (X[:, combin[:, n]] == all_ones)
else:
_bools = np.all(X[:, combin], axis=2)

support = _support(np.array(_bools), rows_count, is_sparse)
_mask = (support >= min_support).reshape(-1)
if any(_mask):
itemset_dict[next_max_itemset] = np.array(combin[_mask])
support_dict[next_max_itemset] = np.array(support[_mask])
max_itemset = next_max_itemset
else:
# Exit condition
break
_bools = np.empty(X.shape[0], dtype=bool)
for itemset in combin:
processed += 1
_bools.fill(True)
for item in itemset:
np.logical_and(_bools, X[:, item], out=_bools)
support = np.count_nonzero(_bools) / X.shape[0]
if support >= min_support:
frequent_itemsets.append(itemset)
frequent_supports.append(support)
if verbose:
print(
'\rProcessed %d combinations | Sampling itemset size %d' %
(processed, next_max_itemset), end="")
if not frequent_itemsets:
# Exit condition
break
itemset_dict[next_max_itemset] = frequent_itemsets
support_dict[next_max_itemset] = frequent_supports
max_itemset = next_max_itemset

all_res = []
for k in sorted(itemset_dict):
Expand Down
2 changes: 1 addition & 1 deletion mlxtend/frequent_patterns/tests/test_fpbase.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,7 @@ def test_low_memory_flag(self):
_ = self.fpalgo(self.df, low_memory=True, verbose=1)

# Only get the last value of the stream to reduce test noise
expect = 'Processing 4 combinations | Sampling itemset size 3\n'
expect = 'Processed 1 combinations | Sampling itemset size 3\n'
out = out.getvalue().split('\r')[-1]
assert out == expect
else:
Expand Down

0 comments on commit 053a9cb

Please sign in to comment.