diff --git a/src/hybrid_reader.c b/src/hybrid_reader.c
index ca720240eb2..e858159549e 100644
--- a/src/hybrid_reader.c
+++ b/src/hybrid_reader.c
@@ -443,11 +443,28 @@ void HybridIterator_Free(struct indexIterator *self) {
rm_free(it);
}
+static IndexIterator* HybridIteratorReducer(HybridIteratorParams *hParams) {
+ IndexIterator* ret = NULL;
+ if (hParams->childIt && hParams->childIt->type == EMPTY_ITERATOR) {
+ ret = hParams->childIt;
+ } else if (hParams->childIt && hParams->childIt->type == WILDCARD_ITERATOR) {
+ //TODO: When new Iterator API (consider READER_ITERATOR with isWildcard flag)
+ hParams->childIt->Free(hParams->childIt);
+ hParams->qParams.searchMode = VECSIM_STANDARD_KNN;
+ hParams->childIt = NULL;
+ }
+ return ret;
+}
+
IndexIterator *NewHybridVectorIterator(HybridIteratorParams hParams, QueryError *status) {
// If searchMode is out of the expected range.
if (hParams.qParams.searchMode < 0 || hParams.qParams.searchMode >= VECSIM_LAST_SEARCHMODE) {
QueryError_SetError(status, QUERY_EGENERIC, "Creating new hybrid vector iterator has failed");
}
+ IndexIterator* ri = HybridIteratorReducer(&hParams);
+ if (ri) {
+ return ri;
+ }
HybridIterator *hi = rm_new(HybridIterator);
hi->lastDocId = 0;
@@ -503,7 +520,7 @@ IndexIterator *NewHybridVectorIterator(HybridIteratorParams hParams, QueryError
hi->returnedResults = array_new(RSIndexResult *, hParams.query.k);
}
- IndexIterator *ri = &hi->base;
+ ri = &hi->base;
ri->ctx = hi;
// This will be changed later to a valid RLookupKey if there is no syntax error in the query,
// by the creation of the metrics loader results processor.
diff --git a/src/iterators/intersection_iterator.c b/src/iterators/intersection_iterator.c
index c1ea0a6ba4c..289ab126204 100644
--- a/src/iterators/intersection_iterator.c
+++ b/src/iterators/intersection_iterator.c
@@ -11,6 +11,7 @@
#include "empty_iterator.h"
#include "union_iterator.h"
#include "index_result.h"
+#include "wildcard_iterator.h"
/**************************** Read + SkipTo Helpers ****************************/
@@ -230,29 +231,89 @@ static int cmpIter(QueryIterator **it1, QueryIterator **it2) {
return (int)(est1 - est2);
}
-// Set estimation for number of results. Returns false if the query is empty (some of the iterators are NULL)
-static bool II_SetEstimation(IntersectionIterator *it) {
+// Set estimation for number of results.
+static void II_SetEstimation(IntersectionIterator *it) {
// Set the expected number of results to the minimum of all iterators.
// If any of the iterators is NULL, we set the expected number to 0
RS_ASSERT(it->num_its); // Ensure there is at least one iterator, so we can set num_expected to SIZE_MAX temporarily
it->num_expected = SIZE_MAX;
for (uint32_t i = 0; i < it->num_its; ++i) {
QueryIterator *cur = it->its[i];
- if (!cur) {
- // If the current iterator is empty, then the entire query will fail
- it->num_expected = 0;
- return false;
- }
size_t amount = cur->NumEstimated(cur);
if (amount < it->num_expected) {
it->num_expected = amount;
}
}
- return true;
+}
+
+/**
+ * Reduce the intersection iterator by applying these rules:
+ * 1. If any of the iterators is an empty iterator, return the empty iterator and update the number of children
+ * 2. Remove all wildcard iterators since they would not contribute to the intersection (Return one of them if all are wildcards)
+ * 3. If there is only one left child iterator, return it
+ * 4. Otherwise, return NULL and let the caller create the intersection iterator
+*/
+static QueryIterator *IntersectionIteratorReducer(QueryIterator **its, size_t *num) {
+ QueryIterator *ret = NULL;
+
+ // Remove all wildcard iterators from the array
+ size_t current_size = *num;
+ size_t write_idx = 0;
+ bool all_wildcards = true;
+ for (size_t read_idx = 0; read_idx < current_size; read_idx++) {
+ if (IsWildcardIterator(its[read_idx])) {
+ if (!all_wildcards || all_wildcards && read_idx != current_size - 1) {
+ // remove all the wildcards in case there are other non-wildcard iterators
+ // avoid removing it in case it's the last one and all are wildcards
+ its[read_idx]->Free(its[read_idx]);
+ }
+ } else {
+ all_wildcards = false;
+ its[write_idx++] = its[read_idx];
+ }
+ }
+ *num = write_idx;
+
+ // Check for empty iterators
+ for (size_t ii = 0; ii < write_idx; ++ii) {
+ if (!its[ii] || its[ii]->type == EMPTY_ITERATOR) {
+ ret = its[ii] ? its[ii] : IT_V2(NewEmptyIterator)();
+ its[ii] = NULL; // Mark as taken
+ break;
+ }
+ }
+
+ if (ret) {
+ // Free all non-NULL iterators
+ for (size_t ii = 0; ii < write_idx; ++ii) {
+ if (its[ii]) {
+ its[ii]->Free(its[ii]);
+ }
+ }
+ } else {
+ // Handle edge cases after wildcard removal
+ if (write_idx == 0) {
+ // All iterators were wildcards, return the last one which was not Freed
+ ret = its[current_size - 1];
+ } else if (write_idx == 1) {
+ // Only one iterator left, return it directly
+ ret = its[0];
+ }
+ }
+
+ if (ret != NULL) {
+ rm_free(its);
+ }
+
+ return ret;
}
QueryIterator *NewIntersectionIterator(QueryIterator **its, size_t num, int max_slop, bool in_order, double weight) {
RS_ASSERT(its && num > 0);
+ QueryIterator *ret = IntersectionIteratorReducer(its, &num);
+ if (ret != NULL) {
+ return ret;
+ }
IntersectionIterator *it = rm_calloc(1, sizeof(*it));
it->its = its;
it->num_its = num;
@@ -260,36 +321,31 @@ QueryIterator *NewIntersectionIterator(QueryIterator **its, size_t num, int max_
it->max_slop = max_slop < 0 ? INT_MAX : max_slop;
it->in_order = in_order;
- bool allValid = II_SetEstimation(it);
+ II_SetEstimation(it);
// Sort children iterators from low count to high count which reduces the number of iterations.
- if (!in_order && allValid) {
+ if (!in_order) {
qsort(its, num, sizeof(*its), (CompareFunc)cmpIter);
}
// bind the iterator calls
- QueryIterator *base = &it->base;
- base->type = INTERSECT_ITERATOR;
- base->atEOF = false;
- base->lastDocId = 0;
- base->current = NewIntersectResult(num, weight);
- base->NumEstimated = II_NumEstimated;
+ ret = &it->base;
+ ret->type = INTERSECT_ITERATOR;
+ ret->atEOF = false;
+ ret->lastDocId = 0;
+ ret->current = NewIntersectResult(num, weight);
+ ret->NumEstimated = II_NumEstimated;
if (max_slop < 0 && !in_order) {
// No slop and no order means every result is relevant, so we can use the fast path
- base->Read = II_Read;
- base->SkipTo = II_SkipTo;
+ ret->Read = II_Read;
+ ret->SkipTo = II_SkipTo;
} else {
// Otherwise, we need to check relevancy
- base->Read = II_Read_CheckRelevancy;
- base->SkipTo = II_SkipTo_CheckRelevancy;
+ ret->Read = II_Read_CheckRelevancy;
+ ret->SkipTo = II_SkipTo_CheckRelevancy;
}
- base->Free = II_Free;
- base->Rewind = II_Rewind;
+ ret->Free = II_Free;
+ ret->Rewind = II_Rewind;
- if (!allValid) {
- // Some of the iterators are NULL, so the intersection will always be empty.
- base->Free(base);
- base = IT_V2(NewEmptyIterator)();
- }
- return base;
+ return ret;
}
diff --git a/src/iterators/inverted_index_iterator.c b/src/iterators/inverted_index_iterator.c
index ab7411c83ba..58d83c5d402 100644
--- a/src/iterators/inverted_index_iterator.c
+++ b/src/iterators/inverted_index_iterator.c
@@ -278,6 +278,7 @@ static QueryIterator *NewInvIndIterator(InvertedIndex *idx, RSIndexResult *res,
it->skipMulti = skipMulti;
it->sctx = sctx;
it->filterCtx = *filterCtx;
+ it->isWildcard = false;
SetCurrentBlockReader(it);
it->base.current = res;
@@ -361,13 +362,13 @@ QueryIterator *NewInvIndIterator_TermQuery(InvertedIndex *idx, const RedisSearch
}
QueryIterator *NewInvIndIterator_GenericQuery(InvertedIndex *idx, const RedisSearchCtx *sctx, t_fieldIndex fieldIndex,
- enum FieldExpirationPredicate predicate) {
+ enum FieldExpirationPredicate predicate, double weight) {
FieldFilterContext fieldCtx = {
.field = {.isFieldMask = false, .value = {.index = fieldIndex}},
.predicate = predicate,
};
IndexDecoderCtx decoderCtx = {.wideMask = RS_FIELDMASK_ALL}; // Also covers the case of a non-wide schema
- RSIndexResult *record = NewVirtualResult(1, RS_FIELDMASK_ALL);
+ RSIndexResult *record = NewVirtualResult(weight, RS_FIELDMASK_ALL);
record->freq = (predicate == FIELD_EXPIRATION_MISSING) ? 0 : 1; // TODO: is this required?
return NewInvIndIterator(idx, record, &fieldCtx, true, sctx, &decoderCtx);
}
diff --git a/src/iterators/inverted_index_iterator.h b/src/iterators/inverted_index_iterator.h
index 4555347c446..5249f15aed9 100644
--- a/src/iterators/inverted_index_iterator.h
+++ b/src/iterators/inverted_index_iterator.h
@@ -40,6 +40,9 @@ typedef struct InvIndIterator {
// Whether to skip multi values from the same doc
bool skipMulti;
+ // Whether this iterator is result of a wildcard query
+ bool isWildcard;
+
union {
struct {
double rangeMin;
@@ -70,7 +73,7 @@ QueryIterator *NewInvIndIterator_TermQuery(InvertedIndex *idx, const RedisSearch
// The returned iterator will yield "virtual" records. For term/numeric indexes, it is best to use
// the specific functions NewInvIndIterator_TermQuery/NewInvIndIterator_NumericQuery
QueryIterator *NewInvIndIterator_GenericQuery(InvertedIndex *idx, const RedisSearchCtx *sctx, t_fieldIndex fieldIndex,
- enum FieldExpirationPredicate predicate);
+ enum FieldExpirationPredicate predicate, double weight);
#ifdef __cplusplus
}
diff --git a/src/iterators/not_iterator.c b/src/iterators/not_iterator.c
index 9ff19a393bd..a74018e3101 100644
--- a/src/iterators/not_iterator.c
+++ b/src/iterators/not_iterator.c
@@ -226,14 +226,40 @@ static IteratorStatus NI_SkipTo_Optimized(QueryIterator *base, t_docId docId) {
return rc;
}
+/*
+ * Reduce the not iterator by applying these rules:
+ * 1. If the child is an empty iterator or NULL, return a wildcard iterator
+ * 2. If the child is a wildcard iterator, return an empty iterator
+ * 3. Otherwise, return NULL and let the caller create the not iterator
+ */
+static QueryIterator* NotIteratorReducer(QueryIterator *it, t_docId maxDocId, double weight, struct timespec timeout, QueryEvalCtx *q) {
+ RS_ASSERT(q);
+ QueryIterator *ret = NULL;
+ if (!it || it->type == EMPTY_ITERATOR) {
+ ret = IT_V2(NewWildcardIterator)(q, weight);
+ } else if (IsWildcardIterator(it)) {
+ ret = IT_V2(NewEmptyIterator)();
+ }
+ if (ret != NULL) {
+ if (it) {
+ it->Free(it);
+ }
+ }
+ return ret;
+}
+
QueryIterator *IT_V2(NewNotIterator)(QueryIterator *it, t_docId maxDocId, double weight, struct timespec timeout, QueryEvalCtx *q) {
+ QueryIterator *ret = NotIteratorReducer(it, maxDocId, weight, timeout, q);
+ if (ret != NULL) {
+ return ret;
+ }
NotIterator *ni = rm_calloc(1, sizeof(*ni));
- QueryIterator *ret = &ni->base;
- bool optimized = q && q->sctx->spec->rule && q->sctx->spec->rule->index_all;
+ ret = &ni->base;
+ bool optimized = q && q->sctx && q->sctx->spec && q->sctx->spec->rule && q->sctx->spec->rule->index_all;
if (optimized) {
- ni->wcii = IT_V2(NewWildcardIterator_Optimized)(q->sctx);
+ ni->wcii = IT_V2(NewWildcardIterator_Optimized)(q->sctx, weight);
}
- ni->child = it ? it : IT_V2(NewEmptyIterator)();
+ ni->child = it;
ni->maxDocId = maxDocId; // Valid for the optimized case as well, since this is the maxDocId of the embedded wildcard iterator
ni->timeoutCtx = (TimeoutCtx){ .timeout = timeout, .counter = 0 };
diff --git a/src/iterators/optional_iterator.c b/src/iterators/optional_iterator.c
index 3c8ee422ef9..38f6008f1f7 100644
--- a/src/iterators/optional_iterator.c
+++ b/src/iterators/optional_iterator.c
@@ -9,6 +9,7 @@
#include "optional_iterator.h"
#include "wildcard_iterator.h"
+#include "inverted_index_iterator.h"
static void OI_Free(QueryIterator *base) {
OptionalIterator *oi = (OptionalIterator *)base;
@@ -173,14 +174,38 @@ static IteratorStatus OI_Read_NotOptimized(QueryIterator *base) {
return ITERATOR_OK;
}
+/**
+ * Reduce the optional iterator by applying these rules:
+ * 1. If the child is an empty iterator or NULL, return a wildcard iterator
+ * 2. If the child is a wildcard iterator, return it
+ * 3. Otherwise, return NULL and let the caller create the optional iterator
+ */
+static QueryIterator* OptionalIteratorReducer(QueryIterator *it, QueryEvalCtx *q, double weight) {
+ QueryIterator *ret = NULL;
+ if (!it || it->type == EMPTY_ITERATOR) {
+ // If the child is NULL, we return a wildcard iterator. All will be virtual hits
+ ret = IT_V2(NewWildcardIterator)(q, weight);
+ if (it) {
+ it->Free(it);
+ }
+ } else if (IsWildcardIterator(it)) {
+ // All will be real hits
+ ret = it;
+ }
+ return ret;
+}
+
// Create a new OPTIONAL iterator - Non-Optimized version.
QueryIterator *IT_V2(NewOptionalIterator)(QueryIterator *it, QueryEvalCtx *q, double weight) {
- RS_ASSERT(it != NULL);
RS_ASSERT(q && q->sctx && q->sctx->spec && q->docTable);
+ QueryIterator *ret = OptionalIteratorReducer(it, q, weight);
+ if (ret != NULL) {
+ return ret;
+ }
OptionalIterator *oi = rm_calloc(1, sizeof(*oi));
bool optimized = q->sctx->spec->rule && q->sctx->spec->rule->index_all;
if (optimized) {
- oi->wcii = IT_V2(NewWildcardIterator_Optimized)(q->sctx);
+ oi->wcii = IT_V2(NewWildcardIterator_Optimized)(q->sctx, weight);
}
oi->child = it;
oi->virt = NewVirtualResult(weight, RS_FIELDMASK_ALL);
@@ -188,7 +213,7 @@ QueryIterator *IT_V2(NewOptionalIterator)(QueryIterator *it, QueryEvalCtx *q, do
oi->virt->freq = 1;
oi->weight = weight;
- QueryIterator *ret = &oi->base;
+ ret = &oi->base;
ret->type = OPTIONAL_ITERATOR;
ret->atEOF = false;
ret->lastDocId = 0;
diff --git a/src/iterators/union_iterator.c b/src/iterators/union_iterator.c
index 4b2b441a3c0..98b27c93388 100644
--- a/src/iterators/union_iterator.c
+++ b/src/iterators/union_iterator.c
@@ -8,6 +8,8 @@
*/
#include "union_iterator.h"
+#include "wildcard_iterator.h"
+#include "empty_iterator.h"
static inline int cmpLastDocId(const void *e1, const void *e2, const void *udata) {
const QueryIterator *it1 = e1, *it2 = e2;
@@ -372,8 +374,59 @@ static void UI_Free(QueryIterator *base) {
rm_free(ui);
}
+/**
+ * Reduce the union iterator by applying these rules:
+ * 1. Remove all empty iterators
+ * 2. If in quick exit mode and any of the iterators is a wildcard iterator, return it and free the rest
+ * 3. Otherwise, return NULL and let the caller create the union iterator
+ */
+static QueryIterator *UnionIteratorReducer(QueryIterator **its, int *num, bool quickExit, double weight, QueryNodeType type, const char *q_str, IteratorsConfig *config) {
+ QueryIterator *ret = NULL;
+ // Let's remove all the empty iterators from the list
+ size_t current_size = *num;
+ size_t write_idx = 0;
+ for (size_t i = 0; i < current_size; ++i) {
+ if (its[i]) {
+ if (its[i]->type != EMPTY_ITERATOR) {
+ its[write_idx++] = its[i];
+ } else {
+ its[i]->Free(its[i]);
+ }
+ }
+ }
+ *num = write_idx;
+ if (quickExit) {
+ for (size_t i = 0; i < write_idx; ++i) {
+ if (IsWildcardIterator(its[i])) {
+ ret = its[i];
+ for (size_t j = 0; j < write_idx; ++j) {
+ if (i != j && its[j]) {
+ its[j]->Free(its[j]);
+ }
+ }
+ break;
+ }
+ }
+ }
+
+ if (write_idx == 1) {
+ ret = its[0];
+ } else if (write_idx == 0) {
+ ret = IT_V2(NewEmptyIterator)();
+ }
+ if (ret != NULL) {
+ rm_free(its);
+ }
+ return ret;
+}
+
QueryIterator *IT_V2(NewUnionIterator)(QueryIterator **its, int num, bool quickExit,
- double weight, QueryNodeType type, const char *q_str, IteratorsConfig *config) {
+ double weight, QueryNodeType type, const char *q_str, IteratorsConfig *config) {
+
+ QueryIterator* ret = UnionIteratorReducer(its, &num, quickExit, weight, type, q_str, config);
+ if (ret != NULL) {
+ return ret;
+ }
// create union context
UnionIterator *ui = rm_calloc(1, sizeof(UnionIterator));
ui->its_orig = its;
@@ -384,14 +437,14 @@ QueryIterator *IT_V2(NewUnionIterator)(QueryIterator **its, int num, bool quickE
ui->q_str = q_str;
// bind the union iterator calls
- QueryIterator *base = &ui->base;
- base->type = UNION_ITERATOR;
- base->atEOF = false;
- base->lastDocId = 0;
- base->current = NewUnionResult(num, weight);
- base->NumEstimated = UI_NumEstimated;
- base->Free = UI_Free;
- base->Rewind = UI_Rewind;
+ ret = &ui->base;
+ ret->type = UNION_ITERATOR;
+ ret->atEOF = false;
+ ret->lastDocId = 0;
+ ret->current = NewUnionResult(num, weight);
+ ret->NumEstimated = UI_NumEstimated;
+ ret->Free = UI_Free;
+ ret->Rewind = UI_Rewind;
// Choose `Read` and `SkipTo` implementations.
// We have 2 factors for the choice:
@@ -401,15 +454,15 @@ QueryIterator *IT_V2(NewUnionIterator)(QueryIterator **its, int num, bool quickE
// Each implementation if fine-tuned for the best performance in its scenario, and relies on the current state
// of the iterator and how it was left by previous API calls, so we can't change implementation mid-execution.
if (num > config->minUnionIterHeap) {
- base->Read = quickExit ? UI_Read_Quick_Heap : UI_Read_Full_Heap;
- base->SkipTo = quickExit ? UI_Skip_Quick_Heap : UI_Skip_Full_Heap;
+ ret->Read = quickExit ? UI_Read_Quick_Heap : UI_Read_Full_Heap;
+ ret->SkipTo = quickExit ? UI_Skip_Quick_Heap : UI_Skip_Full_Heap;
ui->heap_min_id = rm_malloc(heap_sizeof(num));
heap_init(ui->heap_min_id, cmpLastDocId, NULL, num);
} else {
- base->Read = quickExit ? UI_Read_Quick_Flat : UI_Read_Full_Flat;
- base->SkipTo = quickExit ? UI_Skip_Quick_Flat : UI_Skip_Full_Flat;
+ ret->Read = quickExit ? UI_Read_Quick_Flat : UI_Read_Full_Flat;
+ ret->SkipTo = quickExit ? UI_Skip_Quick_Flat : UI_Skip_Full_Flat;
}
UI_SyncIterList(ui);
- return base;
+ return ret;
}
diff --git a/src/iterators/wildcard_iterator.c b/src/iterators/wildcard_iterator.c
index 659d69a2af9..fba50f7a4d9 100644
--- a/src/iterators/wildcard_iterator.c
+++ b/src/iterators/wildcard_iterator.c
@@ -55,14 +55,25 @@ static void WI_Rewind(QueryIterator *base) {
base->lastDocId = 0;
}
+bool IsWildcardIterator(QueryIterator *it) {
+ if (it && it->type == WILDCARD_ITERATOR) {
+ return true;
+ }
+ if (it && it->type == READ_ITERATOR) {
+ InvIndIterator *invIdxIt = (InvIndIterator *)it;
+ return invIdxIt->isWildcard;
+ }
+ return false;
+}
+
/* Create a new wildcard iterator */
-QueryIterator *IT_V2(NewWildcardIterator_NonOptimized)(t_docId maxId, size_t numDocs) {
+QueryIterator *IT_V2(NewWildcardIterator_NonOptimized)(t_docId maxId, size_t numDocs, double weight) {
WildcardIterator *wi = rm_calloc(1, sizeof(*wi));
wi->currentId = 0;
wi->topId = maxId;
wi->numDocs = numDocs;
QueryIterator *ret = &wi->base;
- ret->current = NewVirtualResult(1, RS_FIELDMASK_ALL);
+ ret->current = NewVirtualResult(weight, RS_FIELDMASK_ALL);
ret->current->freq = 1;
ret->atEOF = false;
ret->lastDocId = 0;
@@ -75,24 +86,29 @@ QueryIterator *IT_V2(NewWildcardIterator_NonOptimized)(t_docId maxId, size_t num
return ret;
}
-QueryIterator *IT_V2(NewWildcardIterator_Optimized)(const RedisSearchCtx *sctx) {
+QueryIterator *IT_V2(NewWildcardIterator_Optimized)(const RedisSearchCtx *sctx, double weight) {
RS_ASSERT(sctx->spec->rule->index_all);
+ QueryIterator *ret = NULL;
if (sctx->spec->existingDocs) {
- return NewInvIndIterator_GenericQuery(sctx->spec->existingDocs, sctx,
- RS_INVALID_FIELD_INDEX, FIELD_EXPIRATION_DEFAULT);
+ ret = NewInvIndIterator_GenericQuery(sctx->spec->existingDocs, sctx,
+ RS_INVALID_FIELD_INDEX, FIELD_EXPIRATION_DEFAULT, weight);
+ InvIndIterator *it = (InvIndIterator *)ret;
+ it->isWildcard = true;
} else {
- return IT_V2(NewEmptyIterator)(); // Index all and no index, means the spec is currently empty.
+ ret = IT_V2(NewEmptyIterator)(); // Index all and no index, means the spec is currently empty.
}
+ return ret;
}
// Returns a new wildcard iterator.
// If the spec tracks all existing documents, it will return an iterator over those documents.
// Otherwise, it will return a non-optimized wildcard iterator
-QueryIterator *IT_V2(NewWildcardIterator)(const QueryEvalCtx *q) {
+QueryIterator *IT_V2(NewWildcardIterator)(const QueryEvalCtx *q, double weight) {
+ QueryIterator *ret = NULL;
if (q->sctx->spec->rule->index_all == true) {
- return IT_V2(NewWildcardIterator_Optimized)(q->sctx);
+ return IT_V2(NewWildcardIterator_Optimized)(q->sctx, weight);
} else {
// Non-optimized wildcard iterator, using a simple doc-id increment as its base.
- return IT_V2(NewWildcardIterator_NonOptimized)(q->docTable->maxDocId, q->docTable->size);
+ return IT_V2(NewWildcardIterator_NonOptimized)(q->docTable->maxDocId, q->docTable->size, weight);
}
}
diff --git a/src/iterators/wildcard_iterator.h b/src/iterators/wildcard_iterator.h
index 23bde009872..5e9852b2a9a 100644
--- a/src/iterators/wildcard_iterator.h
+++ b/src/iterators/wildcard_iterator.h
@@ -27,22 +27,25 @@ typedef struct {
* @param maxId - The maxID to return
* @param numDocs - the number of docs to return
*/
-QueryIterator *IT_V2(NewWildcardIterator_NonOptimized)(t_docId maxId, size_t numDocs);
+QueryIterator *IT_V2(NewWildcardIterator_NonOptimized)(t_docId maxId, size_t numDocs, double weight);
/**
* Create a new optimized wildcard iterator.
* This iterator can only be used when the index is configured to index all documents.
* @param sctx - The search context
*/
-QueryIterator *IT_V2(NewWildcardIterator_Optimized)(const RedisSearchCtx *sctx);
+QueryIterator *IT_V2(NewWildcardIterator_Optimized)(const RedisSearchCtx *sctx, double weight);
/**
* Create a new wildcard iterator.
* If possible, it will use the optimized wildcard iterator,
* otherwise it will fall back to the non-optimized version.
* @param q - The query evaluation context
+ * @param weight - The weight of the iterator
*/
-QueryIterator *IT_V2(NewWildcardIterator)(const QueryEvalCtx *q);
+QueryIterator *IT_V2(NewWildcardIterator)(const QueryEvalCtx *q, double weight);
+
+bool IsWildcardIterator(QueryIterator *it);
#ifdef __cplusplus
}
diff --git a/src/query.c b/src/query.c
index 51eda4a7721..cc4ed5ddd72 100644
--- a/src/query.c
+++ b/src/query.c
@@ -912,10 +912,13 @@ static IndexIterator *Query_EvalWildcardNode(QueryEvalCtx *q, QueryNode *qn) {
static IndexIterator *Query_EvalNotNode(QueryEvalCtx *q, QueryNode *qn) {
RS_LOG_ASSERT(qn->type == QN_NOT, "query node type should be not")
+ IndexIterator *child = NULL;
+ bool currently_notSubtree = q->notSubtree;
+ q->notSubtree = true;
+ child = Query_EvalNode(q, qn->children[0]);
+ q->notSubtree = currently_notSubtree;
- return NewNotIterator(qn ? Query_EvalNode(q, qn->children[0]) : NULL,
- q->docTable->maxDocId, qn->opts.weight, q->sctx->time.timeout,
- q);
+ return NewNotIterator(child, q->docTable->maxDocId, qn->opts.weight, q->sctx->time.timeout, q);
}
static IndexIterator *Query_EvalOptionalNode(QueryEvalCtx *q, QueryNode *qn) {
@@ -1089,8 +1092,8 @@ static IndexIterator *Query_EvalUnionNode(QueryEvalCtx *q, QueryNode *qn) {
rm_free(iters);
return ret;
}
-
- IndexIterator *ret = NewUnionIterator(iters, n, 0, qn->opts.weight, QN_UNION, NULL, q->config);
+ bool quickExit = q->notSubtree;
+ IndexIterator *ret = NewUnionIterator(iters, n, quickExit, qn->opts.weight, QN_UNION, NULL, q->config);
return ret;
}
@@ -1477,8 +1480,10 @@ static IndexIterator *Query_EvalTagNode(QueryEvalCtx *q, QueryNode *qn) {
array_free(total_its);
}
}
-
- ret = NewUnionIterator(iters, n, 0, qn->opts.weight, QN_TAG, NULL, q->config);
+ // We want to get results with all the matching children (`quickExit == false`), unless:
+ // 1. We are a `Not` sub-tree, so we only care about the set of IDs
+ bool quickExit = q->notSubtree;
+ ret = NewUnionIterator(iters, n, quickExit, qn->opts.weight, QN_TAG, NULL, q->config);
done:
return ret;
@@ -1602,6 +1607,7 @@ IndexIterator *QAST_Iterate(QueryAST *qast, const RSSearchOptions *opts, RedisSe
.metricRequestsP = &qast->metricRequests,
.reqFlags = reqflags,
.config = &qast->config,
+ .notSubtree = false,
};
IndexIterator *root = Query_EvalNode(&qectx, qast->root);
if (!root) {
diff --git a/src/query_ctx.h b/src/query_ctx.h
index fe09e61da79..0e20c03fffd 100644
--- a/src/query_ctx.h
+++ b/src/query_ctx.h
@@ -24,4 +24,5 @@ typedef struct QueryEvalCtx {
DocTable *docTable;
uint32_t reqFlags;
IteratorsConfig *config;
+ bool notSubtree;
} QueryEvalCtx;
diff --git a/src/query_parser/v1/parser.c b/src/query_parser/v1/parser.c
index 49495f13a55..ed1757007f2 100644
--- a/src/query_parser/v1/parser.c
+++ b/src/query_parser/v1/parser.c
@@ -76,6 +76,27 @@ static int one_not_null(void *a, void *b, void *out) {
}
}
+// optimize NOT nodes: NOT(NOT(A)) = A
+// if the child is a NOT node, return its child instead of creating a double negation
+static inline struct RSQueryNode* not_step(struct RSQueryNode* child) {
+ if (!child) {
+ return NULL;
+ }
+
+ // If the child is a NOT node, return its child (double negation elimination)
+ if (child->type == QN_NOT) {
+ struct RSQueryNode* grandchild = child->children[0];
+ // Detach the grandchild from its parent to prevent it from being freed
+ child->children[0] = NULL;
+ // Free the NOT node (the parent)
+ QueryNode_Free(child);
+ return grandchild;
+ }
+
+ // Otherwise, create a new NOT node
+ return NewNotNode(child);
+}
+
/**************** End of %include directives **********************************/
/* These constants specify the various numeric values for terminal symbols.
***************** Begin token definitions *************************************/
@@ -134,7 +155,7 @@ static int one_not_null(void *a, void *b, void *out) {
** the minor type might be the name of the identifier.
** Each non-terminal can have a different minor type.
** Terminal symbols all have the same minor type, though.
-** This macros defines the minor type for terminal
+** This macros defines the minor type for terminal
** symbols.
** YYMINORTYPE is the data type used for all minor types.
** This is typically a union of many types, one of
@@ -184,8 +205,8 @@ typedef union {
#define YYSTACKDEPTH 100
#endif
#define RSQueryParser_v1_ARG_SDECL QueryParseCtx *ctx ;
-#define RSQueryParser_v1_ARG_PDECL , QueryParseCtx *ctx
-#define RSQueryParser_v1_ARG_PARAM ,ctx
+#define RSQueryParser_v1_ARG_PDECL , QueryParseCtx *ctx
+#define RSQueryParser_v1_ARG_PARAM ,ctx
#define RSQueryParser_v1_ARG_FETCH QueryParseCtx *ctx =yypParser->ctx ;
#define RSQueryParser_v1_ARG_STORE yypParser->ctx =ctx ;
#define RSQueryParser_v1_CTX_SDECL
@@ -224,7 +245,7 @@ typedef union {
/* Next are the tables used to determine what action to take based on the
** current state and lookahead token. These tables are used to implement
** functions that take a state number and lookahead value and return an
-** action integer.
+** action integer.
**
** Suppose the action integer is N. Then the action is determined as
** follows
@@ -368,9 +389,9 @@ static const YYACTIONTYPE yy_default[] = {
};
/********** End of lemon-generated parsing tables *****************************/
-/* The next table maps tokens (terminal symbols) into fallback tokens.
+/* The next table maps tokens (terminal symbols) into fallback tokens.
** If a construct like the following:
-**
+**
** %fallback ID X Y Z.
**
** appears in the grammar, then ID becomes a fallback token for X, Y,
@@ -443,10 +464,10 @@ static char *yyTracePrompt = 0;
#endif /* NDEBUG */
#ifndef NDEBUG
-/*
+/*
** Turn parser tracing on by giving a stream to which to write the trace
** and a prompt to preface each trace message. Tracing is turned off
-** by making either argument NULL
+** by making either argument NULL
**
** Inputs:
**
@@ -471,7 +492,7 @@ void RSQueryParser_v1_Trace(FILE *TraceFILE, char *zTracePrompt){
#if defined(YYCOVERAGE) || !defined(NDEBUG)
/* For tracing shifts, the names of all terminals and nonterminals
** are required. The following table supplies these names */
-static const char *const yyTokenName[] = {
+static const char *const yyTokenName[] = {
/* 0 */ "$",
/* 1 */ "LOWEST",
/* 2 */ "TILDE",
@@ -614,7 +635,7 @@ static int yyGrowStack(yyParser *p){
#endif
p->yystksz = newSize;
}
- return pNew==0;
+ return pNew==0;
}
#endif
@@ -656,7 +677,7 @@ void RSQueryParser_v1_Init(void *yypRawParser RSQueryParser_v1_CTX_PDECL){
}
#ifndef RSQueryParser_v1__ENGINEALWAYSONSTACK
-/*
+/*
** This function allocates a new parser.
** The only argument is a pointer to a function which works like
** malloc.
@@ -683,7 +704,7 @@ void *RSQueryParser_v1_Alloc(void *(*mallocProc)(YYMALLOCARGTYPE) RSQueryParser_
/* The following function deletes the "minor type" or semantic value
** associated with a symbol. The symbol can be either a terminal
** or nonterminal. "yymajor" is the symbol code, and "yypminor" is
-** a pointer to the value to be deleted. The code used to do the
+** a pointer to the value to be deleted. The code used to do the
** deletions is derived from the %destructor and/or %token_destructor
** directives of the input grammar.
*/
@@ -698,7 +719,7 @@ static void yy_destructor(
/* Here is inserted the actions which take place when a
** terminal or non-terminal is destroyed. This can happen
** when the symbol is popped from the stack during a
- ** reduce or during error processing or when a parser is
+ ** reduce or during error processing or when a parser is
** being destroyed before it is finished parsing.
**
** Note: during a reduce, the only symbols destroyed are those
@@ -712,7 +733,7 @@ static void yy_destructor(
case 42: /* modifier */
case 43: /* term */
{
-
+
}
break;
case 29: /* expr */
@@ -722,23 +743,23 @@ static void yy_destructor(
case 35: /* fuzzy */
case 36: /* tag_list */
{
- QueryNode_Free((yypminor->yy75));
+ QueryNode_Free((yypminor->yy75));
}
break;
case 30: /* attribute */
{
- rm_free((char*)(yypminor->yy87).value);
+ rm_free((char*)(yypminor->yy87).value);
}
break;
case 31: /* attribute_list */
{
- array_free_ex((yypminor->yy1), rm_free((char*)((QueryAttribute*)ptr )->value));
+ array_free_ex((yypminor->yy1), rm_free((char*)((QueryAttribute*)ptr )->value));
}
break;
case 37: /* geo_filter */
case 40: /* numeric_range */
{
- QueryParam_Free((yypminor->yy62));
+ QueryParam_Free((yypminor->yy62));
}
break;
case 38: /* modifierlist */
@@ -791,7 +812,7 @@ void RSQueryParser_v1_Finalize(void *p){
}
#ifndef RSQueryParser_v1__ENGINEALWAYSONSTACK
-/*
+/*
** Deallocate and destroy a parser. Destructors are called for
** all stack elements before shutting the parser down.
**
@@ -1011,7 +1032,7 @@ static void yy_shift(
assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) );
}
#endif
-#if YYSTACKDEPTH>0
+#if YYSTACKDEPTH>0
if( yypParser->yytos>yypParser->yystackEnd ){
yypParser->yytos--;
yyStackOverflow(yypParser);
@@ -1418,11 +1439,7 @@ static YYACTIONTYPE yy_reduce(
break;
case 25: /* expr ::= MINUS expr */
{
- if (yymsp[0].minor.yy75) {
- yymsp[-1].minor.yy75 = NewNotNode(yymsp[0].minor.yy75);
- } else {
- yymsp[-1].minor.yy75 = NULL;
- }
+ yymsp[-1].minor.yy75 = not_step(yymsp[0].minor.yy75);
}
break;
case 26: /* expr ::= TILDE expr */
@@ -1801,7 +1818,7 @@ void RSQueryParser_v1_(
(int)(yypParser->yytos - yypParser->yystack));
}
#endif
-#if YYSTACKDEPTH>0
+#if YYSTACKDEPTH>0
if( yypParser->yytos>=yypParser->yystackEnd ){
yyStackOverflow(yypParser);
break;
@@ -1840,7 +1857,7 @@ void RSQueryParser_v1_(
#ifdef YYERRORSYMBOL
/* A syntax error has occurred.
** The response to an error depends upon whether or not the
- ** grammar defines an error token "ERROR".
+ ** grammar defines an error token "ERROR".
**
** This is what we do if the grammar does define ERROR:
**
diff --git a/src/query_parser/v1/parser.y b/src/query_parser/v1/parser.y
index 5528d4f49d3..7e067c17c5b 100644
--- a/src/query_parser/v1/parser.y
+++ b/src/query_parser/v1/parser.y
@@ -91,6 +91,27 @@ static int one_not_null(void *a, void *b, void *out) {
}
}
+// optimize NOT nodes: NOT(NOT(A)) = A
+// if the child is a NOT node, return its child instead of creating a double negation
+static inline struct RSQueryNode* not_step(struct RSQueryNode* child) {
+ if (!child) {
+ return NULL;
+ }
+
+ // If the child is a NOT node, return its child (double negation elimination)
+ if (child->type == QN_NOT) {
+ struct RSQueryNode* grandchild = child->children[0];
+ // Detach the grandchild from its parent to prevent it from being freed
+ child->children[0] = NULL;
+ // Free the NOT node (the parent)
+ QueryNode_Free(child);
+ return grandchild;
+ }
+
+ // Otherwise, create a new NOT node
+ return NewNotNode(child);
+}
+
} // END %include
%extra_argument { QueryParseCtx *ctx }
@@ -359,11 +380,7 @@ termlist(A) ::= termlist(B) STOPWORD . [TERMLIST] {
/////////////////////////////////////////////////////////////////
expr(A) ::= MINUS expr(B) . {
- if (B) {
- A = NewNotNode(B);
- } else {
- A = NULL;
- }
+ A = not_step(B);
}
/////////////////////////////////////////////////////////////////
diff --git a/src/query_parser/v2/parser.c b/src/query_parser/v2/parser.c
index f75c4bd3075..0bada7c1e0c 100644
--- a/src/query_parser/v2/parser.c
+++ b/src/query_parser/v2/parser.c
@@ -104,6 +104,27 @@ static inline struct RSQueryNode* union_step(struct RSQueryNode* B, struct RSQue
return A;
}
+// optimize NOT nodes: NOT(NOT(A)) = A
+// if the child is a NOT node, return its child instead of creating a double negation
+static inline struct RSQueryNode* not_step(struct RSQueryNode* child) {
+ if (!child) {
+ return NULL;
+ }
+
+ // If the child is a NOT node, return its child (double negation elimination)
+ if (child->type == QN_NOT) {
+ struct RSQueryNode* grandchild = child->children[0];
+ // Detach the grandchild from its parent to prevent it from being freed
+ child->children[0] = NULL;
+ // Free the NOT node (the parent)
+ QueryNode_Free(child);
+ return grandchild;
+ }
+
+ // Otherwise, create a new NOT node
+ return NewNotNode(child);
+}
+
static void setup_trace(QueryParseCtx *ctx) {
#ifdef PARSER_DEBUG
void RSQueryParser_Trace(FILE*, char*);
@@ -233,7 +254,7 @@ static inline char *toksep2(char **s, size_t *tokLen) {
** the minor type might be the name of the identifier.
** Each non-terminal can have a different minor type.
** Terminal symbols all have the same minor type, though.
-** This macros defines the minor type for terminal
+** This macros defines the minor type for terminal
** symbols.
** YYMINORTYPE is the data type used for all minor types.
** This is typically a union of many types, one of
@@ -286,8 +307,8 @@ typedef union {
#define YYSTACKDEPTH 256
#endif
#define RSQueryParser_v2_ARG_SDECL QueryParseCtx *ctx ;
-#define RSQueryParser_v2_ARG_PDECL , QueryParseCtx *ctx
-#define RSQueryParser_v2_ARG_PARAM ,ctx
+#define RSQueryParser_v2_ARG_PDECL , QueryParseCtx *ctx
+#define RSQueryParser_v2_ARG_PARAM ,ctx
#define RSQueryParser_v2_ARG_FETCH QueryParseCtx *ctx =yypParser->ctx ;
#define RSQueryParser_v2_ARG_STORE yypParser->ctx =ctx ;
#define RSQueryParser_v2_CTX_SDECL
@@ -327,7 +348,7 @@ typedef union {
/* Next are the tables used to determine what action to take based on the
** current state and lookahead token. These tables are used to implement
** functions that take a state number and lookahead value and return an
-** action integer.
+** action integer.
**
** Suppose the action integer is N. Then the action is determined as
** follows
@@ -610,9 +631,9 @@ static const YYACTIONTYPE yy_default[] = {
};
/********** End of lemon-generated parsing tables *****************************/
-/* The next table maps tokens (terminal symbols) into fallback tokens.
+/* The next table maps tokens (terminal symbols) into fallback tokens.
** If a construct like the following:
-**
+**
** %fallback ID X Y Z.
**
** appears in the grammar, then ID becomes a fallback token for X, Y,
@@ -727,10 +748,10 @@ static char *yyTracePrompt = 0;
#endif /* NDEBUG */
#ifndef NDEBUG
-/*
+/*
** Turn parser tracing on by giving a stream to which to write the trace
** and a prompt to preface each trace message. Tracing is turned off
-** by making either argument NULL
+** by making either argument NULL
**
** Inputs:
**
@@ -755,7 +776,7 @@ void RSQueryParser_v2_Trace(FILE *TraceFILE, char *zTracePrompt){
#if defined(YYCOVERAGE) || !defined(NDEBUG)
/* For tracing shifts, the names of all terminals and nonterminals
** are required. The following table supplies these names */
-static const char *const yyTokenName[] = {
+static const char *const yyTokenName[] = {
/* 0 */ "$",
/* 1 */ "LOWEST",
/* 2 */ "TEXTEXPR",
@@ -985,7 +1006,7 @@ static int yyGrowStack(yyParser *p){
#endif
p->yystksz = newSize;
}
- return pNew==0;
+ return pNew==0;
}
#endif
@@ -1027,7 +1048,7 @@ void RSQueryParser_v2_Init(void *yypRawParser RSQueryParser_v2_CTX_PDECL){
}
#ifndef RSQueryParser_v2__ENGINEALWAYSONSTACK
-/*
+/*
** This function allocates a new parser.
** The only argument is a pointer to a function which works like
** malloc.
@@ -1054,7 +1075,7 @@ void *RSQueryParser_v2_Alloc(void *(*mallocProc)(YYMALLOCARGTYPE) RSQueryParser_
/* The following function deletes the "minor type" or semantic value
** associated with a symbol. The symbol can be either a terminal
** or nonterminal. "yymajor" is the symbol code, and "yypminor" is
-** a pointer to the value to be deleted. The code used to do the
+** a pointer to the value to be deleted. The code used to do the
** deletions is derived from the %destructor and/or %token_destructor
** directives of the input grammar.
*/
@@ -1069,7 +1090,7 @@ static void yy_destructor(
/* Here is inserted the actions which take place when a
** terminal or non-terminal is destroyed. This can happen
** when the symbol is popped from the stack during a
- ** reduce or during error processing or when a parser is
+ ** reduce or during error processing or when a parser is
** being destroyed before it is finished parsing.
**
** Note: during a reduce, the only symbols destroyed are those
@@ -1092,7 +1113,7 @@ static void yy_destructor(
case 74: /* as */
case 75: /* param_size */
{
-
+
}
break;
case 42: /* expr */
@@ -1111,22 +1132,22 @@ static void yy_destructor(
case 58: /* vector_command */
case 59: /* vector_range_command */
{
- QueryNode_Free((yypminor->yy3));
+ QueryNode_Free((yypminor->yy3));
}
break;
case 43: /* attribute */
{
- rm_free((char*)(yypminor->yy79).value);
+ rm_free((char*)(yypminor->yy79).value);
}
break;
case 44: /* attribute_list */
{
- array_free_ex((yypminor->yy41), rm_free((char*)((QueryAttribute*)ptr )->value));
+ array_free_ex((yypminor->yy41), rm_free((char*)((QueryAttribute*)ptr )->value));
}
break;
case 55: /* geo_filter */
{
- QueryParam_Free((yypminor->yy62));
+ QueryParam_Free((yypminor->yy62));
}
break;
case 61: /* vector_attribute_list */
@@ -1192,7 +1213,7 @@ void RSQueryParser_v2_Finalize(void *p){
}
#ifndef RSQueryParser_v2__ENGINEALWAYSONSTACK
-/*
+/*
** Deallocate and destroy a parser. Destructors are called for
** all stack elements before shutting the parser down.
**
@@ -1415,7 +1436,7 @@ static void yy_shift(
assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) );
}
#endif
-#if YYSTACKDEPTH>0
+#if YYSTACKDEPTH>0
if( yypParser->yytos>yypParser->yystackEnd ){
yypParser->yytos--;
yyStackOverflow(yypParser);
@@ -1935,11 +1956,7 @@ static YYACTIONTYPE yy_reduce(
case 35: /* expr ::= MINUS expr */
case 36: /* text_expr ::= MINUS text_expr */ yytestcase(yyruleno==36);
{
- if (yymsp[0].minor.yy3) {
- yymsp[-1].minor.yy3 = NewNotNode(yymsp[0].minor.yy3);
- } else {
- yymsp[-1].minor.yy3 = NULL;
- }
+ yymsp[-1].minor.yy3 = not_step(yymsp[0].minor.yy3);
}
break;
case 37: /* expr ::= TILDE expr */
@@ -2186,7 +2203,7 @@ static YYACTIONTYPE yy_reduce(
} else {
QueryParam *qp = NewNumericFilterQueryParam_WithParams(ctx, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0, 1, 1);
QueryNode* E = NewNumericNode(qp, yymsp[-2].minor.yy150.fs);
- yylhsminor.yy3 = NewNotNode(E);
+ yylhsminor.yy3 = not_step(E);
}
}
yymsp[-2].minor.yy3 = yylhsminor.yy3;
@@ -2786,7 +2803,7 @@ void RSQueryParser_v2_(
(int)(yypParser->yytos - yypParser->yystack));
}
#endif
-#if YYSTACKDEPTH>0
+#if YYSTACKDEPTH>0
if( yypParser->yytos>=yypParser->yystackEnd ){
yyStackOverflow(yypParser);
break;
@@ -2825,7 +2842,7 @@ void RSQueryParser_v2_(
#ifdef YYERRORSYMBOL
/* A syntax error has occurred.
** The response to an error depends upon whether or not the
- ** grammar defines an error token "ERROR".
+ ** grammar defines an error token "ERROR".
**
** This is what we do if the grammar does define ERROR:
**
diff --git a/src/query_parser/v2/parser.y b/src/query_parser/v2/parser.y
index 65bc75b2dbf..a278d13b76c 100644
--- a/src/query_parser/v2/parser.y
+++ b/src/query_parser/v2/parser.y
@@ -147,6 +147,27 @@ static inline struct RSQueryNode* union_step(struct RSQueryNode* B, struct RSQue
return A;
}
+// optimize NOT nodes: NOT(NOT(A)) = A
+// if the child is a NOT node, return its child instead of creating a double negation
+static inline struct RSQueryNode* not_step(struct RSQueryNode* child) {
+ if (!child) {
+ return NULL;
+ }
+
+ // If the child is a NOT node, return its child (double negation elimination)
+ if (child->type == QN_NOT) {
+ struct RSQueryNode* grandchild = child->children[0];
+ // Detach the grandchild from its parent to prevent it from being freed
+ child->children[0] = NULL;
+ // Free the NOT node (the parent)
+ QueryNode_Free(child);
+ return grandchild;
+ }
+
+ // Otherwise, create a new NOT node
+ return NewNotNode(child);
+}
+
static void setup_trace(QueryParseCtx *ctx) {
#ifdef PARSER_DEBUG
void RSQueryParser_Trace(FILE*, char*);
@@ -562,19 +583,11 @@ termlist(A) ::= termlist(B) param_term(C) . [TERMLIST] {
/////////////////////////////////////////////////////////////////
expr(A) ::= MINUS expr(B) . {
- if (B) {
- A = NewNotNode(B);
- } else {
- A = NULL;
- }
+ A = not_step(B);
}
text_expr(A) ::= MINUS text_expr(B) . {
- if (B) {
- A = NewNotNode(B);
- } else {
- A = NULL;
- }
+ A = not_step(B);
}
/////////////////////////////////////////////////////////////////
@@ -836,7 +849,7 @@ expr(A) ::= modifier(B) NOT_EQUAL param_num(C) . {
} else {
QueryParam *qp = NewNumericFilterQueryParam_WithParams(ctx, &C, &C, 1, 1);
QueryNode* E = NewNumericNode(qp, B.fs);
- A = NewNotNode(E);
+ A = not_step(E);
}
}
diff --git a/tests/cpptests/iterator_util.h b/tests/cpptests/iterator_util.h
index 32197ccc099..aed5477d12e 100644
--- a/tests/cpptests/iterator_util.h
+++ b/tests/cpptests/iterator_util.h
@@ -36,7 +36,7 @@ class MockIterator {
std::optional sleepTime; // Sleep for this duration before returning from Read/SkipTo
private:
void Init() {
- base.type = READ_ITERATOR;
+ base.type = MAX_ITERATOR;
base.atEOF = false;
base.lastDocId = 0;
base.current = NewVirtualResult(1, RS_FIELDMASK_ALL);
@@ -113,13 +113,13 @@ class MockIterator {
: docIds({std::forward(args)...}), whenDone(ITERATOR_EOF), nextIndex(0), readCount(0), sleepTime(sleep) {
Init();
}
-
+
template
MockIterator(IteratorStatus st, Args&&... ids_args)
: docIds({std::forward(ids_args)...}), whenDone(st), nextIndex(0), readCount(0), sleepTime(std::nullopt) {
Init();
}
-
+
template
MockIterator(IteratorStatus st, std::chrono::nanoseconds sleep, Args&&... ids_args)
: docIds({std::forward(ids_args)...}), whenDone(st), nextIndex(0), readCount(0), sleepTime(sleep) {
diff --git a/tests/cpptests/micro-benchmarks/benchmark_index_iterator.cpp b/tests/cpptests/micro-benchmarks/benchmark_index_iterator.cpp
index 4afd1746456..ce841b18f0a 100644
--- a/tests/cpptests/micro-benchmarks/benchmark_index_iterator.cpp
+++ b/tests/cpptests/micro-benchmarks/benchmark_index_iterator.cpp
@@ -146,7 +146,7 @@ class BM_IndexIterator : public BM_IndexIterator_Base {
FieldFilterContext fieldCtx = {.field = fieldMaskOrIndex, .predicate = FIELD_EXPIRATION_DEFAULT};
iterator = NewInvIndIterator_NumericQuery(index, nullptr, &fieldCtx, nullptr, -INFINITY, INFINITY);
} else if (flags == Index_DocIdsOnly || flags == (Index_DocIdsOnly | Index_Temporary)) {
- iterator = NewInvIndIterator_GenericQuery(index, nullptr, 0, FIELD_EXPIRATION_DEFAULT);
+ iterator = NewInvIndIterator_GenericQuery(index, nullptr, 0, FIELD_EXPIRATION_DEFAULT, 1.0);
} else {
iterator = NewInvIndIterator_TermQuery(index, nullptr, {true, RS_FIELDMASK_ALL}, nullptr, 1.0);
}
diff --git a/tests/cpptests/micro-benchmarks/benchmark_wildcard_non_optimized_iterator.cpp b/tests/cpptests/micro-benchmarks/benchmark_wildcard_non_optimized_iterator.cpp
index b9c29662fe4..9f447beea86 100644
--- a/tests/cpptests/micro-benchmarks/benchmark_wildcard_non_optimized_iterator.cpp
+++ b/tests/cpptests/micro-benchmarks/benchmark_wildcard_non_optimized_iterator.cpp
@@ -37,7 +37,7 @@ class BM_WildcardIterator : public benchmark::Fixture {
// Initialize iterators based on the test name
if constexpr (std::is_same_v) {
- iterator_base = IT_V2(NewWildcardIterator_NonOptimized)(maxDocId, numDocs);
+ iterator_base = IT_V2(NewWildcardIterator_NonOptimized)(maxDocId, numDocs, 1.0);
} else {
iterator_base = NewWildcardIterator_NonOptimized(maxDocId, numDocs);
}
diff --git a/tests/cpptests/test_cpp_index.cpp b/tests/cpptests/test_cpp_index.cpp
index 16242896e6f..296f28c42bd 100644
--- a/tests/cpptests/test_cpp_index.cpp
+++ b/tests/cpptests/test_cpp_index.cpp
@@ -1639,3 +1639,80 @@ TEST_F(IndexTest, testRawDocId) {
InvertedIndex_Free(idx);
RSGlobalConfig.invertedIndexRawDocidEncoding = previousConfig;
}
+
+// Test HybridIteratorReducer optimization with NULL child iterator
+TEST_F(IndexTest, testHybridIteratorReducerWithEmptyChild) {
+ // Create hybrid params with NULL child iterator
+ size_t n = 100;
+ size_t d = 4;
+ size_t step = 4;
+ size_t max_id = n*step;
+ size_t k = 10;
+
+ VecSimQueryParams queryParams = {0};
+ KNNVectorQuery top_k_query = {.vector = NULL, .vecLen = d, .k = k, .order = BY_SCORE};
+
+ HybridIteratorParams hParams = {
+ .sctx = NULL,
+ .index = NULL,
+ .dim = d,
+ .elementType = VecSimType_FLOAT32,
+ .spaceMetric = VecSimMetric_L2,
+ .query = top_k_query,
+ .qParams = queryParams,
+ .vectorScoreField = (char *)"__v_score",
+ .canTrimDeepResults = true,
+ .childIt = NewEmptyIterator(), // Empty child iterator
+ .filterCtx = NULL
+ };
+
+ QueryError err = {QUERY_OK};
+ IndexIterator *hybridIt = NewHybridVectorIterator(hParams, &err);
+
+ // Verify the iterator was not created due to NULL child
+ ASSERT_FALSE(QueryError_HasError(&err));
+ ASSERT_TRUE(hybridIt == hParams.childIt);
+ ASSERT_EQ(hybridIt->type, EMPTY_ITERATOR);
+ hybridIt->Free(hybridIt);
+}
+
+// Test HybridIteratorReducer optimization with invalid child iterator
+TEST_F(IndexTest, testHybridIteratorReducerWithWildcardChild) {
+ size_t n = 100;
+ size_t d = 4;
+ size_t step = 4;
+ size_t max_id = n*step;
+ size_t k = 10;
+
+ VecSimQueryParams queryParams = {0};
+ KNNVectorQuery top_k_query = {.vector = NULL, .vecLen = d, .k = k, .order = BY_SCORE};
+ FieldFilterContext filterCtx = {.field = {.isFieldMask = false, .value = {.index = RS_INVALID_FIELD_INDEX}}, .predicate = FIELD_EXPIRATION_DEFAULT};
+
+ // Mock the WILDCARD_ITERATOR consideration
+ IndexIterator* wildcardIt = NewEmptyIterator();
+ wildcardIt->type = WILDCARD_ITERATOR;
+
+ HybridIteratorParams hParams = {
+ .sctx = NULL,
+ .index = NULL,
+ .dim = d,
+ .elementType = VecSimType_FLOAT32,
+ .spaceMetric = VecSimMetric_L2,
+ .query = top_k_query,
+ .qParams = queryParams,
+ .vectorScoreField = (char *)"__v_score",
+ .canTrimDeepResults = true,
+ .childIt = wildcardIt,
+ .filterCtx = &filterCtx
+ };
+
+ QueryError err = {QUERY_OK};
+ IndexIterator *hybridIt = NewHybridVectorIterator(hParams, &err);
+
+ // Verify the iterator was not created due to NULL child
+ ASSERT_FALSE(QueryError_HasError(&err));
+ ASSERT_EQ(hybridIt->type, HYBRID_ITERATOR);
+ HybridIterator* hi = (HybridIterator *)hybridIt->ctx;
+ ASSERT_EQ(hi->searchMode, VECSIM_STANDARD_KNN);
+ hybridIt->Free(hybridIt);
+}
diff --git a/tests/cpptests/test_cpp_iterator_index.cpp b/tests/cpptests/test_cpp_iterator_index.cpp
index 2efda44be08..7bc754b3311 100644
--- a/tests/cpptests/test_cpp_iterator_index.cpp
+++ b/tests/cpptests/test_cpp_iterator_index.cpp
@@ -53,7 +53,7 @@ class IndexIteratorTest : public ::testing::TestWithParam {
break;
case INDEX_TYPE_GENERIC:
SetGenericInvIndex();
- it_base = NewInvIndIterator_GenericQuery(idx, nullptr, 0, FIELD_EXPIRATION_DEFAULT);
+ it_base = NewInvIndIterator_GenericQuery(idx, nullptr, 0, FIELD_EXPIRATION_DEFAULT, 1.0);
break;
}
}
@@ -275,13 +275,13 @@ TEST_F(IndexIteratorTestWithSeeker, EOFAfterFiltering) {
ASSERT_TRUE(InvertedIndex_GetDecoder(idx->flags).seeker != nullptr);
auto encoder = InvertedIndex_GetEncoder(idx->flags);
for (t_docId i = 1; i < 1000; ++i) {
- auto res = (RSIndexResult) {
- .docId = i,
- .fieldMask = 1,
- .freq = 1,
- .type = RSResultType::RSResultType_Term,
- };
- InvertedIndex_WriteEntryGeneric(idx, encoder, i, &res);
+ auto res = (RSIndexResult) {
+ .docId = i,
+ .fieldMask = 1,
+ .freq = 1,
+ .type = RSResultType::RSResultType_Term,
+ };
+ InvertedIndex_WriteEntryGeneric(idx, encoder, i, &res);
}
// Create an iterator that reads only entries with field mask 2
QueryIterator *iterator = NewInvIndIterator_TermQuery(idx, nullptr, {.isFieldMask = true, .value = {.mask = 2}}, nullptr, 1.0);
diff --git a/tests/cpptests/test_cpp_iterator_intersection.cpp b/tests/cpptests/test_cpp_iterator_intersection.cpp
index 3eafc1d57e6..73d92cda60a 100644
--- a/tests/cpptests/test_cpp_iterator_intersection.cpp
+++ b/tests/cpptests/test_cpp_iterator_intersection.cpp
@@ -13,350 +13,449 @@
#include "src/iterators/intersection_iterator.h"
#include "src/iterators/inverted_index_iterator.h"
#include "src/inverted_index/inverted_index.h"
+#include "src/iterators/empty_iterator.h"
+#include "src/iterators/wildcard_iterator.h"
#include "src/forward_index.h"
class IntersectionIteratorCommonTest : public ::testing::TestWithParam>> {
protected:
- std::vector> docIds;
- std::vector resultSet;
- QueryIterator *ii_base;
-
- void SetUp() override {
- unsigned numChildren;
- std::tie(numChildren, resultSet) = GetParam();
- // Verify the resultSet is sorted and unique
- std::sort(resultSet.begin(), resultSet.end());
- resultSet.erase(std::unique(resultSet.begin(), resultSet.end()), resultSet.end());
- // Set docIds so the intersection of all children is resultSet.
- // Make sure that some ids are unique to each child
- docIds.resize(numChildren);
- t_docId id = 1;
- for (auto &childIds : docIds) {
- // Copy the resultSet to each child as a base
- childIds = resultSet;
- // Add some unique ids to each child. Mock constructor will ensure that the ids are unique and sorted.
- for (size_t i = 0; i < 100; i++) {
- childIds.push_back(id++);
- }
- }
- // Create children iterators
- auto children = (QueryIterator **)rm_malloc(sizeof(QueryIterator *) * numChildren);
- for (unsigned i = 0; i < numChildren; i++) {
- auto cur = new MockIterator(docIds[i]);
- docIds[i] = cur->docIds; // Ensure that the docIds are unique and sorted
- children[i] = reinterpret_cast(cur);
- }
- // Create an intersection iterator
- ii_base = NewIntersectionIterator(children, numChildren, -1, false, 1.0);
+ std::vector> docIds;
+ std::vector resultSet;
+ QueryIterator *ii_base;
+
+ void SetUp() override {
+ unsigned numChildren;
+ std::tie(numChildren, resultSet) = GetParam();
+ // Verify the resultSet is sorted and unique
+ std::sort(resultSet.begin(), resultSet.end());
+ resultSet.erase(std::unique(resultSet.begin(), resultSet.end()), resultSet.end());
+ // Set docIds so the intersection of all children is resultSet.
+ // Make sure that some ids are unique to each child
+ docIds.resize(numChildren);
+ t_docId id = 1;
+ for (auto &childIds : docIds) {
+ // Copy the resultSet to each child as a base
+ childIds = resultSet;
+ // Add some unique ids to each child. Mock constructor will ensure that the ids are unique and sorted.
+ for (size_t i = 0; i < 100; i++) {
+ childIds.push_back(id++);
+ }
}
- void TearDown() override {
- ii_base->Free(ii_base);
+ // Create children iterators
+ auto children = (QueryIterator **)rm_malloc(sizeof(QueryIterator *) * numChildren);
+ for (unsigned i = 0; i < numChildren; i++) {
+ auto cur = new MockIterator(docIds[i]);
+ docIds[i] = cur->docIds; // Ensure that the docIds are unique and sorted
+ children[i] = reinterpret_cast(cur);
}
+ // Create an intersection iterator
+ ii_base = NewIntersectionIterator(children, numChildren, -1, false, 1.0);
+ }
+ void TearDown() override {
+ ii_base->Free(ii_base);
+ }
};
TEST_P(IntersectionIteratorCommonTest, Read) {
- IntersectionIterator *ii = (IntersectionIterator *)ii_base;
- IteratorStatus rc;
-
- // Verify that the child iterators are sorted correctly by the estimated number of results
- for (uint32_t i = 1; i < ii->num_its; i++) {
- auto prev_est = ii->its[i - 1]->NumEstimated(ii->its[i - 1]);
- auto cur_est = ii->its[i]->NumEstimated(ii->its[i]);
- EXPECT_LE(prev_est, cur_est) << "Child iterators are not sorted by estimated results";
- }
+ IntersectionIterator *ii = (IntersectionIterator *)ii_base;
+ IteratorStatus rc;
- // Test reading until EOF
- size_t i = 0;
- while ((rc = ii_base->Read(ii_base)) == ITERATOR_OK) {
- ASSERT_EQ(ii->base.current->docId, resultSet[i]);
- ASSERT_EQ(ii->base.lastDocId, resultSet[i]);
- ASSERT_FALSE(ii->base.atEOF);
- i++;
- }
- ASSERT_EQ(rc, ITERATOR_EOF);
- ASSERT_TRUE(ii->base.atEOF);
- ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_EOF); // Reading after EOF should return EOF
- ASSERT_EQ(i, resultSet.size()) << "Expected to read " << resultSet.size() << " documents";
-
- size_t expected = SIZE_MAX;
- for (auto &child : docIds) {
- expected = std::min(expected, child.size());
- }
- ASSERT_EQ(ii_base->NumEstimated(ii_base), expected);
+ // Verify that the child iterators are sorted correctly by the estimated number of results
+ for (uint32_t i = 1; i < ii->num_its; i++) {
+ auto prev_est = ii->its[i - 1]->NumEstimated(ii->its[i - 1]);
+ auto cur_est = ii->its[i]->NumEstimated(ii->its[i]);
+ EXPECT_LE(prev_est, cur_est) << "Child iterators are not sorted by estimated results";
+ }
+
+ // Test reading until EOF
+ size_t i = 0;
+ while ((rc = ii_base->Read(ii_base)) == ITERATOR_OK) {
+ ASSERT_EQ(ii->base.current->docId, resultSet[i]);
+ ASSERT_EQ(ii->base.lastDocId, resultSet[i]);
+ ASSERT_FALSE(ii->base.atEOF);
+ i++;
+ }
+ ASSERT_EQ(rc, ITERATOR_EOF);
+ ASSERT_TRUE(ii->base.atEOF);
+ ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_EOF); // Reading after EOF should return EOF
+ ASSERT_EQ(i, resultSet.size()) << "Expected to read " << resultSet.size() << " documents";
+
+ size_t expected = SIZE_MAX;
+ for (auto &child : docIds) {
+ expected = std::min(expected, child.size());
+ }
+ ASSERT_EQ(ii_base->NumEstimated(ii_base), expected);
}
TEST_P(IntersectionIteratorCommonTest, SkipTo) {
- IntersectionIterator *ii = (IntersectionIterator *)ii_base;
- IteratorStatus rc;
- // Test skipping to any id between 1 and the last id
- t_docId i = 1;
- for (t_docId id : resultSet) {
- while (i < id) {
- ii_base->Rewind(ii_base);
- rc = ii_base->SkipTo(ii_base, i);
- ASSERT_EQ(rc, ITERATOR_NOTFOUND);
- ASSERT_EQ(ii->base.lastDocId, id);
- ASSERT_EQ(ii->base.current->docId, id);
- i++;
- }
- ii_base->Rewind(ii_base);
- rc = ii_base->SkipTo(ii_base, id);
- ASSERT_EQ(rc, ITERATOR_OK);
- ASSERT_EQ(ii->base.lastDocId, id);
- ASSERT_EQ(ii->base.current->docId, id);
- i++;
+ IntersectionIterator *ii = (IntersectionIterator *)ii_base;
+ IteratorStatus rc;
+ // Test skipping to any id between 1 and the last id
+ t_docId i = 1;
+ for (t_docId id : resultSet) {
+ while (i < id) {
+ ii_base->Rewind(ii_base);
+ rc = ii_base->SkipTo(ii_base, i);
+ ASSERT_EQ(rc, ITERATOR_NOTFOUND);
+ ASSERT_EQ(ii->base.lastDocId, id);
+ ASSERT_EQ(ii->base.current->docId, id);
+ i++;
}
- // Test reading after skipping to the last id
- ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_EOF);
- ASSERT_EQ(ii_base->SkipTo(ii_base, ii_base->lastDocId + 1), ITERATOR_EOF);
- ASSERT_TRUE(ii->base.atEOF);
-
ii_base->Rewind(ii_base);
- ASSERT_EQ(ii->base.lastDocId, 0);
- ASSERT_FALSE(ii->base.atEOF);
- // Test skipping to all ids that exist
- for (t_docId id : resultSet) {
- rc = ii_base->SkipTo(ii_base, id);
- ASSERT_EQ(rc, ITERATOR_OK);
- ASSERT_EQ(ii->base.lastDocId, id);
- ASSERT_EQ(ii->base.current->docId, id);
- }
+ rc = ii_base->SkipTo(ii_base, id);
+ ASSERT_EQ(rc, ITERATOR_OK);
+ ASSERT_EQ(ii->base.lastDocId, id);
+ ASSERT_EQ(ii->base.current->docId, id);
+ i++;
+ }
+ // Test reading after skipping to the last id
+ ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_EOF);
+ ASSERT_EQ(ii_base->SkipTo(ii_base, ii_base->lastDocId + 1), ITERATOR_EOF);
+ ASSERT_TRUE(ii->base.atEOF);
- // Test skipping to an id that exceeds the last id
- ii_base->Rewind(ii_base);
- ASSERT_EQ(ii->base.lastDocId, 0);
- ASSERT_FALSE(ii->base.atEOF);
- rc = ii_base->SkipTo(ii_base, resultSet.back() + 1);
- ASSERT_EQ(rc, ITERATOR_EOF);
- ASSERT_EQ(ii->base.lastDocId, 0); // we just rewound
- ASSERT_TRUE(ii->base.atEOF);
+ ii_base->Rewind(ii_base);
+ ASSERT_EQ(ii->base.lastDocId, 0);
+ ASSERT_FALSE(ii->base.atEOF);
+ // Test skipping to all ids that exist
+ for (t_docId id : resultSet) {
+ rc = ii_base->SkipTo(ii_base, id);
+ ASSERT_EQ(rc, ITERATOR_OK);
+ ASSERT_EQ(ii->base.lastDocId, id);
+ ASSERT_EQ(ii->base.current->docId, id);
+ }
+
+ // Test skipping to an id that exceeds the last id
+ ii_base->Rewind(ii_base);
+ ASSERT_EQ(ii->base.lastDocId, 0);
+ ASSERT_FALSE(ii->base.atEOF);
+ rc = ii_base->SkipTo(ii_base, resultSet.back() + 1);
+ ASSERT_EQ(rc, ITERATOR_EOF);
+ ASSERT_EQ(ii->base.lastDocId, 0); // we just rewound
+ ASSERT_TRUE(ii->base.atEOF);
}
TEST_P(IntersectionIteratorCommonTest, Rewind) {
- IntersectionIterator *ii = (IntersectionIterator *)ii_base;
- IteratorStatus rc;
- for (int i = 0; i < 5; i++) {
- for (int j = 0; j <= i; j++) {
- ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_OK);
- ASSERT_EQ(ii->base.current->docId, resultSet[j]);
- ASSERT_EQ(ii->base.lastDocId, resultSet[j]);
- }
- ii_base->Rewind(ii_base);
- ASSERT_EQ(ii->base.lastDocId, 0);
- ASSERT_FALSE(ii->base.atEOF);
+ IntersectionIterator *ii = (IntersectionIterator *)ii_base;
+ IteratorStatus rc;
+ for (int i = 0; i < 5; i++) {
+ for (int j = 0; j <= i; j++) {
+ ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_OK);
+ ASSERT_EQ(ii->base.current->docId, resultSet[j]);
+ ASSERT_EQ(ii->base.lastDocId, resultSet[j]);
}
+ ii_base->Rewind(ii_base);
+ ASSERT_EQ(ii->base.lastDocId, 0);
+ ASSERT_FALSE(ii->base.atEOF);
+ }
}
// Parameters for the tests above. We run all the combinations of:
// 1. number of child iterators in {2, 5, 25}
// 2. expected result set, one of the 3 given lists below
INSTANTIATE_TEST_SUITE_P(IntersectionIterator, IntersectionIteratorCommonTest, ::testing::Combine(
- ::testing::Values(2, 5, 25),
- ::testing::Values(
- std::vector{1, 2, 3, 40, 50},
- std::vector{5, 6, 7, 24, 25, 46, 47, 48, 49, 50, 51, 234, 2345, 3456, 4567, 5678, 6789, 7890, 8901, 9012, 12345, 23456, 34567, 45678, 56789},
- std::vector{9, 25, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, 150, 160, 170, 180, 190, 200, 210, 220, 230, 240, 250}
- )
+ ::testing::Values(2, 5, 25),
+ ::testing::Values(
+ std::vector{1, 2, 3, 40, 50},
+ std::vector{5, 6, 7, 24, 25, 46, 47, 48, 49, 50, 51, 234, 2345, 3456, 4567, 5678, 6789, 7890, 8901, 9012, 12345, 23456, 34567, 45678, 56789},
+ std::vector{9, 25, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, 150, 160, 170, 180, 190, 200, 210, 220, 230, 240, 250}
+ )
));
class IntersectionIteratorTest : public ::testing::Test {
protected:
- QueryIterator *ii_base;
- std::map invertedIndexes;
- t_docId num_docs;
+ QueryIterator *ii_base;
+ std::map invertedIndexes;
+ t_docId num_docs;
- void SetUp() override {
- num_docs = 0;
- ii_base = nullptr;
+ void SetUp() override {
+ num_docs = 0;
+ ii_base = nullptr;
+ }
+ void TearDown() override {
+ if (ii_base != nullptr) {
+ ii_base->Free(ii_base);
}
- void TearDown() override {
- if (ii_base != nullptr) {
- ii_base->Free(ii_base);
- }
- for (auto &[_, index] : invertedIndexes) {
- InvertedIndex_Free(index);
- }
+ for (auto &[_, index] : invertedIndexes) {
+ InvertedIndex_Free(index);
}
+ }
public:
- void CreateIntersectionIterator(const std::vector &terms, int max_slop = -1, bool in_order = false) {
- QueryIterator **children = (QueryIterator **)rm_malloc(sizeof(QueryIterator *) * terms.size());
- for (size_t i = 0; i < terms.size(); i++) {
- ASSERT_NE(invertedIndexes.find(terms[i]), invertedIndexes.end()) << "Term " << terms[i] << " not found in inverted indexes";
- children[i] = NewInvIndIterator_TermQuery(invertedIndexes[terms[i]], NULL, {true, RS_FIELDMASK_ALL}, NULL, 1.0);
- }
- ii_base = NewIntersectionIterator(children, terms.size(), max_slop, in_order, 1.0);
+ void CreateIntersectionIterator(const std::vector &terms, int max_slop = -1, bool in_order = false) {
+ QueryIterator **children = (QueryIterator **)rm_malloc(sizeof(QueryIterator *) * terms.size());
+ for (size_t i = 0; i < terms.size(); i++) {
+ ASSERT_NE(invertedIndexes.find(terms[i]), invertedIndexes.end()) << "Term " << terms[i] << " not found in inverted indexes";
+ children[i] = NewInvIndIterator_TermQuery(invertedIndexes[terms[i]], NULL, {true, RS_FIELDMASK_ALL}, NULL, 1.0);
+ }
+ ii_base = NewIntersectionIterator(children, terms.size(), max_slop, in_order, 1.0);
+ }
+ void AddDocument(const std::vector &terms) {
+ size_t dummy;
+ for (auto &term : terms) {
+ if (invertedIndexes.find(term) == invertedIndexes.end()) {
+ // Create a new inverted index for the term if it doesn't exist
+ invertedIndexes[term] = NewInvertedIndex((IndexFlags)(INDEX_DEFAULT_FLAGS), 1, &dummy);
+ }
+ }
+ t_docId docId = ++num_docs;
+ std::map entries;
+ // Add a document to all inverted indexes
+ for (size_t i = 0; i < terms.size(); i++) {
+ // Get (create if not exists) the forward index entry for the term
+ ForwardIndexEntry &entry = entries[terms[i]];
+ entry.docId = docId;
+ entry.freq++;
+ entry.fieldMask = RS_FIELDMASK_ALL;
+ if (entry.vw == NULL) {
+ entry.vw = NewVarintVectorWriter(8);
+ }
+ VVW_Write(entry.vw, i + 1); // Store the term index
}
- void AddDocument(const std::vector &terms) {
- size_t dummy;
- for (auto &term : terms) {
- if (invertedIndexes.find(term) == invertedIndexes.end()) {
- // Create a new inverted index for the term if it doesn't exist
- invertedIndexes[term] = NewInvertedIndex((IndexFlags)(INDEX_DEFAULT_FLAGS), 1, &dummy);
- }
- }
- t_docId docId = ++num_docs;
- std::map entries;
- // Add a document to all inverted indexes
- for (size_t i = 0; i < terms.size(); i++) {
- // Get (create if not exists) the forward index entry for the term
- ForwardIndexEntry &entry = entries[terms[i]];
- entry.docId = docId;
- entry.freq++;
- entry.fieldMask = RS_FIELDMASK_ALL;
- if (entry.vw == NULL) {
- entry.vw = NewVarintVectorWriter(8);
- }
- VVW_Write(entry.vw, i + 1); // Store the term index
- }
- // Write the forward index entries to the inverted indexes
- for (auto &[term, entry] : entries) {
- InvertedIndex *index = invertedIndexes[term];
- IndexEncoder enc = InvertedIndex_GetEncoder(index->flags);
- InvertedIndex_WriteForwardIndexEntry(index, enc, &entry);
- // Free the entry's vector writer
- VVW_Free(entry.vw);
- }
+ // Write the forward index entries to the inverted indexes
+ for (auto &[term, entry] : entries) {
+ InvertedIndex *index = invertedIndexes[term];
+ IndexEncoder enc = InvertedIndex_GetEncoder(index->flags);
+ InvertedIndex_WriteForwardIndexEntry(index, enc, &entry);
+ // Free the entry's vector writer
+ VVW_Free(entry.vw);
}
+ }
};
TEST_F(IntersectionIteratorTest, NullChildren) {
- QueryIterator **children = (QueryIterator **)rm_calloc(2, sizeof(QueryIterator *));
- children[0] = nullptr;
- children[1] = reinterpret_cast(new MockIterator({1UL, 2UL, 3UL}));
- ii_base = NewIntersectionIterator(children, 2, -1, false, 1.0);
- ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_EOF);
- ASSERT_TRUE(ii_base->atEOF);
- ASSERT_EQ(ii_base->NumEstimated(ii_base), 0);
- ASSERT_EQ(ii_base->SkipTo(ii_base, 1), ITERATOR_EOF);
- ASSERT_TRUE(ii_base->atEOF);
- ASSERT_EQ(ii_base->type, EMPTY_ITERATOR);
- ii_base->Free(ii_base);
+ QueryIterator **children = (QueryIterator **)rm_calloc(2, sizeof(QueryIterator *));
+ children[0] = nullptr;
+ children[1] = reinterpret_cast(new MockIterator({1UL, 2UL, 3UL}));
+ ii_base = NewIntersectionIterator(children, 2, -1, false, 1.0);
+ ASSERT_EQ(ii_base->type, EMPTY_ITERATOR);
+ ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_EOF);
+ ASSERT_TRUE(ii_base->atEOF);
+ ASSERT_EQ(ii_base->NumEstimated(ii_base), 0);
+ ASSERT_EQ(ii_base->SkipTo(ii_base, 1), ITERATOR_EOF);
+ ASSERT_TRUE(ii_base->atEOF);
+ ii_base->Free(ii_base);
- children = (QueryIterator **)rm_calloc(2, sizeof(QueryIterator *));
- children[0] = reinterpret_cast(new MockIterator({1UL, 2UL, 3UL}));
- children[1] = nullptr;
- ii_base = NewIntersectionIterator(children, 2, -1, false, 1.0);
- ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_EOF);
- ASSERT_TRUE(ii_base->atEOF);
- ASSERT_EQ(ii_base->NumEstimated(ii_base), 0);
- ASSERT_EQ(ii_base->SkipTo(ii_base, 1), ITERATOR_EOF);
- ASSERT_TRUE(ii_base->atEOF);
- ASSERT_EQ(ii_base->type, EMPTY_ITERATOR);
- // No explicit Free call here, the iterator is freed in the TearDown method
+ children = (QueryIterator **)rm_calloc(2, sizeof(QueryIterator *));
+ children[0] = reinterpret_cast(new MockIterator({1UL, 2UL, 3UL}));
+ children[1] = nullptr;
+ ii_base = NewIntersectionIterator(children, 2, -1, false, 1.0);
+ ASSERT_EQ(ii_base->type, EMPTY_ITERATOR);
+ ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_EOF);
+ ASSERT_TRUE(ii_base->atEOF);
+ ASSERT_EQ(ii_base->NumEstimated(ii_base), 0);
+ ASSERT_EQ(ii_base->SkipTo(ii_base, 1), ITERATOR_EOF);
+ ASSERT_TRUE(ii_base->atEOF);
+ // No explicit Free call here, the iterator is freed in the TearDown method
}
TEST_F(IntersectionIteratorTest, Slop) {
- // Add documents
- AddDocument({"foo", "bar"});
- AddDocument({"foo", "baz"});
- AddDocument({"bar", "foo"});
- AddDocument({"foo", "baz", "bar"});
-
- // Create an intersection iterator with slop
- CreateIntersectionIterator({"foo", "bar"}, 0, false);
- ASSERT_EQ(ii_base->type, INTERSECT_ITERATOR);
- ASSERT_EQ(ii_base->NumEstimated(ii_base), 3); // 3 documents match "bar"
-
- // Read the results. Expected: 1, 3 (slop 0, no order)
- ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_OK);
- ASSERT_EQ(ii_base->current->docId, 1);
- ASSERT_EQ(ii_base->lastDocId, 1);
- ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_OK);
- ASSERT_EQ(ii_base->current->docId, 3);
- ASSERT_EQ(ii_base->lastDocId, 3);
- ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_EOF);
- ASSERT_TRUE(ii_base->atEOF);
- ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_EOF); // Reading after EOF should return EOF again
-
- // Rewind and check again with SkipTo
- ii_base->Rewind(ii_base);
- ASSERT_EQ(ii_base->lastDocId, 0);
- ASSERT_FALSE(ii_base->atEOF);
- ASSERT_EQ(ii_base->SkipTo(ii_base, 1), ITERATOR_OK);
- ASSERT_EQ(ii_base->current->docId, 1);
- ASSERT_EQ(ii_base->lastDocId, 1);
- ASSERT_EQ(ii_base->SkipTo(ii_base, 2), ITERATOR_NOTFOUND);
- ASSERT_EQ(ii_base->current->docId, 3);
- ASSERT_EQ(ii_base->lastDocId, 3);
- ASSERT_EQ(ii_base->SkipTo(ii_base, 4), ITERATOR_EOF);
- ASSERT_TRUE(ii_base->atEOF);
- ASSERT_EQ(ii_base->SkipTo(ii_base, 5), ITERATOR_EOF); // Skipping beyond the last docId should return EOF
- ASSERT_EQ(ii_base->lastDocId, 3); // Last docId should remain unchanged
- ASSERT_TRUE(ii_base->atEOF); // atEOF should remain true
+ // Add documents
+ AddDocument({"foo", "bar"});
+ AddDocument({"foo", "baz"});
+ AddDocument({"bar", "foo"});
+ AddDocument({"foo", "baz", "bar"});
+
+ // Create an intersection iterator with slop
+ CreateIntersectionIterator({"foo", "bar"}, 0, false);
+ ASSERT_EQ(ii_base->type, INTERSECT_ITERATOR);
+ ASSERT_EQ(ii_base->NumEstimated(ii_base), 3); // 3 documents match "bar"
+
+ // Read the results. Expected: 1, 3 (slop 0, no order)
+ ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_OK);
+ ASSERT_EQ(ii_base->current->docId, 1);
+ ASSERT_EQ(ii_base->lastDocId, 1);
+ ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_OK);
+ ASSERT_EQ(ii_base->current->docId, 3);
+ ASSERT_EQ(ii_base->lastDocId, 3);
+ ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_EOF);
+ ASSERT_TRUE(ii_base->atEOF);
+ ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_EOF); // Reading after EOF should return EOF again
+
+ // Rewind and check again with SkipTo
+ ii_base->Rewind(ii_base);
+ ASSERT_EQ(ii_base->lastDocId, 0);
+ ASSERT_FALSE(ii_base->atEOF);
+ ASSERT_EQ(ii_base->SkipTo(ii_base, 1), ITERATOR_OK);
+ ASSERT_EQ(ii_base->current->docId, 1);
+ ASSERT_EQ(ii_base->lastDocId, 1);
+ ASSERT_EQ(ii_base->SkipTo(ii_base, 2), ITERATOR_NOTFOUND);
+ ASSERT_EQ(ii_base->current->docId, 3);
+ ASSERT_EQ(ii_base->lastDocId, 3);
+ ASSERT_EQ(ii_base->SkipTo(ii_base, 4), ITERATOR_EOF);
+ ASSERT_TRUE(ii_base->atEOF);
+ ASSERT_EQ(ii_base->SkipTo(ii_base, 5), ITERATOR_EOF); // Skipping beyond the last docId should return EOF
+ ASSERT_EQ(ii_base->lastDocId, 3); // Last docId should remain unchanged
+ ASSERT_TRUE(ii_base->atEOF); // atEOF should remain true
}
TEST_F(IntersectionIteratorTest, InOrder) {
- // Add documents
- AddDocument({"foo", "bar"});
- AddDocument({"foo", "baz"});
- AddDocument({"bar", "foo"});
- AddDocument({"foo", "baz", "bar"});
-
- // Create an intersection iterator with in-order
- CreateIntersectionIterator({"foo", "bar"}, -1, true);
- ASSERT_EQ(ii_base->type, INTERSECT_ITERATOR);
- ASSERT_EQ(ii_base->NumEstimated(ii_base), 3); // 3 documents match "bar"
-
- // Read the results. Expected: 1, 4 (any slop, in order)
- ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_OK);
- ASSERT_EQ(ii_base->current->docId, 1);
- ASSERT_EQ(ii_base->lastDocId, 1);
- ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_OK);
- ASSERT_EQ(ii_base->current->docId, 4);
- ASSERT_EQ(ii_base->lastDocId, 4);
- ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_EOF);
- ASSERT_TRUE(ii_base->atEOF);
- ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_EOF); // Reading after EOF should return EOF again
-
- // Rewind and check again with SkipTo
- ii_base->Rewind(ii_base);
- ASSERT_EQ(ii_base->lastDocId, 0);
- ASSERT_FALSE(ii_base->atEOF);
- ASSERT_EQ(ii_base->SkipTo(ii_base, 1), ITERATOR_OK);
- ASSERT_EQ(ii_base->current->docId, 1);
- ASSERT_EQ(ii_base->lastDocId, 1);
- ASSERT_EQ(ii_base->SkipTo(ii_base, 2), ITERATOR_NOTFOUND);
- ASSERT_EQ(ii_base->current->docId, 4);
- ASSERT_EQ(ii_base->lastDocId, 4);
- ASSERT_EQ(ii_base->SkipTo(ii_base, 5), ITERATOR_EOF);
- ASSERT_TRUE(ii_base->atEOF);
- ASSERT_EQ(ii_base->SkipTo(ii_base, 6), ITERATOR_EOF); // Skipping beyond the last docId should return EOF
- ASSERT_EQ(ii_base->lastDocId, 4); // Last docId should remain unchanged
- ASSERT_TRUE(ii_base->atEOF); // atEOF should remain true
+ // Add documents
+ AddDocument({"foo", "bar"});
+ AddDocument({"foo", "baz"});
+ AddDocument({"bar", "foo"});
+ AddDocument({"foo", "baz", "bar"});
+
+ // Create an intersection iterator with in-order
+ CreateIntersectionIterator({"foo", "bar"}, -1, true);
+ ASSERT_EQ(ii_base->type, INTERSECT_ITERATOR);
+ ASSERT_EQ(ii_base->NumEstimated(ii_base), 3); // 3 documents match "bar"
+
+ // Read the results. Expected: 1, 4 (any slop, in order)
+ ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_OK);
+ ASSERT_EQ(ii_base->current->docId, 1);
+ ASSERT_EQ(ii_base->lastDocId, 1);
+ ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_OK);
+ ASSERT_EQ(ii_base->current->docId, 4);
+ ASSERT_EQ(ii_base->lastDocId, 4);
+ ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_EOF);
+ ASSERT_TRUE(ii_base->atEOF);
+ ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_EOF); // Reading after EOF should return EOF again
+
+ // Rewind and check again with SkipTo
+ ii_base->Rewind(ii_base);
+ ASSERT_EQ(ii_base->lastDocId, 0);
+ ASSERT_FALSE(ii_base->atEOF);
+ ASSERT_EQ(ii_base->SkipTo(ii_base, 1), ITERATOR_OK);
+ ASSERT_EQ(ii_base->current->docId, 1);
+ ASSERT_EQ(ii_base->lastDocId, 1);
+ ASSERT_EQ(ii_base->SkipTo(ii_base, 2), ITERATOR_NOTFOUND);
+ ASSERT_EQ(ii_base->current->docId, 4);
+ ASSERT_EQ(ii_base->lastDocId, 4);
+ ASSERT_EQ(ii_base->SkipTo(ii_base, 5), ITERATOR_EOF);
+ ASSERT_TRUE(ii_base->atEOF);
+ ASSERT_EQ(ii_base->SkipTo(ii_base, 6), ITERATOR_EOF); // Skipping beyond the last docId should return EOF
+ ASSERT_EQ(ii_base->lastDocId, 4); // Last docId should remain unchanged
+ ASSERT_TRUE(ii_base->atEOF); // atEOF should remain true
}
TEST_F(IntersectionIteratorTest, SlopAndOrder) {
- // Add documents
- AddDocument({"foo", "bar"});
- AddDocument({"foo", "baz"});
- AddDocument({"bar", "foo"});
- AddDocument({"foo", "baz", "bar"});
-
- // Create an intersection iterator with slop and in-order
- CreateIntersectionIterator({"foo", "bar"}, 0, true);
- ASSERT_EQ(ii_base->type, INTERSECT_ITERATOR);
- ASSERT_EQ(ii_base->NumEstimated(ii_base), 3); // 3 documents match "bar"
-
- // Read the results. Expected: 1 (slop 0, in order)
- ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_OK);
- ASSERT_EQ(ii_base->current->docId, 1);
- ASSERT_EQ(ii_base->lastDocId, 1);
- ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_EOF);
- ASSERT_TRUE(ii_base->atEOF);
- ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_EOF); // Reading after EOF should return EOF again
-
- // Rewind and check again with SkipTo
- ii_base->Rewind(ii_base);
- ASSERT_EQ(ii_base->lastDocId, 0);
- ASSERT_FALSE(ii_base->atEOF);
- ASSERT_EQ(ii_base->SkipTo(ii_base, 1), ITERATOR_OK);
- ASSERT_EQ(ii_base->current->docId, 1);
- ASSERT_EQ(ii_base->lastDocId, 1);
- ASSERT_EQ(ii_base->SkipTo(ii_base, 2), ITERATOR_EOF);
- ASSERT_EQ(ii_base->lastDocId, 1); // Last docId should remain unchanged
- ASSERT_TRUE(ii_base->atEOF);
- ASSERT_EQ(ii_base->SkipTo(ii_base, 3), ITERATOR_EOF); // Skipping beyond the last docId should return EOF
- ASSERT_EQ(ii_base->lastDocId, 1); // Last docId should remain unchanged
- ASSERT_TRUE(ii_base->atEOF); // atEOF should remain true
+ // Add documents
+ AddDocument({"foo", "bar"});
+ AddDocument({"foo", "baz"});
+ AddDocument({"bar", "foo"});
+ AddDocument({"foo", "baz", "bar"});
+
+ // Create an intersection iterator with slop and in-order
+ CreateIntersectionIterator({"foo", "bar"}, 0, true);
+ ASSERT_EQ(ii_base->type, INTERSECT_ITERATOR);
+ ASSERT_EQ(ii_base->NumEstimated(ii_base), 3); // 3 documents match "bar"
+
+ // Read the results. Expected: 1 (slop 0, in order)
+ ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_OK);
+ ASSERT_EQ(ii_base->current->docId, 1);
+ ASSERT_EQ(ii_base->lastDocId, 1);
+ ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_EOF);
+ ASSERT_TRUE(ii_base->atEOF);
+ ASSERT_EQ(ii_base->Read(ii_base), ITERATOR_EOF); // Reading after EOF should return EOF again
+
+ // Rewind and check again with SkipTo
+ ii_base->Rewind(ii_base);
+ ASSERT_EQ(ii_base->lastDocId, 0);
+ ASSERT_FALSE(ii_base->atEOF);
+ ASSERT_EQ(ii_base->SkipTo(ii_base, 1), ITERATOR_OK);
+ ASSERT_EQ(ii_base->current->docId, 1);
+ ASSERT_EQ(ii_base->lastDocId, 1);
+ ASSERT_EQ(ii_base->SkipTo(ii_base, 2), ITERATOR_EOF);
+ ASSERT_EQ(ii_base->lastDocId, 1); // Last docId should remain unchanged
+ ASSERT_TRUE(ii_base->atEOF);
+ ASSERT_EQ(ii_base->SkipTo(ii_base, 3), ITERATOR_EOF); // Skipping beyond the last docId should return EOF
+ ASSERT_EQ(ii_base->lastDocId, 1); // Last docId should remain unchanged
+ ASSERT_TRUE(ii_base->atEOF); // atEOF should remain true
+}
+
+
+class IntersectionIteratorReducerTest : public ::testing::Test {};
+
+TEST_F(IntersectionIteratorReducerTest, TestIntersectionWithEmptyChild) {
+ QueryIterator **children = (QueryIterator **)rm_malloc(sizeof(QueryIterator *) * 3);
+ children[0] = (QueryIterator *) new MockIterator({1UL, 2UL, 3UL});
+ children[1] = IT_V2(NewEmptyIterator)();
+ children[2] = (QueryIterator *) new MockIterator({1UL, 2UL, 3UL, 4UL, 5UL});
+
+ size_t num = 3;
+ QueryIterator *ii_base = NewIntersectionIterator(children, num, -1, false, 1.0);
+
+ // Should return an empty iterator when any child is empty
+ ASSERT_EQ(ii_base->type, EMPTY_ITERATOR);
+ ii_base->Free(ii_base);
+}
+
+TEST_F(IntersectionIteratorReducerTest, TestIntersectionWithNULLChild) {
+ QueryIterator **children = (QueryIterator **)rm_malloc(sizeof(QueryIterator *) * 3);
+ children[0] = (QueryIterator *) new MockIterator({1UL, 2UL, 3UL});
+ children[1] = NULL;
+ children[2] = (QueryIterator *) new MockIterator({1UL, 2UL, 3UL, 4UL, 5UL});
+
+ size_t num = 3;
+ QueryIterator *ii_base = NewIntersectionIterator(children, num, -1, false, 1.0);
+
+ // Should return an empty iterator when any child is empty
+ ASSERT_EQ(ii_base->type, EMPTY_ITERATOR);
+ ii_base->Free(ii_base);
+}
+
+TEST_F(IntersectionIteratorReducerTest, TestIntersectionRemovesWildcardChildren) {
+ QueryIterator **children = (QueryIterator **)rm_malloc(sizeof(QueryIterator *) * 4);
+ children[0] = reinterpret_cast(new MockIterator({1UL, 2UL, 3UL}));
+ children[1] = IT_V2(NewWildcardIterator_NonOptimized)(30, 2, 1.0);
+ children[2] = reinterpret_cast(new MockIterator({1UL, 2UL, 3UL}));
+ // Create a READER Iterator and set the `isWildCard` flag so that it is removed by the reducer
+ size_t memsize;
+ InvertedIndex *idx = NewInvertedIndex(static_cast(INDEX_DEFAULT_FLAGS), 1, &memsize);
+ ASSERT_TRUE(idx != nullptr);
+ ASSERT_TRUE(InvertedIndex_GetDecoder(idx->flags).seeker != nullptr);
+ auto encoder = InvertedIndex_GetEncoder(idx->flags);
+ for (t_docId i = 1; i < 1000; ++i) {
+ auto res = (RSIndexResult) {
+ .docId = i,
+ .fieldMask = 1,
+ .freq = 1,
+ .type = RSResultType::RSResultType_Term,
+ };
+ InvertedIndex_WriteEntryGeneric(idx, encoder, i, &res);
+ }
+ // Create an iterator that reads only entries with field mask 2
+ QueryIterator *iterator = NewInvIndIterator_TermQuery(idx, nullptr, {.isFieldMask = true, .value = {.mask = 2}}, nullptr, 1.0);
+ InvIndIterator* invIdxIt = (InvIndIterator *)iterator;
+ invIdxIt->isWildcard = true;
+ children[3] = iterator;
+
+ size_t num = 4;
+ QueryIterator *ii_base = NewIntersectionIterator(children, num, -1, false, 1.0);
+
+ // Should remove wildcard iterators and keep only the other iterators
+ ASSERT_EQ(ii_base->type, INTERSECT_ITERATOR);
+ IntersectionIterator *ii = (IntersectionIterator *)ii_base;
+ ASSERT_EQ(ii->num_its, 2);
+
+ ii_base->Free(ii_base);
+ InvertedIndex_Free(idx);
+}
+
+TEST_F(IntersectionIteratorReducerTest, TestIntersectionAllWildCardChildren) {
+ QueryIterator **children = (QueryIterator **)rm_malloc(sizeof(QueryIterator *) * 4);
+ children[0] = IT_V2(NewWildcardIterator_NonOptimized)(30, 2, 1.0);
+ children[1] = IT_V2(NewWildcardIterator_NonOptimized)(30, 2, 1.0);
+ children[2] = IT_V2(NewWildcardIterator_NonOptimized)(30, 2, 1.0);
+ children[3] = IT_V2(NewWildcardIterator_NonOptimized)(30, 2, 1.0);
+
+ QueryIterator *expected_iter = children[3];
+ size_t num = 4;
+ QueryIterator *ii_base = NewIntersectionIterator(children, num, -1, false, 1.0);
+ ASSERT_EQ(ii_base, expected_iter);
+ ii_base->Free(ii_base);
+}
+
+TEST_F(IntersectionIteratorReducerTest, TestIntersectionWithSingleChild) {
+ QueryIterator **children = (QueryIterator **)rm_malloc(sizeof(QueryIterator *) * 3);
+ children[0] = reinterpret_cast(new MockIterator({1UL, 2UL, 3UL}));
+ children[1] = IT_V2(NewWildcardIterator_NonOptimized)(30, 2, 1.0);
+ children[2] = IT_V2(NewWildcardIterator_NonOptimized)(30, 2, 1.0);
+ auto expected_type = children[0]->type;
+
+ size_t num = 3;
+ QueryIterator *ii_base = NewIntersectionIterator(children, num, -1, false, 1.0);
+
+ ASSERT_EQ(ii_base->type, expected_type);
+ ii_base->Free(ii_base);
}
diff --git a/tests/cpptests/test_cpp_iterator_not.cpp b/tests/cpptests/test_cpp_iterator_not.cpp
index f60b823cf1e..715007dd8d6 100644
--- a/tests/cpptests/test_cpp_iterator_not.cpp
+++ b/tests/cpptests/test_cpp_iterator_not.cpp
@@ -16,6 +16,11 @@
#include
#include "src/iterators/not_iterator.h"
+#include "src/iterators/wildcard_iterator.h"
+#include "src/iterators/empty_iterator.h"
+#include "src/iterators/inverted_index_iterator.h"
+#include "src/inverted_index/inverted_index.h"
+#include "index_utils.h"
class NotIteratorCommonTest : public ::testing::TestWithParam, std::vector, std::optional, bool>> {
protected:
@@ -25,12 +30,8 @@ class NotIteratorCommonTest : public ::testing::TestWithParam opt_max_doc_id;
t_docId maxDocId;
QueryIterator *iterator_base;
+ std::unique_ptr mockQctx;
bool optimized;
- IndexSpec *spec = nullptr;
- RedisSearchCtx *sctx = nullptr;
- QueryEvalCtx *qctx = nullptr;
- DocTable *docTable = nullptr;
-
void SetUp() override {
// Get child document IDs from parameter
std::tie(childDocIds, wcDocIds, opt_max_doc_id, optimized) = GetParam();
@@ -75,38 +76,23 @@ class NotIteratorCommonTest : public ::testing::TestWithParamrule = (SchemaRule*)rm_calloc(1, sizeof(SchemaRule));
- spec->rule->index_all = true;
-
- sctx = (RedisSearchCtx*)rm_calloc(1, sizeof(RedisSearchCtx));
- sctx->spec = spec;
- docTable = (DocTable*)rm_calloc(1, sizeof(DocTable));
- docTable->maxDocId = maxDocId;
- docTable->size = maxDocId;
- qctx = (QueryEvalCtx*)rm_calloc(1, sizeof(QueryEvalCtx));
- qctx->sctx = sctx;
- qctx->docTable = docTable;
-
- iterator_base = IT_V2(NewNotIterator)(child, maxDocId, 1.0, timeout, qctx);
+ if (optimized) {
+ std::vector wildcard = {1, 2, 3};
+ mockQctx = std::make_unique(wildcard);
+ iterator_base = IT_V2(NewNotIterator)(child, maxDocId, 1.0, timeout, &mockQctx->qctx);
NotIterator *ni = (NotIterator *)iterator_base;
ni->wcii->Free(ni->wcii);
ni->wcii = (QueryIterator *) new MockIterator(wcDocIds);
} else {
- iterator_base = IT_V2(NewNotIterator)(child, maxDocId, 1.0, timeout, nullptr);
+ mockQctx = std::make_unique(maxDocId, maxDocId);
+ iterator_base = IT_V2(NewNotIterator)(child, maxDocId, 1.0, timeout, &mockQctx->qctx);
}
}
void TearDown() override {
iterator_base->Free(iterator_base);
- if (spec && spec->rule) rm_free(spec->rule);
- if (spec) rm_free(spec);
- if (sctx) rm_free(sctx);
- if (docTable) rm_free(docTable);
- if (qctx) rm_free(qctx);
}
};
@@ -475,6 +461,9 @@ INSTANTIATE_TEST_SUITE_P(
class NotIteratorSelfTimeoutTest : public NotIteratorCommonTest {
protected:
+ // Add member variable to store the context
+ std::unique_ptr mockQctx;
+
void SetUp() override {
// Get child document IDs from parameter
std::tie(childDocIds, wcDocIds, opt_max_doc_id, optimized) = GetParam();
@@ -506,12 +495,13 @@ class NotIteratorSelfTimeoutTest : public NotIteratorCommonTest {
// Define timeout only once
struct timespec timeout = {0, 1};
- iterator_base = IT_V2(NewNotIterator)(child, maxDocId, 1.0, timeout, nullptr);
- NotIterator *ni = (NotIterator *)iterator_base;
-
- // Store the wildcard iterator in the NotIterator structure instead of directly in QueryIterator
if (optimized) {
- ni->wcii = (QueryIterator *) new MockIterator(wcDocIds);
+ std::vector wildcard = {1, 2, 3};
+ mockQctx = std::make_unique(wildcard);
+ iterator_base = IT_V2(NewNotIterator)(child, maxDocId, 1.0, timeout, &mockQctx->qctx);
+ } else {
+ mockQctx = std::make_unique(maxDocId, maxDocId);
+ iterator_base = IT_V2(NewNotIterator)(child, maxDocId, 1.0, timeout, &mockQctx->qctx);
}
}
@@ -575,10 +565,12 @@ class NotIteratorNoChildTest : public ::testing::Test {
protected:
QueryIterator *iterator_base;
t_docId maxDocId = 50;
+ std::unique_ptr mockQctx;
void SetUp() override {
struct timespec timeout = {LONG_MAX, 999999999}; // Initialize with "infinite" timeout
- iterator_base = IT_V2(NewNotIterator)(nullptr, maxDocId, 1.0, timeout, nullptr);
+ mockQctx = std::make_unique(maxDocId, maxDocId);
+ iterator_base = IT_V2(NewNotIterator)(nullptr, maxDocId, 1.0, timeout, &mockQctx->qctx);
}
void TearDown() override {
@@ -635,3 +627,97 @@ TEST_F(NotIteratorNoChildTest, Rewind) {
ASSERT_FALSE(iterator_base->atEOF);
}
}
+
+class NotIteratorReducerTest : public ::testing::Test {};
+
+TEST_F(NotIteratorReducerTest, TestNotWithNullChild) {
+ // Test rule 1: If the child is NULL, return a wildcard iterator
+ struct timespec timeout = {LONG_MAX, 999999999};
+ t_docId maxDocId = 100;
+
+ MockQueryEvalCtx mockQctx(maxDocId, maxDocId);
+
+ QueryIterator *it = IT_V2(NewNotIterator)(nullptr, maxDocId, 1.0, timeout, &mockQctx.qctx);
+
+ // Should return a wildcard iterator
+ ASSERT_EQ(it->type, WILDCARD_ITERATOR);
+ it->Free(it);
+}
+
+TEST_F(NotIteratorReducerTest, TestNotWithEmptyChild) {
+ // Test rule 1: If the child is an empty iterator, return a wildcard iterator
+ struct timespec timeout = {LONG_MAX, 999999999};
+ t_docId maxDocId = 100;
+
+ MockQueryEvalCtx mockQctx(maxDocId, maxDocId);
+
+ QueryIterator *emptyChild = IT_V2(NewEmptyIterator)();
+ QueryIterator *it = IT_V2(NewNotIterator)(emptyChild, maxDocId, 1.0, timeout, &mockQctx.qctx);
+
+ // Should return a wildcard iterator
+ ASSERT_EQ(it->type, WILDCARD_ITERATOR);
+ it->Free(it);
+}
+
+TEST_F(NotIteratorReducerTest, TestNotWithEmptyChildOptimized) {
+ // Test rule 1: If the child is an empty iterator, return a wildcard iterator
+ struct timespec timeout = {LONG_MAX, 999999999};
+ t_docId maxDocId = 100;
+
+ std::vector wildcard = {1, 2, 3};
+ MockQueryEvalCtx mockQctx(wildcard);
+
+ QueryIterator *emptyChild = IT_V2(NewEmptyIterator)();
+ QueryIterator *it = IT_V2(NewNotIterator)(emptyChild, maxDocId, 1.0, timeout, &mockQctx.qctx);
+
+ // Should return a wildcard iterator
+ ASSERT_EQ(it->type, READ_ITERATOR);
+ it->Free(it);
+}
+
+TEST_F(NotIteratorReducerTest, TestNotWithWildcardChild) {
+ // Test rule 2: If the child is a wildcard iterator, return an empty iterator
+ struct timespec timeout = {LONG_MAX, 999999999};
+ t_docId maxDocId = 100;
+
+ std::vector wildcard = {1, 2, 3};
+ MockQueryEvalCtx mockQctx(wildcard);
+
+ QueryIterator *wildcardChild = IT_V2(NewWildcardIterator_NonOptimized)(maxDocId, maxDocId, 1.0);
+ QueryIterator *it = IT_V2(NewNotIterator)(wildcardChild, maxDocId, 1.0, timeout, &mockQctx.qctx);
+
+ // Should return an empty iterator
+ ASSERT_EQ(it->type, EMPTY_ITERATOR);
+ it->Free(it);
+}
+
+TEST_F(NotIteratorReducerTest, TestNotWithReaderWildcardChild) {
+ // Test rule 2: If the child is a wildcard iterator, return an empty iterator
+ struct timespec timeout = {LONG_MAX, 999999999};
+ t_docId maxDocId = 100;
+ size_t memsize;
+ InvertedIndex *idx = NewInvertedIndex(static_cast(INDEX_DEFAULT_FLAGS), 1, &memsize);
+ ASSERT_TRUE(idx != nullptr);
+ ASSERT_TRUE(InvertedIndex_GetDecoder(idx->flags).seeker != nullptr);
+ auto encoder = InvertedIndex_GetEncoder(idx->flags);
+ for (t_docId i = 1; i < 1000; ++i) {
+ auto res = (RSIndexResult) {
+ .docId = i,
+ .fieldMask = 1,
+ .freq = 1,
+ .type = RSResultType::RSResultType_Term,
+ };
+ InvertedIndex_WriteEntryGeneric(idx, encoder, i, &res);
+ }
+ // Create an iterator that reads only entries with field mask 2
+ QueryIterator *wildcardChild = NewInvIndIterator_TermQuery(idx, nullptr, {.isFieldMask = true, .value = {.mask = 2}}, nullptr, 1.0);
+ InvIndIterator* invIdxIt = (InvIndIterator *)wildcardChild;
+ invIdxIt->isWildcard = true;
+ MockQueryEvalCtx mockQctx(maxDocId, maxDocId);
+ QueryIterator *it = IT_V2(NewNotIterator)(wildcardChild, maxDocId, 1.0, timeout, &mockQctx.qctx);
+
+ // Should return an empty iterator
+ ASSERT_EQ(it->type, EMPTY_ITERATOR);
+ it->Free(it);
+ InvertedIndex_Free(idx);
+}
diff --git a/tests/cpptests/test_cpp_iterator_optional.cpp b/tests/cpptests/test_cpp_iterator_optional.cpp
index 9e897263e41..37b340e2fa0 100644
--- a/tests/cpptests/test_cpp_iterator_optional.cpp
+++ b/tests/cpptests/test_cpp_iterator_optional.cpp
@@ -17,6 +17,9 @@
#include "src/iterators/empty_iterator.h"
#include "iterator_util.h"
#include "index_utils.h"
+#include "src/iterators/wildcard_iterator.h"
+#include "src/iterators/inverted_index_iterator.h"
+#include "src/inverted_index/inverted_index.h"
// Test optional iterator
@@ -321,8 +324,7 @@ TEST_F(OptionalIteratorWithEmptyChildTest, ReadAllVirtualResults) {
ASSERT_EQ(iterator_base->lastDocId, i);
// All hits should be virtual
- OptionalIterator *oi = (OptionalIterator *)iterator_base;
- ASSERT_EQ(iterator_base->current, oi->virt);
+ ASSERT_EQ(iterator_base->current->type, RSResultType_Virtual);
ASSERT_EQ(iterator_base->current->weight, weight);
ASSERT_EQ(iterator_base->current->freq, 1);
ASSERT_EQ(iterator_base->current->fieldMask, RS_FIELDMASK_ALL);
@@ -343,8 +345,7 @@ TEST_F(OptionalIteratorWithEmptyChildTest, SkipToVirtualHits) {
ASSERT_EQ(iterator_base->lastDocId, target);
// Should be virtual hit
- OptionalIterator *oi = (OptionalIterator *)iterator_base;
- ASSERT_EQ(iterator_base->current, oi->virt);
+ ASSERT_EQ(iterator_base->current->type, RSResultType_Virtual);
ASSERT_EQ(iterator_base->current->weight, weight);
}
}
@@ -361,13 +362,10 @@ TEST_F(OptionalIteratorWithEmptyChildTest, RewindBehavior) {
ASSERT_EQ(iterator_base->lastDocId, 0);
ASSERT_FALSE(iterator_base->atEOF);
- OptionalIterator *oi = (OptionalIterator *)iterator_base;
- ASSERT_EQ(oi->virt->docId, 0);
-
// After Rewind, should be able to read from the beginning
ASSERT_EQ(iterator_base->Read(iterator_base), ITERATOR_OK);
ASSERT_EQ(iterator_base->current->docId, 1);
- ASSERT_EQ(iterator_base->current, oi->virt);
+ ASSERT_EQ(iterator_base->current->type, RSResultType_Virtual);
}
TEST_F(OptionalIteratorWithEmptyChildTest, EOFBehavior) {
@@ -377,8 +375,7 @@ TEST_F(OptionalIteratorWithEmptyChildTest, EOFBehavior) {
ASSERT_EQ(iterator_base->lastDocId, maxDocId);
// Should be virtual hit
- OptionalIterator *oi = (OptionalIterator *)iterator_base;
- ASSERT_EQ(iterator_base->current, oi->virt);
+ ASSERT_EQ(iterator_base->current->type, RSResultType_Virtual);
// Next read should return EOF
ASSERT_EQ(iterator_base->Read(iterator_base), ITERATOR_EOF);
@@ -393,8 +390,7 @@ TEST_F(OptionalIteratorWithEmptyChildTest, VirtualResultProperties) {
// Test that virtual results have correct properties
ASSERT_EQ(iterator_base->Read(iterator_base), ITERATOR_OK);
- OptionalIterator *oi = (OptionalIterator *)iterator_base;
- ASSERT_EQ(iterator_base->current, oi->virt);
+ ASSERT_EQ(iterator_base->current->type, RSResultType_Virtual);
ASSERT_EQ(iterator_base->current->docId, 1);
ASSERT_EQ(iterator_base->current->weight, weight);
ASSERT_EQ(iterator_base->current->freq, 1);
@@ -441,6 +437,7 @@ class OptionalIteratorOptimized : public ::testing::TestWithParamtype == WILDCARD_ITERATOR);
+
+ // Read first document and check properties
+ ASSERT_EQ(it->Read(it), ITERATOR_OK);
+ ASSERT_EQ(it->current->docId, 1);
+ ASSERT_EQ(it->current->weight, weight);
+ ASSERT_EQ(it->current->type, RSResultType_Virtual);
+
+ it->Free(it);
+}
+
+TEST_F(OptionalIteratorReducerTest, TestOptionalWithEmptyChild) {
+ // Test rule 1: If the child is an empty iterator, return a wildcard iterator
+ t_docId maxDocId = 100;
+ size_t numDocs = 50;
+ double weight = 2.0;
+
+ // Create a mock QueryEvalCtx
+ MockQueryEvalCtx ctx(maxDocId, numDocs);
+
+ // Create empty child iterator
+ QueryIterator *emptyChild = IT_V2(NewEmptyIterator)();
+
+ // Create optional iterator with empty child
+ QueryIterator *it = IT_V2(NewOptionalIterator)(emptyChild, &ctx.qctx, weight);
+
+ // Verify iterator type
+ ASSERT_TRUE(it->type == WILDCARD_ITERATOR);
+
+ // Read first document and check properties
+ ASSERT_EQ(it->Read(it), ITERATOR_OK);
+ ASSERT_EQ(it->current->docId, 1);
+ ASSERT_EQ(it->current->weight, weight);
+ ASSERT_EQ(it->current->type, RSResultType_Virtual);
+
+ it->Free(it);
+}
+
+TEST_F(OptionalIteratorReducerTest, TestOptionalWithWildcardChild) {
+ // Test rule 2: If the child is a wildcard iterator, return it directly
+ t_docId maxDocId = 100;
+ size_t numDocs = 50;
+ double childWeight = 3.0;
+
+ // Create a mock QueryEvalCtx
+ MockQueryEvalCtx ctx(maxDocId, numDocs);
+
+ // Create wildcard child iterator
+ QueryIterator *wildcardChild = IT_V2(NewWildcardIterator_NonOptimized)(maxDocId, numDocs, childWeight);
+
+ // Create optional iterator with wildcard child - should return the child directly
+ QueryIterator *it = IT_V2(NewOptionalIterator)(wildcardChild, &ctx.qctx, 2.0);
+
+ // Verify it's the same iterator (optimization returns child directly)
+ ASSERT_TRUE(it->type == WILDCARD_ITERATOR);
+ ASSERT_EQ(it, wildcardChild);
+
+ // Read first document and check properties - should have child's weight
+ ASSERT_EQ(it->Read(it), ITERATOR_OK);
+ ASSERT_EQ(it->current->docId, 1);
+ ASSERT_EQ(it->current->weight, childWeight);
+ ASSERT_EQ(it->current->type, RSResultType_Virtual);
+
+ it->Free(it);
+}
+
+TEST_F(OptionalIteratorReducerTest, TestOptionalWithReaderWildcardChild) {
+ t_docId maxDocId = 100;
+ size_t numDocs = 50;
+ double childWeight = 3.0;
+
+ // Create a mock QueryEvalCtx
+ MockQueryEvalCtx ctx(maxDocId, numDocs);
+ size_t memsize;
+ InvertedIndex *idx = NewInvertedIndex(static_cast(INDEX_DEFAULT_FLAGS), 1, &memsize);
+ ASSERT_TRUE(idx != nullptr);
+ ASSERT_TRUE(InvertedIndex_GetDecoder(idx->flags).seeker != nullptr);
+ auto encoder = InvertedIndex_GetEncoder(idx->flags);
+ for (t_docId i = 1; i < 1000; ++i) {
+ auto res = (RSIndexResult) {
+ .docId = i,
+ .fieldMask = 1,
+ .freq = 1,
+ .type = RSResultType::RSResultType_Term,
+ };
+ InvertedIndex_WriteEntryGeneric(idx, encoder, i, &res);
+ }
+ // Create an iterator that reads only entries with field mask 2
+ QueryIterator *wildcardChild = NewInvIndIterator_TermQuery(idx, nullptr, {.isFieldMask = true, .value = {.mask = 2}}, nullptr, 1.0);
+ InvIndIterator* invIdxIt = (InvIndIterator *)wildcardChild;
+ invIdxIt->isWildcard = true;
+
+ // Create optional iterator with wildcard child - should return the child directly
+ QueryIterator *it = IT_V2(NewOptionalIterator)(wildcardChild, &ctx.qctx, 2.0);
+
+ // Verify it's the same iterator (optimization returns child directly)
+ ASSERT_TRUE(it->type == READ_ITERATOR);
+ ASSERT_EQ(it, wildcardChild);
+ it->Free(it);
+ InvertedIndex_Free(idx);
+}
diff --git a/tests/cpptests/test_cpp_iterator_union.cpp b/tests/cpptests/test_cpp_iterator_union.cpp
index b15c898c781..ccfa7cb19b8 100644
--- a/tests/cpptests/test_cpp_iterator_union.cpp
+++ b/tests/cpptests/test_cpp_iterator_union.cpp
@@ -10,66 +10,70 @@
#include "iterator_util.h"
#include "src/iterators/union_iterator.h"
+#include "src/iterators/empty_iterator.h"
+#include "src/iterators/wildcard_iterator.h"
+#include "src/iterators/inverted_index_iterator.h"
+#include "src/inverted_index/inverted_index.h"
class UnionIteratorCommonTest : public ::testing::TestWithParam>> {
protected:
- std::vector> docIds;
- std::vector resultSet;
- QueryIterator *ui_base;
-
- void SetUp() override {
- ASSERT_EQ(RSGlobalConfig.iteratorsConfigParams.minUnionIterHeap, 20) <<
- "If we ever change the default threshold for using heaps, we need to modify the tests "
- "here so they still check both flat and heap alternatives.";
-
- auto [numChildren, quickExit, union_res] = GetParam();
- // Set resultSet to the expected union result
- resultSet = union_res;
- // Set docIds so the union of all children is union_res.
- // Make sure that some ids are repeated in some children
- docIds.resize(numChildren);
- for (size_t i = 0; i < union_res.size(); i++) {
- for (unsigned j = 0; j < numChildren; j++) {
- if (j % (i + 1) == 0) {
- docIds[j].push_back(union_res[i]);
- }
- }
- }
- // Create children iterators
- auto children = (QueryIterator **)rm_malloc(sizeof(QueryIterator *) * numChildren);
- for (unsigned i = 0; i < numChildren; i++) {
- children[i] = (QueryIterator *) new MockIterator(docIds[i]);
+ std::vector> docIds;
+ std::vector resultSet;
+ QueryIterator *ui_base;
+
+ void SetUp() override {
+ ASSERT_EQ(RSGlobalConfig.iteratorsConfigParams.minUnionIterHeap, 20) <<
+ "If we ever change the default threshold for using heaps, we need to modify the tests "
+ "here so they still check both flat and heap alternatives.";
+
+ auto [numChildren, quickExit, union_res] = GetParam();
+ // Set resultSet to the expected union result
+ resultSet = union_res;
+ // Set docIds so the union of all children is union_res.
+ // Make sure that some ids are repeated in some children
+ docIds.resize(numChildren);
+ for (size_t i = 0; i < union_res.size(); i++) {
+ for (unsigned j = 0; j < numChildren; j++) {
+ if (j % (i + 1) == 0) {
+ docIds[j].push_back(union_res[i]);
}
- // Create a union iterator
- ui_base = IT_V2(NewUnionIterator)(children, numChildren, quickExit, 1.0, QN_UNION, NULL, &RSGlobalConfig.iteratorsConfigParams);
+ }
}
- void TearDown() override {
- ui_base->Free(ui_base);
+ // Create children iterators
+ auto children = (QueryIterator **)rm_malloc(sizeof(QueryIterator *) * numChildren);
+ for (unsigned i = 0; i < numChildren; i++) {
+ children[i] = (QueryIterator *) new MockIterator(docIds[i]);
}
+ // Create a union iterator
+ ui_base = IT_V2(NewUnionIterator)(children, numChildren, quickExit, 1.0, QN_UNION, NULL, &RSGlobalConfig.iteratorsConfigParams);
+ }
+ void TearDown() override {
+ ui_base->Free(ui_base);
+ }
};
TEST_P(UnionIteratorCommonTest, Read) {
- UnionIterator *ui = (UnionIterator *)ui_base;
- IteratorStatus rc;
+ UnionIterator *ui = (UnionIterator *)ui_base;
+ IteratorStatus rc;
- // Test reading until EOF
- size_t i = 0;
- while ((rc = ui_base->Read(ui_base)) == ITERATOR_OK) {
- ASSERT_EQ(ui->base.current->docId, resultSet[i]);
- ASSERT_EQ(ui->base.lastDocId, resultSet[i]);
- ASSERT_FALSE(ui->base.atEOF);
- i++;
- }
- ASSERT_EQ(rc, ITERATOR_EOF);
- ASSERT_TRUE(ui->base.atEOF);
- ASSERT_EQ(ui_base->Read(ui_base), ITERATOR_EOF); // Reading after EOF should return EOF
- ASSERT_EQ(i, resultSet.size()) << "Expected to read " << resultSet.size() << " documents";
+ // Test reading until EOF
+ size_t i = 0;
+ while ((rc = ui_base->Read(ui_base)) == ITERATOR_OK) {
+ ASSERT_EQ(ui->base.current->docId, resultSet[i]);
+ ASSERT_EQ(ui->base.lastDocId, resultSet[i]);
+ ASSERT_FALSE(ui->base.atEOF);
+ i++;
+ }
+ ASSERT_EQ(rc, ITERATOR_EOF);
+ ASSERT_TRUE(ui->base.atEOF);
+ ASSERT_EQ(ui_base->Read(ui_base), ITERATOR_EOF); // Reading after EOF should return EOF
+ ASSERT_EQ(i, resultSet.size()) << "Expected to read " << resultSet.size() << " documents";
- size_t expected = 0;
- for (auto &child : docIds) {
- expected += child.size();
- }
- ASSERT_EQ(ui_base->NumEstimated(ui_base), expected);
+ size_t expected = 0;
+ for (auto &child : docIds) {
+ expected += child.size();
+ }
+ ASSERT_EQ(ui_base->NumEstimated(ui_base), expected);
}
TEST_P(UnionIteratorCommonTest, SkipTo) {
@@ -120,18 +124,18 @@ TEST_P(UnionIteratorCommonTest, SkipTo) {
}
TEST_P(UnionIteratorCommonTest, Rewind) {
- UnionIterator *ui = (UnionIterator *)ui_base;
- IteratorStatus rc;
- for (int i = 0; i < 5; i++) {
- for (int j = 0; j <= i; j++) {
- ASSERT_EQ(ui_base->Read(ui_base), ITERATOR_OK);
- ASSERT_EQ(ui->base.current->docId, resultSet[j]);
- ASSERT_EQ(ui->base.lastDocId, resultSet[j]);
- }
- ui_base->Rewind(ui_base);
- ASSERT_EQ(ui->base.lastDocId, 0);
- ASSERT_FALSE(ui->base.atEOF);
+ UnionIterator *ui = (UnionIterator *)ui_base;
+ IteratorStatus rc;
+ for (int i = 0; i < 5; i++) {
+ for (int j = 0; j <= i; j++) {
+ ASSERT_EQ(ui_base->Read(ui_base), ITERATOR_OK);
+ ASSERT_EQ(ui->base.current->docId, resultSet[j]);
+ ASSERT_EQ(ui->base.lastDocId, resultSet[j]);
}
+ ui_base->Rewind(ui_base);
+ ASSERT_EQ(ui->base.lastDocId, 0);
+ ASSERT_FALSE(ui->base.atEOF);
+ }
}
// Parameters for the tests above. We run all the combinations of:
@@ -139,78 +143,78 @@ TEST_P(UnionIteratorCommonTest, Rewind) {
// 2. quick mode (true/false)
// 3. expected result set, one of the 3 given lists below
INSTANTIATE_TEST_SUITE_P(UnionIteratorP, UnionIteratorCommonTest, ::testing::Combine(
- ::testing::Values(2, 5, 25),
- ::testing::Bool(),
- ::testing::Values(
- std::vector{1, 2, 3, 40, 50},
- std::vector{5, 6, 7, 24, 25, 46, 47, 48, 49, 50, 51, 234, 2345, 3456, 4567, 5678, 6789, 7890, 8901, 9012, 12345, 23456, 34567, 45678, 56789},
- std::vector{9, 25, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, 150, 160, 170, 180, 190, 200, 210, 220, 230, 240, 250}
- )
+ ::testing::Values(2, 5, 25),
+ ::testing::Bool(),
+ ::testing::Values(
+ std::vector{1, 2, 3, 40, 50},
+ std::vector{5, 6, 7, 24, 25, 46, 47, 48, 49, 50, 51, 234, 2345, 3456, 4567, 5678, 6789, 7890, 8901, 9012, 12345, 23456, 34567, 45678, 56789},
+ std::vector{9, 25, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, 150, 160, 170, 180, 190, 200, 210, 220, 230, 240, 250}
+ )
));
class UnionIteratorEdgesTest : public ::testing::TestWithParam> {
protected:
- QueryIterator *ui_base;
-
- void SetUp() override {
- auto [numChildren, quickExit, sparse_ids] = GetParam();
- auto children = (QueryIterator **)rm_malloc(sizeof(QueryIterator *) * numChildren);
- for (unsigned i = 0; i < numChildren; i++) {
- MockIterator *it;
- if (sparse_ids) {
- it = new MockIterator(10UL, 20UL, 30UL, 40UL, 50UL);
- } else {
- it = new MockIterator(1UL, 2UL, 3UL, 4UL, 5UL);
- }
- children[i] = (QueryIterator *) it;
- }
- // Create a union iterator
- ui_base = IT_V2(NewUnionIterator)(children, numChildren, quickExit, 1.0, QN_UNION, NULL, &RSGlobalConfig.iteratorsConfigParams);
- }
- void TearDown() override {
- ui_base->Free(ui_base);
+ QueryIterator *ui_base;
+
+ void SetUp() override {
+ auto [numChildren, quickExit, sparse_ids] = GetParam();
+ auto children = (QueryIterator **)rm_malloc(sizeof(QueryIterator *) * numChildren);
+ for (unsigned i = 0; i < numChildren; i++) {
+ MockIterator *it;
+ if (sparse_ids) {
+ it = new MockIterator(10UL, 20UL, 30UL, 40UL, 50UL);
+ } else {
+ it = new MockIterator(1UL, 2UL, 3UL, 4UL, 5UL);
+ }
+ children[i] = (QueryIterator *) it;
}
+ // Create a union iterator
+ ui_base = IT_V2(NewUnionIterator)(children, numChildren, quickExit, 1.0, QN_UNION, NULL, &RSGlobalConfig.iteratorsConfigParams);
+ }
+ void TearDown() override {
+ ui_base->Free(ui_base);
+ }
- void TimeoutChildTest(int childIdx) {
- UnionIterator *ui = (UnionIterator *)ui_base;
- auto [numChildren, quickExit, sparse_ids] = GetParam();
-
- auto child = reinterpret_cast(ui->its[childIdx]);
- child->whenDone = ITERATOR_TIMEOUT;
- child->docIds.clear();
-
- auto rc = ui_base->Read(ui_base);
- if (!quickExit || sparse_ids) {
- // Usually, the first read will detect the timeout
- ASSERT_EQ(rc, ITERATOR_TIMEOUT);
- } else {
- // If quickExit is enabled and we have a dense range of ids, we may not read from the timed-out child
- ASSERT_TRUE(rc == ITERATOR_OK || rc == ITERATOR_TIMEOUT);
- // We still expect the first non-ok status to be TIMEOUT
- while (rc == ITERATOR_OK) {
- rc = ui_base->Read(ui_base);
- }
- ASSERT_EQ(rc, ITERATOR_TIMEOUT);
- }
+ void TimeoutChildTest(int childIdx) {
+ UnionIterator *ui = (UnionIterator *)ui_base;
+ auto [numChildren, quickExit, sparse_ids] = GetParam();
- ui_base->Rewind(ui_base);
+ auto child = reinterpret_cast(ui->its[childIdx]);
+ child->whenDone = ITERATOR_TIMEOUT;
+ child->docIds.clear();
- // Test skipping with a timeout child
- t_docId next = 1;
- rc = ui_base->SkipTo(ui_base, next);
- if (!quickExit || sparse_ids) {
- // Usually, the first read will detect the timeout
- ASSERT_EQ(rc, ITERATOR_TIMEOUT);
- } else {
- // If quickExit is enabled and we have a dense range of ids, we may not read from the timed-out child
- ASSERT_TRUE(rc == ITERATOR_OK || rc == ITERATOR_TIMEOUT);
- // We still expect the first non-ok status to be TIMEOUT
- while (rc == ITERATOR_OK) {
- rc = ui_base->SkipTo(ui_base, ++next);
- }
- ASSERT_EQ(rc, ITERATOR_TIMEOUT);
- }
+ auto rc = ui_base->Read(ui_base);
+ if (!quickExit || sparse_ids) {
+ // Usually, the first read will detect the timeout
+ ASSERT_EQ(rc, ITERATOR_TIMEOUT);
+ } else {
+ // If quickExit is enabled and we have a dense range of ids, we may not read from the timed-out child
+ ASSERT_TRUE(rc == ITERATOR_OK || rc == ITERATOR_TIMEOUT);
+ // We still expect the first non-ok status to be TIMEOUT
+ while (rc == ITERATOR_OK) {
+ rc = ui_base->Read(ui_base);
+ }
+ ASSERT_EQ(rc, ITERATOR_TIMEOUT);
}
+
+ ui_base->Rewind(ui_base);
+
+ // Test skipping with a timeout child
+ t_docId next = 1;
+ rc = ui_base->SkipTo(ui_base, next);
+ if (!quickExit || sparse_ids) {
+ // Usually, the first read will detect the timeout
+ ASSERT_EQ(rc, ITERATOR_TIMEOUT);
+ } else {
+ // If quickExit is enabled and we have a dense range of ids, we may not read from the timed-out child
+ ASSERT_TRUE(rc == ITERATOR_OK || rc == ITERATOR_TIMEOUT);
+ // We still expect the first non-ok status to be TIMEOUT
+ while (rc == ITERATOR_OK) {
+ rc = ui_base->SkipTo(ui_base, ++next);
+ }
+ ASSERT_EQ(rc, ITERATOR_TIMEOUT);
+ }
+ }
};
// Run the test in the case where the first child times out
@@ -241,38 +245,118 @@ INSTANTIATE_TEST_SUITE_P(UnionIteratorEdgesP, UnionIteratorEdgesTest, ::testing:
class UnionIteratorSingleTest : public ::testing::Test {};
TEST_F(UnionIteratorSingleTest, ReuseResults) {
- QueryIterator **children = (QueryIterator **)rm_malloc(sizeof(QueryIterator *) * 2);
- MockIterator *it1 = new MockIterator(3UL);
- MockIterator *it2 = new MockIterator(2UL);
- children[0] = (QueryIterator *)it1;
- children[1] = (QueryIterator *)it2;
- // Create a union iterator
- IteratorsConfig config = RSGlobalConfig.iteratorsConfigParams;
- config.minUnionIterHeap = INT64_MAX; // Ensure we don't use the heap
- QueryIterator *ui_base = IT_V2(NewUnionIterator)(children, 2, true, 1.0, QN_UNION, NULL, &config);
- ASSERT_EQ(ui_base->NumEstimated(ui_base), it1->docIds.size() + it2->docIds.size());
-
- ASSERT_EQ(ui_base->Read(ui_base), ITERATOR_OK);
- ASSERT_EQ(ui_base->lastDocId, 2);
- ASSERT_EQ(it1->base.lastDocId, 3);
- ASSERT_EQ(it2->base.lastDocId, 2);
- ASSERT_EQ(it1->readCount, 1);
- ASSERT_EQ(it2->readCount, 1);
-
- ASSERT_EQ(ui_base->Read(ui_base), ITERATOR_OK);
- ASSERT_EQ(ui_base->lastDocId, 3);
- ASSERT_EQ(it1->base.lastDocId, 3);
- ASSERT_EQ(it2->base.lastDocId, 2);
- ASSERT_EQ(it1->readCount, 1) << "it1 should not be read again";
- ASSERT_FALSE(it1->base.atEOF);
- ASSERT_EQ(it2->readCount, 1) << "it2 should not be read again";
- ASSERT_FALSE(it2->base.atEOF);
+ QueryIterator **children = (QueryIterator **)rm_malloc(sizeof(QueryIterator *) * 2);
+ MockIterator *it1 = new MockIterator(3UL);
+ MockIterator *it2 = new MockIterator(2UL);
+ children[0] = (QueryIterator *)it1;
+ children[1] = (QueryIterator *)it2;
+ // Create a union iterator
+ IteratorsConfig config = RSGlobalConfig.iteratorsConfigParams;
+ config.minUnionIterHeap = INT64_MAX; // Ensure we don't use the heap
+ QueryIterator *ui_base = IT_V2(NewUnionIterator)(children, 2, true, 1.0, QN_UNION, NULL, &config);
+ ASSERT_EQ(ui_base->NumEstimated(ui_base), it1->docIds.size() + it2->docIds.size());
- ASSERT_EQ(ui_base->Read(ui_base), ITERATOR_EOF);
- ASSERT_EQ(it1->readCount, 2) << "it1 should be read again";
- ASSERT_TRUE(it1->base.atEOF);
- ASSERT_EQ(it2->readCount, 2) << "it2 should be read again";
- ASSERT_TRUE(it2->base.atEOF);
+ ASSERT_EQ(ui_base->Read(ui_base), ITERATOR_OK);
+ ASSERT_EQ(ui_base->lastDocId, 2);
+ ASSERT_EQ(it1->base.lastDocId, 3);
+ ASSERT_EQ(it2->base.lastDocId, 2);
+ ASSERT_EQ(it1->readCount, 1);
+ ASSERT_EQ(it2->readCount, 1);
- ui_base->Free(ui_base);
+ ASSERT_EQ(ui_base->Read(ui_base), ITERATOR_OK);
+ ASSERT_EQ(ui_base->lastDocId, 3);
+ ASSERT_EQ(it1->base.lastDocId, 3);
+ ASSERT_EQ(it2->base.lastDocId, 2);
+ ASSERT_EQ(it1->readCount, 1) << "it1 should not be read again";
+ ASSERT_FALSE(it1->base.atEOF);
+ ASSERT_EQ(it2->readCount, 1) << "it2 should not be read again";
+ ASSERT_FALSE(it2->base.atEOF);
+
+ ASSERT_EQ(ui_base->Read(ui_base), ITERATOR_EOF);
+ ASSERT_EQ(it1->readCount, 2) << "it1 should be read again";
+ ASSERT_TRUE(it1->base.atEOF);
+ ASSERT_EQ(it2->readCount, 2) << "it2 should be read again";
+ ASSERT_TRUE(it2->base.atEOF);
+
+ ui_base->Free(ui_base);
+}
+
+
+class UnionIteratorReducerTest : public ::testing::Test {};
+
+TEST_F(UnionIteratorReducerTest, TestUnionRemovesEmptyChildren) {
+ QueryIterator **children = (QueryIterator **)rm_malloc(sizeof(QueryIterator *) * 4);
+ children[0] = nullptr;
+ children[1] = reinterpret_cast(new MockIterator({1UL, 2UL, 3UL}));
+ children[2] = IT_V2(NewEmptyIterator)();
+ children[3] = reinterpret_cast(new MockIterator({1UL, 2UL, 3UL}));
+ QueryIterator *ui_base = IT_V2(NewUnionIterator)(children, 4, false, 1.0, QN_UNION, NULL, &RSGlobalConfig.iteratorsConfigParams);
+ ASSERT_EQ(ui_base->type, UNION_ITERATOR);
+ UnionIterator *ui = (UnionIterator *)ui_base;
+ ASSERT_EQ(ui->num, 2);
+ ui_base->Free(ui_base);
+}
+
+TEST_F(UnionIteratorReducerTest, TestUnionRemovesAllEmptyChildren) {
+ QueryIterator **children = (QueryIterator **)rm_malloc(sizeof(QueryIterator *) * 4);
+ children[0] = nullptr;
+ children[1] = IT_V2(NewEmptyIterator)();
+ children[2] = IT_V2(NewEmptyIterator)();
+ children[3] = nullptr;
+ QueryIterator *ui_base = IT_V2(NewUnionIterator)(children, 4, false, 1.0, QN_UNION, NULL, &RSGlobalConfig.iteratorsConfigParams);
+ ASSERT_EQ(ui_base->type, EMPTY_ITERATOR);
+ ui_base->Free(ui_base);
+}
+
+TEST_F(UnionIteratorReducerTest, TestUnionRemovesEmptyChildrenOnlyOneLeft) {
+ QueryIterator **children = (QueryIterator **)rm_malloc(sizeof(QueryIterator *) * 4);
+ children[0] = nullptr;
+ children[1] = reinterpret_cast(new MockIterator({1UL, 2UL, 3UL}));
+ children[2] = IT_V2(NewEmptyIterator)();
+ children[3] = nullptr;
+ QueryIterator* expected_iter = children[1];
+ QueryIterator *ui_base = IT_V2(NewUnionIterator)(children, 4, false, 1.0, QN_UNION, NULL, &RSGlobalConfig.iteratorsConfigParams);
+ ASSERT_EQ(ui_base, expected_iter);
+ ui_base->Free(ui_base);
+}
+
+TEST_F(UnionIteratorReducerTest, TestUnionQuickWithWildcard) {
+ QueryIterator **children = (QueryIterator **)rm_malloc(sizeof(QueryIterator *) * 4);
+ children[0] = reinterpret_cast(new MockIterator({1UL, 2UL, 3UL}));
+ children[1] = IT_V2(NewWildcardIterator_NonOptimized)(30, 2, 1.0);
+ children[2] = nullptr;
+ children[3] = IT_V2(NewEmptyIterator)();
+ QueryIterator *ui_base = IT_V2(NewUnionIterator)(children, 4, true, 1.0, QN_UNION, NULL, &RSGlobalConfig.iteratorsConfigParams);
+ ASSERT_EQ(ui_base->type, WILDCARD_ITERATOR);
+ ui_base->Free(ui_base);
+}
+
+TEST_F(UnionIteratorReducerTest, TestUnionQuickWithReaderWildcard) {
+ QueryIterator **children = (QueryIterator **)rm_malloc(sizeof(QueryIterator *) * 4);
+ size_t memsize;
+ InvertedIndex *idx = NewInvertedIndex(static_cast(INDEX_DEFAULT_FLAGS), 1, &memsize);
+ ASSERT_TRUE(idx != nullptr);
+ ASSERT_TRUE(InvertedIndex_GetDecoder(idx->flags).seeker != nullptr);
+ auto encoder = InvertedIndex_GetEncoder(idx->flags);
+ for (t_docId i = 1; i < 1000; ++i) {
+ auto res = (RSIndexResult) {
+ .docId = i,
+ .fieldMask = 1,
+ .freq = 1,
+ .type = RSResultType::RSResultType_Term,
+ };
+ InvertedIndex_WriteEntryGeneric(idx, encoder, i, &res);
+ }
+ // Create an iterator that reads only entries with field mask 2
+ QueryIterator *iterator = NewInvIndIterator_TermQuery(idx, nullptr, {.isFieldMask = true, .value = {.mask = 2}}, nullptr, 1.0);
+ InvIndIterator* invIdxIt = (InvIndIterator *)iterator;
+ invIdxIt->isWildcard = true;
+ children[0] = reinterpret_cast(new MockIterator({1UL, 2UL, 3UL}));
+ children[1] = iterator;
+ children[2] = nullptr;
+ children[3] = IT_V2(NewEmptyIterator)();
+ QueryIterator *ui_base = IT_V2(NewUnionIterator)(children, 4, true, 1.0, QN_UNION, NULL, &RSGlobalConfig.iteratorsConfigParams);
+ ASSERT_EQ(ui_base->type, READ_ITERATOR);
+ ui_base->Free(ui_base);
+ InvertedIndex_Free(idx);
}
diff --git a/tests/cpptests/test_cpp_iterator_wildcard.cpp b/tests/cpptests/test_cpp_iterator_wildcard.cpp
index 66f864727a7..cd5a5fdfd83 100644
--- a/tests/cpptests/test_cpp_iterator_wildcard.cpp
+++ b/tests/cpptests/test_cpp_iterator_wildcard.cpp
@@ -17,9 +17,10 @@ class WildcardIteratorTest : public ::testing::Test {
QueryIterator *iterator_base;
const t_docId maxDocId = 100;
const size_t numDocs = 50;
+ const double weight = 2.0;
void SetUp() override {
- iterator_base = IT_V2(NewWildcardIterator_NonOptimized)(maxDocId, numDocs);
+ iterator_base = IT_V2(NewWildcardIterator_NonOptimized)(maxDocId, numDocs, weight);
}
void TearDown() override {
@@ -37,6 +38,7 @@ TEST_F(WildcardIteratorTest, InitialState) {
ASSERT_FALSE(iterator_base->atEOF);
ASSERT_EQ(iterator_base->lastDocId, 0);
ASSERT_EQ(iterator_base->type, WILDCARD_ITERATOR);
+ ASSERT_EQ(iterator_base->current->weight, weight);
// Test NumEstimated returns the correct number of docs
ASSERT_EQ(iterator_base->NumEstimated(iterator_base), numDocs);
@@ -48,6 +50,7 @@ TEST_F(WildcardIteratorTest, Read) {
ASSERT_EQ(iterator_base->Read(iterator_base), ITERATOR_OK);
ASSERT_EQ(iterator_base->current->docId, i);
ASSERT_EQ(iterator_base->lastDocId, i);
+ ASSERT_EQ(iterator_base->current->weight, weight);
}
// After reading all docs, should return EOF
@@ -67,6 +70,7 @@ TEST_F(WildcardIteratorTest, SkipTo) {
ASSERT_EQ(iterator_base->SkipTo(iterator_base, target), ITERATOR_OK);
ASSERT_EQ(iterator_base->current->docId, target);
ASSERT_EQ(iterator_base->lastDocId, target);
+ ASSERT_EQ(iterator_base->current->weight, weight);
}
// Skip beyond maxDocId should return EOF
@@ -82,17 +86,20 @@ TEST_F(WildcardIteratorTest, Rewind) {
ASSERT_EQ(iterator_base->current->docId, 10);
ASSERT_EQ(((WildcardIterator *)iterator_base)->currentId, 10);
ASSERT_EQ(iterator_base->lastDocId, 10);
+ ASSERT_EQ(iterator_base->current->weight, weight);
// Test that Rewind resets the iterator
iterator_base->Rewind(iterator_base);
ASSERT_EQ(((WildcardIterator *)iterator_base)->currentId, 0);
ASSERT_EQ(iterator_base->lastDocId, 0);
ASSERT_FALSE(iterator_base->atEOF);
+ ASSERT_EQ(iterator_base->current->weight, weight);
// After Rewind, should be able to read from the beginning
ASSERT_EQ(iterator_base->Read(iterator_base), ITERATOR_OK);
ASSERT_EQ(iterator_base->current->docId, 1);
ASSERT_EQ(iterator_base->lastDocId, 1);
+ ASSERT_EQ(iterator_base->current->weight, weight);
}
TEST_F(WildcardIteratorTest, ReadAfterSkip) {
@@ -106,6 +113,7 @@ TEST_F(WildcardIteratorTest, ReadAfterSkip) {
ASSERT_EQ(iterator_base->Read(iterator_base), ITERATOR_OK);
ASSERT_EQ(iterator_base->current->docId, i);
ASSERT_EQ(iterator_base->lastDocId, i);
+ ASSERT_EQ(iterator_base->current->weight, weight);
}
// After reading all docs, should return EOF
@@ -118,11 +126,12 @@ TEST_F(WildcardIteratorTest, ResultProperties) {
ASSERT_EQ(iterator_base->current->docId, 1);
ASSERT_EQ(iterator_base->current->freq, 1);
ASSERT_EQ(iterator_base->current->fieldMask, RS_FIELDMASK_ALL);
+ ASSERT_EQ(iterator_base->current->weight, weight);
}
TEST_F(WildcardIteratorTest, ZeroDocuments) {
// Create a wildcard iterator with zero documents
- QueryIterator *emptyIterator = IT_V2(NewWildcardIterator_NonOptimized)(0, 0);
+ QueryIterator *emptyIterator = IT_V2(NewWildcardIterator_NonOptimized)(0, 0, weight);
// Should immediately return EOF on read
ASSERT_EQ(emptyIterator->Read(emptyIterator), ITERATOR_EOF);
diff --git a/tests/cpptests/test_cpp_query.cpp b/tests/cpptests/test_cpp_query.cpp
index 9096edcb682..17d2a910597 100644
--- a/tests/cpptests/test_cpp_query.cpp
+++ b/tests/cpptests/test_cpp_query.cpp
@@ -705,6 +705,54 @@ TEST_F(QueryTest, testPureNegative) {
IndexSpec_RemoveFromGlobals(ref, false);
}
+TEST_F(QueryTest, testDoubleNegationOptimization) {
+ // Test that NOT(NOT(A)) = A optimization works
+ static const char *args[] = {"SCHEMA", "title", "text", "weight", "0.1", "body", "text", "weight", "2.0"};
+ QueryError err = {QueryErrorCode(0)};
+ StrongRef ref = IndexSpec_ParseC("idx", args, sizeof(args) / sizeof(const char *), &err);
+ RedisSearchCtx ctx = SEARCH_CTX_STATIC(NULL, (IndexSpec *)StrongRef_Get(ref));
+
+ // Test v1 parser
+ {
+ QASTCXX ast;
+ ast.setContext(&ctx);
+ ASSERT_TRUE(ast.parse("--hello", 1)) << ast.getError();
+ QueryNode *n = ast.root;
+ ASSERT_TRUE(n != NULL);
+ // Should be optimized to just a token node, not a double NOT
+ ASSERT_EQ(n->type, QN_TOKEN);
+ ASSERT_STREQ("hello", n->tn.str);
+ }
+
+ // Test v2 parser
+ {
+ QASTCXX ast;
+ ast.setContext(&ctx);
+ ASSERT_TRUE(ast.parse("--hello", 2)) << ast.getError();
+ QueryNode *n = ast.root;
+ ASSERT_TRUE(n != NULL);
+ // Should be optimized to just a token node, not a double NOT
+ ASSERT_EQ(n->type, QN_TOKEN);
+ ASSERT_STREQ("hello", n->tn.str);
+ }
+
+ // Test triple negation: ---hello should be -hello
+ {
+ QASTCXX ast;
+ ast.setContext(&ctx);
+ ASSERT_TRUE(ast.parse("---hello", 2)) << ast.getError();
+ QueryNode *n = ast.root;
+ ASSERT_TRUE(n != NULL);
+ // Should be optimized to a single NOT node
+ ASSERT_EQ(n->type, QN_NOT);
+ ASSERT_TRUE(QueryNode_GetChild(n, 0) != NULL);
+ ASSERT_EQ(QueryNode_GetChild(n, 0)->type, QN_TOKEN);
+ ASSERT_STREQ("hello", QueryNode_GetChild(n, 0)->tn.str);
+ }
+
+ IndexSpec_RemoveFromGlobals(ref, false);
+}
+
TEST_F(QueryTest, testGeoQuery_v1) {
static const char *args[] = {"SCHEMA", "title", "text", "loc", "geo"};
QueryError err = {QueryErrorCode(0)};
diff --git a/tests/pytests/test_issues.py b/tests/pytests/test_issues.py
index 07b60a6da0e..2c6ebabc025 100644
--- a/tests/pytests/test_issues.py
+++ b/tests/pytests/test_issues.py
@@ -180,11 +180,13 @@ def test_issue1880(env):
# test with a term which does not exist
excepted_res = ['Type', 'INTERSECT', 'Counter', 0, 'Child iterators', [
- None,
- ['Type', 'TEXT', 'Term', 'world', 'Counter', 0, 'Size', 1],
- ['Type', 'TEXT', 'Term', 'hello', 'Counter', 0, 'Size', 2]]]
- res3 = env.cmd('FT.PROFILE', 'idx', 'SEARCH', 'QUERY', 'hello new world')
+ None,
+ ['Type', 'TEXT', 'Term', 'world', 'Counter', 0, 'Size', 1],
+ ['Type', 'TEXT', 'Term', 'hello', 'Counter', 0, 'Size', 2]]]
+
+ # excepted_res = ['Type', 'EMPTY', 'Counter', 0] (After Optimization and new iterators are used, it should change to this)
+ res3 = env.cmd('FT.PROFILE', 'idx', 'SEARCH', 'QUERY', 'hello new world')
env.assertEqual(res3[1][1][0][3], excepted_res)
def test_issue1932(env):
diff --git a/tests/pytests/test_profile.py b/tests/pytests/test_profile.py
index c9ba132c965..8a15480bbcd 100644
--- a/tests/pytests/test_profile.py
+++ b/tests/pytests/test_profile.py
@@ -430,8 +430,9 @@ def testNotIterator(env):
'Iterators profile',
['Type', 'INTERSECT', 'Counter', 1, 'Child iterators',
[['Type', 'TEXT', 'Term', 'foo', 'Counter', 1, 'Size', 1],
- ['Type', 'NOT', 'Counter', 1, 'Child iterator',
+ ['Type', 'NOT', 'Counter', 1, 'Child iterator',
['Type', 'EMPTY', 'Counter', 0]]]],
+ #['Type', 'WILDCARD', 'Counter', 1]]], (After Optimization and new iterators are used, NOT ITERATOR should change to this)
'Result processors profile',
[['Type', 'Index', 'Counter', 1],
['Type', 'Scorer', 'Counter', 1],