From b1a104d987ce1a8bf399f00f781b988bf23d6332 Mon Sep 17 00:00:00 2001 From: madschemas <155993105+MadSchemas@users.noreply.github.com> Date: Thu, 5 Sep 2024 10:54:34 +0300 Subject: [PATCH] Update to version v4.17.1 --- CMakeLists.txt | 4 +- bindings/consts.go | 2 +- changelog.md | 24 +++ cjson/decoder.go | 20 ++- cpp_src/CMakeLists.txt | 12 +- .../cmd/reindexer_tool/commandsexecutor.cc | 13 +- .../cmd/reindexer_tool/commandsprocessor.cc | 5 +- cpp_src/core/ft/ft_fast/selecter.cc | 47 +++--- cpp_src/core/ft/ft_fast/selecter.h | 9 +- cpp_src/core/idset.h | 1 + cpp_src/core/index/index.cc | 2 +- cpp_src/core/index/index.h | 4 +- cpp_src/core/index/indextext/fastindextext.cc | 142 +++++++++++++++--- cpp_src/core/index/indextext/fastindextext.h | 7 +- .../core/index/indextext/fuzzyindextext.cc | 3 +- cpp_src/core/index/indextext/fuzzyindextext.h | 2 +- cpp_src/core/index/indextext/indextext.cc | 9 +- cpp_src/core/index/indextext/indextext.h | 6 +- cpp_src/core/namespace/namespaceimpl.cc | 68 +++++---- cpp_src/core/namespace/namespaceimpl.h | 1 + cpp_src/core/nsselecter/nsselecter.cc | 6 +- .../nsselecter/selectiteratorcontainer.cc | 36 +++-- .../core/nsselecter/selectiteratorcontainer.h | 12 +- cpp_src/core/nsselecter/substitutionhelpers.h | 24 ++- cpp_src/core/payload/fieldsset.cc | 6 + cpp_src/core/payload/fieldsset.h | 11 +- cpp_src/core/payload/payloadfieldtype.cc | 14 ++ cpp_src/core/payload/payloadfieldtype.h | 1 + cpp_src/core/query/query.cc | 7 + cpp_src/core/query/query.h | 11 +- cpp_src/core/query/queryentry.cc | 11 ++ cpp_src/core/query/queryentry.h | 60 ++++---- cpp_src/core/query/sql/sqlencoder.cc | 3 - cpp_src/core/selectfunc/ctx/ftctx.h | 2 + cpp_src/debug/backtrace.h | 1 + cpp_src/gtests/tests/fixtures/queries_api.h | 12 +- cpp_src/gtests/tests/unit/grpcclient_test.cc | 2 +- cpp_src/gtests/tests/unit/index_tuple_test.cc | 22 ++- cpp_src/server/CMakeLists.txt | 2 +- cpp_src/server/contrib/server.md | 2 +- cpp_src/server/contrib/server.yml | 2 +- cpp_src/tools/scope_guard.h | 20 +++ cpp_src/tools/string_regexp_functions.h | 2 +- cpp_src/vendor/libbacktrace/dwarf.c | 4 + test/composite_indexes_test.go | 76 ++++++++++ test/encdec_test.go | 4 + 46 files changed, 554 insertions(+), 180 deletions(-) create mode 100644 cpp_src/tools/scope_guard.h diff --git a/CMakeLists.txt b/CMakeLists.txt index df3d3831e..68580f3d5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,7 @@ +cmake_minimum_required(VERSION 3.10..3.13) + project(reindexer) -cmake_minimum_required(VERSION 3.10) + enable_testing() set(CMAKE_DISABLE_IN_SOURCE_BUILD ON) set(CMAKE_POSITION_INDEPENDENT_CODE ON) diff --git a/bindings/consts.go b/bindings/consts.go index 941eefda1..5a959d560 100644 --- a/bindings/consts.go +++ b/bindings/consts.go @@ -2,7 +2,7 @@ package bindings const CInt32Max = int(^uint32(0) >> 1) -const ReindexerVersion = "v4.17.0" +const ReindexerVersion = "v4.17.1" // public go consts from type_consts.h and reindexer_ctypes.h const ( diff --git a/changelog.md b/changelog.md index b6564b547..2c483d7d4 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,27 @@ +# Version 4.17.1 *beta* (05.09.2024) +## Core +- [fix] Fixed pagination for `MERGE` fulltext queries. Previously those queries could return duplicates on different pages due to missing ordering guarantees +- [fix] Fixed fields filters serialization for `DISTINCT` aggregations (affects SQL logging only) +- [fix] Temporary disabled default values logic for indexed fields from v4.16.0 - this logic may cause stability issues and will be reworked in further releases +- [fix] Add extra check for composites indexes substitution +- [fix] Fix composites substitution after indexes update + +## Reindexer tool +- [fix] Fixed crash in `\quit`-command after previous errors +- [fix] Fixed crash in `git bash` on `Windows` platforms + +## Face +- [fea] Added RU language in UI +- [fea] Changed BM25 settings layout +- [fea] Added new documentation links +- [fix] Fixed database deletion +- [fix] Fixed default value of the stop words in fulltext config +- [fix] Fixed default value of the rtree-indexes in item creation tab +- [fix] Fixed last namespace displaying +- [fix] Fixed the issue with removing of 'Statistics -> Replication' table on the page reloading +- [fix] Renamed 'Stat(ms)' field on Replication page +- [fix] Fixed XSS vulnerability in table view + # Version 4.17.0 *beta* (16.08.2024) ## Core - [fea] Updated [logging library](https://github.com/gabime/spdlog) to v1.14.1 and [formatting library](https://github.com/fmtlib/fmt) to v11.0.2 diff --git a/cjson/decoder.go b/cjson/decoder.go index 6fbd73e4f..1f5371ef4 100644 --- a/cjson/decoder.go +++ b/cjson/decoder.go @@ -313,6 +313,7 @@ func (dec *Decoder) decodeSlice(pl *payloadIface, rdser *Serializer, v *reflect. var ptr unsafe.Pointer k := v.Kind() + offset := 0 switch k { case reflect.Slice: @@ -331,7 +332,11 @@ func (dec *Decoder) decodeSlice(pl *payloadIface, rdser *Serializer, v *reflect. // offset is 0 // No concatenation for the fixed size arrays default: - panic(fmt.Errorf("can not convert '%s' to 'array'", v.Type().Kind().String())) + if count == 0 { // Allows empty slice for any scalar type (using default value) + return + } else { + panic(fmt.Errorf("can not convert '%s' to 'array'", v.Type().Kind().String())) + } } if subtag != TAG_OBJECT { @@ -583,6 +588,7 @@ func (dec *Decoder) decodeValue(pl *payloadIface, rdser *Serializer, v reflect.V ctagName := ctag.Name() k := v.Kind() + initialV := v if k == reflect.Ptr { if v.IsNil() { v.Set(reflect.New(v.Type().Elem())) @@ -641,6 +647,7 @@ func (dec *Decoder) decodeValue(pl *payloadIface, rdser *Serializer, v reflect.V } else { panic(fmt.Errorf("err: intf=%s, name='%s' %s", v.Type().Name(), dec.state.tagsMatcher.tag2name(ctagName), ctag.Dump())) } + initialV = v k = v.Kind() if k == reflect.Ptr { if v.IsNil() { @@ -659,8 +666,12 @@ func (dec *Decoder) decodeValue(pl *payloadIface, rdser *Serializer, v reflect.V switch ctagType { case TAG_ARRAY: count := int(rdser.GetVarUInt()) - pl.getArray(int(ctagField), *cnt, count, v) - *cnt += count + if k == reflect.Slice || k == reflect.Array || count != 0 { // Allows empty slice for any scalar type (using default value) + pl.getArray(int(ctagField), *cnt, count, v) + *cnt += count + } else { + initialV.Set(reflect.Zero(initialV.Type())) // Set nil to scalar pointers, intialized with empty arrays + } default: pl.getValue(int(ctagField), *cnt, v) (*cnt)++ @@ -670,6 +681,9 @@ func (dec *Decoder) decodeValue(pl *payloadIface, rdser *Serializer, v reflect.V switch ctagType { case TAG_ARRAY: dec.decodeSlice(pl, rdser, &v, fieldsoutcnt, cctagsPath) + if v.Kind() != reflect.Array && v.Kind() != reflect.Slice && v.Kind() != reflect.Interface { + initialV.Set(reflect.Zero(initialV.Type())) // Set nil to scalar pointers, intialized with empty arrays + } case TAG_OBJECT: for dec.decodeValue(pl, rdser, v, fieldsoutcnt, cctagsPath) { } diff --git a/cpp_src/CMakeLists.txt b/cpp_src/CMakeLists.txt index b89ed671b..0377cae6d 100644 --- a/cpp_src/CMakeLists.txt +++ b/cpp_src/CMakeLists.txt @@ -45,7 +45,7 @@ else() option(LINK_RESOURCES "Link web resources as binary data" ON) endif() -set (REINDEXER_VERSION_DEFAULT "4.17.0") +set (REINDEXER_VERSION_DEFAULT "4.17.1") if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE "RelWithDebInfo") @@ -297,7 +297,11 @@ else() endif() # Static LevelDB v1.23 is built with -fno-rtti by default. To inherit our logger from leveldb's logger, this file must be built with -fno-rtti to -set_source_files_properties (${REINDEXER_SOURCE_PATH}/core/storage/leveldblogger.cc PROPERTIES COMPILE_FLAGS "-fno-rtti") +if(MSVC) + set_source_files_properties (${REINDEXER_SOURCE_PATH}/core/storage/leveldblogger.cc PROPERTIES COMPILE_FLAGS "/GR-") +else() + set_source_files_properties (${REINDEXER_SOURCE_PATH}/core/storage/leveldblogger.cc PROPERTIES COMPILE_FLAGS "-fno-rtti") +endif() list(APPEND REINDEXER_LIBRARIES reindexer) add_library(${TARGET} STATIC ${HDRS} ${SRCS} ${VENDORS}) @@ -357,6 +361,7 @@ else() GIT_TAG "1.1.8" CMAKE_ARGS -DSNAPPY_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_BINARY_DIR} -DCMAKE_INSTALL_LIBDIR=${CMAKE_CURRENT_BINARY_DIR} + -DCMAKE_POSITION_INDEPENDENT_CODE=ON ) include_directories(${CMAKE_CURRENT_BINARY_DIR}/include) link_directories(${CMAKE_CURRENT_BINARY_DIR}) @@ -443,9 +448,10 @@ if(NOT LevelDB_LIBRARY OR NOT LevelDB_INCLUDE_DIR OR WITH_TSAN) GIT_TAG "master" CMAKE_ARGS -DLEVELDB_BUILD_TESTS=OFF -DLEVELDB_BUILD_BENCHMARKS=OFF -DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_BINARY_DIR} - "-DCMAKE_CXX_FLAGS=-fPIC -I${CMAKE_CURRENT_BINARY_DIR}/include" + "-DCMAKE_CXX_FLAGS=-I${CMAKE_CURRENT_BINARY_DIR}/include" -DCMAKE_EXE_LINKER_FLAGS=-L${CMAKE_CURRENT_BINARY_DIR} -DCMAKE_INSTALL_LIBDIR=${CMAKE_CURRENT_BINARY_DIR} + -DCMAKE_POSITION_INDEPENDENT_CODE=ON ) if(NOT SNAPPY_FOUND) add_dependencies(leveldb_lib snappy_lib) diff --git a/cpp_src/cmd/reindexer_tool/commandsexecutor.cc b/cpp_src/cmd/reindexer_tool/commandsexecutor.cc index 9699451eb..9863cf4fb 100644 --- a/cpp_src/cmd/reindexer_tool/commandsexecutor.cc +++ b/cpp_src/cmd/reindexer_tool/commandsexecutor.cc @@ -59,11 +59,14 @@ Error CommandsExecutor::GetSuggestions(const std::string& input, st template Error CommandsExecutor::Stop() { - GenericCommand cmd([this] { - stop(true); - return Error{}; - }); - auto err = execCommand(cmd); + Error err; + if (GetStatus().running) { + GenericCommand cmd([this] { + stop(true); + return Error{}; + }); + err = execCommand(cmd); + } if (err.ok() && executorThr_.joinable()) { executorThr_.join(); } diff --git a/cpp_src/cmd/reindexer_tool/commandsprocessor.cc b/cpp_src/cmd/reindexer_tool/commandsprocessor.cc index 0971dd0fe..078452693 100644 --- a/cpp_src/cmd/reindexer_tool/commandsprocessor.cc +++ b/cpp_src/cmd/reindexer_tool/commandsprocessor.cc @@ -177,10 +177,7 @@ bool CommandsProcessor::Run(const std::string& command, const std:: template Error CommandsProcessor::stop() { - if (executor_.GetStatus().running) { - return executor_.Stop(); - } - return Error(); + return executor_.Stop(); } template class CommandsProcessor; diff --git a/cpp_src/core/ft/ft_fast/selecter.cc b/cpp_src/core/ft/ft_fast/selecter.cc index c02d82ed7..e0bef67a1 100644 --- a/cpp_src/core/ft/ft_fast/selecter.cc +++ b/cpp_src/core/ft/ft_fast/selecter.cc @@ -120,8 +120,8 @@ void Selecter::prepareVariants(std::vector& variants, RV // RX_NO_INLINE just for build test purpose. Do not expect any effect here template template -RX_NO_INLINE MergeData Selecter::Process(FtDSLQuery&& dsl, bool inTransaction, FtMergeStatuses::Statuses&& mergeStatuses, - const RdxContext& rdxCtx) { +RX_NO_INLINE MergeData Selecter::Process(FtDSLQuery&& dsl, bool inTransaction, FtSortType ftSortType, + FtMergeStatuses::Statuses&& mergeStatuses, const RdxContext& rdxCtx) { FtSelectContext ctx; ctx.rawResults.reserve(dsl.size()); // STEP 2: Search dsl terms for each variant @@ -213,11 +213,11 @@ RX_NO_INLINE MergeData Selecter::Process(FtDSLQuery&& dsl, bool inTransa const auto maxMergedSize = std::min(size_t(holder_.cfg_->mergeLimit), ctx.totalORVids); if (maxMergedSize < 0xFFFF) { - return mergeResultsBmType(std::move(results), ctx.totalORVids, synonymsBounds, inTransaction, std::move(mergeStatuses), - rdxCtx); + return mergeResultsBmType(std::move(results), ctx.totalORVids, synonymsBounds, inTransaction, ftSortType, + std::move(mergeStatuses), rdxCtx); } else if (maxMergedSize < 0xFFFFFFFF) { - return mergeResultsBmType(std::move(results), ctx.totalORVids, synonymsBounds, inTransaction, std::move(mergeStatuses), - rdxCtx); + return mergeResultsBmType(std::move(results), ctx.totalORVids, synonymsBounds, inTransaction, ftSortType, + std::move(mergeStatuses), rdxCtx); } else { assertrx_throw(false); } @@ -227,17 +227,17 @@ RX_NO_INLINE MergeData Selecter::Process(FtDSLQuery&& dsl, bool inTransa template template MergeData Selecter::mergeResultsBmType(std::vector&& results, size_t totalORVids, - const std::vector& synonymsBounds, bool inTransaction, + const std::vector& synonymsBounds, bool inTransaction, FtSortType ftSortType, FtMergeStatuses::Statuses&& mergeStatuses, const RdxContext& rdxCtx) { switch (holder_.cfg_->bm25Config.bm25Type) { case FtFastConfig::Bm25Config::Bm25Type::rx: - return mergeResults(std::move(results), totalORVids, synonymsBounds, inTransaction, + return mergeResults(std::move(results), totalORVids, synonymsBounds, inTransaction, ftSortType, std::move(mergeStatuses), rdxCtx); case FtFastConfig::Bm25Config::Bm25Type::classic: - return mergeResults(std::move(results), totalORVids, synonymsBounds, inTransaction, + return mergeResults(std::move(results), totalORVids, synonymsBounds, inTransaction, ftSortType, std::move(mergeStatuses), rdxCtx); case FtFastConfig::Bm25Config::Bm25Type::wordCount: - return mergeResults(std::move(results), totalORVids, synonymsBounds, inTransaction, + return mergeResults(std::move(results), totalORVids, synonymsBounds, inTransaction, ftSortType, std::move(mergeStatuses), rdxCtx); } assertrx_throw(false); @@ -1284,7 +1284,7 @@ bool Selecter::TyposHandler::isWordFitMaxLettPerm(const std::string_view template template MergeData Selecter::mergeResults(std::vector&& rawResults, size_t maxMergedSize, - const std::vector& synonymsBounds, bool inTransaction, + const std::vector& synonymsBounds, bool inTransaction, FtSortType ftSortType, FtMergeStatuses::Statuses&& mergeStatuses, const RdxContext& rdxCtx) { const auto& vdocs = holder_.vdocs_; MergeData merged; @@ -1388,9 +1388,19 @@ MergeData Selecter::mergeResults(std::vector&& rawRes merged.maxRank = m.proc; } } - - boost::sort::pdqsort_branchless(merged.begin(), merged.end(), - [](const MergeInfo& lhs, const MergeInfo& rhs) noexcept { return lhs.proc > rhs.proc; }); + switch (ftSortType) { + case FtSortType::RankOnly: { + boost::sort::pdqsort_branchless(merged.begin(), merged.end(), + [](const MergeInfo& lhs, const MergeInfo& rhs) noexcept { return lhs.proc > rhs.proc; }); + return merged; + } + case FtSortType::RankAndID: { + return merged; + } + case FtSortType::ExternalExpression: + throw Error(errLogic, "FtSortType::ExternalExpression not implemented."); + break; + } return merged; } @@ -1431,13 +1441,14 @@ void Selecter::printVariants(const FtSelectContext& ctx, const TextSearc } template class Selecter; -template MergeData Selecter::Process(FtDSLQuery&&, bool, FtMergeStatuses::Statuses&&, +template MergeData Selecter::Process(FtDSLQuery&&, bool, FtSortType, FtMergeStatuses::Statuses&&, const RdxContext&); -template MergeData Selecter::Process(FtDSLQuery&&, bool, FtMergeStatuses::Statuses&&, +template MergeData Selecter::Process(FtDSLQuery&&, bool, FtSortType, FtMergeStatuses::Statuses&&, const RdxContext&); template class Selecter; -template MergeData Selecter::Process(FtDSLQuery&&, bool, FtMergeStatuses::Statuses&&, const RdxContext&); -template MergeData Selecter::Process(FtDSLQuery&&, bool, FtMergeStatuses::Statuses&&, +template MergeData Selecter::Process(FtDSLQuery&&, bool, FtSortType, FtMergeStatuses::Statuses&&, + const RdxContext&); +template MergeData Selecter::Process(FtDSLQuery&&, bool, FtSortType, FtMergeStatuses::Statuses&&, const RdxContext&); } // namespace reindexer diff --git a/cpp_src/core/ft/ft_fast/selecter.h b/cpp_src/core/ft/ft_fast/selecter.h index 9cb546d53..0e159f484 100644 --- a/cpp_src/core/ft/ft_fast/selecter.h +++ b/cpp_src/core/ft/ft_fast/selecter.h @@ -1,6 +1,7 @@ #pragma once #include "core/ft/ftdsl.h" #include "core/ft/idrelset.h" +#include "core/selectfunc/ctx/ftctx.h" #include "dataholder.h" namespace reindexer { @@ -53,7 +54,8 @@ class Selecter { }; template - MergeData Process(FtDSLQuery&& dsl, bool inTransaction, FtMergeStatuses::Statuses&& mergeStatuses, const RdxContext&); + MergeData Process(FtDSLQuery&& dsl, bool inTransaction, FtSortType ftSortType, FtMergeStatuses::Statuses&& mergeStatuses, + const RdxContext&); private: struct TextSearchResult { @@ -178,7 +180,7 @@ class Selecter { template MergeData mergeResults(std::vector&& rawResults, size_t maxMergedSize, const std::vector& synonymsBounds, - bool inTransaction, FtMergeStatuses::Statuses&& mergeStatuses, const RdxContext&); + bool inTransaction, FtSortType ftSortType, FtMergeStatuses::Statuses&& mergeStatuses, const RdxContext&); template void mergeIteration(TextSearchResults& rawRes, index_t rawResIndex, FtMergeStatuses::Statuses& mergeStatuses, MergeData& merged, @@ -244,7 +246,8 @@ class Selecter { template MergeData mergeResultsBmType(std::vector&& results, size_t totalORVids, const std::vector& synonymsBounds, - bool inTransaction, FtMergeStatuses::Statuses&& mergeStatuses, const RdxContext& rdxCtx); + bool inTransaction, FtSortType ftSortType, FtMergeStatuses::Statuses&& mergeStatuses, + const RdxContext& rdxCtx); void debugMergeStep(const char* msg, int vid, float normBm25, float normDist, int finalRank, int prevRank); template diff --git a/cpp_src/core/idset.h b/cpp_src/core/idset.h index af1b7d44b..d6235ac90 100644 --- a/cpp_src/core/idset.h +++ b/cpp_src/core/idset.h @@ -35,6 +35,7 @@ class IdSetPlain : protected base_idset { using base_idset::rend; using base_idset::const_reverse_iterator; using base_idset::const_iterator; + using base_idset::operator[]; enum EditMode { Ordered, // Keep idset ordered, and ready to select (insert is slow O(logN)+O(N)) diff --git a/cpp_src/core/index/index.cc b/cpp_src/core/index/index.cc index c7b8a6151..c2fa3a9a9 100644 --- a/cpp_src/core/index/index.cc +++ b/cpp_src/core/index/index.cc @@ -86,7 +86,7 @@ void Index::dump(S& os, std::string_view step, std::string_view offset) const { payloadType_.Dump(os, step, newOffset); if (IsComposite(type_)) { os << ",\n" << newOffset << "fields: "; - fields_.Dump(os); + fields_.Dump(os, FieldsSet::DumpWithMask::Yes); } os << ",\n" << newOffset << "sortedIdxCount: " << sortedIdxCount_ << ",\n" diff --git a/cpp_src/core/index/index.h b/cpp_src/core/index/index.h index 5b6aac3e5..7e4875c6e 100644 --- a/cpp_src/core/index/index.h +++ b/cpp_src/core/index/index.h @@ -34,7 +34,8 @@ class Index { forceComparator(0), unbuiltSortOrders(0), indexesNotOptimized(0), - inTransaction{0} {} + inTransaction{0}, + ftSortType(0) {} unsigned itemsCountInNamespace; int maxIterations; unsigned distinct : 1; @@ -43,6 +44,7 @@ class Index { unsigned unbuiltSortOrders : 1; unsigned indexesNotOptimized : 1; unsigned inTransaction : 1; + unsigned ftSortType : 2; }; using KeyEntry = reindexer::KeyEntry; using KeyEntryPlain = reindexer::KeyEntry; diff --git a/cpp_src/core/index/indextext/fastindextext.cc b/cpp_src/core/index/indextext/fastindextext.cc index 7e56292de..fdded3b1f 100644 --- a/cpp_src/core/index/indextext/fastindextext.cc +++ b/cpp_src/core/index/indextext/fastindextext.cc @@ -3,8 +3,8 @@ #include "core/ft/filters/kblayout.h" #include "core/ft/filters/synonyms.h" #include "core/ft/filters/translit.h" -#include "core/ft/ft_fast/selecter.h" #include "estl/contexted_locks.h" +#include "sort/pdqsort.hpp" #include "tools/clock.h" #include "tools/logger.h" @@ -124,10 +124,60 @@ IndexMemStat FastIndexText::GetMemStat(const RdxContext& ctx) { ret.idsetCache = this->cache_ft_ ? this->cache_ft_->GetMemStat() : LRUCacheMemStat(); return ret; } +template +MergeData::iterator FastIndexText::unstableRemoveIf(MergeData& md, int minRelevancy, double scalingFactor, size_t& releventDocs, + int& cnt) { + if (md.empty()) { + return md.begin(); + } + auto& holder = *this->holder_; + auto first = md.begin(); + auto last = md.end(); + while (true) { + while (true) { + if (first == last) { + return first; + } + auto& vdoc = holder.vdocs_[first->id]; + if (!vdoc.keyEntry) { + break; + } + first->proc *= scalingFactor; + if (first->proc < minRelevancy) { + break; + } + assertrx_throw(!vdoc.keyEntry->Unsorted().empty()); + cnt += vdoc.keyEntry->Sorted(0).size(); + ++releventDocs; + + ++first; + } + while (true) { + --last; + if (first == last) { + return first; + } + if (!holder.vdocs_[last->id].keyEntry) { + continue; + } + last->proc *= scalingFactor; + if (last->proc >= minRelevancy) { + break; + } + } + auto& vdoc = holder.vdocs_[last->id]; + assertrx_throw(!vdoc.keyEntry->Unsorted().empty()); + cnt += vdoc.keyEntry->Sorted(0).size(); + ++releventDocs; + + *first = std::move(*last); + ++first; + } +} template -IdSet::Ptr FastIndexText::Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTransaction, FtMergeStatuses&& statuses, - FtUseExternStatuses useExternSt, const RdxContext& rdxCtx) { +IdSet::Ptr FastIndexText::Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTransaction, FtSortType ftSortType, + FtMergeStatuses&& statuses, FtUseExternStatuses useExternSt, const RdxContext& rdxCtx) { fctx->GetData()->extraWordSymbols_ = this->getConfig()->extraWordSymbols; fctx->GetData()->isWordPositions_ = true; @@ -138,9 +188,11 @@ IdSet::Ptr FastIndexText::Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTr assertrx_throw(d); Selecter selecter{*d, this->Fields().size(), fctx->NeedArea(), holder_->cfg_->maxAreasInDoc}; if (useExternSt == FtUseExternStatuses::No) { - mergeData = selecter.Process(std::move(dsl), inTransaction, std::move(statuses.statuses), rdxCtx); + mergeData = selecter.Process(std::move(dsl), inTransaction, ftSortType, + std::move(statuses.statuses), rdxCtx); } else { - mergeData = selecter.Process(std::move(dsl), inTransaction, std::move(statuses.statuses), rdxCtx); + mergeData = selecter.Process(std::move(dsl), inTransaction, ftSortType, + std::move(statuses.statuses), rdxCtx); } break; } @@ -149,9 +201,11 @@ IdSet::Ptr FastIndexText::Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTr assertrx_throw(d); Selecter selecter{*d, this->Fields().size(), fctx->NeedArea(), holder_->cfg_->maxAreasInDoc}; if (useExternSt == FtUseExternStatuses::No) { - mergeData = selecter.Process(std::move(dsl), inTransaction, std::move(statuses.statuses), rdxCtx); + mergeData = selecter.Process(std::move(dsl), inTransaction, ftSortType, + std::move(statuses.statuses), rdxCtx); } else { - mergeData = selecter.Process(std::move(dsl), inTransaction, std::move(statuses.statuses), rdxCtx); + mergeData = selecter.Process(std::move(dsl), inTransaction, ftSortType, + std::move(statuses.statuses), rdxCtx); } break; } @@ -170,19 +224,32 @@ IdSet::Ptr FastIndexText::Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTr const double scalingFactor = mergeData.maxRank > 255 ? 255.0 / mergeData.maxRank : 1.0; const int minRelevancy = getConfig()->minRelevancy * 100 * scalingFactor; size_t releventDocs = 0; - for (auto& vid : mergeData) { - auto& vdoc = holder.vdocs_[vid.id]; - if (!vdoc.keyEntry) { - continue; - } - vid.proc *= scalingFactor; - if (vid.proc <= minRelevancy) { + switch (ftSortType) { + case FtSortType::RankAndID: { + auto itF = unstableRemoveIf(mergeData, minRelevancy, scalingFactor, releventDocs, cnt); + mergeData.erase(itF, mergeData.end()); break; } + case FtSortType::RankOnly: { + for (auto& vid : mergeData) { + auto& vdoc = holder.vdocs_[vid.id]; + if (!vdoc.keyEntry) { + continue; + } + vid.proc *= scalingFactor; + if (vid.proc <= minRelevancy) { + break; + } - assertrx_throw(!vdoc.keyEntry->Unsorted().empty()); - cnt += vdoc.keyEntry->Sorted(0).size(); - ++releventDocs; + assertrx_throw(!vdoc.keyEntry->Unsorted().empty()); + cnt += vdoc.keyEntry->Sorted(0).size(); + ++releventDocs; + } + break; + } + case FtSortType::ExternalExpression: { + throw Error(errLogic, "FtSortType::ExternalExpression not implemented."); + } } mergedIds->reserve(cnt); @@ -238,6 +305,44 @@ IdSet::Ptr FastIndexText::Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTr logPrintf(LogInfo, "Relevancy(%d): %s", fctx->Size(), str); } assertrx_throw(mergedIds->size() == fctx->Size()); + if (ftSortType == FtSortType::RankAndID) { + std::vector sortIds; + size_t nItems = mergedIds->size(); + sortIds.reserve(mergedIds->size()); + for (size_t i = 0; i < nItems; i++) { + sortIds.emplace_back(i); + } + std::vector& proc = fctx->GetData()->proc_; + boost::sort::pdqsort(sortIds.begin(), sortIds.end(), [&proc, mergedIds](size_t i1, size_t i2) { + int p1 = proc[i1]; + int p2 = proc[i2]; + if (p1 > p2) { + return true; + } else if (p1 < p2) { + return false; + } else { + return (*mergedIds)[i1] < (*mergedIds)[i2]; + } + }); + + for (size_t i = 0; i < nItems; i++) { + auto vm = (*mergedIds)[i]; + auto vp = proc[i]; + size_t j = i; + while (true) { + size_t k = sortIds[j]; + sortIds[j] = j; + if (k == i) { + break; + } + (*mergedIds)[j] = (*mergedIds)[k]; + proc[j] = proc[k]; + j = k; + } + (*mergedIds)[j] = vm; + proc[j] = vp; + } + } return mergedIds; } template @@ -351,11 +456,12 @@ template template RX_ALWAYS_INLINE void FastIndexText::appendMergedIds(MergeData& mergeData, size_t releventDocs, F&& appender) { auto& holder = *this->holder_; - for (size_t i = 0; i < releventDocs; ++i) { + for (size_t i = 0; i < releventDocs;) { auto& vid = mergeData[i]; auto& vdoc = holder.vdocs_[vid.id]; if (vdoc.keyEntry) { appender(vdoc.keyEntry->Sorted(0).begin(), vdoc.keyEntry->Sorted(0).end(), vid); + i++; } } } diff --git a/cpp_src/core/index/indextext/fastindextext.h b/cpp_src/core/index/indextext/fastindextext.h index 054afa650..8e902d945 100644 --- a/cpp_src/core/index/indextext/fastindextext.h +++ b/cpp_src/core/index/indextext/fastindextext.h @@ -2,6 +2,7 @@ #include "core/ft/config/ftfastconfig.h" #include "core/ft/ft_fast/dataholder.h" +#include "core/ft/ft_fast/selecter.h" #include "indextext.h" namespace reindexer { @@ -31,8 +32,8 @@ class FastIndexText : public IndexText { // Creates uncommited copy return std::make_unique>(*this); } - IdSet::Ptr Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTransaction, FtMergeStatuses&&, FtUseExternStatuses, - const RdxContext&) override final; + IdSet::Ptr Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTransactionbool, FtSortType ftSortType, FtMergeStatuses&&, + FtUseExternStatuses, const RdxContext&) override final; IndexMemStat GetMemStat(const RdxContext&) override final; Variant Upsert(const Variant& key, IdType id, bool& clearCache) override final; void Delete(const Variant& key, IdType id, StringsHolder&, bool& clearCache) override final; @@ -55,6 +56,8 @@ class FastIndexText : public IndexText { template void appendMergedIds(MergeData& merged, size_t releventDocs, F&& appender); + MergeData::iterator unstableRemoveIf(MergeData& md, const int minRelevancy, double scalingFactor, size_t& releventDocs, int& cnt); + std::unique_ptr holder_; }; diff --git a/cpp_src/core/index/indextext/fuzzyindextext.cc b/cpp_src/core/index/indextext/fuzzyindextext.cc index a3ea56d02..148db2e0c 100644 --- a/cpp_src/core/index/indextext/fuzzyindextext.cc +++ b/cpp_src/core/index/indextext/fuzzyindextext.cc @@ -5,9 +5,10 @@ namespace reindexer { template -IdSet::Ptr FuzzyIndexText::Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTransaction, FtMergeStatuses&&, +IdSet::Ptr FuzzyIndexText::Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTransaction, FtSortType ftSortType, FtMergeStatuses&&, FtUseExternStatuses withExternSt, const RdxContext& rdxCtx) { assertrx_throw(withExternSt == FtUseExternStatuses::No); + (void)ftSortType; (void)withExternSt; auto result = engine_.Search(dsl, inTransaction, rdxCtx); diff --git a/cpp_src/core/index/indextext/fuzzyindextext.h b/cpp_src/core/index/indextext/fuzzyindextext.h index 0c5d90846..d92a0a210 100644 --- a/cpp_src/core/index/indextext/fuzzyindextext.h +++ b/cpp_src/core/index/indextext/fuzzyindextext.h @@ -23,7 +23,7 @@ class FuzzyIndexText : public IndexText { abort(); } std::unique_ptr Clone() const override final { return std::make_unique>(*this); } - IdSet::Ptr Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTransaction, FtMergeStatuses&&, FtUseExternStatuses, + IdSet::Ptr Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTransaction, FtSortType ftSortType, FtMergeStatuses&&, FtUseExternStatuses, const RdxContext&) override final; Variant Upsert(const Variant& key, IdType id, bool& clearCache) override final { this->isBuilt_ = false; diff --git a/cpp_src/core/index/indextext/indextext.cc b/cpp_src/core/index/indextext/indextext.cc index 6ba1425ed..695456a7e 100644 --- a/cpp_src/core/index/indextext/indextext.cc +++ b/cpp_src/core/index/indextext/indextext.cc @@ -118,7 +118,7 @@ SelectKeyResults IndexText::SelectKey(const VariantArray& keys, CondType cond } } return doSelectKey(keys, needPutCache ? std::optional{std::move(ckey)} : std::nullopt, std::move(mergeStatuses), - FtUseExternStatuses::No, opts.inTransaction, std::move(ftctx), rdxCtx); + FtUseExternStatuses::No, opts.inTransaction, FtSortType(opts.ftSortType), std::move(ftctx), rdxCtx); } template @@ -135,7 +135,7 @@ SelectKeyResults IndexText::resultFromCache(const VariantArray& keys, FtIdSet template SelectKeyResults IndexText::doSelectKey(const VariantArray& keys, const std::optional& ckey, FtMergeStatuses&& mergeStatuses, FtUseExternStatuses useExternSt, bool inTransaction, - FtCtx::Ptr ftctx, const RdxContext& rdxCtx) { + FtSortType ftSortType, FtCtx::Ptr ftctx, const RdxContext& rdxCtx) { if rx_unlikely (cfg_->logLevel >= LogInfo) { logPrintf(LogInfo, "Searching for '%s' in '%s' %s", keys[0].As(), this->payloadType_ ? this->payloadType_->Name() : "", ckey ? "(will cache)" : ""); @@ -145,7 +145,7 @@ SelectKeyResults IndexText::doSelectKey(const VariantArray& keys, const std:: FtDSLQuery dsl(this->ftFields_, this->cfg_->stopWords, this->cfg_->extraWordSymbols); dsl.parse(keys[0].As()); - IdSet::Ptr mergedIds = Select(ftctx, std::move(dsl), inTransaction, std::move(mergeStatuses), useExternSt, rdxCtx); + IdSet::Ptr mergedIds = Select(ftctx, std::move(dsl), inTransaction, ftSortType, std::move(mergeStatuses), useExternSt, rdxCtx); SelectKeyResult res; if (mergedIds) { bool need_put = (useExternSt == FtUseExternStatuses::No) && ckey.has_value(); @@ -189,7 +189,8 @@ SelectKeyResults IndexText::SelectKey(const VariantArray& keys, CondType cond if rx_unlikely (keys.size() < 1 || (condition != CondEq && condition != CondSet)) { throw Error(errParams, "Full text index (%s) support only EQ or SET condition with 1 or 2 parameter", Index::Name()); } - return doSelectKey(keys, std::nullopt, std::move(preselect), FtUseExternStatuses::Yes, opts.inTransaction, prepareFtCtx(ctx), rdxCtx); + return doSelectKey(keys, std::nullopt, std::move(preselect), FtUseExternStatuses::Yes, opts.inTransaction, FtSortType(opts.ftSortType), + prepareFtCtx(ctx), rdxCtx); } template diff --git a/cpp_src/core/index/indextext/indextext.h b/cpp_src/core/index/indextext/indextext.h index 65f54d7b0..52b094da8 100644 --- a/cpp_src/core/index/indextext/indextext.h +++ b/cpp_src/core/index/indextext/indextext.h @@ -33,8 +33,8 @@ class IndexText : public IndexUnordered { SelectKeyResults SelectKey(const VariantArray& keys, CondType, Index::SelectOpts, const BaseFunctionCtx::Ptr&, FtPreselectT&&, const RdxContext&) override; void UpdateSortedIds(const UpdateSortedContext&) override {} - virtual IdSet::Ptr Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTransaction, FtMergeStatuses&&, FtUseExternStatuses, - const RdxContext&) = 0; + virtual IdSet::Ptr Select(FtCtx::Ptr fctx, FtDSLQuery&& dsl, bool inTransaction, FtSortType ftSortType, FtMergeStatuses&&, + FtUseExternStatuses, const RdxContext&) = 0; void SetOpts(const IndexOpts& opts) override; void Commit() override final { // Do nothing @@ -67,7 +67,7 @@ class IndexText : public IndexUnordered { virtual void commitFulltextImpl() = 0; FtCtx::Ptr prepareFtCtx(const BaseFunctionCtx::Ptr&); SelectKeyResults doSelectKey(const VariantArray& keys, const std::optional&, FtMergeStatuses&&, - FtUseExternStatuses useExternSt, bool inTransaction, FtCtx::Ptr, const RdxContext&); + FtUseExternStatuses useExternSt, bool inTransaction, FtSortType ftSortType, FtCtx::Ptr, const RdxContext&); SelectKeyResults resultFromCache(const VariantArray& keys, FtIdSetCache::Iterator&&, FtCtx::Ptr&); void build(const RdxContext& rdxCtx); diff --git a/cpp_src/core/namespace/namespaceimpl.cc b/cpp_src/core/namespace/namespaceimpl.cc index 61e27fca3..ce171f51b 100644 --- a/cpp_src/core/namespace/namespaceimpl.cc +++ b/cpp_src/core/namespace/namespaceimpl.cc @@ -4,7 +4,7 @@ #include #include #include "core/cjson/cjsondecoder.h" -#include "core/cjson/defaultvaluecoder.h" +// #include "core/cjson/defaultvaluecoder.h" #include "core/cjson/jsonbuilder.h" #include "core/cjson/uuid_recoders.h" #include "core/formatters/lsn_fmt.h" @@ -26,6 +26,7 @@ #include "tools/fsops.h" #include "tools/hardware_concurrency.h" #include "tools/logger.h" +#include "tools/scope_guard.h" #include "tools/timetools.h" #include "wal/walselecter.h" @@ -339,7 +340,6 @@ class NamespaceImpl::RollBack_recreateCompositeIndexes final : private RollBackB public: RollBack_recreateCompositeIndexes(NamespaceImpl& ns, size_t startIdx, size_t count) : ns_{ns}, startIdx_{startIdx} { indexes_.reserve(count); - std::swap(ns_.indexesToComposites_, indexesToComposites_); } ~RollBack_recreateCompositeIndexes() override { RollBack(); @@ -361,7 +361,6 @@ class NamespaceImpl::RollBack_recreateCompositeIndexes final : private RollBackB for (size_t i = 0, s = indexes_.size(); i < s; ++i) { std::swap(ns_.indexes_[i + startIdx_], indexes_[i]); } - std::swap(ns_.indexesToComposites_, indexesToComposites_); Disable(); } void SaveIndex(std::unique_ptr&& idx) { indexes_.emplace_back(std::move(idx)); } @@ -376,7 +375,6 @@ class NamespaceImpl::RollBack_recreateCompositeIndexes final : private RollBackB private: NamespaceImpl& ns_; std::vector> indexes_; - fast_hash_map> indexesToComposites_; size_t startIdx_; }; @@ -411,10 +409,6 @@ NamespaceImpl::RollBack_recreateCompositeIndexes NamespaceImpl::re auto newIndex{Index::New(indexDef, PayloadType{payloadType_}, FieldsSet{fields}, config_.cacheConfig)}; rollbacker.SaveIndex(std::move(index)); std::swap(index, newIndex); - - for (auto field : fields) { - indexesToComposites_[field].emplace_back(i); - } } } return rollbacker; @@ -553,15 +547,17 @@ NamespaceImpl::RollBack_updateItems NamespaceImpl::updateItems(con } else { recoder = std::make_unique(changedField); } - } else { - const auto& indexToUpdate = indexes_[changedField]; - if (!IsComposite(indexToUpdate->Type()) && !indexToUpdate->Opts().IsSparse()) { - auto tagsNames = pickJsonPath(fld); - if (!tagsNames.empty()) { - recoder = std::make_unique(name_, fld, std::move(tagsNames), changedField); - } - } } + // TODO: This logic must be reenabled after #1353. Now it's potentially unsafe + // else { + // const auto& indexToUpdate = indexes_[changedField]; + // if (!IsComposite(indexToUpdate->Type()) && !indexToUpdate->Opts().IsSparse()) { + // auto tagsNames = pickJsonPath(fld); + // if (!tagsNames.empty()) { + // recoder = std::make_unique(name_, fld, std::move(tagsNames), changedField); + // } + // } + // } } rollbacker.SaveTuple(); @@ -788,6 +784,10 @@ void NamespaceImpl::dropIndex(const IndexDef& index, bool disableTmVersionInc) { throw Error(errParams, errMsg, index.name_); } + // Guard approach is a bit suboptimal, but simpler + const auto compositesMappingGuard = + MakeScopeGuard([this] { indexesToComposites_.clear(); }, [this] { rebuildIndexesToCompositeMapping(); }); + int fieldIdx = itIdxName->second; std::unique_ptr& indexToRemove = indexes_[fieldIdx]; if (!IsComposite(indexToRemove->Type()) && indexToRemove->Opts().IsSparse()) { @@ -826,15 +826,7 @@ void NamespaceImpl::dropIndex(const IndexDef& index, bool disableTmVersionInc) { idx->SetFields(std::move(newFields)); } - const bool isComposite = IsComposite(indexToRemove->Type()); - if (isComposite) { - for (auto& v : indexesToComposites_) { - const auto f = std::find(v.second.begin(), v.second.end(), fieldIdx); - if (f != v.second.end()) { - v.second.erase(f); - } - } - } else if (!indexToRemove->Opts().IsSparse()) { + if (!IsComposite(indexToRemove->Type()) && !indexToRemove->Opts().IsSparse()) { PayloadType oldPlType = payloadType_; payloadType_.Drop(index.name_); tagsMatcher_.UpdatePayloadType(payloadType_, disableTmVersionInc ? NeedChangeTmVersion::No : NeedChangeTmVersion::Increment); @@ -1164,6 +1156,10 @@ void NamespaceImpl::addIndex(const IndexDef& indexDef, bool disableTmVersionInc, indexes_[currentPKIndex->second]->Name()); } + // Guard approach is a bit suboptimal, but simpler + const auto compositesMappingGuard = + MakeScopeGuard([this] { indexesToComposites_.clear(); }, [this] { rebuildIndexesToCompositeMapping(); }); + if (IsComposite(indexDef.Type())) { verifyCompositeIndex(indexDef); addCompositeIndex(indexDef); @@ -1304,10 +1300,6 @@ void NamespaceImpl::addCompositeIndex(const IndexDef& indexDef) { } } - for (auto field : fields) { - indexesToComposites_[field].emplace_back(idxPos); - } - updateSortedIdxCount(); insertIndex_rollbacker.Disable(); } @@ -2741,6 +2733,8 @@ bool NamespaceImpl::loadIndexesFromStorage() { addIndex(indexDef, false); } catch (const Error& e) { err = e; + } catch (std::exception& e) { + err = Error(errLogic, "Exception: '%s'", e.what()); } } if (!err.ok()) { @@ -3559,4 +3553,20 @@ int64_t NamespaceImpl::correctMaxIterationsIdSetPreResult(int64_t maxIterationsI return res; } +void NamespaceImpl::rebuildIndexesToCompositeMapping() noexcept { + // The only possible exception here is bad_alloc, but required memory footprint is really small + const auto beg = indexes_.firstCompositePos(); + const auto end = beg + indexes_.compositeIndexesSize(); + fast_hash_map> indexesToComposites; + for (auto i = beg; i < end; ++i) { + const auto& index = indexes_[i]; + assertrx(IsComposite(index->Type())); + const auto& fields = index->Fields(); + for (auto field : fields) { + indexesToComposites[field].emplace_back(i); + } + } + indexesToComposites_ = std::move(indexesToComposites); +} + } // namespace reindexer diff --git a/cpp_src/core/namespace/namespaceimpl.h b/cpp_src/core/namespace/namespaceimpl.h index f860b74b2..54ed29df6 100644 --- a/cpp_src/core/namespace/namespaceimpl.h +++ b/cpp_src/core/namespace/namespaceimpl.h @@ -541,6 +541,7 @@ class NamespaceImpl final : public intrusive_atomic_rc_base { // NOLINT(*perfor bool SortOrdersBuilt() const noexcept { return optimizationState_.load(std::memory_order_acquire) == OptimizationCompleted; } int64_t correctMaxIterationsIdSetPreResult(int64_t maxIterationsIdSetPreResult) const; + void rebuildIndexesToCompositeMapping() noexcept; IndexesStorage indexes_; fast_hash_map indexesNames_; diff --git a/cpp_src/core/nsselecter/nsselecter.cc b/cpp_src/core/nsselecter/nsselecter.cc index 4461b9808..3d2d76486 100644 --- a/cpp_src/core/nsselecter/nsselecter.cc +++ b/cpp_src/core/nsselecter/nsselecter.cc @@ -195,7 +195,11 @@ void NsSelecter::operator()(LocalQueryResults& result, SelectCtxWithJoinPreSelec ctx.preSelect.Result().payload); } - qres.PrepareIteratorsForSelectLoop(qPreproc, ctx.sortingContext.sortId(), isFt, *ns_, fnc_, ft_ctx_, rdxCtx); + qres.PrepareIteratorsForSelectLoop(qPreproc, ctx.sortingContext.sortId(), isFt, + (isFt && (ctx.isMergeQuery == IsMergeQuery::Yes || (!ctx.query.GetMergeQueries().empty()))) + ? FtSortType::RankAndID + : FtSortType::RankOnly, + *ns_, fnc_, ft_ctx_, rdxCtx); explain.AddSelectTime(); diff --git a/cpp_src/core/nsselecter/selectiteratorcontainer.cc b/cpp_src/core/nsselecter/selectiteratorcontainer.cc index 070ffebbb..d7ffa1d64 100644 --- a/cpp_src/core/nsselecter/selectiteratorcontainer.cc +++ b/cpp_src/core/nsselecter/selectiteratorcontainer.cc @@ -279,14 +279,15 @@ void SelectIteratorContainer::processField(FieldsComparator& fc, const QueryFiel } SelectKeyResults SelectIteratorContainer::processQueryEntry(const QueryEntry& qe, bool enableSortIndexOptimize, const NamespaceImpl& ns, - unsigned sortId, bool isQueryFt, SelectFunction::Ptr& selectFnc, - bool& isIndexFt, bool& isIndexSparse, FtCtx::Ptr& ftCtx, - QueryPreprocessor& qPreproc, const RdxContext& rdxCtx) { + unsigned sortId, bool isQueryFt, FtSortType ftSortType, + SelectFunction::Ptr& selectFnc, bool& isIndexFt, bool& isIndexSparse, + FtCtx::Ptr& ftCtx, QueryPreprocessor& qPreproc, const RdxContext& rdxCtx) { auto& index = ns.indexes_[qe.IndexNo()]; isIndexFt = IsFullText(index->Type()); isIndexSparse = index->Opts().IsSparse(); Index::SelectOpts opts; + opts.ftSortType = ftSortType; opts.itemsCountInNamespace = ns.itemsCount(); if (!ns.SortOrdersBuilt()) { opts.disableIdSetCache = 1; @@ -512,15 +513,15 @@ std::vector SelectIteratorContainer::pr return result; } -void SelectIteratorContainer::PrepareIteratorsForSelectLoop(QueryPreprocessor& qPreproc, unsigned sortId, bool isFt, +void SelectIteratorContainer::PrepareIteratorsForSelectLoop(QueryPreprocessor& qPreproc, unsigned sortId, bool isFt, FtSortType ftSortType, const NamespaceImpl& ns, SelectFunction::Ptr& selectFnc, FtCtx::Ptr& ftCtx, const RdxContext& rdxCtx) { - prepareIteratorsForSelectLoop(qPreproc, 0, qPreproc.Size(), sortId, isFt, ns, selectFnc, ftCtx, rdxCtx); + prepareIteratorsForSelectLoop(qPreproc, 0, qPreproc.Size(), sortId, isFt, ftSortType, ns, selectFnc, ftCtx, rdxCtx); } bool SelectIteratorContainer::prepareIteratorsForSelectLoop(QueryPreprocessor& qPreproc, size_t begin, size_t end, unsigned sortId, - bool isQueryFt, const NamespaceImpl& ns, SelectFunction::Ptr& selectFnc, - FtCtx::Ptr& ftCtx, const RdxContext& rdxCtx) { + bool isQueryFt, FtSortType ftSortType, const NamespaceImpl& ns, + SelectFunction::Ptr& selectFnc, FtCtx::Ptr& ftCtx, const RdxContext& rdxCtx) { const auto& queries = qPreproc.GetQueryEntries(); auto equalPositions = prepareEqualPositions(queries, begin, end); bool sortIndexFound = false; @@ -534,7 +535,7 @@ bool SelectIteratorContainer::prepareIteratorsForSelectLoop(QueryPreprocessor& q [&](const QueryEntriesBracket&) { OpenBracket(op); const bool contFT = - prepareIteratorsForSelectLoop(qPreproc, i + 1, next, sortId, isQueryFt, ns, selectFnc, ftCtx, rdxCtx); + prepareIteratorsForSelectLoop(qPreproc, i + 1, next, sortId, isQueryFt, ftSortType, ns, selectFnc, ftCtx, rdxCtx); if (contFT && (op == OpOr || (next < end && queries.GetOperation(next) == OpOr))) { throw Error(errLogic, "OR operation is not allowed with bracket containing fulltext index"); } @@ -560,8 +561,8 @@ bool SelectIteratorContainer::prepareIteratorsForSelectLoop(QueryPreprocessor& q } sortIndexFound = true; } - selectResults = processQueryEntry(qe, enableSortIndexOptimize, ns, sortId, isQueryFt, selectFnc, isIndexFt, - isIndexSparse, ftCtx, qPreproc, rdxCtx); + selectResults = processQueryEntry(qe, enableSortIndexOptimize, ns, sortId, isQueryFt, ftSortType, selectFnc, + isIndexFt, isIndexSparse, ftCtx, qPreproc, rdxCtx); } else { auto strictMode = ns.config_.strictMode; if (ctx_) { @@ -664,16 +665,21 @@ bool SelectIteratorContainer::checkIfSatisfyAllConditions(iterator begin, iterat } bool lastFinish = false; const bool lastResult = it->Visit( - [&](SelectIteratorsBracket&) { + [&] RX_PRE_LMBD_ALWAYS_INLINE(SelectIteratorsBracket&) RX_POST_LMBD_ALWAYS_INLINE { return checkIfSatisfyAllConditions(it.begin(), it.end(), pv, &lastFinish, rowId, properRowId, match); }, - [&](SelectIterator& sit) { return checkIfSatisfyCondition(sit, &lastFinish, rowId); }, - [&](JoinSelectIterator& jit) { return checkIfSatisfyCondition(jit, pv, properRowId, match); }, + [&] RX_PRE_LMBD_ALWAYS_INLINE(SelectIterator & sit) + RX_POST_LMBD_ALWAYS_INLINE { return checkIfSatisfyCondition(sit, &lastFinish, rowId); }, + [&] /*RX_PRE_LMBD_ALWAYS_INLINE*/ (JoinSelectIterator & jit) /*RX_POST_LMBD_ALWAYS_INLINE*/ { + return checkIfSatisfyCondition(jit, pv, properRowId, match); + }, Restricted>{}( - [&pv, properRowId](auto& c) { return c.Compare(pv, properRowId); }), - [](AlwaysTrue&) noexcept { return true; }); + [&pv, properRowId] /*RX_PRE_LMBD_ALWAYS_INLINE*/ (auto& c) /*RX_POST_LMBD_ALWAYS_INLINE*/ { + return c.Compare(pv, properRowId); + }), + [] RX_PRE_LMBD_ALWAYS_INLINE(AlwaysTrue&) RX_POST_LMBD_ALWAYS_INLINE noexcept { return true; }); if (op == OpOr) { result |= lastResult; currentFinish &= (!result && lastFinish); diff --git a/cpp_src/core/nsselecter/selectiteratorcontainer.h b/cpp_src/core/nsselecter/selectiteratorcontainer.h index 4f23b5446..5ec4cc240 100644 --- a/cpp_src/core/nsselecter/selectiteratorcontainer.h +++ b/cpp_src/core/nsselecter/selectiteratorcontainer.h @@ -52,8 +52,8 @@ class SelectIteratorContainer void CheckFirstQuery(); // Let iterators choose most effecive algorith void SetExpectMaxIterations(int expectedIterations); - void PrepareIteratorsForSelectLoop(QueryPreprocessor&, unsigned sortId, bool isFt, const NamespaceImpl&, SelectFunction::Ptr&, - FtCtx::Ptr&, const RdxContext&); + void PrepareIteratorsForSelectLoop(QueryPreprocessor&, unsigned sortId, bool isFt, FtSortType ftSortType, const NamespaceImpl&, + SelectFunction::Ptr&, FtCtx::Ptr&, const RdxContext&); template bool Process(PayloadValue&, bool* finish, IdType* rowId, IdType, bool match); @@ -92,8 +92,8 @@ class SelectIteratorContainer } private: - bool prepareIteratorsForSelectLoop(QueryPreprocessor&, size_t begin, size_t end, unsigned sortId, bool isFt, const NamespaceImpl&, - SelectFunction::Ptr&, FtCtx::Ptr&, const RdxContext&); + bool prepareIteratorsForSelectLoop(QueryPreprocessor&, size_t begin, size_t end, unsigned sortId, bool isFt, FtSortType ftSortType, + const NamespaceImpl&, SelectFunction::Ptr&, FtCtx::Ptr&, const RdxContext&); void sortByCost(span indexes, span costs, unsigned from, unsigned to, int expectedIterations); double fullCost(span indexes, unsigned i, unsigned from, unsigned to, int expectedIterations) const noexcept; double cost(span indexes, unsigned cur, int expectedIterations) const noexcept; @@ -117,8 +117,8 @@ class SelectIteratorContainer SelectKeyResults processQueryEntry(const QueryEntry& qe, const NamespaceImpl& ns, StrictMode strictMode); SelectKeyResults processQueryEntry(const QueryEntry& qe, bool enableSortIndexOptimize, const NamespaceImpl& ns, unsigned sortId, - bool isQueryFt, SelectFunction::Ptr& selectFnc, bool& isIndexFt, bool& isIndexSparse, FtCtx::Ptr&, - QueryPreprocessor& qPreproc, const RdxContext&); + bool isQueryFt, FtSortType ftSortType, SelectFunction::Ptr& selectFnc, bool& isIndexFt, + bool& isIndexSparse, FtCtx::Ptr&, QueryPreprocessor& qPreproc, const RdxContext&); template void processField(FieldsComparator&, const QueryField&, const NamespaceImpl&) const; void processJoinEntry(const JoinQueryEntry&, OpType); diff --git a/cpp_src/core/nsselecter/substitutionhelpers.h b/cpp_src/core/nsselecter/substitutionhelpers.h index 3c89f197b..e605fb678 100644 --- a/cpp_src/core/nsselecter/substitutionhelpers.h +++ b/cpp_src/core/nsselecter/substitutionhelpers.h @@ -2,6 +2,7 @@ #include "core/index/index.h" #include "core/namespace/namespaceimpl.h" +#include "tools/logger.h" namespace reindexer { @@ -36,11 +37,32 @@ class CompositeSearcher { void Add(int field, const std::vector& composites, unsigned entry) { assertrx_throw(entry < std::numeric_limits::max()); + const auto compositesBeg = ns_.indexes_.firstCompositePos(); + const auto compositesEnd = compositesBeg + ns_.indexes_.compositeIndexesSize(); for (auto composite : composites) { - const auto idxType = ns_.indexes_[composite]->Type(); + if rx_unlikely (composite < compositesBeg || composite >= compositesEnd) { + // TODO: this may be removed later (somewhere around v3.31/v3.32) after some extra investigations (relates to #1830) + logFmt(LogError, + ": Unexpected composite index identifier during substitution attempt: {}. Composites range is " + "[{}, {});\n(field: {}; {})", + composite, compositesBeg, compositesEnd, field, ns_.payloadType_.Field(field).ToString()); + assertrx_dbg(false); + continue; + } + auto compositePtr = ns_.indexes_[composite].get(); + const auto idxType = compositePtr->Type(); if (idxType != IndexCompositeBTree && idxType != IndexCompositeHash) { continue; } + if (auto& idxFields = compositePtr->Fields(); !idxFields.contains(field)) { + // TODO: this may be removed later (somewhere around v3.31/v3.32) after some extra investigations (relates to #1830) + logFmt(LogError, + ": Unexpected field {} in composite index {}:{} during substitution attempt. Actual composite " + "fields: {}", + field, composite, compositePtr->Name(), idxFields.ToString(FieldsSet::DumpWithMask::No)); + assertrx_dbg(false); + continue; + } bool found = false; for (auto& d : d_) { if (d.idx == composite) { diff --git a/cpp_src/core/payload/fieldsset.cc b/cpp_src/core/payload/fieldsset.cc index 6f929ab16..3a2dfeee7 100644 --- a/cpp_src/core/payload/fieldsset.cc +++ b/cpp_src/core/payload/fieldsset.cc @@ -13,6 +13,12 @@ FieldsSet::FieldsSet(const TagsMatcher& tagsMatcher, const h_vector - void Dump(T& os) const { + void Dump(T& os, DumpWithMask withMask) const { const DumpFieldsPath fieldsPathDumper{os}; os << "{["; for (auto b = begin(), it = b, e = end(); it != e; ++it) { @@ -234,7 +236,11 @@ class FieldsSet : protected base_fields_set { } os << *it; } - os << "], mask: " << mask_ << ", tagsPaths: ["; + os << "], "; + if (withMask == DumpWithMask::Yes) { + os << "mask: " << mask_ << ", "; + } + os << "tagsPaths: ["; for (auto b = tagsPaths_.cbegin(), it = b, e = tagsPaths_.cend(); it != e; ++it) { if (it != b) { os << ", "; @@ -251,6 +257,7 @@ class FieldsSet : protected base_fields_set { } os << "]}"; } + std::string ToString(DumpWithMask withMask) const; private: template diff --git a/cpp_src/core/payload/payloadfieldtype.cc b/cpp_src/core/payload/payloadfieldtype.cc index 36d83f02f..a66ce588f 100644 --- a/cpp_src/core/payload/payloadfieldtype.cc +++ b/cpp_src/core/payload/payloadfieldtype.cc @@ -1,4 +1,5 @@ #include "payloadfieldtype.h" +#include #include "core/keyvalue/p_string.h" #include "core/keyvalue/uuid.h" #include "estl/one_of.h" @@ -39,4 +40,17 @@ size_t PayloadFieldType::Alignof() const noexcept { }); } +std::string PayloadFieldType::ToString() const { + std::stringstream ss; + ss << "{ type: " << type_.Name() << ", name: " << name_ << ", jsonpaths: ["; + for (auto jit = jsonPaths_.cbegin(); jit != jsonPaths_.cend(); ++jit) { + if (jit != jsonPaths_.cbegin()) { + ss << ", "; + } + ss << *jit; + } + ss << "] }"; + return ss.str(); +} + } // namespace reindexer diff --git a/cpp_src/core/payload/payloadfieldtype.h b/cpp_src/core/payload/payloadfieldtype.h index b4cb5588c..438b3563b 100644 --- a/cpp_src/core/payload/payloadfieldtype.h +++ b/cpp_src/core/payload/payloadfieldtype.h @@ -24,6 +24,7 @@ class PayloadFieldType { const std::vector& JsonPaths() const& noexcept { return jsonPaths_; } const std::vector& JsonPaths() && = delete; void AddJsonPath(const std::string& jsonPath) { jsonPaths_.push_back(jsonPath); } + std::string ToString() const; private: KeyValueType type_; diff --git a/cpp_src/core/query/query.cc b/cpp_src/core/query/query.cc index 651dacd30..77576a215 100644 --- a/cpp_src/core/query/query.cc +++ b/cpp_src/core/query/query.cc @@ -168,6 +168,13 @@ void Query::Join(JoinedQuery&& jq) & { adoptNested(joinQueries_.back()); } +void Query::checkSetObjectValue(const Variant& value) const { + if (!value.Type().Is()) { + throw Error(errLogic, "Unexpected variant type in SetObject: %s. Expecting KeyValueType::String with JSON-content", + value.Type().Name()); + } +} + VariantArray Query::deserializeValues(Serializer& ser, CondType cond) { VariantArray values; auto cnt = ser.GetVarUint(); diff --git a/cpp_src/core/query/query.h b/cpp_src/core/query/query.h index 8a401357e..63fbb9cb3 100644 --- a/cpp_src/core/query/query.h +++ b/cpp_src/core/query/query.h @@ -406,11 +406,8 @@ class Query { /// @param hasExpressions - true: value has expressions in it template >* = nullptr> Query& SetObject(Str&& field, VariantArray value, bool hasExpressions = false) & { - for (auto& it : value) { - if (!it.Type().Is()) { - throw Error(errLogic, "Unexpected variant type in SetObject: %s. Expecting KeyValueType::String with JSON-content", - it.Type().Name()); - } + for (const auto& it : value) { + checkSetObjectValue(it); } updateFields_.emplace_back(std::forward(field), std::move(value), FieldModeSetJson, hasExpressions); return *this; @@ -693,7 +690,8 @@ class Query { throw Error(errConflict, kAggregationWithSelectFieldsMsgError); } selectFilter_.insert(selectFilter_.begin(), l.begin(), l.end()); - selectFilter_.erase(std::remove_if(selectFilter_.begin(), selectFilter_.end(), [](const auto& s) { return s == "*"sv; }), + selectFilter_.erase( + std::remove_if(selectFilter_.begin(), selectFilter_.end(), [](const auto& s) { return s == "*"sv || s.empty(); }), selectFilter_.end()); return *this; } @@ -975,6 +973,7 @@ class Query { using OnHelper = OnHelperTempl; using OnHelperR = OnHelperTempl; + void checkSetObjectValue(const Variant& value) const; void deserialize(Serializer& ser, bool& hasJoinConditions); VariantArray deserializeValues(Serializer&, CondType); void checkSubQueryNoData() const; diff --git a/cpp_src/core/query/queryentry.cc b/cpp_src/core/query/queryentry.cc index 6e2d8fcb6..73feb9b4a 100644 --- a/cpp_src/core/query/queryentry.cc +++ b/cpp_src/core/query/queryentry.cc @@ -270,6 +270,12 @@ std::string BetweenFieldsQueryEntry::Dump() const { return std::string{ser.Slice()}; } +void BetweenFieldsQueryEntry::checkCondition(CondType cond) const { + if (cond == CondAny || cond == CondEmpty || cond == CondDWithin) { + throw Error{errLogic, "Condition '%s' is inapplicable between two fields", CondTypeToStr(cond)}; + } +} + void QueryEntries::serialize(CondType cond, const VariantArray& values, WrSerializer& ser) { ser.PutVarUint(cond); if (cond == CondDWithin) { @@ -910,6 +916,11 @@ std::string SubQueryFieldEntry::Dump(const std::vector& subQueries) const return ss.str(); } +void SubQueryFieldEntry::checkCondition(CondType cond) const { + if (cond == CondAny || cond == CondEmpty) { + throw Error{errQueryExec, "Condition %s with field and subquery", cond == CondAny ? "Any" : "Empty"}; + } +} template void QueryEntries::dump(size_t level, const_iterator begin, const_iterator end, const std::vector& joinedSelectors, const std::vector& subQueries, WrSerializer& ser) { diff --git a/cpp_src/core/query/queryentry.h b/cpp_src/core/query/queryentry.h index 0dd734f80..e699cc183 100644 --- a/cpp_src/core/query/queryentry.h +++ b/cpp_src/core/query/queryentry.h @@ -20,8 +20,8 @@ using ConstPayload = PayloadIface; class TagsMatcher; struct JoinQueryEntry { - JoinQueryEntry(size_t joinIdx) noexcept : joinIndex{joinIdx} {} - size_t joinIndex; + explicit JoinQueryEntry(size_t joinIdx) noexcept : joinIndex{joinIdx} {} + size_t joinIndex{std::numeric_limits::max()}; bool operator==(const JoinQueryEntry& other) const noexcept { return joinIndex == other.joinIndex; } bool operator!=(const JoinQueryEntry& other) const noexcept { return !operator==(other); } @@ -38,8 +38,6 @@ class QueryField { template explicit QueryField(Str&& fieldName) noexcept : fieldName_{std::forward(fieldName)} {} - QueryField(std::string&& fieldName, int idxNo, FieldsSet fields, KeyValueType fieldType, - std::vector&& compositeFieldsTypes); QueryField(QueryField&&) noexcept = default; QueryField(const QueryField&) = default; QueryField& operator=(QueryField&&) noexcept = default; @@ -160,9 +158,7 @@ class QueryEntry : private QueryField { [[nodiscard]] std::string Dump() const; [[nodiscard]] std::string DumpBrief() const; - [[nodiscard]] bool IsInjectedFromMain() const noexcept { return injectedFrom_ == InjectedFromMain; } [[nodiscard]] bool IsInjectedFrom(size_t joinedQueryNo) const noexcept { return injectedFrom_ == joinedQueryNo; } - void InjectedFrom(size_t joinedQueryNo) noexcept { injectedFrom_ = joinedQueryNo; } auto Values() const&& = delete; auto FieldData() const&& = delete; @@ -172,9 +168,9 @@ class QueryEntry : private QueryField { void verifyNotIgnoringEmptyValues() const { VerifyQueryEntryValues(condition_, values_); } VariantArray values_; - CondType condition_; - bool distinct_ = false; - size_t injectedFrom_ = NotInjected; + CondType condition_{CondAny}; + bool distinct_{false}; + size_t injectedFrom_{NotInjected}; }; class BetweenFieldsQueryEntry { @@ -182,9 +178,7 @@ class BetweenFieldsQueryEntry { template BetweenFieldsQueryEntry(StrL&& fstIdx, CondType cond, StrR&& sndIdx) : leftField_{std::forward(fstIdx)}, rightField_{std::forward(sndIdx)}, condition_{cond} { - if (condition_ == CondAny || condition_ == CondEmpty || condition_ == CondDWithin) { - throw Error{errLogic, "Condition '%s' is inapplicable between two fields", CondTypeToStr(condition_)}; - } + checkCondition(cond); } [[nodiscard]] bool operator==(const BetweenFieldsQueryEntry&) const noexcept; @@ -224,9 +218,11 @@ class BetweenFieldsQueryEntry { auto RightFieldData() const&& = delete; private: + void checkCondition(CondType cond) const; + QueryField leftField_; QueryField rightField_; - CondType condition_; + CondType condition_{CondAny}; }; struct AlwaysFalse {}; @@ -264,9 +260,9 @@ class SubQueryEntry { auto Values() const&& = delete; private: - CondType condition_; + CondType condition_{CondAny}; // index of Query in Query::subQueries_ - size_t queryIndex_; + size_t queryIndex_{std::numeric_limits::max()}; VariantArray values_; }; @@ -274,9 +270,7 @@ class SubQueryFieldEntry { public: template SubQueryFieldEntry(Str&& field, CondType cond, size_t qIdx) : field_{std::forward(field)}, condition_{cond}, queryIndex_{qIdx} { - if (cond == CondAny || cond == CondEmpty) { - throw Error{errQueryExec, "Condition %s with field and subquery", cond == CondAny ? "Any" : "Empty"}; - } + checkCondition(cond); } [[nodiscard]] const std::string& FieldName() const& noexcept { return field_; } [[nodiscard]] std::string&& FieldName() && noexcept { return std::move(field_); } @@ -291,10 +285,12 @@ class SubQueryFieldEntry { auto FieldName() const&& = delete; private: + void checkCondition(CondType cond) const; + std::string field_; - CondType condition_; + CondType condition_{CondAny}; // index of Query in Query::subQueries_ - size_t queryIndex_; + size_t queryIndex_{std::numeric_limits::max()}; }; class UpdateEntry { @@ -319,8 +315,8 @@ class UpdateEntry { private: std::string column_; VariantArray values_; - FieldModifyMode mode_ = FieldModeSet; - bool isExpression_ = false; + FieldModifyMode mode_{FieldModeSet}; + bool isExpression_{false}; }; class QueryJoinEntry { @@ -379,11 +375,11 @@ class QueryJoinEntry { private: QueryField leftField_; QueryField rightField_; - const OpType op_; - const CondType condition_; - const bool reverseNamespacesOrder_; ///< controls SQL encoding order - ///< false: mainNs.index Condition joinNs.joinIndex - ///< true: joinNs.joinIndex Invert(Condition) mainNs.index + const OpType op_{OpOr}; + const CondType condition_{CondAny}; + const bool reverseNamespacesOrder_{false}; ///< controls SQL encoding order + ///< false: mainNs.index Condition joinNs.joinIndex + ///< true: joinNs.joinIndex Invert(Condition) mainNs.index }; enum class InjectionDirection : bool { IntoMain, FromMain }; @@ -393,7 +389,7 @@ class QueryEntries : public ExpressionTree { using Base = ExpressionTree; - QueryEntries(Base&& b) : Base{std::move(b)} {} + explicit QueryEntries(Base&& b) : Base{std::move(b)} {} public: QueryEntries() = default; @@ -402,7 +398,6 @@ class QueryEntries : public ExpressionTree& subQueries) const { serialize(cbegin(), cend(), ser, subQueries); } bool CheckIfSatisfyConditions(const ConstPayload& pl) const { return checkIfSatisfyConditions(cbegin(), cend(), pl); } static bool CheckIfSatisfyCondition(const VariantArray& lValues, CondType, const VariantArray& rValues); @@ -422,7 +417,6 @@ class QueryEntries : public ExpressionTree& subQueries); static void serialize(CondType, const VariantArray& values, WrSerializer&); static bool checkIfSatisfyConditions(const_iterator begin, const_iterator end, const ConstPayload&); @@ -473,11 +467,11 @@ class AggregateEntry { void SetOffset(unsigned); private: - AggType type_; + AggType type_{AggUnknown}; h_vector fields_; SortingEntries sortingEntries_; - unsigned limit_ = QueryEntry::kDefaultLimit; - unsigned offset_ = QueryEntry::kDefaultOffset; + unsigned limit_{QueryEntry::kDefaultLimit}; + unsigned offset_{QueryEntry::kDefaultOffset}; }; } // namespace reindexer diff --git a/cpp_src/core/query/sql/sqlencoder.cc b/cpp_src/core/query/sql/sqlencoder.cc index f4ad1a51a..180014c60 100644 --- a/cpp_src/core/query/sql/sqlencoder.cc +++ b/cpp_src/core/query/sql/sqlencoder.cc @@ -207,9 +207,6 @@ WrSerializer& SQLEncoder::GetSQL(WrSerializer& ser, bool stripArgs) const { } } else { for (const auto& filter : query_.SelectFilters()) { - if (filter == distinctIndex) { - continue; - } if (needComma) { ser << ", "; } else { diff --git a/cpp_src/core/selectfunc/ctx/ftctx.h b/cpp_src/core/selectfunc/ctx/ftctx.h index 2031cb8d5..636ccc42a 100644 --- a/cpp_src/core/selectfunc/ctx/ftctx.h +++ b/cpp_src/core/selectfunc/ctx/ftctx.h @@ -8,6 +8,8 @@ namespace reindexer { +enum FtSortType { RankOnly, RankAndID, ExternalExpression }; + class FtCtx : public BaseFunctionCtx { public: typedef intrusive_ptr Ptr; diff --git a/cpp_src/debug/backtrace.h b/cpp_src/debug/backtrace.h index 6d367a6ab..d55d9bd38 100644 --- a/cpp_src/debug/backtrace.h +++ b/cpp_src/debug/backtrace.h @@ -2,6 +2,7 @@ #include #include +#include namespace reindexer { namespace debug { diff --git a/cpp_src/gtests/tests/fixtures/queries_api.h b/cpp_src/gtests/tests/fixtures/queries_api.h index 1844006f6..06b35b5b1 100644 --- a/cpp_src/gtests/tests/fixtures/queries_api.h +++ b/cpp_src/gtests/tests/fixtures/queries_api.h @@ -237,7 +237,8 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { template void ExecuteAndVerifyWithSql(Q&& query) { ExecuteAndVerify(query); - Query queryFromSql = Query::FromSQL(query.GetSQL()); + Query queryFromSql = Query::FromSQL(query.GetSQL()).Strict(query.GetStrictMode()).Debug(query.GetDebugLevel()); + ASSERT_EQ(query, queryFromSql); ExecuteAndVerify(std::move(queryFromSql)); } @@ -257,7 +258,8 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { template void ExecuteAndVerifyWithSql(Q&& query, QueryResults& qr) { ExecuteAndVerify(query, qr); - Query queryFromSql = Query::FromSQL(query.GetSQL()); + Query queryFromSql = Query::FromSQL(query.GetSQL()).Strict(query.GetStrictMode()).Debug(query.GetDebugLevel()); + ASSERT_EQ(query, queryFromSql); qr.Clear(); ExecuteAndVerify(std::move(queryFromSql), qr); } @@ -733,6 +735,12 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { .Where(kFieldNameGenre, CondEq, randomGenre) .Distinct(distinct.c_str()) .Sort(kFieldNameYear, true)); + + ExecuteAndVerifyWithSql(Query(default_namespace) + .Select({distinct.c_str()}) + .Distinct(distinct.c_str()) + .Where(kFieldNameGenre, CondEq, randomGenre) + .Sort(kFieldNameYear, true)); } } diff --git a/cpp_src/gtests/tests/unit/grpcclient_test.cc b/cpp_src/gtests/tests/unit/grpcclient_test.cc index 7c0e5edec..f61870ea2 100644 --- a/cpp_src/gtests/tests/unit/grpcclient_test.cc +++ b/cpp_src/gtests/tests/unit/grpcclient_test.cc @@ -75,7 +75,7 @@ TEST_F(GrpcClientApi, SelectJSON) { for (auto field : object) { name = std::string_view(field.key); const auto& fieldValue(field.value); - if (name == "id") { + if (name == "id" || name == "age") { ASSERT_TRUE(fieldValue.getTag() == gason::JSON_NUMBER); } else if (name == "joined_test_namespace2") { ASSERT_TRUE(fieldValue.getTag() == gason::JSON_ARRAY); diff --git a/cpp_src/gtests/tests/unit/index_tuple_test.cc b/cpp_src/gtests/tests/unit/index_tuple_test.cc index b0e8b7631..e89040f4c 100644 --- a/cpp_src/gtests/tests/unit/index_tuple_test.cc +++ b/cpp_src/gtests/tests/unit/index_tuple_test.cc @@ -236,7 +236,7 @@ class IndexTupleTest : public ReindexerApi { } }; -TEST_F(IndexTupleTest, ScalarTest) { +TEST_F(IndexTupleTest, DISABLED_ScalarTest) { static const std::string ns = "testNSScalar"; CreateEmptyNamespace(ns); @@ -251,7 +251,7 @@ TEST_F(IndexTupleTest, ScalarTest) { ValidateReloadState(rt.reindexer, ns, R"({"id":%d,"text":"","int":0})", "reload ns (ScalarTest)"); } -TEST_F(IndexTupleTest, ScalarNestedTest) { +TEST_F(IndexTupleTest, DISABLED_ScalarNestedTest) { static const std::string ns = "testNSNested"; CreateEmptyNamespace(ns); @@ -329,7 +329,8 @@ TEST_F(IndexTupleTest, NestedUpdateTest) { "reload ns (NestedUpdateTest)"); } -TEST_F(IndexTupleTest, ArrayTest) { +// TODO: This test must be reenabled after #1353 +TEST_F(IndexTupleTest, DISABLED_ArrayTest) { static const std::string ns = "testNSArray"; CreateEmptyNamespace(ns); @@ -365,7 +366,8 @@ TEST_F(IndexTupleTest, ArrayTest) { "reload ns (ArrayTest)"); } -TEST_F(IndexTupleTest, ArrayNestedTest) { +// TODO: This test must be reenabled after #1353 +TEST_F(IndexTupleTest, DISABLED_ArrayNestedTest) { static const std::string ns = "testNSArrayObj"; CreateNamespace(ns); @@ -439,7 +441,8 @@ TEST_F(IndexTupleTest, ArrayInToArrayTest) { "reload ns (ArrayInToArrayTest)"); } -TEST_F(IndexTupleTest, NestedOrderingTest) { +// TODO: This test must be reenabled after #1353 +TEST_F(IndexTupleTest, DISABLED_NestedOrderingTest) { static const std::string ns = "testNSNestedOrdering"; CreateEmptyNamespace(ns); @@ -457,7 +460,8 @@ TEST_F(IndexTupleTest, NestedOrderingTest) { "reload ns (NestedDiffOrderingTest)"); } -TEST_F(IndexTupleTest, NullTest) { +// TODO: This test must be reenabled after #1353 +TEST_F(IndexTupleTest, DISABLED_NullTest) { static const std::string ns = "testNSNull"; CreateEmptyNamespace(ns); @@ -483,7 +487,8 @@ TEST_F(IndexTupleTest, NullTest) { "null values test"); } -TEST_F(IndexTupleTest, FailTest) { +// TODO: This test must be reenabled after #1353 +TEST_F(IndexTupleTest, DISABLED_FailTest) { static const std::string ns = "testNSFail"; CreateEmptyNamespace(ns); @@ -494,7 +499,8 @@ TEST_F(IndexTupleTest, FailTest) { ValidateReloadState(rt.reindexer, ns, R"({"id":%d,"obj":{"nest":0},"idx":false})", "reload ns (FailTest)"); } -TEST_F(IndexTupleTest, NestedArrayTest) { +// TODO: This test must be reenabled after #1353 +TEST_F(IndexTupleTest, DISABLED_NestedArrayTest) { static const std::string ns = "testNSNestedArray"; CreateArrayNamespace(ns); diff --git a/cpp_src/server/CMakeLists.txt b/cpp_src/server/CMakeLists.txt index 693857c78..e48132a5b 100644 --- a/cpp_src/server/CMakeLists.txt +++ b/cpp_src/server/CMakeLists.txt @@ -4,7 +4,7 @@ cmake_minimum_required(VERSION 3.10) project(reindexer_server_library) set (SWAGGER_VERSION "2.x") -set (GH_FACE_VERSION "4.16.1") +set (GH_FACE_VERSION "4.17.1") set (GH_FACE_TAG "v${GH_FACE_VERSION}") set (TARGET reindexer_server_library) set (SERVER_LIB_DIR ${PROJECT_BINARY_DIR} PARENT_SCOPE) diff --git a/cpp_src/server/contrib/server.md b/cpp_src/server/contrib/server.md index 366c687b3..57bdc4424 100644 --- a/cpp_src/server/contrib/server.md +++ b/cpp_src/server/contrib/server.md @@ -137,7 +137,7 @@ Reindexer is compact, fast and it does not have heavy dependencies. ### Version information -*Version* : 4.17.0 +*Version* : 4.17.1 ### License information diff --git a/cpp_src/server/contrib/server.yml b/cpp_src/server/contrib/server.yml index 9c7e88ff7..affaff3b4 100644 --- a/cpp_src/server/contrib/server.yml +++ b/cpp_src/server/contrib/server.yml @@ -4,7 +4,7 @@ info: **Reindexer** is an embeddable, in-memory, document-oriented database with a high-level Query builder interface. Reindexer's goal is to provide fast search with complex queries. Reindexer is compact, fast and it does not have heavy dependencies. - version: "4.17.0" + version: "4.17.1" title: "Reindexer REST API" license: name: "Apache 2.0" diff --git a/cpp_src/tools/scope_guard.h b/cpp_src/tools/scope_guard.h new file mode 100644 index 000000000..e5631c9bc --- /dev/null +++ b/cpp_src/tools/scope_guard.h @@ -0,0 +1,20 @@ +#pragma once + +namespace reindexer { + +template +class ScopeGuard { +public: + ScopeGuard(F1&& onConstruct, F2&& onDestruct) noexcept : onDestruct_(std::move(onDestruct)) { onConstruct(); } + ~ScopeGuard() { onDestruct_(); } + +private: + F2 onDestruct_; +}; + +template +ScopeGuard MakeScopeGuard(F1&& onConstruct, F2&& onDestruct) noexcept { + return ScopeGuard(std::move(onConstruct), std::move(onDestruct)); +} + +} // namespace reindexer diff --git a/cpp_src/tools/string_regexp_functions.h b/cpp_src/tools/string_regexp_functions.h index 5bd7951e2..06cebe92c 100644 --- a/cpp_src/tools/string_regexp_functions.h +++ b/cpp_src/tools/string_regexp_functions.h @@ -1,7 +1,7 @@ #pragma once +#include #include -#include namespace reindexer { diff --git a/cpp_src/vendor/libbacktrace/dwarf.c b/cpp_src/vendor/libbacktrace/dwarf.c index 86f29e4fc..b760353f4 100644 --- a/cpp_src/vendor/libbacktrace/dwarf.c +++ b/cpp_src/vendor/libbacktrace/dwarf.c @@ -1,5 +1,7 @@ +#ifndef _MSC_VER #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wswitch-enum" +#endif #ifdef __linux__ /* dwarf.c -- Get file/line information from DWARF for backtraces. @@ -3419,4 +3421,6 @@ int backtrace_dwarf_add(struct backtrace_state *state, uintptr_t base_address, c int ___dwarf_c_dummy_suppress_warning; #endif +#ifndef _MSC_VER #pragma GCC diagnostic pop +#endif diff --git a/test/composite_indexes_test.go b/test/composite_indexes_test.go index 1c1c14b66..00f5ad2dd 100644 --- a/test/composite_indexes_test.go +++ b/test/composite_indexes_test.go @@ -909,6 +909,82 @@ func TestCompositeIndexesSubstitution(t *testing.T) { }, }, "") }) + + t.Run("substitution after other indexes drop", func(t *testing.T) { + checkSubstitution := func() { + it := DB.Query(ns). + Where("first1", reindexer.EQ, item.First1). + Where("third", reindexer.EQ, item.Third). + Explain().Exec(t) + require.NoError(t, it.Error()) + defer it.Close() + require.Equal(t, it.Count(), 1) + explainRes, err := it.GetExplainResults() + require.NoError(t, err) + require.NotNil(t, explainRes) + + printExplainRes(explainRes) + checkExplain(t, explainRes.Selectors, []expectedExplain{ + { + Field: "first1+third", + FieldType: "indexed", + Method: "index", + Keys: 1, + Matched: 1, + }, + }, "") + } + + err := DB.DropIndex(ns, "second1+second2") + require.NoError(t, err) + // Check substitution right after composite deletion + checkSubstitution() + + err = DB.DropIndex(ns, "second1") + require.NoError(t, err) + err = DB.DropIndex(ns, "second2") + require.NoError(t, err) + // Check substitution after other indexes deletion + checkSubstitution() + }) + + t.Run("no substitution after current index drop", func(t *testing.T) { + err := DB.DropIndex(ns, "first1+third") + require.NoError(t, err) + it := DB.Query(ns). + Where("first1", reindexer.EQ, item.First1). + Where("third", reindexer.EQ, item.Third). + Explain().Exec(t) + require.NoError(t, it.Error()) + defer it.Close() + require.Equal(t, it.Count(), 1) + explainRes, err := it.GetExplainResults() + require.NoError(t, err) + require.NotNil(t, explainRes) + + printExplainRes(explainRes) + checkExplain(t, explainRes.Selectors, []expectedExplain{ + { + Field: "-scan", + Method: "scan", + Matched: 1, + }, + { + Field: "first1", + FieldType: "indexed", + Method: "scan", + Matched: 1, + Comparators: 1, + }, + { + Field: "third", + FieldType: "indexed", + Method: "scan", + Matched: 1, + Comparators: 1, + }, + }, "") + }) } func TestCompositeIndexesBestSubstitution(t *testing.T) { diff --git a/test/encdec_test.go b/test/encdec_test.go index dcf50adcd..dbcb0bb63 100644 --- a/test/encdec_test.go +++ b/test/encdec_test.go @@ -330,6 +330,8 @@ func TestEncDec(t *testing.T) { } func TestSingleElemToSlice(t *testing.T) { + t.Parallel() + ns := "test_single_elem_slice" item := SingleElemSliceItem{ ID: 1, @@ -359,6 +361,8 @@ func TestSingleElemToSlice(t *testing.T) { } func TestSlicesConcatenation(t *testing.T) { + t.Parallel() + ns := "test_slices_concatenation" item := SlicesConcatenationItem{ ID: 1,