From 6c81946d295ebf54f76fd171739e96e6185d4ed9 Mon Sep 17 00:00:00 2001 From: reindexer-bot <@> Date: Mon, 2 Oct 2023 23:11:50 +0000 Subject: [PATCH] [fix] Fix deadlock in builtin go logger #1571 --- bindings/builtin/builtin.go | 21 +- bindings/builtinserver/config/config.go | 10 +- changelog.md | 2 - cjson/decoder.go | 2 +- cpp_src/CMakeLists.txt | 9 +- cpp_src/client/cororeindexer.cc | 2 +- cpp_src/client/cororeindexer.h | 2 +- cpp_src/client/cororpcclient.cc | 9 +- cpp_src/client/cororpcclient.h | 2 +- cpp_src/client/item.h | 8 +- cpp_src/client/queryresults.cc | 5 - cpp_src/client/queryresults.h | 26 +- cpp_src/client/reindexer.cc | 2 +- cpp_src/client/reindexer.h | 2 +- cpp_src/client/rpcclient.cc | 11 +- cpp_src/client/rpcclient.h | 2 +- cpp_src/client/rpcclientmock.cc | 6 +- .../test/test_storage_compatibility.sh | 194 +++++++ .../cmd/reindexer_tool/commandsexecutor.cc | 71 ++- cpp_src/cmd/reindexer_tool/commandsexecutor.h | 6 +- .../cmd/reindexer_tool/commandsprocessor.cc | 13 +- cpp_src/core/cjson/fieldextractor.h | 4 +- cpp_src/core/cjson/tagsmatcherimpl.h | 33 +- cpp_src/core/comparator.cc | 11 +- cpp_src/core/comparator.h | 8 +- cpp_src/core/comparatorimpl.h | 6 +- cpp_src/core/compositearraycomparator.cc | 40 +- cpp_src/core/compositearraycomparator.h | 14 +- cpp_src/core/dbconfig.cc | 13 +- cpp_src/core/dbconfig.h | 16 +- cpp_src/core/defnsconfigs.h | 1 - cpp_src/core/expressiontree.h | 18 +- cpp_src/core/idset.cc | 10 - cpp_src/core/idset.h | 28 +- cpp_src/core/index/index.cc | 23 +- cpp_src/core/index/index.h | 24 +- cpp_src/core/index/indexordered.cc | 25 +- cpp_src/core/index/indexordered.h | 6 +- cpp_src/core/index/indexstore.cc | 61 +- cpp_src/core/index/indexstore.h | 7 +- cpp_src/core/index/indextext/fastindextext.cc | 12 +- cpp_src/core/index/indextext/fastindextext.h | 9 +- cpp_src/core/index/indextext/fieldsgetter.h | 1 - .../core/index/indextext/fuzzyindextext.cc | 7 +- cpp_src/core/index/indextext/fuzzyindextext.h | 8 +- cpp_src/core/index/indextext/indextext.cc | 13 +- cpp_src/core/index/indextext/indextext.h | 5 +- cpp_src/core/index/indexunordered.cc | 76 +-- cpp_src/core/index/indexunordered.h | 4 +- cpp_src/core/index/payload_map.h | 29 +- cpp_src/core/index/rtree/indexrtree.cc | 25 +- cpp_src/core/index/rtree/indexrtree.h | 6 +- cpp_src/core/index/string_map.h | 4 +- cpp_src/core/index/ttlindex.cc | 7 +- cpp_src/core/index/ttlindex.h | 10 +- cpp_src/core/index/uuid_index.cc | 4 +- cpp_src/core/index/uuid_index.h | 6 +- cpp_src/core/item.cc | 1 - cpp_src/core/itemmodifier.cc | 2 +- cpp_src/core/keyvalue/variant.cc | 44 +- cpp_src/core/keyvalue/variant.h | 15 +- cpp_src/core/namespace/asyncstorage.cc | 1 - cpp_src/core/namespace/itemsloader.h | 1 + cpp_src/core/namespace/namespace.cc | 10 +- cpp_src/core/namespace/namespace.h | 9 +- cpp_src/core/namespace/namespaceimpl.cc | 105 ++-- cpp_src/core/namespace/namespaceimpl.h | 15 +- cpp_src/core/nsselecter/aggregator.cc | 11 +- cpp_src/core/nsselecter/fieldscomparator.h | 13 +- cpp_src/core/nsselecter/joinedselector.cc | 164 +++--- cpp_src/core/nsselecter/joinedselector.h | 22 +- cpp_src/core/nsselecter/joinedselectormock.h | 2 +- cpp_src/core/nsselecter/nsselecter.cc | 33 +- cpp_src/core/nsselecter/querypreprocessor.cc | 474 ++++++++-------- cpp_src/core/nsselecter/querypreprocessor.h | 17 +- .../nsselecter/selectiteratorcontainer.cc | 176 +++--- .../core/nsselecter/selectiteratorcontainer.h | 6 +- cpp_src/core/payload/fieldsset.h | 83 ++- cpp_src/core/payload/payloadiface.cc | 152 +++-- cpp_src/core/payload/payloadiface.h | 5 + cpp_src/core/payload/payloadvalue.cc | 20 +- cpp_src/core/payload/payloadvalue.h | 16 +- cpp_src/core/query/dsl/dslencoder.cc | 48 +- cpp_src/core/query/dsl/dslparser.cc | 57 +- cpp_src/core/query/query.cc | 133 ++--- cpp_src/core/query/query.h | 459 ++++++++------- cpp_src/core/query/queryentry.cc | 292 +++++++--- cpp_src/core/query/queryentry.h | 332 +++++++---- cpp_src/core/query/sql/sqlencoder.cc | 104 ++-- cpp_src/core/query/sql/sqlencoder.h | 1 - cpp_src/core/query/sql/sqlparser.cc | 98 ++-- cpp_src/core/query/sql/sqlsuggester.h | 2 - cpp_src/core/queryresults/queryresults.cc | 12 - cpp_src/core/queryresults/queryresults.h | 29 +- cpp_src/core/queryresults/tableviewbuilder.h | 4 +- cpp_src/core/rdxcontext.h | 18 +- cpp_src/core/reindexerimpl.cc | 155 +++-- cpp_src/core/storage/leveldbstorage.cc | 7 +- cpp_src/core/storage/rocksdbstorage.cc | 7 +- cpp_src/core/txstats.h | 2 +- cpp_src/core/type_consts_helpers.cc | 4 +- cpp_src/core/type_consts_helpers.h | 24 +- .../gtests/bench/fixtures/api_tv_composite.cc | 51 +- cpp_src/gtests/bench/fixtures/ft_fixture.cc | 77 ++- cpp_src/gtests/bench/fixtures/ft_fixture.h | 2 +- cpp_src/gtests/bench/ft_bench.cc | 5 +- cpp_src/gtests/bench/reindexer_bench.cc | 5 +- cpp_src/gtests/tests/API/base_tests.cc | 16 +- cpp_src/gtests/tests/fixtures/ft_api.cc | 9 +- cpp_src/gtests/tests/fixtures/ft_api.h | 2 +- .../gtests/tests/fixtures/fuzzing/fuzzing.h | 15 +- .../gtests/tests/fixtures/fuzzing/index.cc | 54 +- cpp_src/gtests/tests/fixtures/fuzzing/index.h | 41 +- cpp_src/gtests/tests/fixtures/fuzzing/ns.cc | 154 +++-- cpp_src/gtests/tests/fixtures/fuzzing/ns.h | 14 +- .../tests/fixtures/fuzzing/ns_scheme.cc | 240 +++++++- .../gtests/tests/fixtures/fuzzing/ns_scheme.h | 188 +------ .../tests/fixtures/fuzzing/query_generator.cc | 18 +- .../tests/fixtures/fuzzing/query_generator.h | 7 +- .../fixtures/fuzzing/random_generator.cc | 425 ++++++++++---- .../tests/fixtures/fuzzing/random_generator.h | 182 +++--- .../gtests/tests/fixtures/fuzzing/types.cc | 90 +++ cpp_src/gtests/tests/fixtures/fuzzing/types.h | 29 + cpp_src/gtests/tests/fixtures/get_pk_api.h | 2 +- .../tests/fixtures/item_move_semantics_api.h | 30 +- .../tests/fixtures/join_on_conditions_api.h | 6 +- cpp_src/gtests/tests/fixtures/queries_api.cc | 164 ++++-- cpp_src/gtests/tests/fixtures/queries_api.h | 177 +++--- .../gtests/tests/fixtures/queries_verifier.h | 153 +++-- cpp_src/gtests/tests/fixtures/reindexer_api.h | 4 - .../gtests/tests/fixtures/servercontrol.cc | 22 + cpp_src/gtests/tests/fixtures/servercontrol.h | 3 +- .../gtests/tests/fixtures/storage_lazy_load.h | 5 +- cpp_src/gtests/tests/fixtures/ttl_index_api.h | 3 +- cpp_src/gtests/tests/fuzzing/fuzzing.cc | 82 ++- .../gtests/tests/unit/composite_indexes_api.h | 12 +- .../tests/unit/composite_indexes_test.cc | 14 +- cpp_src/gtests/tests/unit/dsl_parser_test.cc | 91 +-- cpp_src/gtests/tests/unit/ft/ft_generic.cc | 6 +- cpp_src/gtests/tests/unit/ft/ft_stress.cc | 10 +- cpp_src/gtests/tests/unit/join_test.cc | 134 ++--- cpp_src/gtests/tests/unit/namespace_test.cc | 93 +-- cpp_src/gtests/tests/unit/queries_test.cc | 45 +- .../unit/replication_master_master_test.cc | 99 ++++ cpp_src/gtests/tests/unit/rpcclient_test.cc | 82 ++- .../gtests/tests/unit/selector_plan_test.cc | 6 +- .../gtests/tests/unit/value_by_json_path.cc | 101 ++-- cpp_src/gtests/tools.h | 64 +++ cpp_src/net/http/router.cc | 3 + cpp_src/net/http/router.h | 2 +- cpp_src/replicator/replicator.cc | 8 +- cpp_src/replicator/walrecord.cc | 5 +- cpp_src/replicator/walselecter.cc | 21 +- cpp_src/server/config.cc | 12 +- cpp_src/server/config.h | 3 +- cpp_src/server/contrib/server.md | 2 +- cpp_src/server/contrib/server.yml | 133 ++++- cpp_src/server/grpc/reindexerservice.cc | 2 +- cpp_src/server/httpserver.cc | 214 ++++--- cpp_src/server/httpserver.h | 5 +- cpp_src/server/rpcserver.cc | 2 +- cpp_src/tools/errors.h | 10 +- cpp_src/tools/logger.cc | 5 - cpp_src/tools/stringstools.cc | 19 +- cpp_src/tools/stringstools.h | 32 +- cpp_src/tools/verifying_updater.h | 30 + dsl/dsl.go | 5 +- reindexer_impl.go | 2 +- test/multiple_json_paths_test.go | 532 ++++++++++++++++++ test/queries_test.go | 42 +- test/query_test.go | 19 +- 171 files changed, 5234 insertions(+), 3070 deletions(-) create mode 100644 cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh create mode 100644 cpp_src/gtests/tests/fixtures/fuzzing/types.cc create mode 100644 cpp_src/gtests/tests/fixtures/fuzzing/types.h create mode 100644 cpp_src/tools/verifying_updater.h create mode 100644 test/multiple_json_paths_test.go diff --git a/bindings/builtin/builtin.go b/bindings/builtin/builtin.go index bfd8b17ed..59eacc4a9 100644 --- a/bindings/builtin/builtin.go +++ b/bindings/builtin/builtin.go @@ -31,8 +31,13 @@ type Logger interface { Printf(level int, fmt string, msg ...interface{}) } -var logger Logger +// Separate mutexes for logger object itself and for reindexer_enable_logger call: +// logMtx provides safe access to the logger +// logEnableMtx provides atomic logic for (enable + set) and (disable + reset) procedures var logMtx sync.RWMutex +var logEnableMtx sync.Mutex +var logger Logger + var enableDebug bool var bufPool sync.Pool @@ -600,18 +605,24 @@ func CGoLogger(level int, msg string) { } } -func (binding *Builtin) EnableLogger(log bindings.Logger) { +func (binding *Builtin) setLogger(log bindings.Logger) { logMtx.Lock() defer logMtx.Unlock() logger = log +} + +func (binding *Builtin) EnableLogger(log bindings.Logger) { + logEnableMtx.Lock() + defer logEnableMtx.Unlock() + binding.setLogger(log) C.reindexer_enable_go_logger() } func (binding *Builtin) DisableLogger() { - logMtx.Lock() - defer logMtx.Unlock() + logEnableMtx.Lock() + defer logEnableMtx.Unlock() C.reindexer_disable_go_logger() - logger = nil + binding.setLogger(nil) } func (binding *Builtin) ReopenLogFiles() error { diff --git a/bindings/builtinserver/config/config.go b/bindings/builtinserver/config/config.go index 30ffe0516..8b3cf0783 100644 --- a/bindings/builtinserver/config/config.go +++ b/bindings/builtinserver/config/config.go @@ -14,10 +14,12 @@ type StorageConf struct { } type NetConf struct { - HTTPAddr string `yaml:"httpaddr"` - RPCAddr string `yaml:"rpcaddr"` - WebRoot string `yaml:"webroot"` - Security bool `yaml:"security"` + HTTPAddr string `yaml:"httpaddr"` + RPCAddr string `yaml:"rpcaddr"` + WebRoot string `yaml:"webroot"` + Security bool `yaml:"security"` + HttpReadTimeoutSec int `yaml:"http_read_timeout,omitempty"` + HttpWriteTimeoutSec int `yaml:"http_write_timeout,omitempty"` } type LoggerConf struct { diff --git a/changelog.md b/changelog.md index 9c3b0da69..479e723fb 100644 --- a/changelog.md +++ b/changelog.md @@ -1905,5 +1905,3 @@ - [ref] EnableStorage method was deprecated - [fix] Query builder did not reset opOR after InnerJoin -## Misc - diff --git a/cjson/decoder.go b/cjson/decoder.go index 3cc9bd183..dc9667ced 100644 --- a/cjson/decoder.go +++ b/cjson/decoder.go @@ -711,7 +711,7 @@ func (dec *Decoder) Decode(cjson []byte, dest interface{}) (err error) { } }() - fieldsoutcnt := make([]int, 64, 64) + fieldsoutcnt := make([]int, MaxIndexes) ctagsPath := make([]int, 0, 8) dec.decodeValue(nil, ser, reflect.ValueOf(dest), fieldsoutcnt, ctagsPath) diff --git a/cpp_src/CMakeLists.txt b/cpp_src/CMakeLists.txt index df73c53cd..4fb239201 100644 --- a/cpp_src/CMakeLists.txt +++ b/cpp_src/CMakeLists.txt @@ -89,6 +89,10 @@ endif () set (EXTRA_FLAGS "") +if (WITH_ASAN AND WITH_TSAN) + message(FATAL_ERROR "You cannot use the ASAN and TSAN options at the same time, CMake will exit.") +endif() + if (WITH_ASAN) set (EXTRA_FLAGS "-fsanitize=address") add_definitions(-DREINDEX_WITH_ASAN) @@ -695,6 +699,7 @@ if (NOT WIN32) SET(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME "server") SET(DIST_INCLUDE_FILES "tools/errors.h" "tools/serializer.h" "tools/varint.h" "tools/stringstools.h" "tools/customhash.h" "tools/assertrx.h" "tools/jsonstring.h" + "tools/verifying_updater.h" "core/reindexer.h" "core/type_consts.h" "core/item.h" "core/payload/payloadvalue.h" "core/payload/payloadiface.h" "core/indexopts.h" "core/namespacedef.h" "core/keyvalue/variant.h" "core/keyvalue/geometry.h" "core/sortingprioritiestable.h" "core/rdxcontext.h" "core/activity_context.h" "core/type_consts_helpers.h" "core/payload/fieldsset.h" "core/payload/payloadtype.h" @@ -702,8 +707,8 @@ if (NOT WIN32) "core/query/query.h" "core/query/queryentry.h" "core/queryresults/queryresults.h" "core/indexdef.h" "core/queryresults/aggregationresult.h" "core/queryresults/itemref.h" "core/namespace/stringsholder.h" "core/keyvalue/key_string.h" "core/key_value_type.h" "core/keyvalue/uuid.h" "core/expressiontree.h" "core/lsn.h" "core/cjson/tagspath.h" "core/cjson/ctag.h" - "estl/cow.h" "estl/overloaded.h" "estl/one_of.h" "estl/h_vector.h" "estl/mutex.h" "estl/intrusive_ptr.h" "estl/trivial_reverse_iterator.h" - "estl/span.h" "estl/chunk.h" "estl/fast_hash_traits.h" "estl/debug_macros.h" "estl/defines.h" + "estl/cow.h" "estl/overloaded.h" "estl/one_of.h" "estl/h_vector.h" "estl/mutex.h" "estl/intrusive_ptr.h" "estl/trivial_reverse_iterator.h" + "estl/span.h" "estl/chunk.h" "estl/fast_hash_traits.h" "estl/debug_macros.h" "estl/defines.h" "client/reindexer.h" "client/item.h" "client/reindexerconfig.h" "client/queryresults.h" "client/resultserializer.h" "client/internalrdxcontext.h" "client/transaction.h" "client/cororeindexer.h" "client/coroqueryresults.h" "client/corotransaction.h" diff --git a/cpp_src/client/cororeindexer.cc b/cpp_src/client/cororeindexer.cc index 4cb747cf5..4794f2ae6 100644 --- a/cpp_src/client/cororeindexer.cc +++ b/cpp_src/client/cororeindexer.cc @@ -33,7 +33,7 @@ CoroReindexer& CoroReindexer::operator=(CoroReindexer&& rdx) noexcept { Error CoroReindexer::Connect(const std::string& dsn, dynamic_loop& loop, const client::ConnectOpts& opts) { return impl_->Connect(dsn, loop, opts); } -Error CoroReindexer::Stop() { return impl_->Stop(); } +void CoroReindexer::Stop() { impl_->Stop(); } Error CoroReindexer::AddNamespace(const NamespaceDef& nsDef) { return impl_->AddNamespace(nsDef, ctx_); } Error CoroReindexer::OpenNamespace(std::string_view nsName, const StorageOpts& storage) { return impl_->OpenNamespace(nsName, ctx_, storage); diff --git a/cpp_src/client/cororeindexer.h b/cpp_src/client/cororeindexer.h index da211a6aa..9d9410902 100644 --- a/cpp_src/client/cororeindexer.h +++ b/cpp_src/client/cororeindexer.h @@ -46,7 +46,7 @@ class CoroReindexer { /// @param opts - Connect options. May contaion any of
Error Connect(const std::string &dsn, dynamic_loop &loop, const client::ConnectOpts &opts = client::ConnectOpts()); /// Stop - shutdown connector - Error Stop(); + void Stop(); /// Open or create namespace /// @param nsName - Name of namespace /// @param opts - Storage options. Can be one of
diff --git a/cpp_src/client/cororpcclient.cc b/cpp_src/client/cororpcclient.cc index 91ab0114b..9b3236b68 100644 --- a/cpp_src/client/cororpcclient.cc +++ b/cpp_src/client/cororpcclient.cc @@ -46,13 +46,12 @@ Error CoroRPCClient::Connect(const std::string& dsn, ev::dynamic_loop& loop, con return errOK; } -Error CoroRPCClient::Stop() { +void CoroRPCClient::Stop() { terminate_ = true; conn_.Stop(); resubWg_.wait(); loop_ = nullptr; terminate_ = false; - return errOK; } Error CoroRPCClient::AddNamespace(const NamespaceDef& nsDef, const InternalRdxContext& ctx) { @@ -236,7 +235,7 @@ Error CoroRPCClient::Delete(const Query& query, CoroQueryResults& result, const query.Serialize(ser); NsArray nsArray; - query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q._namespace)); }); + query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q.NsName())); }); result = CoroQueryResults(&conn_, std::move(nsArray), 0, config_.FetchAmount, config_.RequestTimeout); @@ -257,7 +256,7 @@ Error CoroRPCClient::Update(const Query& query, CoroQueryResults& result, const query.Serialize(ser); NsArray nsArray; - query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q._namespace)); }); + query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q.NsName())); }); result = CoroQueryResults(&conn_, std::move(nsArray), 0, config_.FetchAmount, config_.RequestTimeout); @@ -322,7 +321,7 @@ Error CoroRPCClient::selectImpl(const Query& query, CoroQueryResults& result, se } NsArray nsArray; query.Serialize(qser); - query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q._namespace)); }); + query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q.NsName())); }); h_vector vers; for (auto& ns : nsArray) { vers.push_back(ns->tagsMatcher_.version() ^ ns->tagsMatcher_.stateToken()); diff --git a/cpp_src/client/cororpcclient.h b/cpp_src/client/cororpcclient.h index 8c217aa3d..50278e8fd 100644 --- a/cpp_src/client/cororpcclient.h +++ b/cpp_src/client/cororpcclient.h @@ -36,7 +36,7 @@ class CoroRPCClient { ~CoroRPCClient(); Error Connect(const std::string &dsn, ev::dynamic_loop &loop, const client::ConnectOpts &opts); - Error Stop(); + void Stop(); Error OpenNamespace(std::string_view nsName, const InternalRdxContext &ctx, const StorageOpts &opts = StorageOpts().Enabled().CreateIfMissing()); diff --git a/cpp_src/client/item.h b/cpp_src/client/item.h index 3f1bbd684..618398a5c 100644 --- a/cpp_src/client/item.h +++ b/cpp_src/client/item.h @@ -60,13 +60,13 @@ class Item { /// Get status of item /// @return data slice with JSON. Returned slice is allocated in temporary Item's buffer, and can be invalidated by any next operation /// with Item - Error Status() { return status_; } + Error Status() const noexcept { return status_; } /// Get internal ID of item /// @return ID of item int GetID() const noexcept { return id_; } - /// Get internal version of item - /// @return version of item - int NumFields(); + /// Get count of indexed fields + /// @return count of indexed fields + int NumFields() const noexcept; /// Set additional percepts for modify operation /// @param precepts - strings in format "fieldName=Func()" void SetPrecepts(const std::vector &precepts); diff --git a/cpp_src/client/queryresults.cc b/cpp_src/client/queryresults.cc index 7f0b070b1..83b1755c3 100644 --- a/cpp_src/client/queryresults.cc +++ b/cpp_src/client/queryresults.cc @@ -114,8 +114,6 @@ void QueryResults::fetchNextResults() { rawResult_.assign(rawResult.begin() + ser.Pos(), rawResult.end()); } -QueryResults::~QueryResults() {} - h_vector QueryResults::GetNamespaces() const { h_vector ret; ret.reserve(nsArray_.size()); @@ -310,8 +308,5 @@ QueryResults::Iterator &QueryResults::Iterator::operator++() { return *this; } -bool QueryResults::Iterator::operator!=(const Iterator &other) const { return idx_ != other.idx_; } -bool QueryResults::Iterator::operator==(const Iterator &other) const { return idx_ == other.idx_; } - } // namespace client } // namespace reindexer diff --git a/cpp_src/client/queryresults.h b/cpp_src/client/queryresults.h index 97c2a1835..e8b059568 100644 --- a/cpp_src/client/queryresults.h +++ b/cpp_src/client/queryresults.h @@ -26,7 +26,7 @@ class QueryResults { QueryResults(int fetchFlags = 0); QueryResults(const QueryResults&) = delete; QueryResults(QueryResults&&) noexcept; - ~QueryResults(); + ~QueryResults() = default; QueryResults& operator=(const QueryResults&) = delete; QueryResults& operator=(QueryResults&& obj) noexcept; @@ -40,10 +40,10 @@ class QueryResults { bool IsRaw(); std::string_view GetRaw(); Iterator& operator++(); - Error Status() { return qr_->status_; } - bool operator!=(const Iterator&) const; - bool operator==(const Iterator&) const; - Iterator& operator*() { return *this; } + Error Status() const noexcept { return qr_->status_; } + bool operator==(const Iterator& other) const noexcept { return idx_ == other.idx_; } + bool operator!=(const Iterator& other) const noexcept { return !operator==(other); } + Iterator& operator*() noexcept { return *this; } void readNext(); void getJSONFromCJSON(std::string_view cjson, WrSerializer& wrser, bool withHdrLen = true); @@ -55,13 +55,15 @@ class QueryResults { Iterator begin() const { return Iterator{this, 0, 0, 0, {}}; } Iterator end() const { return Iterator{this, queryParams_.qcount, 0, 0, {}}; } - size_t Count() const { return queryParams_.qcount; } - int TotalCount() const { return queryParams_.totalcount; } - bool HaveRank() const { return queryParams_.flags & kResultsWithRank; } - bool NeedOutputRank() const { return queryParams_.flags & kResultsNeedOutputRank; } - const std::string& GetExplainResults() const { return queryParams_.explainResults; } - const std::vector& GetAggregationResults() const { return queryParams_.aggResults; } - Error Status() { return status_; } + size_t Count() const noexcept { return queryParams_.qcount; } + int TotalCount() const noexcept { return queryParams_.totalcount; } + bool HaveRank() const noexcept { return queryParams_.flags & kResultsWithRank; } + bool NeedOutputRank() const noexcept { return queryParams_.flags & kResultsNeedOutputRank; } + const std::string& GetExplainResults() const& noexcept { return queryParams_.explainResults; } + const std::string& GetExplainResults() const&& = delete; + const std::vector& GetAggregationResults() const& noexcept { return queryParams_.aggResults; } + const std::vector& GetAggregationResults() const&& = delete; + Error Status() const noexcept { return status_; } h_vector GetNamespaces() const; bool IsCacheEnabled() const { return queryParams_.flags & kResultsWithItemID; } diff --git a/cpp_src/client/reindexer.cc b/cpp_src/client/reindexer.cc index e28041713..248b25cb0 100644 --- a/cpp_src/client/reindexer.cc +++ b/cpp_src/client/reindexer.cc @@ -29,7 +29,7 @@ Error Reindexer::Connect(const std::string& dsn, const client::ConnectOpts& opts Error Reindexer::Connect(const std::vector>& connectData) { return impl_->Connect(connectData); } -Error Reindexer::Stop() { return impl_->Stop(); } +void Reindexer::Stop() { impl_->Stop(); } Error Reindexer::AddNamespace(const NamespaceDef& nsDef) { return impl_->AddNamespace(nsDef, ctx_); } Error Reindexer::OpenNamespace(std::string_view nsName, const StorageOpts& storage) { return impl_->OpenNamespace(nsName, ctx_, storage); } Error Reindexer::DropNamespace(std::string_view nsName) { return impl_->DropNamespace(nsName, ctx_); } diff --git a/cpp_src/client/reindexer.h b/cpp_src/client/reindexer.h index 705de78e3..8a8780f2e 100644 --- a/cpp_src/client/reindexer.h +++ b/cpp_src/client/reindexer.h @@ -48,7 +48,7 @@ class Reindexer { /// @param connectData - list of server dsn + it's ConnectOpts Error Connect(const std::vector> &connectData); /// Stop - shutdown connector - Error Stop(); + void Stop(); /// Open or create namespace /// @param nsName - Name of namespace /// @param opts - Storage options. Can be one of
diff --git a/cpp_src/client/rpcclient.cc b/cpp_src/client/rpcclient.cc index 29b6f5ccc..3c7d7089c 100644 --- a/cpp_src/client/rpcclient.cc +++ b/cpp_src/client/rpcclient.cc @@ -76,8 +76,8 @@ Error RPCClient::Connect(const std::vector vers; for (auto& ns : nsArray) { shared_lock lck(ns->lck_); diff --git a/cpp_src/client/rpcclient.h b/cpp_src/client/rpcclient.h index 238b16a12..eefb2cf03 100644 --- a/cpp_src/client/rpcclient.h +++ b/cpp_src/client/rpcclient.h @@ -41,7 +41,7 @@ class RPCClient { Error Connect(const std::string &dsn, const client::ConnectOpts &opts); Error Connect(const std::vector> &connectData); - Error Stop(); + void Stop(); Error OpenNamespace(std::string_view nsName, const InternalRdxContext &ctx, const StorageOpts &opts = StorageOpts().Enabled().CreateIfMissing()); diff --git a/cpp_src/client/rpcclientmock.cc b/cpp_src/client/rpcclientmock.cc index f2cb5635e..5c0d4b6ef 100644 --- a/cpp_src/client/rpcclientmock.cc +++ b/cpp_src/client/rpcclientmock.cc @@ -37,7 +37,7 @@ Error RPCClientMock::Delete(const Query& query, QueryResults& result, const Inte auto conn = getConn(); NsArray nsArray; - query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q._namespace)); }); + query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q.NsName())); }); result = QueryResults(conn, std::move(nsArray), nullptr, 0, config_.FetchAmount, config_.RequestTimeout); @@ -69,7 +69,7 @@ Error RPCClientMock::Update(const Query& query, QueryResults& result, const Inte auto conn = getConn(); NsArray nsArray; - query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q._namespace)); }); + query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q.NsName())); }); result = QueryResults(conn, std::move(nsArray), nullptr, 0, config_.FetchAmount, config_.RequestTimeout); @@ -308,7 +308,7 @@ Error RPCClientMock::selectImpl(const Query& query, QueryResults& result, cproto NsArray nsArray; query.Serialize(qser); - query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q._namespace)); }); + query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q.NsName())); }); h_vector vers; for (auto& ns : nsArray) { shared_lock lck(ns->lck_); diff --git a/cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh b/cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh new file mode 100644 index 000000000..2c014c6a5 --- /dev/null +++ b/cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh @@ -0,0 +1,194 @@ +#!/bin/bash +# Task: https://github.com/restream/reindexer/-/issues/1188 +set -e + +function KillAndRemoveServer { + local pid=$1 + kill $pid + wait $pid + yum remove -y 'reindexer*' > /dev/null +} + +function WaitForDB { + # wait until DB is loaded + set +e # disable "exit on error" so the script won't stop when DB's not loaded yet + is_connected=$(reindexer_tool --dsn $ADDRESS --command '\databases list'); + while [[ $is_connected != "test" ]] + do + sleep 2 + is_connected=$(reindexer_tool --dsn $ADDRESS --command '\databases list'); + done + set -e +} + +function CompareNamespacesLists { + local ns_list_actual=$1 + local ns_list_expected=$2 + local pid=$3 + + diff=$(echo ${ns_list_actual[@]} ${ns_list_expected[@]} | tr ' ' '\n' | sort | uniq -u) # compare in any order + if [ "$diff" == "" ]; then + echo "## PASS: namespaces list not changed" + else + echo "##### FAIL: namespaces list was changed" + echo "expected: $ns_list_expected" + echo "actual: $ns_list_actual" + KillAndRemoveServer $pid; + exit 1 + fi +} + +function CompareMemstats { + local actual=$1 + local expected=$2 + local pid=$3 + diff=$(echo ${actual[@]} ${expected[@]} | tr ' ' '\n' | sed 's/\(.*\),$/\1/' | sort | uniq -u) # compare in any order + if [ "$diff" == "" ]; then + echo "## PASS: memstats not changed" + else + echo "##### FAIL: memstats was changed" + echo "expected: $expected" + echo "actual: $actual" + KillAndRemoveServer $pid; + exit 1 + fi +} + + +RX_SERVER_CURRENT_VERSION_RPM="$(basename build/reindexer-*server*.rpm)" +VERSION_FROM_RPM=$(echo "$RX_SERVER_CURRENT_VERSION_RPM" | grep -o '.*server-..') +VERSION=$(echo ${VERSION_FROM_RPM: -2:1}) # one-digit version + +if [ $VERSION == 3 ]; then + LATEST_RELEASE=$(curl -s http://repo.restream.ru/itv-api-ng/7/x86_64/ | grep -o '>reindexer-server-.*.rpm' | tail -n1 | cut -c 2-) + namespaces_list_expected=$'purchase_options_ext_dict\nchild_account_recommendations\n#config\n#activitystats\nradio_channels\ncollections\n#namespaces\nwp_imports_tasks\nepg_genres\nrecom_media_items_personal\nrecom_epg_archive_default\n#perfstats\nrecom_epg_live_default\nmedia_view_templates\nasset_video_servers\nwp_tasks_schedule\nadmin_roles\n#clientsstats\nrecom_epg_archive_personal\nrecom_media_items_similars\nmenu_items\naccount_recommendations\nkaraoke_items\nmedia_items\nbanners\n#queriesperfstats\nrecom_media_items_default\nrecom_epg_live_personal\nservices\n#memstats\nchannels\nmedia_item_recommendations\nwp_tasks_tasks\nepg' +elif [ $VERSION == 4 ]; then + LATEST_RELEASE=$(curl -s http://repo.restream.ru/itv-api-ng/7/x86_64/ | grep -o '>reindexer-4-server-.*.rpm' | tail -n1 | cut -c 2-) + # replicationstats ns added for v4 + namespaces_list_expected=$'purchase_options_ext_dict\nchild_account_recommendations\n#config\n#activitystats\n#replicationstats\nradio_channels\ncollections\n#namespaces\nwp_imports_tasks\nepg_genres\nrecom_media_items_personal\nrecom_epg_archive_default\n#perfstats\nrecom_epg_live_default\nmedia_view_templates\nasset_video_servers\nwp_tasks_schedule\nadmin_roles\n#clientsstats\nrecom_epg_archive_personal\nrecom_media_items_similars\nmenu_items\naccount_recommendations\nkaraoke_items\nmedia_items\nbanners\n#queriesperfstats\nrecom_media_items_default\nrecom_epg_live_personal\nservices\n#memstats\nchannels\nmedia_item_recommendations\nwp_tasks_tasks\nepg' +else + echo "Unknown version" + exit 1 +fi + +echo "## downloading latest release rpm file: $LATEST_RELEASE" +curl "http://repo.itv.restr.im/itv-api-ng/7/x86_64/$LATEST_RELEASE" --output $LATEST_RELEASE; +echo "## downloading example DB" +curl "https://git.restream.ru/MaksimKravchuk/reindexer_testdata/-/raw/master/big.zip" --output big.zip; +unzip -o big.zip # unzips into mydb_big.rxdump; + +ADDRESS="cproto://127.0.0.1:6534/" +DB_NAME="test" + +memstats_expected=$'[ +{"replication":{"data_hash":24651210926,"data_count":3}}, +{"replication":{"data_hash":6252344969,"data_count":1}}, +{"replication":{"data_hash":37734732881,"data_count":28}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":1024095024522,"data_count":1145}}, +{"replication":{"data_hash":8373644068,"data_count":1315}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":7404222244,"data_count":97}}, +{"replication":{"data_hash":94132837196,"data_count":4}}, +{"replication":{"data_hash":1896088071,"data_count":2}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":-672103903,"data_count":33538}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":6833710705,"data_count":1}}, +{"replication":{"data_hash":5858155773472,"data_count":4500}}, +{"replication":{"data_hash":-473221280268823592,"data_count":65448}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":8288213744,"data_count":3}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":354171024786967,"data_count":3941}}, +{"replication":{"data_hash":-6520334670,"data_count":35886}}, +{"replication":{"data_hash":112772074632,"data_count":281}}, +{"replication":{"data_hash":-12679568198538,"data_count":1623116}} +] +Returned 27 rows' + +echo "##### Forward compatibility test #####" + +DB_PATH=$(pwd)"/rx_db" + +echo "Database: "$DB_PATH + +echo "## installing latest release: $LATEST_RELEASE" +yum install -y $LATEST_RELEASE > /dev/null; +# run RX server with disabled logging +reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +reindexer_tool --dsn $ADDRESS$DB_NAME -f mydb_big.rxdump --createdb; +sleep 1; + +namespaces_1=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_1; +CompareNamespacesLists "${namespaces_1[@]}" "${namespaces_list_expected[@]}" $server_pid; + +memstats_1=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats'); +CompareMemstats "${memstats_1[@]}" "${memstats_expected[@]}" $server_pid; + +KillAndRemoveServer $server_pid; + +echo "## installing current version: $RX_SERVER_CURRENT_VERSION_RPM" +yum install -y build/*.rpm > /dev/null; +reindexer_server -l0 --corelog=none --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +WaitForDB + +namespaces_2=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_2; +CompareNamespacesLists "${namespaces_2[@]}" "${namespaces_1[@]}" $server_pid; + +memstats_2=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats'); +CompareMemstats "${memstats_2[@]}" "${memstats_1[@]}" $server_pid; + +KillAndRemoveServer $server_pid; +rm -rf $DB_PATH; +sleep 1; + +echo "##### Backward compatibility test #####" + +echo "## installing current version: $RX_SERVER_CURRENT_VERSION_RPM" +yum install -y build/*.rpm > /dev/null; +reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +reindexer_tool --dsn $ADDRESS$DB_NAME -f mydb_big.rxdump --createdb; +sleep 1; + +namespaces_3=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_3; +CompareNamespacesLists "${namespaces_3[@]}" "${namespaces_list_expected[@]}" $server_pid; + +memstats_3=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats'); +CompareMemstats "${memstats_3[@]}" "${memstats_expected[@]}" $server_pid; + +KillAndRemoveServer $server_pid; + +echo "## installing latest release: $LATEST_RELEASE" +yum install -y $LATEST_RELEASE > /dev/null; +reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +WaitForDB + +namespaces_4=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_4; +CompareNamespacesLists "${namespaces_4[@]}" "${namespaces_3[@]}" $server_pid; + +memstats_4=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats'); +CompareMemstats "${memstats_4[@]}" "${memstats_3[@]}" $server_pid; + +KillAndRemoveServer $server_pid; +rm -rf $DB_PATH; diff --git a/cpp_src/cmd/reindexer_tool/commandsexecutor.cc b/cpp_src/cmd/reindexer_tool/commandsexecutor.cc index 8509155f5..eee2a4f62 100644 --- a/cpp_src/cmd/reindexer_tool/commandsexecutor.cc +++ b/cpp_src/cmd/reindexer_tool/commandsexecutor.cc @@ -47,19 +47,18 @@ Error CommandsExecutor::Run(const std::string& } template -void CommandsExecutor::GetSuggestions(const std::string& input, std::vector& suggestions) { +Error CommandsExecutor::GetSuggestions(const std::string& input, std::vector& suggestions) { OutParamCommand> cmd( - [this, &input](std::vector& suggestions) { - getSuggestions(input, suggestions); - return errOK; - }, - suggestions); - execCommand(cmd); + [this, &input](std::vector& suggestions) { return getSuggestions(input, suggestions); }, suggestions); + return execCommand(cmd); } template Error CommandsExecutor::Stop() { - GenericCommand cmd([this] { return stop(true); }); + GenericCommand cmd([this] { + stop(true); + return Error{}; + }); auto err = execCommand(cmd); if (err.ok() && executorThr_.joinable()) { executorThr_.join(); @@ -475,15 +474,14 @@ Error CommandsExecutor::processImpl(const std::string& command) noe } template <> -Error CommandsExecutor::stop(bool terminate) { +void CommandsExecutor::stop(bool terminate) { if (terminate) { stopCh_.close(); } - return Error(); } template <> -Error CommandsExecutor::stop(bool terminate) { +void CommandsExecutor::stop(bool terminate) { if (terminate) { stopCh_.close(); } @@ -491,11 +489,17 @@ Error CommandsExecutor::stop(bool terminate) { } template -void CommandsExecutor::getSuggestions(const std::string& input, std::vector& suggestions) { - if (!input.empty() && input[0] != '\\') db().GetSqlSuggestions(input, input.length() - 1, suggestions); +Error CommandsExecutor::getSuggestions(const std::string& input, std::vector& suggestions) { + if (!input.empty() && input[0] != '\\') { + auto err = db().GetSqlSuggestions(input, input.length() - 1, suggestions); + if (!err.ok()) { + return err; + } + } if (suggestions.empty()) { addCommandsSuggestions(input, suggestions); } + return {}; } template @@ -761,7 +765,8 @@ Error CommandsExecutor::commandDump(const std::string& command) { return Error(errCanceled, "Canceled"); } wrser << "\\UPSERT " << reindexer::escapeString(nsDef.name) << ' '; - it.GetJSON(wrser, false); + err = it.GetJSON(wrser, false); + if (!err.ok()) return err; wrser << '\n'; if (wrser.Len() > 0x100000) { output_() << wrser.Slice(); @@ -847,9 +852,11 @@ Error CommandsExecutor::commandMeta(const std::string& command) { auto nsName = reindexer::unescapeString(parser.NextToken()); std::vector allMeta; auto err = db().EnumMeta(nsName, allMeta); + if (!err.ok()) return err; for (auto& metaKey : allMeta) { std::string metaData; - db().GetMeta(nsName, metaKey, metaData); + err = db().GetMeta(nsName, metaKey, metaData); + if (!err.ok()) return err; output_() << metaKey << " = " << metaData << std::endl; } return err; @@ -917,15 +924,18 @@ Error CommandsExecutor::commandBench(const std::string& command) { LineParser parser(command); parser.NextToken(); - int benchTime = reindexer::stoi(parser.NextToken()); - if (benchTime == 0) benchTime = kBenchDefaultTime; + const std::string_view benchTimeToken = parser.NextToken(); + const int benchTime = benchTimeToken.empty() ? kBenchDefaultTime : reindexer::stoi(benchTimeToken); - db().DropNamespace(kBenchNamespace); + auto err = db().DropNamespace(kBenchNamespace); + if (!err.ok() && err.code() != errNotFound) { + return err; + } NamespaceDef nsDef(kBenchNamespace); nsDef.AddIndex("id", "hash", "int", IndexOpts().PK()); - auto err = db().AddNamespace(nsDef); + err = db().AddNamespace(nsDef); if (!err.ok()) return err; output_() << "Seeding " << kBenchItemsCount << " documents to bench namespace..." << std::endl; @@ -1014,19 +1024,17 @@ Error CommandsExecutor::commandProcessDatabase return Error(); } else if (subCommand == "use") { std::string currentDsn = getCurrentDsn() + std::string(parser.NextToken()); - Error err = stop(false); - if (!err.ok()) return err; - err = db().Connect(currentDsn, loop_); + stop(false); + auto err = db().Connect(currentDsn, loop_); if (err.ok()) err = db().Status(); if (err.ok()) output_() << "Succesfully connected to " << currentDsn << std::endl; return err; } else if (subCommand == "create") { auto dbName = parser.NextToken(); std::string currentDsn = getCurrentDsn() + std::string(dbName); - Error err = stop(false); - if (!err.ok()) return err; + stop(false); output_() << "Creating database '" << dbName << "'" << std::endl; - err = db().Connect(currentDsn, loop_, reindexer::client::ConnectOpts().CreateDBIfMissing()); + auto err = db().Connect(currentDsn, loop_, reindexer::client::ConnectOpts().CreateDBIfMissing()); if (!err.ok()) { std::cerr << "Error on database '" << dbName << "' creation" << std::endl; return err; @@ -1145,11 +1153,14 @@ std::function CommandsExecutor Error Run(const std::string& dsn, const Args&... args); - void GetSuggestions(const std::string& input, std::vector& suggestions); + Error GetSuggestions(const std::string& input, std::vector& suggestions); Error Stop(); Error Process(const std::string& command); Error FromFile(std::istream& in); @@ -78,8 +78,8 @@ class CommandsExecutor : public reindexer::IUpdatesObserver { std::vector& suggestions); Error processImpl(const std::string& command) noexcept; - Error stop(bool terminate); - void getSuggestions(const std::string& input, std::vector& suggestions); + void stop(bool terminate); + Error getSuggestions(const std::string& input, std::vector& suggestions); Error commandSelect(const std::string& command); Error commandUpsert(const std::string& command); Error commandUpdateSQL(const std::string& command); diff --git a/cpp_src/cmd/reindexer_tool/commandsprocessor.cc b/cpp_src/cmd/reindexer_tool/commandsprocessor.cc index 17ce39931..f084ea387 100644 --- a/cpp_src/cmd/reindexer_tool/commandsprocessor.cc +++ b/cpp_src/cmd/reindexer_tool/commandsprocessor.cc @@ -3,8 +3,8 @@ #include #include "client/cororeindexer.h" #include "core/reindexer.h" -#include "tableviewscroller.h" #include "tools/fsops.h" +#include "tools/terminalutils.h" namespace reindexer_tool { @@ -30,9 +30,13 @@ template void CommandsProcessor::setCompletionCallback(T& rx, void (T::*set_completion_callback)(new_v_callback_t const&)) { (rx.*set_completion_callback)([this](std::string const& input, int) -> replxx::Replxx::completions_t { std::vector completions; - executor_.GetSuggestions(input, completions); + const auto err = executor_.GetSuggestions(input, completions); replxx::Replxx::completions_t result; - for (const std::string& suggestion : completions) result.emplace_back(suggestion); + if (err.ok()) { + for (const std::string& suggestion : completions) { + result.emplace_back(suggestion); + } + } return result; }); } @@ -43,7 +47,8 @@ void CommandsProcessor::setCompletionCallback(T& rx, void (T::*set_ (rx.*set_completion_callback)( [this](std::string const& input, int, void*) -> replxx::Replxx::completions_t { std::vector completions; - executor_.GetSuggestions(input, completions); + const auto err = executor_.GetSuggestions(input, completions); + if (!err.ok()) return {}; return completions; }, nullptr); diff --git a/cpp_src/core/cjson/fieldextractor.h b/cpp_src/core/cjson/fieldextractor.h index 6b11f179a..43ac72d64 100644 --- a/cpp_src/core/cjson/fieldextractor.h +++ b/cpp_src/core/cjson/fieldextractor.h @@ -16,7 +16,7 @@ class FieldsExtractor { }; FieldsExtractor() = default; - FieldsExtractor(VariantArray *va, KeyValueType expectedType, int expectedPathDepth, FieldsSet *filter = nullptr, + FieldsExtractor(VariantArray *va, KeyValueType expectedType, int expectedPathDepth, const FieldsSet *filter, FieldParams *params = nullptr) noexcept : values_(va), expectedType_(expectedType), expectedPathDepth_(expectedPathDepth), filter_(filter), params_(params) {} FieldsExtractor(FieldsExtractor &&other) = default; @@ -176,7 +176,7 @@ class FieldsExtractor { VariantArray *values_ = nullptr; KeyValueType expectedType_{KeyValueType::Undefined{}}; int expectedPathDepth_ = 0; - FieldsSet *filter_; + const FieldsSet *filter_; FieldParams *params_; }; diff --git a/cpp_src/core/cjson/tagsmatcherimpl.h b/cpp_src/core/cjson/tagsmatcherimpl.h index e96109868..07037b2ee 100644 --- a/cpp_src/core/cjson/tagsmatcherimpl.h +++ b/cpp_src/core/cjson/tagsmatcherimpl.h @@ -4,7 +4,6 @@ #include #include -#include "core/keyvalue/key_string.h" #include "core/payload/payloadtype.h" #include "core/payload/payloadtypeimpl.h" #include "ctag.h" @@ -75,25 +74,29 @@ class TagsMatcherImpl { if (content == "*"sv) { node.MarkAllItems(true); } else { - int index = stoi(content); - if (index == 0 && content != "0"sv && ev) { - VariantArray values = ev(content); - if (values.size() != 1) { - throw Error(errParams, "Index expression_ has wrong syntax: '%s'", content); + auto index = try_stoi(content); + if (!index) { + if (ev) { + VariantArray values = ev(content); + if (values.size() != 1) { + throw Error(errParams, "Index expression_ has wrong syntax: '%s'", content); + } + values.front().Type().EvaluateOneOf( + [](OneOf) noexcept {}, + [&](OneOf) { + throw Error(errParams, "Wrong type of index: '%s'", content); + }); + node.SetExpression(content); + index = values.front().As(); + } else { + throw Error(errParams, "Can't convert '%s' to number", content); } - values.front().Type().EvaluateOneOf( - [](OneOf) noexcept {}, - [&](OneOf) { - throw Error(errParams, "Wrong type of index: '%s'", content); - }); - node.SetExpression(content); - index = values.front().As(); } if (index < 0) { throw Error(errLogic, "Array index value cannot be negative"); } - node.SetIndex(index); + node.SetIndex(*index); } field = field.substr(0, openBracketPos); } diff --git a/cpp_src/core/comparator.cc b/cpp_src/core/comparator.cc index 08d001339..c105f53f2 100644 --- a/cpp_src/core/comparator.cc +++ b/cpp_src/core/comparator.cc @@ -82,16 +82,11 @@ void Comparator::Bind(const PayloadType &type, int field) { } } -void Comparator::BindEqualPosition(int field, const VariantArray &val, CondType cond) { cmpEqualPosition.BindField(field, val, cond); } - -void Comparator::BindEqualPosition(const TagsPath &tagsPath, const VariantArray &val, CondType cond) { - cmpEqualPosition.BindField(tagsPath, val, cond); -} - bool Comparator::isNumericComparison(const VariantArray &values) const { if (valuesType_.Is() || values.empty()) return false; const KeyValueType keyType{values.front().Type()}; - return !valuesType_.IsSame(keyType) && (valuesType_.Is() || keyType.Is()); + return !valuesType_.IsSame(keyType) && ((valuesType_.Is() && !keyType.Is()) || + (keyType.Is() && !valuesType_.Is())); } bool Comparator::Compare(const PayloadValue &data, int rowId) { @@ -191,7 +186,7 @@ void Comparator::ExcludeDistinct(const PayloadValue &data, int rowId) { } else { // Exclude field from payload by offset (fast path) - assertrx(!type_.Is()); + assertrx_throw(!type_.Is()); // Check if we have column (rawData_), then go to fastest path with column if (rawData_) return excludeDistinct(rawData_ + rowId * sizeof_); diff --git a/cpp_src/core/comparator.h b/cpp_src/core/comparator.h index 3760077c8..7b1a05445 100644 --- a/cpp_src/core/comparator.h +++ b/cpp_src/core/comparator.h @@ -15,8 +15,10 @@ class Comparator : public ComparatorVars { bool Compare(const PayloadValue &lhs, int rowId); void ExcludeDistinct(const PayloadValue &, int rowId); void Bind(const PayloadType &type, int field); - void BindEqualPosition(int field, const VariantArray &val, CondType cond); - void BindEqualPosition(const TagsPath &tagsPath, const VariantArray &val, CondType cond); + template + void BindEqualPosition(F &&field, const VariantArray &val, CondType cond) { + cmpEqualPosition.BindField(std::forward(field), val, cond); + } void ClearDistinct() { cmpInt.ClearDistinct(); cmpBool.ClearDistinct(); @@ -116,7 +118,7 @@ class Comparator : public ComparatorVars { ComparatorImpl cmpComposite; ComparatorImpl cmpGeom; ComparatorImpl cmpUuid; - CompositeArrayComparator cmpEqualPosition; + EqualPositionComparator cmpEqualPosition; KeyValueType valuesType_{KeyValueType::Undefined{}}; }; diff --git a/cpp_src/core/comparatorimpl.h b/cpp_src/core/comparatorimpl.h index 41f0aa486..5da4466bc 100644 --- a/cpp_src/core/comparatorimpl.h +++ b/cpp_src/core/comparatorimpl.h @@ -349,10 +349,12 @@ class ComparatorImpl { void SetValues(CondType cond, const VariantArray &values, const ComparatorVars &vars) { if (cond == CondSet) { valuesSet_.reset(new intrusive_atomic_rc_wrapper( - values.size(), hash_composite(vars.payloadType_, vars.fields_), equal_composite(vars.payloadType_, vars.fields_))); + values.size(), hash_composite(PayloadType{vars.payloadType_}, FieldsSet{vars.fields_}), + equal_composite(PayloadType{vars.payloadType_}, FieldsSet{vars.fields_}))); } else if (cond == CondAllSet) { valuesSet_.reset(new intrusive_atomic_rc_wrapper( - values.size(), hash_composite(vars.payloadType_, vars.fields_), equal_composite(vars.payloadType_, vars.fields_))); + values.size(), hash_composite(PayloadType{vars.payloadType_}, FieldsSet{vars.fields_}), + equal_composite(PayloadType{vars.payloadType_}, FieldsSet{vars.fields_}))); allSetValuesSet_.reset(new intrusive_atomic_rc_wrapper>{}); } diff --git a/cpp_src/core/compositearraycomparator.cc b/cpp_src/core/compositearraycomparator.cc index 6e58fdf98..4ffb48e83 100644 --- a/cpp_src/core/compositearraycomparator.cc +++ b/cpp_src/core/compositearraycomparator.cc @@ -2,37 +2,29 @@ #include namespace reindexer { -CompositeArrayComparator::CompositeArrayComparator() {} +void EqualPositionComparator::BindField(int field, const VariantArray &values, CondType cond) { bindField(field, values, cond); } -void CompositeArrayComparator::BindField(int field, const VariantArray &values, CondType condType) { +void EqualPositionComparator::BindField(const FieldsPath &fieldPath, const VariantArray &values, CondType cond) { + bindField(fieldPath, values, cond); +} + +template +void EqualPositionComparator::bindField(F field, const VariantArray &values, CondType cond) { fields_.push_back(field); Context &ctx = ctx_.emplace_back(); - ctx.cond = condType; - ctx.cmpBool.SetValues(condType, values); - ctx.cmpInt.SetValues(condType, values); - ctx.cmpInt64.SetValues(condType, values); - ctx.cmpString.SetValues(condType, values, CollateOpts()); - ctx.cmpDouble.SetValues(condType, values); - ctx.cmpUuid.SetValues(condType, values); + ctx.cond = cond; + ctx.cmpBool.SetValues(cond, values); + ctx.cmpInt.SetValues(cond, values); + ctx.cmpInt64.SetValues(cond, values); + ctx.cmpString.SetValues(cond, values, CollateOpts()); + ctx.cmpDouble.SetValues(cond, values); + ctx.cmpUuid.SetValues(cond, values); assertrx(ctx_.size() == fields_.size()); } -void CompositeArrayComparator::BindField(const TagsPath &tagsPath, const VariantArray &values, CondType condType) { - fields_.push_back(tagsPath); - Context &ctx = ctx_.emplace_back(); - - ctx.cond = condType; - ctx.cmpBool.SetValues(condType, values); - ctx.cmpInt.SetValues(condType, values); - ctx.cmpInt64.SetValues(condType, values); - ctx.cmpString.SetValues(condType, values, CollateOpts()); - ctx.cmpDouble.SetValues(condType, values); - ctx.cmpUuid.SetValues(condType, values); -} - -bool CompositeArrayComparator::Compare(const PayloadValue &pv, const ComparatorVars &vars) { +bool EqualPositionComparator::Compare(const PayloadValue &pv, const ComparatorVars &vars) { ConstPayload pl(vars.payloadType_, pv); size_t len = INT_MAX; @@ -64,7 +56,7 @@ bool CompositeArrayComparator::Compare(const PayloadValue &pv, const ComparatorV return false; } -bool CompositeArrayComparator::compareField(size_t field, const Variant &v, const ComparatorVars &vars) { +bool EqualPositionComparator::compareField(size_t field, const Variant &v, const ComparatorVars &vars) { return v.Type().EvaluateOneOf( [&](KeyValueType::Bool) { return ctx_[field].cmpBool.Compare(ctx_[field].cond, static_cast(v)); }, [&](KeyValueType::Int) { return ctx_[field].cmpInt.Compare(ctx_[field].cond, static_cast(v)); }, diff --git a/cpp_src/core/compositearraycomparator.h b/cpp_src/core/compositearraycomparator.h index d646ed6b4..a58735dc6 100644 --- a/cpp_src/core/compositearraycomparator.h +++ b/cpp_src/core/compositearraycomparator.h @@ -5,17 +5,19 @@ namespace reindexer { -class CompositeArrayComparator { +class EqualPositionComparator { public: - CompositeArrayComparator(); + EqualPositionComparator() noexcept = default; - void BindField(int field, const VariantArray &values, CondType condType); - void BindField(const TagsPath &tagsPath, const VariantArray &values, CondType condType); - bool Compare(const PayloadValue &pv, const ComparatorVars &vars); + void BindField(int field, const VariantArray &, CondType); + void BindField(const FieldsPath &, const VariantArray &, CondType); + bool Compare(const PayloadValue &, const ComparatorVars &); bool IsBinded() { return !ctx_.empty(); } private: - bool compareField(size_t field, const Variant &v, const ComparatorVars &vars); + bool compareField(size_t field, const Variant &, const ComparatorVars &); + template + void bindField(F field, const VariantArray &, CondType); struct Context { CondType cond; diff --git a/cpp_src/core/dbconfig.cc b/cpp_src/core/dbconfig.cc index d2b4aa462..bd43caa6d 100644 --- a/cpp_src/core/dbconfig.cc +++ b/cpp_src/core/dbconfig.cc @@ -45,8 +45,7 @@ Error DBConfigProvider::FromJSON(const gason::JsonNode &root) { LongTxLoggingParams{profilingNode["long_queries_logging"]["transaction"]["threshold_us"].As(), profilingNode["long_queries_logging"]["transaction"]["avg_step_threshold_us"].As()}); } - auto it = handlers_.find(ProfilingConf); - if (it != handlers_.end()) (it->second)(); + if (handlers_[ProfilingConf]) (handlers_[ProfilingConf])(); } auto &namespacesNode = root["namespaces"]; @@ -75,8 +74,7 @@ Error DBConfigProvider::FromJSON(const gason::JsonNode &root) { data.syncStorageFlushLimit = nsNode["sync_storage_flush_limit"].As(data.syncStorageFlushLimit, 0); namespacesData_.emplace(nsNode["namespace"].As(), std::move(data)); // NOLINT(performance-move-const-arg) } - auto it = handlers_.find(NamespaceDataConf); - if (it != handlers_.end()) (it->second)(); + if (handlers_[NamespaceDataConf]) (handlers_[NamespaceDataConf])(); } auto &replicationNode = root["replication"]; @@ -84,8 +82,7 @@ Error DBConfigProvider::FromJSON(const gason::JsonNode &root) { auto err = replicationData_.FromJSON(replicationNode); if (!err.ok()) return err; - auto it = handlers_.find(ReplicationConf); - if (it != handlers_.end()) (it->second)(); + if (handlers_[ReplicationConf]) (handlers_[ReplicationConf])(); } return errOK; } catch (const Error &err) { @@ -105,11 +102,11 @@ ReplicationConfigData DBConfigProvider::GetReplicationConfig() { return replicationData_; } -bool DBConfigProvider::GetNamespaceConfig(const std::string &nsName, NamespaceConfigData &data) { +bool DBConfigProvider::GetNamespaceConfig(std::string_view nsName, NamespaceConfigData &data) { shared_lock lk(mtx_); auto it = namespacesData_.find(nsName); if (it == namespacesData_.end()) { - it = namespacesData_.find("*"); + it = namespacesData_.find(std::string_view("*")); } if (it == namespacesData_.end()) { data = {}; diff --git a/cpp_src/core/dbconfig.h b/cpp_src/core/dbconfig.h index cbb4b1bed..05d3745f3 100644 --- a/cpp_src/core/dbconfig.h +++ b/cpp_src/core/dbconfig.h @@ -2,7 +2,7 @@ #include #include -#include +#include "estl/fast_hash_map.h" #include "estl/fast_hash_set.h" #include "estl/mutex.h" #include "estl/shared_mutex.h" @@ -18,7 +18,13 @@ class JsonBuilder; class RdxContext; class WrSerializer; -enum ConfigType { ProfilingConf, NamespaceDataConf, ReplicationConf }; +enum ConfigType { + ProfilingConf = 0, + NamespaceDataConf, + ReplicationConf, + // + kConfigTypesTotalCount +}; class LongQueriesLoggingParams { public: @@ -133,7 +139,7 @@ class DBConfigProvider { void setHandler(ConfigType cfgType, std::function handler); ReplicationConfigData GetReplicationConfig(); - bool GetNamespaceConfig(const std::string &nsName, NamespaceConfigData &data); + bool GetNamespaceConfig(std::string_view nsName, NamespaceConfigData &data); LongQueriesLoggingParams GetSelectLoggingParams() const noexcept { return profilingData_.longSelectLoggingParams.load(std::memory_order_relaxed); } @@ -150,8 +156,8 @@ class DBConfigProvider { private: ProfilingConfigData profilingData_; ReplicationConfigData replicationData_; - std::unordered_map namespacesData_; - std::unordered_map> handlers_; + fast_hash_map namespacesData_; + std::array, kConfigTypesTotalCount> handlers_; shared_timed_mutex mtx_; }; diff --git a/cpp_src/core/defnsconfigs.h b/cpp_src/core/defnsconfigs.h index 36bcd02b4..702604c84 100644 --- a/cpp_src/core/defnsconfigs.h +++ b/cpp_src/core/defnsconfigs.h @@ -114,7 +114,6 @@ const std::vector kSystemNsDefs = { .AddIndex("last_sec_avg_lock_time_us", "-", "int64", IndexOpts().Dense()) .AddIndex("latency_stddev", "-", "double", IndexOpts().Dense()), NamespaceDef(kNamespacesNamespace, StorageOpts()).AddIndex(kNsNameField, "hash", "string", IndexOpts().PK()), - NamespaceDef(kPerfStatsNamespace, StorageOpts()).AddIndex(kNsNameField, "hash", "string", IndexOpts().PK()), NamespaceDef(kMemStatsNamespace, StorageOpts()) .AddIndex(kNsNameField, "hash", "string", IndexOpts().PK()) .AddIndex("items_count", "-", "int64", IndexOpts().Dense()) diff --git a/cpp_src/core/expressiontree.h b/cpp_src/core/expressiontree.h index 6f4147452..0eeaced61 100644 --- a/cpp_src/core/expressiontree.h +++ b/cpp_src/core/expressiontree.h @@ -283,9 +283,10 @@ class ExpressionTree { }; public: - Node() : storage_{SubTree{1}} {} + Node() : storage_{std::in_place_type, 1} {} template - Node(OperationType op, size_t s, Args&&... args) : storage_{SubTree{s, std::forward(args)...}}, operation{op} {} + Node(OperationType op, size_t s, Args&&... args) + : storage_{std::in_place_type, s, std::forward(args)...}, operation{op} {} template Node(OperationType op, T&& v) : storage_{std::forward(v)}, operation{op} {} Node(const Node& other) : storage_{other.storage_}, operation{other.operation} {} @@ -363,6 +364,10 @@ class ExpressionTree { void SetValue(T&& v) { storage_ = std::forward(v); } + template + void Emplace(Args&&... args) { + storage_.template emplace(std::forward(args)...); + } private: Storage storage_; @@ -443,6 +448,15 @@ class ExpressionTree { } container_.emplace_back(op, v); } + /// Appends value to the last openned subtree + template + void Append(OperationType op, Args&&... args) { + for (unsigned i : activeBrackets_) { + assertrx(i < container_.size()); + container_[i].Append(); + } + container_.emplace_back(op, T{std::forward(args)...}); + } class const_iterator; /// Appends all nodes from the interval to the last openned subtree void Append(const_iterator begin, const_iterator end) { diff --git a/cpp_src/core/idset.cc b/cpp_src/core/idset.cc index c8576fc94..da7047de7 100644 --- a/cpp_src/core/idset.cc +++ b/cpp_src/core/idset.cc @@ -1,18 +1,8 @@ #include "core/idset.h" -#include #include "tools/errors.h" namespace reindexer { -void IdSet::Commit() { - if (!size() && set_) { - resize(0); - for (auto id : *set_) push_back(id); - } - - usingBtree_.store(false, std::memory_order_release); -} - std::string IdSetPlain::Dump() const { std::string buf = "["; diff --git a/cpp_src/core/idset.h b/cpp_src/core/idset.h index 6c193f9c4..81522fcef 100644 --- a/cpp_src/core/idset.h +++ b/cpp_src/core/idset.h @@ -74,7 +74,7 @@ class IdSetPlain : protected base_idset { std::string Dump() const; protected: - IdSetPlain(base_idset &&idset) : base_idset(std::move(idset)) {} + IdSetPlain(base_idset &&idset) noexcept : base_idset(std::move(idset)) {} }; std::ostream &operator<<(std::ostream &, const IdSetPlain &); @@ -87,7 +87,7 @@ class IdSet : public IdSetPlain { public: using Ptr = intrusive_ptr>; - IdSet() : usingBtree_(false) {} + IdSet() noexcept : usingBtree_(false) {} IdSet(const IdSet &other) : IdSetPlain(other), set_(!other.set_ ? nullptr : new base_idsetset(*other.set_)), usingBtree_(other.usingBtree_.load()) {} IdSet(IdSet &&other) noexcept : IdSetPlain(std::move(other)), set_(std::move(other.set_)), usingBtree_(other.usingBtree_.load()) {} @@ -194,18 +194,24 @@ class IdSet : public IdSetPlain { base_idset::erase(d.first, d.second); return d.second - d.first; } else { - resize(0); + clear(); usingBtree_.store(true, std::memory_order_release); return set_->erase(id); } - return 0; } - void Commit(); - bool IsCommited() const { return !usingBtree_.load(std::memory_order_acquire); } - bool IsEmpty() const { return empty() && (!set_ || set_->empty()); } - size_t Size() const { return usingBtree_.load(std::memory_order_acquire) ? set_->size() : size(); } - size_t BTreeSize() const { return set_ ? sizeof(*set_.get()) + set_->size() * sizeof(int) : 0; } - const base_idsetset *BTree() const { return set_.get(); } + void Commit() { + if (!size() && set_) { + reserve(set_->size()); + for (auto id : *set_) push_back(id); + } + + usingBtree_.store(false, std::memory_order_release); + } + bool IsCommited() const noexcept { return !usingBtree_.load(std::memory_order_acquire); } + bool IsEmpty() const noexcept { return empty() && (!set_ || set_->empty()); } + size_t Size() const noexcept { return usingBtree_.load(std::memory_order_acquire) ? set_->size() : size(); } + size_t BTreeSize() const noexcept { return set_ ? sizeof(*set_.get()) + set_->size() * sizeof(int) : 0; } + const base_idsetset *BTree() const noexcept { return set_.get(); } void ReserveForSorted(int sortedIdxCount) { reserve(((set_ ? set_->size() : size())) * (sortedIdxCount + 1)); } protected: @@ -214,7 +220,7 @@ class IdSet : public IdSetPlain { template friend class BtreeIndexReverseIteratorImpl; - IdSet(base_idset &&idset) : IdSetPlain(std::move(idset)), usingBtree_(false) {} + IdSet(base_idset &&idset) noexcept : IdSetPlain(std::move(idset)), usingBtree_(false) {} std::unique_ptr set_; std::atomic usingBtree_; diff --git a/cpp_src/core/index/index.cc b/cpp_src/core/index/index.cc index 552033fc2..dcaeda32f 100644 --- a/cpp_src/core/index/index.cc +++ b/cpp_src/core/index/index.cc @@ -1,5 +1,4 @@ #include "index.h" -#include "core/namespacedef.h" #include "indexordered.h" #include "indextext/fastindextext.h" #include "indextext/fuzzyindextext.h" @@ -10,8 +9,8 @@ namespace reindexer { -Index::Index(const IndexDef& idef, PayloadType payloadType, const FieldsSet& fields) - : type_(idef.Type()), name_(idef.name_), opts_(idef.opts_), payloadType_(std::move(payloadType)), fields_(fields) { +Index::Index(const IndexDef& idef, PayloadType&& payloadType, FieldsSet&& fields) + : type_(idef.Type()), name_(idef.name_), opts_(idef.opts_), payloadType_(std::move(payloadType)), fields_(std::move(fields)) { logPrintf(LogTrace, "Index::Index ('%s',%s,%s) %s%s%s", idef.name_, idef.indexType_, idef.fieldType_, idef.opts_.IsPK() ? ",pk" : "", idef.opts_.IsDense() ? ",dense" : "", idef.opts_.IsArray() ? ",array" : ""); } @@ -28,38 +27,38 @@ Index::Index(const Index& obj) selectKeyType_(obj.selectKeyType_), sortedIdxCount_(obj.sortedIdxCount_) {} -std::unique_ptr Index::New(const IndexDef& idef, PayloadType payloadType, const FieldsSet& fields) { +std::unique_ptr Index::New(const IndexDef& idef, PayloadType&& payloadType, FieldsSet&& fields) { switch (idef.Type()) { case IndexStrBTree: case IndexIntBTree: case IndexDoubleBTree: case IndexInt64BTree: case IndexCompositeBTree: - return IndexOrdered_New(idef, std::move(payloadType), fields); + return IndexOrdered_New(idef, std::move(payloadType), std::move(fields)); case IndexStrHash: case IndexIntHash: case IndexInt64Hash: case IndexCompositeHash: - return IndexUnordered_New(idef, std::move(payloadType), fields); + return IndexUnordered_New(idef, std::move(payloadType), std::move(fields)); case IndexIntStore: case IndexStrStore: case IndexInt64Store: case IndexDoubleStore: case IndexBool: case IndexUuidStore: - return IndexStore_New(idef, std::move(payloadType), fields); + return IndexStore_New(idef, std::move(payloadType), std::move(fields)); case IndexFastFT: case IndexCompositeFastFT: - return FastIndexText_New(idef, std::move(payloadType), fields); + return FastIndexText_New(idef, std::move(payloadType), std::move(fields)); case IndexFuzzyFT: case IndexCompositeFuzzyFT: - return FuzzyIndexText_New(idef, std::move(payloadType), fields); + return FuzzyIndexText_New(idef, std::move(payloadType), std::move(fields)); case IndexTtl: - return TtlIndex_New(idef, std::move(payloadType), fields); + return TtlIndex_New(idef, std::move(payloadType), std::move(fields)); case ::IndexRTree: - return IndexRTree_New(idef, std::move(payloadType), fields); + return IndexRTree_New(idef, std::move(payloadType), std::move(fields)); case IndexUuidHash: - return IndexUuid_New(idef, std::move(payloadType), fields); + return IndexUuid_New(idef, std::move(payloadType), std::move(fields)); } throw Error(errParams, "Ivalid index type %d for index '%s'", idef.Type(), idef.name_); } diff --git a/cpp_src/core/index/index.h b/cpp_src/core/index/index.h index 136a97fdf..b1d3f2d24 100644 --- a/cpp_src/core/index/index.h +++ b/cpp_src/core/index/index.h @@ -12,7 +12,6 @@ #include "core/payload/payloadiface.h" #include "core/perfstatcounter.h" #include "core/selectkeyresult.h" -#include "core/type_consts_helpers.h" #include "ft_preselect.h" #include "indexiterator.h" @@ -46,7 +45,7 @@ class Index { using KeyEntry = reindexer::KeyEntry; using KeyEntryPlain = reindexer::KeyEntry; - Index(const IndexDef& idef, PayloadType payloadType, const FieldsSet& fields); + Index(const IndexDef& idef, PayloadType&& payloadType, FieldsSet&& fields); Index(const Index&); Index& operator=(const Index&) = delete; virtual ~Index() = default; @@ -82,20 +81,23 @@ class Index { virtual bool IsDestroyPartSupported() const noexcept { return false; } virtual void AddDestroyTask(tsl::detail_sparse_hash::ThreadTaskQueue&) {} - const PayloadType& GetPayloadType() const { return payloadType_; } - void UpdatePayloadType(PayloadType payloadType) { payloadType_ = std::move(payloadType); } + const PayloadType& GetPayloadType() const& { return payloadType_; } + const PayloadType& GetPayloadType() const&& = delete; + void UpdatePayloadType(PayloadType&& payloadType) { payloadType_ = std::move(payloadType); } - static std::unique_ptr New(const IndexDef& idef, PayloadType payloadType, const FieldsSet& fields_); + static std::unique_ptr New(const IndexDef& idef, PayloadType&& payloadType, FieldsSet&& fields_); KeyValueType KeyType() const { return keyType_; } KeyValueType SelectKeyType() const { return selectKeyType_; } - const FieldsSet& Fields() const { return fields_; } - const std::string& Name() const { return name_; } + const FieldsSet& Fields() const& noexcept { return fields_; } + const FieldsSet& Fields() const&& = delete; + const std::string& Name() const& noexcept { return name_; } + const std::string& Name() const&& = delete; IndexType Type() const { return type_; } const std::vector& SortOrders() const { return sortOrders_; } const IndexOpts& Opts() const { return opts_; } virtual void SetOpts(const IndexOpts& opts) { opts_ = opts; } - virtual void SetFields(FieldsSet&& fields) { fields_ = std::move(fields); } + void SetFields(FieldsSet&& fields) { fields_ = std::move(fields); } [[nodiscard]] SortType SortId() const noexcept { return sortId_; } virtual void SetSortedIdxCount(int sortedIdxCount) { sortedIdxCount_ = sortedIdxCount; } virtual FtMergeStatuses GetFtMergeStatuses(const RdxContext&) { @@ -140,8 +142,12 @@ class Index { IndexOpts opts_; // Payload type of items mutable PayloadType payloadType_; - // Fields in index. Valid only for composite indexes + +private: + // Fields in index FieldsSet fields_; + +protected: // Perfstat counter PerfStatCounterMT commitPerfCounter_; PerfStatCounterMT selectPerfCounter_; diff --git a/cpp_src/core/index/indexordered.cc b/cpp_src/core/index/indexordered.cc index 828030b1b..70ae1be0f 100644 --- a/cpp_src/core/index/indexordered.cc +++ b/cpp_src/core/index/indexordered.cc @@ -56,10 +56,6 @@ SelectKeyResults IndexOrdered::SelectKey(const VariantArray &keys, CondType c return IndexUnordered::SelectKey(keys, condition, sortId, opts, ctx, rdxCtx); } - if (keys.size() < 1) { - throw Error(errParams, "For condition required at least 1 argument, but provided 0"); - } - SelectKeyResult res; auto startIt = this->idx_map.begin(); auto endIt = this->idx_map.end(); @@ -81,7 +77,6 @@ SelectKeyResults IndexOrdered::SelectKey(const VariantArray &keys, CondType c if (startIt == this->idx_map.end()) startIt = this->idx_map.upper_bound(static_cast(key1)); break; case CondRange: { - if (keys.size() != 2) throw Error(errParams, "For ranged query reuqired 2 arguments, but provided %d", keys.size()); const auto &key2 = keys[1]; startIt = this->idx_map.find(static_cast(key1)); @@ -207,18 +202,19 @@ IndexIterator::Ptr IndexOrdered::CreateIterator() const { } template -static std::unique_ptr IndexOrdered_New(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) { +static std::unique_ptr IndexOrdered_New(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields) { switch (idef.Type()) { case IndexIntBTree: - return std::unique_ptr{new IndexOrdered>(idef, std::move(payloadType), fields)}; + return std::unique_ptr{new IndexOrdered>(idef, std::move(payloadType), std::move(fields))}; case IndexInt64BTree: - return std::unique_ptr{new IndexOrdered>(idef, std::move(payloadType), fields)}; + return std::unique_ptr{ + new IndexOrdered>(idef, std::move(payloadType), std::move(fields))}; case IndexStrBTree: - return std::unique_ptr{new IndexOrdered>(idef, std::move(payloadType), fields)}; + return std::unique_ptr{new IndexOrdered>(idef, std::move(payloadType), std::move(fields))}; case IndexDoubleBTree: - return std::unique_ptr{new IndexOrdered>(idef, std::move(payloadType), fields)}; + return std::unique_ptr{new IndexOrdered>(idef, std::move(payloadType), std::move(fields))}; case IndexCompositeBTree: - return std::unique_ptr{new IndexOrdered>(idef, std::move(payloadType), fields)}; + return std::unique_ptr{new IndexOrdered>(idef, std::move(payloadType), std::move(fields))}; case IndexStrHash: case IndexIntHash: case IndexInt64Hash: @@ -242,9 +238,10 @@ static std::unique_ptr IndexOrdered_New(const IndexDef &idef, PayloadType } // NOLINTBEGIN(*cplusplus.NewDeleteLeaks) -std::unique_ptr IndexOrdered_New(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) { - return (idef.opts_.IsPK() || idef.opts_.IsDense()) ? IndexOrdered_New(idef, std::move(payloadType), fields) - : IndexOrdered_New(idef, std::move(payloadType), fields); +std::unique_ptr IndexOrdered_New(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields) { + return (idef.opts_.IsPK() || idef.opts_.IsDense()) + ? IndexOrdered_New(idef, std::move(payloadType), std::move(fields)) + : IndexOrdered_New(idef, std::move(payloadType), std::move(fields)); } // NOLINTEND(*cplusplus.NewDeleteLeaks) diff --git a/cpp_src/core/index/indexordered.h b/cpp_src/core/index/indexordered.h index e8dc6d229..96230f7cb 100644 --- a/cpp_src/core/index/indexordered.h +++ b/cpp_src/core/index/indexordered.h @@ -10,8 +10,8 @@ class IndexOrdered : public IndexUnordered { using ref_type = typename IndexUnordered::ref_type; using key_type = typename IndexUnordered::key_type; - IndexOrdered(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) - : IndexUnordered(idef, std::move(payloadType), fields) {} + IndexOrdered(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields) + : IndexUnordered(idef, std::move(payloadType), std::move(fields)) {} SelectKeyResults SelectKey(const VariantArray &keys, CondType condition, SortType stype, Index::SelectOpts opts, const BaseFunctionCtx::Ptr &ctx, const RdxContext &) override; @@ -22,6 +22,6 @@ class IndexOrdered : public IndexUnordered { bool IsOrdered() const noexcept override { return true; } }; -std::unique_ptr IndexOrdered_New(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields); +std::unique_ptr IndexOrdered_New(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields); } // namespace reindexer diff --git a/cpp_src/core/index/indexstore.cc b/cpp_src/core/index/indexstore.cc index 27fe6239f..0ac5d6594 100644 --- a/cpp_src/core/index/indexstore.cc +++ b/cpp_src/core/index/indexstore.cc @@ -7,8 +7,8 @@ namespace reindexer { template <> -IndexStore::IndexStore(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) - : Index(idef, std::move(payloadType), fields) { +IndexStore::IndexStore(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields) + : Index(idef, std::move(payloadType), std::move(fields)) { keyType_ = selectKeyType_ = KeyValueType::Double{}; opts_.Array(true); } @@ -68,8 +68,8 @@ Variant IndexStore::Upsert(const Variant &key, IdType /*id*/, bool template Variant IndexStore::Upsert(const Variant &key, IdType id, bool & /*clearCache*/) { - if (!opts_.IsArray() && !opts_.IsDense() && !opts_.IsSparse() && key.Type().Is()) { - idx_data.resize(std::max(id + 1, int(idx_data.size()))); + if (!opts_.IsArray() && !opts_.IsDense() && !opts_.IsSparse() && !key.Type().Is()) { + idx_data.resize(std::max(id + 1, IdType(idx_data.size()))); idx_data[id] = static_cast(key); } return Variant(key); @@ -106,43 +106,8 @@ SelectKeyResults IndexStore::SelectKey(const VariantArray &keys, CondType con if (condition == CondAny && !this->opts_.IsArray() && !this->opts_.IsSparse() && !sopts.distinct) throw Error(errParams, "The 'NOT NULL' condition is suported only by 'sparse' or 'array' indexes"); - // TODO: it may be necessary to remove or change this switch after QueryEntry refactoring - switch (condition) { - case CondAny: - if (!this->opts_.IsArray() && !this->opts_.IsSparse() && !sopts.distinct) { - throw Error(errParams, "The 'NOT NULL' condition is suported only by 'sparse' or 'array' indexes"); - } - break; - case CondEmpty: - if (!this->opts_.IsArray() && !this->opts_.IsSparse()) { - throw Error(errParams, "The 'is NULL' condition is suported only by 'sparse' or 'array' indexes"); - } - break; - case CondAllSet: - case CondSet: - case CondEq: - break; - case CondRange: - case CondDWithin: - if (keys.size() != 2) { - throw Error(errParams, "For condition %s required exactly 2 arguments, but provided %d", CondTypeToStr(condition), - keys.size()); - } - break; - case CondLt: - case CondLe: - case CondGt: - case CondGe: - case CondLike: - if (keys.size() != 1) { - throw Error(errParams, "For condition %s required exactly 1 argument, but provided %d", CondTypeToStr(condition), - keys.size()); - } - break; - } - - res.comparators_.push_back(Comparator(condition, KeyType(), keys, opts_.IsArray(), sopts.distinct, payloadType_, fields_, - idx_data.size() ? idx_data.data() : nullptr, opts_.collateOpts_)); + res.comparators_.emplace_back(condition, KeyType(), keys, opts_.IsArray(), bool(sopts.distinct), payloadType_, Fields(), + idx_data.size() ? idx_data.data() : nullptr, opts_.collateOpts_); return SelectKeyResults(std::move(res)); } @@ -183,20 +148,20 @@ void IndexStore::AddDestroyTask(tsl::detail_sparse_hash::ThreadTaskQueue &q) (void)q; } -std::unique_ptr IndexStore_New(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) { +std::unique_ptr IndexStore_New(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields) { switch (idef.Type()) { case IndexBool: - return std::unique_ptr{new IndexStore(idef, std::move(payloadType), fields)}; + return std::unique_ptr{new IndexStore(idef, std::move(payloadType), std::move(fields))}; case IndexIntStore: - return std::unique_ptr{new IndexStore(idef, std::move(payloadType), fields)}; + return std::unique_ptr{new IndexStore(idef, std::move(payloadType), std::move(fields))}; case IndexInt64Store: - return std::unique_ptr{new IndexStore(idef, std::move(payloadType), fields)}; + return std::unique_ptr{new IndexStore(idef, std::move(payloadType), std::move(fields))}; case IndexDoubleStore: - return std::unique_ptr{new IndexStore(idef, std::move(payloadType), fields)}; + return std::unique_ptr{new IndexStore(idef, std::move(payloadType), std::move(fields))}; case IndexStrStore: - return std::unique_ptr{new IndexStore(idef, std::move(payloadType), fields)}; + return std::unique_ptr{new IndexStore(idef, std::move(payloadType), std::move(fields))}; case IndexUuidStore: - return std::unique_ptr{new IndexStore(idef, std::move(payloadType), fields)}; + return std::unique_ptr{new IndexStore(idef, std::move(payloadType), std::move(fields))}; case IndexStrHash: case IndexStrBTree: case IndexIntBTree: diff --git a/cpp_src/core/index/indexstore.h b/cpp_src/core/index/indexstore.h index 832c11a4f..457ae89e5 100644 --- a/cpp_src/core/index/indexstore.h +++ b/cpp_src/core/index/indexstore.h @@ -8,7 +8,8 @@ namespace reindexer { template class IndexStore : public Index { public: - IndexStore(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) : Index(idef, std::move(payloadType), fields) { + IndexStore(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields) + : Index(idef, std::move(payloadType), std::move(fields)) { static T a; keyType_ = selectKeyType_ = Variant(a).Type(); } @@ -46,8 +47,8 @@ class IndexStore : public Index { }; template <> -IndexStore::IndexStore(const IndexDef &, PayloadType, const FieldsSet &); +IndexStore::IndexStore(const IndexDef &, PayloadType &&, FieldsSet &&); -std::unique_ptr IndexStore_New(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields_); +std::unique_ptr IndexStore_New(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&); } // namespace reindexer diff --git a/cpp_src/core/index/indextext/fastindextext.cc b/cpp_src/core/index/indextext/fastindextext.cc index 36db2dc49..d69307131 100644 --- a/cpp_src/core/index/indextext/fastindextext.cc +++ b/cpp_src/core/index/indextext/fastindextext.cc @@ -135,7 +135,7 @@ IdSet::Ptr FastIndexText::Select(FtCtx::Ptr fctx, FtDSLQuery &&dsl, bool inTr fctx->GetData()->extraWordSymbols_ = this->getConfig()->extraWordSymbols; fctx->GetData()->isWordPositions_ = true; - auto mergeData = this->holder_->Select(std::move(dsl), this->fields_.size(), fctx->NeedArea(), getConfig()->maxAreasInDoc, + auto mergeData = this->holder_->Select(std::move(dsl), this->Fields().size(), fctx->NeedArea(), getConfig()->maxAreasInDoc, inTransaction, std::move(statuses.statuses), useExternSt, rdxCtx); // convert vids(uniq documents id) to ids (real ids) IdSet::Ptr mergedIds = make_intrusive>(); @@ -232,7 +232,7 @@ void FastIndexText::commitFulltextImpl() { } auto tm1 = high_resolution_clock::now(); - this->holder_->Process(this->fields_.size(), !this->opts_.IsDense()); + this->holder_->Process(this->Fields().size(), !this->opts_.IsDense()); if (this->holder_->NeedClear(this->tracker_.isCompleteUpdated())) { this->tracker_.clear(); } @@ -373,12 +373,14 @@ reindexer::FtPreselectT FastIndexText::FtPreselect(const RdxContext &rdxCtx) std::vector(holder_->rowId2Vdoc_.size(), false), &holder_->rowId2Vdoc_}; } -std::unique_ptr FastIndexText_New(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) { +std::unique_ptr FastIndexText_New(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields) { switch (idef.Type()) { case IndexFastFT: - return std::unique_ptr{new FastIndexText>(idef, std::move(payloadType), fields)}; + return std::unique_ptr{ + new FastIndexText>(idef, std::move(payloadType), std::move(fields))}; case IndexCompositeFastFT: - return std::unique_ptr{new FastIndexText>(idef, std::move(payloadType), fields)}; + return std::unique_ptr{ + new FastIndexText>(idef, std::move(payloadType), std::move(fields))}; case IndexStrHash: case IndexStrBTree: case IndexIntBTree: diff --git a/cpp_src/core/index/indextext/fastindextext.h b/cpp_src/core/index/indextext/fastindextext.h index 5a6ebb50d..6a94d3f21 100644 --- a/cpp_src/core/index/indextext/fastindextext.h +++ b/cpp_src/core/index/indextext/fastindextext.h @@ -2,8 +2,6 @@ #include "core/ft/config/ftfastconfig.h" #include "core/ft/ft_fast/dataholder.h" -#include "core/ft/ft_fast/dataprocessor.h" -#include "core/ft/typos.h" #include "indextext.h" namespace reindexer { @@ -22,7 +20,8 @@ class FastIndexText : public IndexText { this->CommitFulltext(); } - FastIndexText(const IndexDef& idef, PayloadType payloadType, const FieldsSet& fields) : Base(idef, std::move(payloadType), fields) { + FastIndexText(const IndexDef& idef, PayloadType&& payloadType, FieldsSet&& fields) + : Base(idef, std::move(payloadType), std::move(fields)) { initConfig(); } std::unique_ptr Clone() const override { return std::unique_ptr{new FastIndexText(*this)}; } @@ -40,7 +39,7 @@ class FastIndexText : public IndexText { reindexer::FtPreselectT FtPreselect(const RdxContext& rdxCtx) override final; bool EnablePreselectBeforeFt() const override final { return getConfig()->enablePreselectBeforeFt; } -protected: +private: void commitFulltextImpl() override final; FtFastConfig* getConfig() const noexcept { return dynamic_cast(this->cfg_.get()); } void initConfig(const FtFastConfig* = nullptr); @@ -53,6 +52,6 @@ class FastIndexText : public IndexText { std::unique_ptr holder_; }; -std::unique_ptr FastIndexText_New(const IndexDef& idef, PayloadType payloadType, const FieldsSet& fields); +std::unique_ptr FastIndexText_New(const IndexDef& idef, PayloadType&& payloadType, FieldsSet&& fields); } // namespace reindexer diff --git a/cpp_src/core/index/indextext/fieldsgetter.h b/cpp_src/core/index/indextext/fieldsgetter.h index 791932b27..3af05f79f 100644 --- a/cpp_src/core/index/indextext/fieldsgetter.h +++ b/cpp_src/core/index/indextext/fieldsgetter.h @@ -1,6 +1,5 @@ #pragma once #include "core/ft/usingcontainer.h" -#include "core/index/payload_map.h" #include "core/payload/fieldsset.h" #include "vendor/utf8cpp/utf8.h" diff --git a/cpp_src/core/index/indextext/fuzzyindextext.cc b/cpp_src/core/index/indextext/fuzzyindextext.cc index 339ac1252..9ccba834f 100644 --- a/cpp_src/core/index/indextext/fuzzyindextext.cc +++ b/cpp_src/core/index/indextext/fuzzyindextext.cc @@ -65,13 +65,14 @@ void FuzzyIndexText::CreateConfig(const FtFuzzyConfig* cfg) { this->cfg_->parse(this->opts_.config, this->ftFields_); } -std::unique_ptr FuzzyIndexText_New(const IndexDef& idef, PayloadType payloadType, const FieldsSet& fields) { +std::unique_ptr FuzzyIndexText_New(const IndexDef& idef, PayloadType&& payloadType, FieldsSet&& fields) { switch (idef.Type()) { case IndexFuzzyFT: - return std::unique_ptr{new FuzzyIndexText>(idef, std::move(payloadType), fields)}; + return std::unique_ptr{ + new FuzzyIndexText>(idef, std::move(payloadType), std::move(fields))}; case IndexCompositeFuzzyFT: return std::unique_ptr{ - new FuzzyIndexText>(idef, std::move(payloadType), fields)}; + new FuzzyIndexText>(idef, std::move(payloadType), std::move(fields))}; case IndexStrHash: case IndexStrBTree: case IndexIntBTree: diff --git a/cpp_src/core/index/indextext/fuzzyindextext.h b/cpp_src/core/index/indextext/fuzzyindextext.h index d1a94780a..b536e4ef0 100644 --- a/cpp_src/core/index/indextext/fuzzyindextext.h +++ b/cpp_src/core/index/indextext/fuzzyindextext.h @@ -1,9 +1,6 @@ #pragma once -#include "core/ft/config/ftfastconfig.h" #include "core/ft/ft_fuzzy/searchengine.h" -#include "core/ft/ftsetcashe.h" -#include "core/ft/idrelset.h" #include "indextext.h" namespace reindexer { @@ -15,7 +12,8 @@ class FuzzyIndexText : public IndexText { public: FuzzyIndexText(const FuzzyIndexText& other) : Base(other) { CreateConfig(other.getConfig()); } - FuzzyIndexText(const IndexDef& idef, PayloadType payloadType, const FieldsSet& fields) : Base(idef, std::move(payloadType), fields) { + FuzzyIndexText(const IndexDef& idef, PayloadType&& payloadType, FieldsSet&& fields) + : Base(idef, std::move(payloadType), std::move(fields)) { CreateConfig(); } @@ -49,6 +47,6 @@ class FuzzyIndexText : public IndexText { std::vector vdocs_; }; -std::unique_ptr FuzzyIndexText_New(const IndexDef& idef, PayloadType payloadType, const FieldsSet& fields); +std::unique_ptr FuzzyIndexText_New(const IndexDef& idef, PayloadType&& payloadType, FieldsSet&& fields); } // namespace reindexer diff --git a/cpp_src/core/index/indextext/indextext.cc b/cpp_src/core/index/indextext/indextext.cc index 4a8a646db..5748ffe7a 100644 --- a/cpp_src/core/index/indextext/indextext.cc +++ b/cpp_src/core/index/indextext/indextext.cc @@ -19,16 +19,17 @@ void IndexText::initSearchers() { size_t jsonPathIdx = 0; if (this->payloadType_) { - for (unsigned i = 0; i < this->fields_.size(); i++) { - auto fieldIdx = this->fields_[i]; + const auto &fields = this->Fields(); + for (unsigned i = 0, s = fields.size(); i < s; i++) { + auto fieldIdx = fields[i]; if (fieldIdx == IndexValueType::SetByJsonPath) { - assertrx(jsonPathIdx < this->fields_.getJsonPathsLength()); - ftFields_.emplace(this->fields_.getJsonPath(jsonPathIdx++), i); + assertrx(jsonPathIdx < fields.getJsonPathsLength()); + ftFields_.emplace(fields.getJsonPath(jsonPathIdx++), i); } else { ftFields_.emplace(this->payloadType_->Field(fieldIdx).Name(), i); } } - if rx_unlikely (ftFields_.size() != this->fields_.size()) { + if rx_unlikely (ftFields_.size() != fields.size()) { throw Error(errParams, "Composite fulltext index '%s' contains duplicated fields", this->name_); } if rx_unlikely (ftFields_.size() > kMaxFtCompositeFields) { @@ -178,7 +179,7 @@ SelectKeyResults IndexText::SelectKey(const VariantArray &keys, CondType cond template FieldsGetter IndexText::Getter() { - return FieldsGetter(this->fields_, this->payloadType_, this->KeyType()); + return FieldsGetter(this->Fields(), this->payloadType_, this->KeyType()); } template class IndexText>; diff --git a/cpp_src/core/index/indextext/indextext.h b/cpp_src/core/index/indextext/indextext.h index d09c61a26..07c4061ef 100644 --- a/cpp_src/core/index/indextext/indextext.h +++ b/cpp_src/core/index/indextext/indextext.h @@ -8,7 +8,6 @@ #include "core/ft/ftsetcashe.h" #include "core/index/indexunordered.h" #include "core/selectfunc/ctx/ftctx.h" -#include "estl/fast_hash_map.h" #include "estl/shared_mutex.h" #include "fieldsgetter.h" @@ -20,8 +19,8 @@ class IndexText : public IndexUnordered { public: IndexText(const IndexText& other); - IndexText(const IndexDef& idef, PayloadType payloadType, const FieldsSet& fields) - : IndexUnordered(idef, std::move(payloadType), fields), cache_ft_(std::make_shared()) { + IndexText(const IndexDef& idef, PayloadType&& payloadType, FieldsSet&& fields) + : IndexUnordered(idef, std::move(payloadType), std::move(fields)), cache_ft_(std::make_shared()) { this->selectKeyType_ = KeyValueType::String{}; initSearchers(); } diff --git a/cpp_src/core/index/indexunordered.cc b/cpp_src/core/index/indexunordered.cc index b311fe25f..8ea5c9e82 100644 --- a/cpp_src/core/index/indexunordered.cc +++ b/cpp_src/core/index/indexunordered.cc @@ -3,7 +3,6 @@ #include "core/index/payload_map.h" #include "core/index/string_map.h" #include "core/indexdef.h" -#include "core/keyvalue/uuid.h" #include "core/rdxcontext.h" #include "rtree/greenesplitter.h" #include "rtree/linearsplitter.h" @@ -18,55 +17,52 @@ namespace reindexer { constexpr int kMaxIdsForDistinct = 500; template -IndexUnordered::IndexUnordered(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) - : Base(idef, std::move(payloadType), fields), idx_map() { +IndexUnordered::IndexUnordered(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields) + : Base(idef, std::move(payloadType), std::move(fields)), idx_map() { static_assert(!(is_str_map_v || is_payload_map_v)); } template <> -IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) - : Base(idef, std::move(payloadType), fields), idx_map(idef.opts_.collateOpts_) {} +IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields) + : Base(idef, std::move(payloadType), std::move(fields)), idx_map(idef.opts_.collateOpts_) {} template <> -IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) - : Base(idef, std::move(payloadType), fields), idx_map(idef.opts_.collateOpts_) {} +IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields) + : Base(idef, std::move(payloadType), std::move(fields)), idx_map(idef.opts_.collateOpts_) {} template <> -IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) - : Base(idef, std::move(payloadType), fields), idx_map(idef.opts_.collateOpts_) {} +IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields) + : Base(idef, std::move(payloadType), std::move(fields)), idx_map(idef.opts_.collateOpts_) {} template <> -IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType payloadType, - const FieldsSet &fields) - : Base(idef, std::move(payloadType), fields), idx_map(idef.opts_.collateOpts_) {} +IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields) + : Base(idef, std::move(payloadType), std::move(fields)), idx_map(idef.opts_.collateOpts_) {} template <> -IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) - : Base(idef, std::move(payloadType), fields), idx_map(idef.opts_.collateOpts_) {} +IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields) + : Base(idef, std::move(payloadType), std::move(fields)), idx_map(idef.opts_.collateOpts_) {} template <> -IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType payloadType, - const FieldsSet &fields) - : Base(idef, payloadType, fields), idx_map(std::move(payloadType), Base::Fields()) {} +IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields) + : Base(idef, std::move(payloadType), std::move(fields)), idx_map(PayloadType{Base::GetPayloadType()}, FieldsSet{Base::Fields()}) {} template <> -IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType payloadType, - const FieldsSet &fields) - : Base(idef, payloadType, fields), idx_map(std::move(payloadType), Base::Fields()) {} +IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType &&payloadType, + FieldsSet &&fields) + : Base(idef, std::move(payloadType), std::move(fields)), idx_map(PayloadType{Base::GetPayloadType()}, FieldsSet{Base::Fields()}) {} template <> -IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType payloadType, - const FieldsSet &fields) - : Base(idef, payloadType, fields), idx_map(std::move(payloadType), Base::Fields()) {} +IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType &&payloadType, + FieldsSet &&fields) + : Base(idef, std::move(payloadType), std::move(fields)), idx_map(PayloadType{Base::GetPayloadType()}, FieldsSet{Base::Fields()}) {} template <> -IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) - : Base(idef, payloadType, fields), idx_map(std::move(payloadType), Base::Fields()) {} +IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields) + : Base(idef, std::move(payloadType), std::move(fields)), idx_map(Base::GetPayloadType(), Base::Fields()) {} template <> -IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType payloadType, - const FieldsSet &fields) - : Base(idef, payloadType, fields), idx_map(std::move(payloadType), Base::Fields()) {} +IndexUnordered>::IndexUnordered(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields) + : Base(idef, std::move(payloadType), std::move(fields)), idx_map(Base::GetPayloadType(), Base::Fields()) {} template bool IndexUnordered::HoldsStrings() const noexcept { @@ -184,8 +180,8 @@ void IndexUnordered::Delete(const Variant &key, IdType id, StringsHolder &str clearCache = true; // TODO: we have to implement removal of composite indexes (doesn't work right now) assertf(this->opts_.IsArray() || this->Opts().IsSparse() || delcnt, "Delete unexists id from index '%s' id=%d,key=%s (%s)", this->name_, - id, key.As(this->payloadType_, this->fields_), - Variant(keyIt->first).As(this->payloadType_, this->fields_)); + id, key.As(this->payloadType_, this->Fields()), + Variant(keyIt->first).As(this->payloadType_, this->Fields())); if (keyIt->second.Unsorted().IsEmpty()) { this->tracker_.markDeleted(keyIt); @@ -434,17 +430,20 @@ void IndexUnordered::AddDestroyTask(tsl::detail_sparse_hash::ThreadTaskQueue } template -static std::unique_ptr IndexUnordered_New(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) { +static std::unique_ptr IndexUnordered_New(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields) { switch (idef.Type()) { case IndexIntHash: - return std::unique_ptr{new IndexUnordered>(idef, std::move(payloadType), fields)}; + return std::unique_ptr{ + new IndexUnordered>(idef, std::move(payloadType), std::move(fields))}; case IndexInt64Hash: return std::unique_ptr{ - new IndexUnordered>(idef, std::move(payloadType), fields)}; + new IndexUnordered>(idef, std::move(payloadType), std::move(fields))}; case IndexStrHash: - return std::unique_ptr{new IndexUnordered>(idef, std::move(payloadType), fields)}; + return std::unique_ptr{ + new IndexUnordered>(idef, std::move(payloadType), std::move(fields))}; case IndexCompositeHash: - return std::unique_ptr{new IndexUnordered>(idef, std::move(payloadType), fields)}; + return std::unique_ptr{ + new IndexUnordered>(idef, std::move(payloadType), std::move(fields))}; case IndexStrBTree: case IndexIntBTree: case IndexInt64BTree: @@ -469,9 +468,10 @@ static std::unique_ptr IndexUnordered_New(const IndexDef &idef, PayloadTy } // NOLINTBEGIN(*cplusplus.NewDeleteLeaks) -std::unique_ptr IndexUnordered_New(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) { - return (idef.opts_.IsPK() || idef.opts_.IsDense()) ? IndexUnordered_New(idef, std::move(payloadType), fields) - : IndexUnordered_New(idef, std::move(payloadType), fields); +std::unique_ptr IndexUnordered_New(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields) { + return (idef.opts_.IsPK() || idef.opts_.IsDense()) + ? IndexUnordered_New(idef, std::move(payloadType), std::move(fields)) + : IndexUnordered_New(idef, std::move(payloadType), std::move(fields)); } // NOLINTEND(*cplusplus.NewDeleteLeaks) diff --git a/cpp_src/core/index/indexunordered.h b/cpp_src/core/index/indexunordered.h index 05cc89d95..987af9712 100644 --- a/cpp_src/core/index/indexunordered.h +++ b/cpp_src/core/index/indexunordered.h @@ -26,7 +26,7 @@ class IndexUnordered : public IndexStore> { typename T::key_type>::type>::type>::type; using key_type = StoreIndexKeyType; - IndexUnordered(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields); + IndexUnordered(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields); IndexUnordered(const IndexUnordered &other); Variant Upsert(const Variant &key, IdType id, bool &chearCache) override; @@ -72,6 +72,6 @@ class IndexUnordered : public IndexStore> { constexpr inline unsigned maxSelectivityPercentForIdset() noexcept { return 30u; } -std::unique_ptr IndexUnordered_New(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields); +std::unique_ptr IndexUnordered_New(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields); } // namespace reindexer diff --git a/cpp_src/core/index/payload_map.h b/cpp_src/core/index/payload_map.h index 91132989f..357b671a3 100644 --- a/cpp_src/core/index/payload_map.h +++ b/cpp_src/core/index/payload_map.h @@ -12,7 +12,7 @@ namespace reindexer { class PayloadValueWithHash : public PayloadValue { public: PayloadValueWithHash() noexcept : PayloadValue() {} - PayloadValueWithHash(PayloadValue pv, const PayloadType &pt, const FieldsSet &fields) + PayloadValueWithHash(PayloadValue &&pv, const PayloadType &pt, const FieldsSet &fields) : PayloadValue(std::move(pv)), hash_(ConstPayload(pt, *static_cast(this)).GetHash(fields)) {} PayloadValueWithHash(const PayloadValueWithHash &o) noexcept : PayloadValue(o), hash_(o.hash_) {} PayloadValueWithHash(PayloadValueWithHash &&o) noexcept : PayloadValue(std::move(o)), hash_(o.hash_) {} @@ -29,7 +29,8 @@ class PayloadValueWithHash : public PayloadValue { struct equal_composite { using is_transparent = void; - equal_composite(PayloadType type, const FieldsSet &fields) : type_(std::move(type)), fields_(fields) {} + template + equal_composite(PT &&type, FS &&fields) : type_(std::forward(type)), fields_(std::forward(fields)) {} bool operator()(const PayloadValue &lhs, const PayloadValue &rhs) const { assertrx(type_); return ConstPayload(type_, lhs).IsEQ(rhs, fields_); @@ -50,7 +51,8 @@ struct equal_composite { FieldsSet fields_; }; struct hash_composite { - hash_composite(PayloadType type, const FieldsSet &fields) : type_(std::move(type)), fields_(fields) {} + template + hash_composite(PT &&type, FS &&fields) : type_(std::forward(type)), fields_(std::forward(fields)) {} size_t operator()(const PayloadValueWithHash &s) const { return s.GetHash(); } size_t operator()(const PayloadValue &s) const { assertrx(type_); @@ -61,7 +63,7 @@ struct hash_composite { }; struct less_composite { - less_composite(PayloadType type, const FieldsSet &fields) : type_(std::move(type)), fields_(fields) {} + less_composite(PayloadType &&type, FieldsSet &&fields) : type_(std::move(type)), fields_(std::move(fields)) {} bool operator()(const PayloadValue &lhs, const PayloadValue &rhs) const { assertrx(type_); assertrx(!lhs.IsFree()); @@ -78,7 +80,7 @@ class payload_str_fields_helper; template <> class payload_str_fields_helper { protected: - payload_str_fields_helper(PayloadType payloadType, const FieldsSet &fields) : payload_type_(std::move(payloadType)) { + payload_str_fields_helper(PayloadType &&payloadType, const FieldsSet &fields) : payload_type_(std::move(payloadType)) { if (fields.getTagsPathsLength() || fields.getJsonPathsLength()) { str_fields_.push_back(0); } @@ -153,13 +155,13 @@ class unordered_payload_map static_assert(std::is_nothrow_move_constructible>::value, "Nothrow movebale key and value required"); - unordered_payload_map(size_t size, PayloadType pt, const FieldsSet &f) - : base_hash_map(size, hash_composite(pt, f), equal_composite(pt, f)), - payload_str_fields_helper(pt, f), + unordered_payload_map(size_t size, PayloadType &&pt, FieldsSet &&f) + : base_hash_map(size, hash_composite(PayloadType{pt}, FieldsSet{f}), equal_composite(PayloadType{pt}, FieldsSet{f})), + payload_str_fields_helper(PayloadType{pt}, f), payloadType_(std::move(pt)), - fields_(f) {} + fields_(std::move(f)) {} - unordered_payload_map(PayloadType pt, const FieldsSet &f) : unordered_payload_map(1000, std::move(pt), f) {} + unordered_payload_map(PayloadType &&pt, FieldsSet &&f) : unordered_payload_map(1000, std::move(pt), std::move(f)) {} unordered_payload_map(const unordered_payload_map &other) : base_hash_map(other), payload_str_fields_helper(other), payloadType_(other.payloadType_), fields_(other.fields_) { @@ -185,7 +187,7 @@ class unordered_payload_map } template std::pair emplace(const PayloadValue &pl, V &&v) { - PayloadValueWithHash key(pl, payloadType_, fields_); + PayloadValueWithHash key(PayloadValue{pl}, payloadType_, fields_); auto res = base_hash_map::emplace(std::move(key), std::forward(v)); if (res.second) this->add_ref(res.first->first); return res; @@ -207,7 +209,7 @@ class unordered_payload_map } T1 &operator[](const PayloadValue &k) { - PayloadValueWithHash key(k, payloadType_, fields_); + PayloadValueWithHash key(PayloadValue{k}, payloadType_, fields_); return base_hash_map::operator[](std::move(key)); } T1 &operator[](PayloadValue &&k) { @@ -247,7 +249,8 @@ class payload_map : private btree::btree_map, using payload_str_fields_helper::have_str_fields; payload_map(PayloadType payloadType, const FieldsSet &fields) - : base_tree_map(less_composite(payloadType, fields)), payload_str_fields_helper(std::move(payloadType), fields) {} + : base_tree_map(less_composite(PayloadType{payloadType}, FieldsSet{fields})), + payload_str_fields_helper(std::move(payloadType), fields) {} payload_map(const payload_map &other) : base_tree_map(other), payload_str_fields_helper(other) { for (auto &item : *this) this->add_ref(const_cast(item.first)); } diff --git a/cpp_src/core/index/rtree/indexrtree.cc b/cpp_src/core/index/rtree/indexrtree.cc index 555db9012..d0b3f7327 100644 --- a/cpp_src/core/index/rtree/indexrtree.cc +++ b/cpp_src/core/index/rtree/indexrtree.cc @@ -108,8 +108,8 @@ void IndexRTree::Delete(const Varia (void)delcnt; // TODO: we have to implement removal of composite indexes (doesn't work right now) assertf(this->Opts().IsSparse() || delcnt, "Delete unexists id from index '%s' id=%d,key=%s (%s)", this->name_, id, - Variant(keys).template As(this->payloadType_, this->fields_), - Variant(keyIt->first).As(this->payloadType_, this->fields_)); + Variant(keys).template As(this->payloadType_, this->Fields()), + Variant(keyIt->first).As(this->payloadType_, this->Fields())); if (keyIt->second.Unsorted().IsEmpty()) { this->tracker_.markDeleted(keyIt); @@ -120,36 +120,39 @@ void IndexRTree::Delete(const Varia } } -std::unique_ptr IndexRTree_New(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) { +std::unique_ptr IndexRTree_New(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields) { switch (idef.opts_.RTreeType()) { case IndexOpts::Linear: if (idef.opts_.IsPK() || idef.opts_.IsDense()) { return std::unique_ptr{ - new IndexRTree(idef, std::move(payloadType), fields)}; + new IndexRTree(idef, std::move(payloadType), std::move(fields))}; } else { - return std::unique_ptr{new IndexRTree(idef, std::move(payloadType), fields)}; + return std::unique_ptr{ + new IndexRTree(idef, std::move(payloadType), std::move(fields))}; } case IndexOpts::Quadratic: if (idef.opts_.IsPK() || idef.opts_.IsDense()) { return std::unique_ptr{ - new IndexRTree(idef, std::move(payloadType), fields)}; + new IndexRTree(idef, std::move(payloadType), std::move(fields))}; } else { return std::unique_ptr{ - new IndexRTree(idef, std::move(payloadType), fields)}; + new IndexRTree(idef, std::move(payloadType), std::move(fields))}; } case IndexOpts::Greene: if (idef.opts_.IsPK() || idef.opts_.IsDense()) { return std::unique_ptr{ - new IndexRTree(idef, std::move(payloadType), fields)}; + new IndexRTree(idef, std::move(payloadType), std::move(fields))}; } else { - return std::unique_ptr{new IndexRTree(idef, std::move(payloadType), fields)}; + return std::unique_ptr{ + new IndexRTree(idef, std::move(payloadType), std::move(fields))}; } case IndexOpts::RStar: if (idef.opts_.IsPK() || idef.opts_.IsDense()) { return std::unique_ptr{ - new IndexRTree(idef, std::move(payloadType), fields)}; + new IndexRTree(idef, std::move(payloadType), std::move(fields))}; } else { - return std::unique_ptr{new IndexRTree(idef, std::move(payloadType), fields)}; + return std::unique_ptr{ + new IndexRTree(idef, std::move(payloadType), std::move(fields))}; } default: assertrx(0); diff --git a/cpp_src/core/index/rtree/indexrtree.h b/cpp_src/core/index/rtree/indexrtree.h index effdbd04d..5cca0a247 100644 --- a/cpp_src/core/index/rtree/indexrtree.h +++ b/cpp_src/core/index/rtree/indexrtree.h @@ -11,8 +11,8 @@ class IndexRTree : public IndexUnordered; public: - IndexRTree(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) - : IndexUnordered{idef, std::move(payloadType), fields} {} + IndexRTree(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields) + : IndexUnordered{idef, std::move(payloadType), std::move(fields)} {} SelectKeyResults SelectKey(const VariantArray &keys, CondType, SortType, Index::SelectOpts, const BaseFunctionCtx::Ptr &, const RdxContext &) override; @@ -24,6 +24,6 @@ class IndexRTree : public IndexUnordered Clone() const override { return std::unique_ptr{new IndexRTree(*this)}; } }; -std::unique_ptr IndexRTree_New(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields); +std::unique_ptr IndexRTree_New(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields); } // namespace reindexer diff --git a/cpp_src/core/index/string_map.h b/cpp_src/core/index/string_map.h index 98114e386..ae60f9221 100644 --- a/cpp_src/core/index/string_map.h +++ b/cpp_src/core/index/string_map.h @@ -2,15 +2,13 @@ #include "core/keyvalue/key_string.h" #include "core/keyvalue/uuid.h" -#include "core/payload/payloadtype.h" +#include "core/namespace/stringsholder.h" #include "cpp-btree/btree_map.h" #include "sparse-map/sparse_map.h" #include "tools/stringstools.h" namespace reindexer { -class FieldsSet; - struct less_key_string { using is_transparent = void; diff --git a/cpp_src/core/index/ttlindex.cc b/cpp_src/core/index/ttlindex.cc index 11e1a8726..300ccdeea 100644 --- a/cpp_src/core/index/ttlindex.cc +++ b/cpp_src/core/index/ttlindex.cc @@ -15,11 +15,12 @@ void UpdateExpireAfter(Index *i, int64_t v) { } } -std::unique_ptr TtlIndex_New(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) { +std::unique_ptr TtlIndex_New(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields) { if (idef.opts_.IsPK() || idef.opts_.IsDense()) { - return std::unique_ptr{new TtlIndex>(idef, std::move(payloadType), fields)}; + return std::unique_ptr{ + new TtlIndex>(idef, std::move(payloadType), std::move(fields))}; } - return std::unique_ptr{new TtlIndex>(idef, std::move(payloadType), fields)}; + return std::unique_ptr{new TtlIndex>(idef, std::move(payloadType), std::move(fields))}; } } // namespace reindexer diff --git a/cpp_src/core/index/ttlindex.h b/cpp_src/core/index/ttlindex.h index acc0b1a72..f062c1e1a 100644 --- a/cpp_src/core/index/ttlindex.h +++ b/cpp_src/core/index/ttlindex.h @@ -10,9 +10,9 @@ namespace reindexer { template class TtlIndex : public IndexOrdered { public: - TtlIndex(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) - : IndexOrdered(idef, std::move(payloadType), fields), expireAfter_(idef.expireAfter_) {} - TtlIndex(const TtlIndex &other) : IndexOrdered(other), expireAfter_(other.expireAfter_) {} + TtlIndex(const IndexDef& idef, PayloadType&& payloadType, FieldsSet&& fields) + : IndexOrdered(idef, std::move(payloadType), std::move(fields)), expireAfter_(idef.expireAfter_) {} + TtlIndex(const TtlIndex& other) : IndexOrdered(other), expireAfter_(other.expireAfter_) {} int64_t GetTTLValue() const noexcept override { return expireAfter_; } std::unique_ptr Clone() const override { return std::unique_ptr{new TtlIndex(*this)}; } void UpdateExpireAfter(int64_t v) noexcept { expireAfter_ = v; } @@ -22,7 +22,7 @@ class TtlIndex : public IndexOrdered { int64_t expireAfter_ = 0; }; -std::unique_ptr TtlIndex_New(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields); -void UpdateExpireAfter(Index *i, int64_t v); +std::unique_ptr TtlIndex_New(const IndexDef& idef, PayloadType&& payloadType, FieldsSet&& fields); +void UpdateExpireAfter(Index* i, int64_t v); } // namespace reindexer diff --git a/cpp_src/core/index/uuid_index.cc b/cpp_src/core/index/uuid_index.cc index 57ff52cb4..5e80fbac4 100644 --- a/cpp_src/core/index/uuid_index.cc +++ b/cpp_src/core/index/uuid_index.cc @@ -10,8 +10,8 @@ void UuidIndex::Upsert(VariantArray &result, const VariantArray &keys, IdType id } } -std::unique_ptr IndexUuid_New(const IndexDef &idef, PayloadType payloadType, const FieldsSet &fields) { - return std::unique_ptr{new UuidIndex{idef, std::move(payloadType), fields}}; +std::unique_ptr IndexUuid_New(const IndexDef &idef, PayloadType &&payloadType, FieldsSet &&fields) { + return std::unique_ptr{new UuidIndex{idef, std::move(payloadType), std::move(fields)}}; } } // namespace reindexer diff --git a/cpp_src/core/index/uuid_index.h b/cpp_src/core/index/uuid_index.h index 1352c3f43..68d740b9e 100644 --- a/cpp_src/core/index/uuid_index.h +++ b/cpp_src/core/index/uuid_index.h @@ -1,7 +1,6 @@ #pragma once #include "core/index/indexunordered.h" -#include "core/keyvalue/uuid.h" namespace reindexer { @@ -9,12 +8,13 @@ class UuidIndex : public IndexUnordered using Base = IndexUnordered>; public: - UuidIndex(const IndexDef& idef, PayloadType payloadType, const FieldsSet& fields) : Base{idef, std::move(payloadType), fields} {} + UuidIndex(const IndexDef& idef, PayloadType&& payloadType, FieldsSet&& fields) + : Base{idef, std::move(payloadType), std::move(fields)} {} std::unique_ptr Clone() const override { return std::unique_ptr{new UuidIndex{*this}}; } using Base::Upsert; void Upsert(VariantArray& result, const VariantArray& keys, IdType id, bool& clearCache) override; // TODO delete this after #1353 }; -std::unique_ptr IndexUuid_New(const IndexDef& idef, PayloadType payloadType, const FieldsSet& fields); +std::unique_ptr IndexUuid_New(const IndexDef&, PayloadType&&, FieldsSet&&); } // namespace reindexer diff --git a/cpp_src/core/item.cc b/cpp_src/core/item.cc index 7840e7853..5ec196050 100644 --- a/cpp_src/core/item.cc +++ b/cpp_src/core/item.cc @@ -3,7 +3,6 @@ #include "core/itemimpl.h" #include "core/keyvalue/p_string.h" #include "core/namespace/namespace.h" -#include "core/rdxcontext.h" #include "tools/catch_and_return.h" namespace reindexer { diff --git a/cpp_src/core/itemmodifier.cc b/cpp_src/core/itemmodifier.cc index fb2cfa52c..5f46fc0be 100644 --- a/cpp_src/core/itemmodifier.cc +++ b/cpp_src/core/itemmodifier.cc @@ -1,9 +1,9 @@ #include "itemmodifier.h" +#include "core/itemimpl.h" #include "core/namespace/namespaceimpl.h" #include "core/query/expressionevaluator.h" #include "core/selectfunc/functionexecutor.h" #include "index/index.h" -#include "tools/logger.h" namespace reindexer { diff --git a/cpp_src/core/keyvalue/variant.cc b/cpp_src/core/keyvalue/variant.cc index 71d80b12e..19faf5114 100644 --- a/cpp_src/core/keyvalue/variant.cc +++ b/cpp_src/core/keyvalue/variant.cc @@ -559,8 +559,14 @@ Variant Variant::convert(KeyValueType type, const PayloadType *payloadType, cons Variant &Variant::convert(KeyValueType type, const PayloadType *payloadType, const FieldsSet *fields) & { if (isUuid()) { type.EvaluateOneOf([&](KeyValueType::Uuid) noexcept {}, [&](KeyValueType::String) { *this = Variant{std::string{Uuid{*this}}}; }, + [&](KeyValueType::Composite) { + assertrx_throw(payloadType && fields); + Variant tmp{VariantArray{std::move(*this)}}; + tmp.convertToComposite(*payloadType, *fields); + *this = std::move(tmp); + }, [type](OneOf) { + KeyValueType::Tuple, KeyValueType::Undefined, KeyValueType::Null>) { throw Error(errParams, "Can't convert Variant from type '%s' to type '%s'", KeyValueType{KeyValueType::Uuid{}}.Name(), type.Name()); }); @@ -572,12 +578,22 @@ Variant &Variant::convert(KeyValueType type, const PayloadType *payloadType, con [&](KeyValueType::Int64) { *this = Variant(As()); }, [&](KeyValueType::Double) { *this = Variant(As()); }, [&](KeyValueType::String) { *this = Variant(As()); }, [&](KeyValueType::Composite) { - if (variant_.type.Is()) { - assertrx(payloadType && fields); - convertToComposite(payloadType, fields); - } else { - throw Error(errParams, "Can't convert Variant from type '%s' to type '%s'", variant_.type.Name(), type.Name()); - } + variant_.type.EvaluateOneOf( + [&](KeyValueType::Tuple) { + assertrx(payloadType && fields); + convertToComposite(*payloadType, *fields); + }, + [](KeyValueType::Composite) noexcept {}, + [&](OneOf) { + assertrx(payloadType && fields); + Variant tmp{VariantArray{std::move(*this)}}; + tmp.convertToComposite(*payloadType, *fields); + *this = std::move(tmp); + }, + [&](OneOf) { + throw Error(errParams, "Can't convert Variant from type '%s' to type '%s'", variant_.type.Name(), type.Name()); + }); }, [&](KeyValueType::Uuid) { *this = Variant{As()}; }, [&](OneOf) { @@ -586,7 +602,7 @@ Variant &Variant::convert(KeyValueType type, const PayloadType *payloadType, con return *this; } -void Variant::convertToComposite(const PayloadType *payloadType, const FieldsSet *fields) { +void Variant::convertToComposite(const PayloadType &payloadType, const FieldsSet &fields) { assertrx(!isUuid()); assertrx(variant_.type.Is() && variant_.hold == 1); key_string val = *cast(); @@ -594,24 +610,24 @@ void Variant::convertToComposite(const PayloadType *payloadType, const FieldsSet if (variant_.hold == 1) free(); // Alloc usual payloadvalue + extra memory for hold string - auto &pv = *new (cast()) PayloadValue(payloadType->TotalSize() + val->size()); + auto &pv = *new (cast()) PayloadValue(payloadType.TotalSize() + val->size()); variant_.hold = 1; variant_.type = KeyValueType::Composite{}; // Copy serializer buffer with strings to extra payloadvalue memory - char *data = reinterpret_cast(pv.Ptr() + payloadType->TotalSize()); + char *data = reinterpret_cast(pv.Ptr() + payloadType.TotalSize()); memcpy(data, val->data(), val->size()); Serializer ser(std::string_view(data, val->size())); size_t count = ser.GetVarUint(); - if (count != fields->size()) { - throw Error(errLogic, "Invalid count of arguments for composite index, expected %d, got %d", fields->size(), count); + if (count != fields.size()) { + throw Error(errLogic, "Invalid count of arguments for composite index, expected %d, got %d", fields.size(), count); } - Payload pl(*payloadType, pv); + Payload pl(payloadType, pv); - for (auto field : *fields) { + for (auto field : fields) { if (field != IndexValueType::SetByJsonPath) { pl.Set(field, ser.GetVariant()); } else { diff --git a/cpp_src/core/keyvalue/variant.h b/cpp_src/core/keyvalue/variant.h index a4296aacb..f4d495587 100644 --- a/cpp_src/core/keyvalue/variant.h +++ b/cpp_src/core/keyvalue/variant.h @@ -144,7 +144,7 @@ class Variant { private: bool isUuid() const noexcept { return uuid_.isUuid != 0; } - void convertToComposite(const PayloadType *, const FieldsSet *); + void convertToComposite(const PayloadType &, const FieldsSet &); void free() noexcept; void copy(const Variant &other); template @@ -212,6 +212,8 @@ template <> std::string Variant::As() const; class VariantArray : public h_vector { + using Base = h_vector; + public: VariantArray() noexcept = default; explicit VariantArray(Point) noexcept; @@ -225,9 +227,14 @@ class VariantArray : public h_vector { return std::move(*this); } void MarkObject() noexcept { isObjectValue = true; } - using h_vector::h_vector; - using h_vector::operator==; - using h_vector::operator!=; + using Base::Base; + using Base::operator==; + using Base::operator!=; + template + void clear() noexcept { + isArrayValue = isObjectValue = false; + Base::clear(); + } size_t Hash() const noexcept { size_t ret = this->size(); for (size_t i = 0; i < this->size(); ++i) ret = (ret * 127) ^ this->at(i).Hash(); diff --git a/cpp_src/core/namespace/asyncstorage.cc b/cpp_src/core/namespace/asyncstorage.cc index 4f2aa4091..d94d0b08c 100644 --- a/cpp_src/core/namespace/asyncstorage.cc +++ b/cpp_src/core/namespace/asyncstorage.cc @@ -52,7 +52,6 @@ void AsyncStorage::Destroy() { throwOnStorageCopy(); if (storage_) { - tryReopenStorage(); clearUpdates(); storage_->Destroy(path_); reset(); diff --git a/cpp_src/core/namespace/itemsloader.h b/cpp_src/core/namespace/itemsloader.h index 0432a9310..3d39c27c8 100644 --- a/cpp_src/core/namespace/itemsloader.h +++ b/cpp_src/core/namespace/itemsloader.h @@ -1,6 +1,7 @@ #pragma once #include +#include "core/itemimpl.h" #include "namespaceimpl.h" namespace reindexer { diff --git a/cpp_src/core/namespace/namespace.cc b/cpp_src/core/namespace/namespace.cc index 3bc79b243..55a8e70a7 100644 --- a/cpp_src/core/namespace/namespace.cc +++ b/cpp_src/core/namespace/namespace.cc @@ -105,10 +105,14 @@ bool Namespace::needNamespaceCopy(const NamespaceImpl::Ptr& ns, const Transactio void Namespace::doRename(const Namespace::Ptr& dst, const std::string& newName, const std::string& storagePath, const RdxContext& ctx) { std::string dbpath; const auto flushOpts = StorageFlushOpts().WithImmediateReopen(); - awaitMainNs(ctx)->storage_.Flush(flushOpts); auto lck = handleInvalidation(NamespaceImpl::wLock)(ctx); - auto& srcNs = *atomicLoadMainNs(); // -V758 - srcNs.storage_.Flush(flushOpts); // Repeat flush, to raise any disk errors before attempt to close storage + auto srcNsPtr = atomicLoadMainNs(); + auto& srcNs = *srcNsPtr; + srcNs.storage_.Flush(flushOpts); // Repeat flush, to raise any disk errors before attempt to close storage + auto storageStatus = srcNs.storage_.GetStatusCached(); + if (!storageStatus.err.ok()) { + throw Error(storageStatus.err.code(), "Unable to flush storage before rename: %s", storageStatus.err.what()); + } NamespaceImpl::Mutex* dstMtx = nullptr; NamespaceImpl::Ptr dstNs; if (dst) { diff --git a/cpp_src/core/namespace/namespace.h b/cpp_src/core/namespace/namespace.h index 71864740d..55dc59548 100644 --- a/cpp_src/core/namespace/namespace.h +++ b/cpp_src/core/namespace/namespace.h @@ -131,7 +131,12 @@ class Namespace { } void OnConfigUpdated(DBConfigProvider &configProvider, const RdxContext &ctx) { NamespaceConfigData configData; - configProvider.GetNamespaceConfig(GetName(ctx), configData); + const auto nsName = GetName(ctx); + std::string_view realNsName(nsName); + if (isTmpNamespaceNameFast(nsName)) { + realNsName = demangleTmpNamespaceName(realNsName); + } + configProvider.GetNamespaceConfig(realNsName, configData); startCopyPolicyTxSize_.store(configData.startCopyPolicyTxSize, std::memory_order_relaxed); copyPolicyMultiplier_.store(configData.copyPolicyMultiplier, std::memory_order_relaxed); txSizeToAlwaysCopy_.store(configData.txSizeToAlwaysCopy, std::memory_order_relaxed); @@ -211,7 +216,7 @@ class Namespace { ns->tryForceFlush(std::move(locker)); } else if constexpr (std::is_same_v) { auto params = longUpdDelLoggingParams_.load(std::memory_order_relaxed); - const bool isEnabled = params.thresholdUs >= 0 && !isSystemNamespaceNameFast(v._namespace); + const bool isEnabled = params.thresholdUs >= 0 && !isSystemNamespaceNameFast(v.NsName()); auto statCalculator = QueryStatCalculator(long_actions::MakeLogger(v, std::move(params)), isEnabled); auto locker = statCalculator.CreateLock(*ns, &NamespaceImpl::wLock, ctx.rdxContext); calc.LockHit(); diff --git a/cpp_src/core/namespace/namespaceimpl.cc b/cpp_src/core/namespace/namespaceimpl.cc index 7848af01e..cf1d776ad 100644 --- a/cpp_src/core/namespace/namespaceimpl.cc +++ b/cpp_src/core/namespace/namespaceimpl.cc @@ -386,8 +386,8 @@ NamespaceImpl::RollBack_recreateCompositeIndexes NamespaceImpl::re indexDef.opts_ = index->Opts(); indexDef.FromType(index->Type()); - createFieldsSet>(indexDef.name_, index->Type(), index->Fields(), fields); - auto newIndex{Index::New(indexDef, payloadType_, fields)}; + createCompositeFieldsSet>(indexDef.name_, index->Fields(), fields); + auto newIndex{Index::New(indexDef, PayloadType{payloadType_}, FieldsSet{fields})}; rollbacker.SaveIndex(std::move(index)); std::swap(index, newIndex); @@ -425,7 +425,7 @@ class NamespaceImpl::RollBack_updateItems : private RollBackBase { } rollbacker_recreateCompositeIndexes_.RollBack(); for (auto& idx : ns_.indexes_) { - idx->UpdatePayloadType(ns_.payloadType_); + idx->UpdatePayloadType(PayloadType{ns_.payloadType_}); } Disable(); } @@ -478,7 +478,7 @@ NamespaceImpl::RollBack_updateItems NamespaceImpl::updateItems(con repl_.dataHash, itemsDataSize_}; for (auto& idx : indexes_) { - idx->UpdatePayloadType(payloadType_); + idx->UpdatePayloadType(PayloadType{payloadType_}); } VariantArray skrefsDel, skrefsUps; @@ -745,22 +745,28 @@ void NamespaceImpl::verifyCompositeIndex(const IndexDef& indexDef) const { throw Error{errParams, "Composite index cannot be sparse. Use non-sparse composite instead"}; } for (const auto& jp : indexDef.jsonPaths_) { - const auto it = indexesNames_.find(jp); - if (it == indexesNames_.end()) { + int idx; + if (!getIndexByName(jp, idx)) { if (!IsFullText(indexDef.Type())) { throw Error(errParams, "Composite indexes over non-indexed field ('%s') are not supported yet (except for full-text indexes). Create " "at least column index('-') over each field inside the composite index", jp); } - continue; - } - const auto& idx = indexes_[it->second]; - if (idx->IsUuid() && type != IndexCompositeHash) { - throw Error{errParams, "Only hash index allowed on UUID field"}; - } - if (IsComposite(idx->Type())) { - throw Error(errParams, "Cannot create composite index '%s' over the other composite '%s'", indexDef.name_, idx->Name()); + } else { + const auto& index = *indexes_[idx]; + if (index.Opts().IsSparse()) { + throw Error(errParams, "Composite indexes over sparse indexed field ('%s') are not supported yet", jp); + } + if (type != IndexCompositeHash && index.IsUuid()) { + throw Error{errParams, "Only hash index allowed on UUID field"}; + } + if (index.Opts().IsArray() && !IsFullText(type)) { + throw Error(errParams, "Cannot add array subindex '%s' to not fulltext composite index '%s'", jp, indexDef.name_); + } + if (IsComposite(index.Type())) { + throw Error(errParams, "Cannot create composite index '%s' over the other composite '%s'", indexDef.name_, index.Name()); + } } } } @@ -852,7 +858,7 @@ void NamespaceImpl::verifyUpdateIndex(const IndexDef& indexDef) const { throw Error(errParams, "Sparse index must have exactly 1 JSON-path, but %d paths found for '%s'", indexDef.jsonPaths_.size(), indexDef.name_); } - const auto newSparseIndex = std::unique_ptr(Index::New(indexDef, payloadType_, {})); + const auto newSparseIndex = std::unique_ptr(Index::New(indexDef, PayloadType{payloadType_}, {})); } else { FieldsSet changedFields{idxNameIt->second}; PayloadType newPlType = payloadType_; @@ -1025,7 +1031,7 @@ void NamespaceImpl::addIndex(const IndexDef& indexDef) { TagsPath tagsPath = tagsMatcher_.path2tag(jsonPaths[0], true); assertrx(tagsPath.size() > 0); fields.push_back(std::move(tagsPath)); - auto newIndex = Index::New(indexDef, payloadType_, fields); + auto newIndex = Index::New(indexDef, PayloadType{payloadType_}, std::move(fields)); rollbacker.RollBacker_insertIndex(insertIndex(std::move(newIndex), idxNo, indexName)); ++sparseIndexesCount_; rollbacker.NeedDecreaseSparseIndexCount(); @@ -1038,7 +1044,7 @@ void NamespaceImpl::addIndex(const IndexDef& indexDef) { tagsMatcher_.UpdatePayloadType(payloadType_); rollbacker.NeedResetPayloadTypeInTagsMatcher(); newIndex->SetFields(FieldsSet(idxNo)); - newIndex->UpdatePayloadType(payloadType_); + newIndex->UpdatePayloadType(PayloadType{payloadType_}); FieldsSet changedFields{0, idxNo}; rollbacker.RollBacker_insertIndex(insertIndex(std::move(newIndex), idxNo, indexName)); @@ -1094,27 +1100,18 @@ IndexDef NamespaceImpl::getIndexDefinition(const std::string& indexName) const { void NamespaceImpl::verifyUpdateCompositeIndex(const IndexDef& indexDef) const { verifyCompositeIndex(indexDef); - IndexType type = indexDef.Type(); - - for (auto& jsonPathOrSubIdx : indexDef.jsonPaths_) { - auto idxNameIt = indexesNames_.find(jsonPathOrSubIdx); - if (idxNameIt != indexesNames_.end() && !indexes_[idxNameIt->second]->Opts().IsSparse() && - indexes_[idxNameIt->second]->Opts().IsArray() && (type == IndexCompositeBTree || type == IndexCompositeHash)) { - throw Error(errParams, "Cannot add array subindex '%s' to composite index '%s'", jsonPathOrSubIdx, indexDef.name_); - } - } - const auto newIndex = std::unique_ptr(Index::New(indexDef, payloadType_, {})); + const auto newIndex = std::unique_ptr(Index::New(indexDef, PayloadType{payloadType_}, {})); } void NamespaceImpl::addCompositeIndex(const IndexDef& indexDef) { const auto& indexName = indexDef.name_; FieldsSet fields; - createFieldsSet(indexName, indexDef.Type(), indexDef.jsonPaths_, fields); + createCompositeFieldsSet(indexName, indexDef.jsonPaths_, fields); assertrx(indexesNames_.find(indexName) == indexesNames_.end()); const int idxPos = indexes_.size(); - auto insertIndex_rollbacker{insertIndex(Index::New(indexDef, payloadType_, fields), idxPos, indexName)}; + auto insertIndex_rollbacker{insertIndex(Index::New(indexDef, PayloadType{payloadType_}, FieldsSet{fields}), idxPos, indexName)}; auto indexesCacheCleaner{GetIndexesCacheCleaner()}; for (IdType rowId = 0; rowId < int(items_.size()); rowId++) { @@ -1134,7 +1131,7 @@ void NamespaceImpl::addCompositeIndex(const IndexDef& indexDef) { } template -void NamespaceImpl::createFieldsSet(const std::string& idxName, IndexType type, const PathsT& paths, FieldsSet& fields) { +void NamespaceImpl::createCompositeFieldsSet(const std::string& idxName, const PathsT& paths, FieldsSet& fields) { fields.clear(); const JsonPathsContainerT* jsonPaths = nullptr; @@ -1149,22 +1146,19 @@ void NamespaceImpl::createFieldsSet(const std::string& idxName, IndexType type, } for (const auto& jsonPathOrSubIdx : *jsonPaths) { - auto idxNameIt = indexesNames_.find(jsonPathOrSubIdx); - if (idxNameIt == indexesNames_.end() || idxName == jsonPathOrSubIdx) { + int idx; + if (!getScalarIndexByName(jsonPathOrSubIdx, idx) /* || idxName == jsonPathOrSubIdx*/) { // TODO may be uncomment TagsPath tagsPath = tagsMatcher_.path2tag(jsonPathOrSubIdx, true); if (tagsPath.empty()) { throw Error(errLogic, "Unable to get or create json-path '%s' for composite index '%s'", jsonPathOrSubIdx, idxName); } fields.push_back(tagsPath); fields.push_back(jsonPathOrSubIdx); - } else if (indexes_[idxNameIt->second]->Opts().IsSparse() && !indexes_[idxNameIt->second]->Opts().IsArray()) { - fields.push_back(jsonPathOrSubIdx); - fields.push_back(indexes_[idxNameIt->second]->Fields().getTagsPath(0)); } else { - if (indexes_[idxNameIt->second]->Opts().IsArray() && (type == IndexCompositeBTree || type == IndexCompositeHash)) { - throw Error(errParams, "Cannot add array subindex '%s' to composite index '%s'", jsonPathOrSubIdx, idxName); - } - fields.push_back(idxNameIt->second); + const auto& idxFields = indexes_[idx]->Fields(); + assertrx_throw(idxFields.size() == 1); + assertrx_throw(idxFields[0] >= 0); + fields.push_back(idxFields[0]); } } @@ -1192,6 +1186,16 @@ int NamespaceImpl::getIndexByNameOrJsonPath(std::string_view index) const { } } +int NamespaceImpl::getScalarIndexByName(std::string_view index) const { + int idx; + if (getIndexByName(index, idx)) { + if (idx < indexes_.firstCompositePos()) { + return idx; + } + } + throw Error(errParams, "Index '%s' not found in '%s'", index, name_); +} + bool NamespaceImpl::getIndexByName(std::string_view name, int& index) const { auto it = indexesNames_.find(name); if (it == indexesNames_.end()) return false; @@ -1211,6 +1215,17 @@ bool NamespaceImpl::getIndexByNameOrJsonPath(std::string_view name, int& index) return false; } +bool NamespaceImpl::getScalarIndexByName(std::string_view name, int& index) const { + int idx; + if (getIndexByName(name, idx)) { + if (idx < indexes_.firstCompositePos()) { + index = idx; + return true; + } + } + return false; +} + bool NamespaceImpl::getSparseIndexByJsonPath(std::string_view jsonPath, int& index) const { // FIXME: Try to merge getIndexByNameOrJsonPath and getSparseIndexByJsonPath if it's possible for (int i = indexes_.firstSparsePos(), end = indexes_.firstSparsePos() + indexes_.sparseIndexesSize(); i < end; ++i) { @@ -1442,7 +1457,8 @@ void NamespaceImpl::doDelete(const Query& q, QueryResults& result, const NsConte WrSerializer cjson; for (auto it : result) { cjson.Reset(); - it.GetCJSON(cjson, false); + const auto err = it.GetCJSON(cjson, false); + if (!err.ok()) throw err; const WALRecord wrec{WalItemModify, cjson.Slice(), tagsMatcher_.version(), ModeDelete, ctx.inTransaction}; processWalRecord(wrec, ctx.rdxContext); } @@ -1489,7 +1505,8 @@ void NamespaceImpl::doTruncate(const NsContext& ctx) { itemsDataSize_ = 0; for (size_t i = 0; i < indexes_.size(); ++i) { const IndexOpts opts = indexes_[i]->Opts(); - std::unique_ptr newIdx{Index::New(getIndexDefinition(i), indexes_[i]->GetPayloadType(), indexes_[i]->Fields())}; + std::unique_ptr newIdx{ + Index::New(getIndexDefinition(i), PayloadType{indexes_[i]->GetPayloadType()}, FieldsSet{indexes_[i]->Fields()})}; newIdx->SetOpts(opts); std::swap(indexes_[i], newIdx); removeIndex(newIdx); @@ -1908,12 +1925,10 @@ void NamespaceImpl::optimizeIndexes(const NsContext& ctx) { rlck = rLock(ctx.rdxContext); } - if (isSystem()) return; - if (!lastUpdateTime || !config_.optimizationTimeout || now - lastUpdateTime < config_.optimizationTimeout) { + if (isSystem() || repl_.temporary || !indexes_.size()) { return; } - - if (!indexes_.size()) { + if (!lastUpdateTime || !config_.optimizationTimeout || now - lastUpdateTime < config_.optimizationTimeout) { return; } diff --git a/cpp_src/core/namespace/namespaceimpl.h b/cpp_src/core/namespace/namespaceimpl.h index 46fa4d1e1..cc05dfa34 100644 --- a/cpp_src/core/namespace/namespaceimpl.h +++ b/cpp_src/core/namespace/namespaceimpl.h @@ -20,11 +20,10 @@ #include "core/schema.h" #include "core/storage/idatastorage.h" #include "core/storage/storagetype.h" -#include "core/transactionimpl.h" +#include "core/transaction.h" #include "estl/contexted_locks.h" #include "estl/fast_hash_map.h" #include "estl/shared_mutex.h" -#include "estl/smart_lock.h" #include "estl/syncpool.h" #include "replicator/updatesobserver.h" #include "replicator/waltracker.h" @@ -258,8 +257,10 @@ class NamespaceImpl : public intrusive_atomic_rc_base { // NOLINT(*performance. int getIndexByName(std::string_view index) const; int getIndexByNameOrJsonPath(std::string_view name) const; + int getScalarIndexByName(std::string_view name) const; bool getIndexByName(std::string_view name, int &index) const; bool getIndexByNameOrJsonPath(std::string_view name, int &index) const; + bool getScalarIndexByName(std::string_view name, int &index) const; bool getSparseIndexByJsonPath(std::string_view jsonPath, int &index) const; void FillResult(QueryResults &result, const IdSet &ids) const; @@ -281,6 +282,12 @@ class NamespaceImpl : public intrusive_atomic_rc_base { // NOLINT(*performance. int getNsNumber() const { return schema_ ? schema_->GetProtobufNsNumber() : 0; } IndexesCacheCleaner GetIndexesCacheCleaner() { return IndexesCacheCleaner{*this}; } void SetDestroyFlag() { dbDestroyed_ = true; } + Error FlushStorage(const RdxContext &ctx) { + const auto flushOpts = StorageFlushOpts().WithImmediateReopen(); + auto lck = rLock(ctx); + storage_.Flush(flushOpts); + return storage_.GetStatusCached().err; + } private: struct SysRecordsVersions { @@ -345,7 +352,7 @@ class NamespaceImpl : public intrusive_atomic_rc_base { // NOLINT(*performance. void addIndex(const IndexDef &indexDef); void addCompositeIndex(const IndexDef &indexDef); template - void createFieldsSet(const std::string &idxName, IndexType type, const PathsT &paths, FieldsSet &fields); + void createCompositeFieldsSet(const std::string &idxName, const PathsT &paths, FieldsSet &fields); void verifyCompositeIndex(const IndexDef &indexDef) const; template void verifyAddIndex(const IndexDef &indexDef, GetNameF &&) const; @@ -431,7 +438,7 @@ class NamespaceImpl : public intrusive_atomic_rc_base { // NOLINT(*performance. NamespaceImpl(const NamespaceImpl &src, AsyncStorage::FullLockT &storageLock); - bool isSystem() const { return !name_.empty() && name_[0] == '#'; } + bool isSystem() const { return isSystemNamespaceNameFast(name_); } IdType createItem(size_t realSize); void checkApplySlaveUpdate(bool v); diff --git a/cpp_src/core/nsselecter/aggregator.cc b/cpp_src/core/nsselecter/aggregator.cc index 81194c7d3..837bebab0 100644 --- a/cpp_src/core/nsselecter/aggregator.cc +++ b/cpp_src/core/nsselecter/aggregator.cc @@ -26,8 +26,13 @@ static void copy(It begin, It end, std::vector &facets, const Field ConstPayload pl(payloadType, begin->first); VariantArray va; if (fields[i] == IndexValueType::SetByJsonPath) { - const TagsPath &tagsPath = fields.getTagsPath(tagPathIdx++); - pl.GetByJsonPath(tagsPath, va, KeyValueType::Undefined{}); + if (fields.isTagsPathIndexed(tagPathIdx)) { + const IndexedTagsPath &tagsPath = fields.getIndexedTagsPath(tagPathIdx++); + pl.GetByJsonPath(tagsPath, va, KeyValueType::Undefined{}); + } else { + const TagsPath &tagsPath = fields.getTagsPath(tagPathIdx++); + pl.GetByJsonPath(tagsPath, va, KeyValueType::Undefined{}); + } if (va.IsObjectValue()) { throw Error(errQueryExec, "Cannot aggregate object field"); } @@ -221,7 +226,7 @@ Aggregator::Aggregator(const PayloadType &payloadType, const FieldsSet &fields, } } else { if (sort.empty()) { - facets_ = std::make_unique(MultifieldUnorderedMap{payloadType_, fields_}); + facets_ = std::make_unique(MultifieldUnorderedMap{PayloadType{payloadType_}, FieldsSet{fields_}}); } else { facets_ = std::make_unique(MultifieldOrderedMap{MultifieldComparator{sort, fields_, payloadType_}}); } diff --git a/cpp_src/core/nsselecter/fieldscomparator.h b/cpp_src/core/nsselecter/fieldscomparator.h index dfe375bd0..cda7dd1ff 100644 --- a/cpp_src/core/nsselecter/fieldscomparator.h +++ b/cpp_src/core/nsselecter/fieldscomparator.h @@ -18,13 +18,13 @@ class FieldsComparator { const std::string& Name() const&& = delete; std::string Dump() const { return Name(); } int GetMatchedCount() const noexcept { return matchedCount_; } - void SetLeftField(const TagsPath& tpath) { - setField(tpath, ctx_[0].lCtx_); + void SetLeftField(const FieldsSet& fields) { + setField(fields, ctx_[0].lCtx_); leftFieldSet = true; } - void SetRightField(const TagsPath& tpath) { + void SetRightField(const FieldsSet& fields) { assertrx(leftFieldSet); - setField(tpath, ctx_[0].rCtx_); + setField(fields, ctx_[0].rCtx_); } void SetLeftField(const FieldsSet& fset, KeyValueType type, bool isArray) { if (type.Is()) { @@ -67,6 +67,11 @@ class FieldsComparator { }; void setField(const TagsPath& tpath, FieldContext& fctx) { fctx.fields_.push_back(tpath); } + void setField(const FieldsSet& fields, FieldContext& fctx) { + assertrx_throw(fields.size() == 1); + assertrx_throw(fields[0] == IndexValueType::SetByJsonPath); + setField(fields.getTagsPath(0), fctx); + } void setField(FieldContext& fctx, FieldsSet fset, KeyValueType type, bool isArray) { fctx.fields_ = std::move(fset); fctx.type_ = type; diff --git a/cpp_src/core/nsselecter/joinedselector.cc b/cpp_src/core/nsselecter/joinedselector.cc index 84fbe4b8b..7492f23a5 100644 --- a/cpp_src/core/nsselecter/joinedselector.cc +++ b/cpp_src/core/nsselecter/joinedselector.cc @@ -3,6 +3,7 @@ #include "core/namespace/namespaceimpl.h" #include "core/queryresults/joinresults.h" #include "nsselecter.h" +#include "vendor/sparse-map/sparse_set.h" constexpr size_t kMaxIterationsScaleForInnerJoinOptimization = 100; @@ -53,8 +54,8 @@ void JoinedSelector::selectFromPreResultValues(QueryResults &joinItemR, const Qu for (const ItemRef &item : preResult_->values) { auto &v = item.Value(); assertrx(!v.IsFree()); - if (query.entries.CheckIfSatisfyConditions({preResult_->values.payloadType, v}, preResult_->values.tagsMatcher)) { - if (++matched > query.count) break; + if (query.entries.CheckIfSatisfyConditions({preResult_->values.payloadType, v})) { + if (++matched > query.Limit()) break; found = true; joinItemR.Add(item); } @@ -76,21 +77,12 @@ bool JoinedSelector::Process(IdType rowId, int nsId, ConstPayload payload, bool std::unique_ptr itemQueryCopy; Query *itemQueryPtr = &itemQuery_; for (auto &je : joinQuery_.joinEntries_) { - const bool nonIndexedField = (je.idxNo == IndexValueType::SetByJsonPath); - if (nonIndexedField) { - VariantArray &values = itemQueryPtr->entries.Get(i).values; - const KeyValueType type{values.empty() ? KeyValueType::Undefined{} : values[0].Type()}; - payload.GetByJsonPath(je.index_, leftNs_->tagsMatcher_, values, type); - } else { - const auto &index = *leftNs_->indexes_[je.idxNo]; - const auto &fields = index.Fields(); - if (fields.getJsonPathsLength() == 0) { - payload.Get(fields[0], itemQueryPtr->entries.Get(i).values); - } else { - payload.GetByJsonPath(fields.getTagsPath(0), itemQueryPtr->entries.Get(i).values, index.KeyType()); - } + QueryEntry &qentry = itemQueryPtr->entries.Get(i); + { + auto keyValues = qentry.UpdatableValues(QueryEntry::IgnoreEmptyValues{}); + payload.GetByFieldsSet(je.LeftFields(), keyValues, je.LeftFieldType(), je.LeftCompositeFieldsTypes()); } - if (itemQueryPtr->entries.Get(i).values.empty()) { + if (qentry.Values().empty()) { if (itemQueryPtr == &itemQuery_) { itemQueryCopy = std::unique_ptr{new Query(itemQuery_)}; itemQueryPtr = itemQueryCopy.get(); @@ -99,7 +91,7 @@ bool JoinedSelector::Process(IdType rowId, int nsId, ConstPayload payload, bool } ++i; } - itemQueryPtr->Limit(match ? joinQuery_.count : 0); + itemQueryPtr->Limit(match ? joinQuery_.Limit() : 0); bool found = false; bool matchedAtLeastOnce = false; @@ -122,58 +114,70 @@ bool JoinedSelector::Process(IdType rowId, int nsId, ConstPayload payload, bool return matchedAtLeastOnce; } -template -void JoinedSelector::readValuesFromRightNs(VariantArray &values, const KeyValueType leftIndexType, [[maybe_unused]] int rightIdxNo, - [[maybe_unused]] std::string_view rightIndex) const { - std::unordered_set set; - VariantArray buffer; - for (IdType rowId : preResult_->ids) { - if (rightNs_->items_[rowId].IsFree()) continue; - buffer.clear(); - const ConstPayload pl{rightNs_->payloadType_, rightNs_->items_[rowId]}; - if constexpr (byJsonPath) { - pl.GetByJsonPath(rightIndex, rightNs_->tagsMatcher_, buffer, leftIndexType); - } else { - pl.Get(rightIdxNo, buffer); +template +VariantArray JoinedSelector::readValuesOfRightNsFrom(const Cont &data, const Fn &createPayload, const QueryJoinEntry &entry, + const PayloadType &pt) const { + const auto rightFieldType = entry.RightFieldType(); + const auto leftFieldType = entry.LeftFieldType(); + VariantArray res; + if (rightFieldType.Is()) { + unordered_payload_set set(data.size(), hash_composite(pt, entry.RightFields()), equal_composite(pt, entry.RightFields())); + for (const auto &v : data) { + const auto pl = createPayload(v); + if (pl) { + set.insert(*pl->Value()); + } } - if (!leftIndexType.Is() && !leftIndexType.Is()) { - for (Variant &v : buffer) set.insert(std::move(v.convert(leftIndexType))); - } else { - for (Variant &v : buffer) set.insert(std::move(v)); + res.reserve(set.size()); + for (auto &s : set) { + res.emplace_back(std::move(s)); } - } - values.reserve(set.size()); - std::move(set.begin(), set.end(), std::back_inserter(values)); -} - -template -void JoinedSelector::readValuesFromPreResult(VariantArray &values, const KeyValueType leftIndexType, int rightIdxNo, - std::string_view rightIndex) const { - std::unordered_set set; - VariantArray buffer; - for (const ItemRef &item : preResult_->values) { - buffer.clear(); - assertrx(!item.Value().IsFree()); - const ConstPayload pl{preResult_->values.payloadType, item.Value()}; - if constexpr (byJsonPath) { - pl.GetByJsonPath(rightIndex, preResult_->values.tagsMatcher, buffer, leftIndexType); - (void)rightIdxNo; - } else { - pl.Get(rightIdxNo, buffer); - (void)rightIndex; + } else { + tsl::sparse_set set(data.size()); + for (const auto &v : data) { + const auto pl = createPayload(v); + if (!pl) { + continue; + } + pl->GetByFieldsSet(entry.RightFields(), res, entry.RightFieldType(), entry.RightCompositeFieldsTypes()); + if (!leftFieldType.Is() && !leftFieldType.Is()) { + for (Variant &v : res) set.insert(std::move(v.convert(leftFieldType))); + } else { + for (Variant &v : res) set.insert(std::move(v)); + } } - if (!leftIndexType.Is() && !leftIndexType.Is()) { - for (Variant &v : buffer) set.insert(std::move(v.convert(leftIndexType))); - } else { - for (Variant &v : buffer) set.insert(std::move(v)); + res.clear(); + for (auto &s : set) { + res.emplace_back(std::move(s)); } } - values.reserve(set.size()); - std::move(set.begin(), set.end(), std::back_inserter(values)); + return res; +} + +VariantArray JoinedSelector::readValuesFromRightNs(const QueryJoinEntry &entry) const { + return readValuesOfRightNsFrom( + preResult_->ids, + [this](IdType rowId) -> std::optional { + const auto &item = rightNs_->items_[rowId]; + if (item.IsFree()) { + return std::nullopt; + } + return ConstPayload{rightNs_->payloadType_, item}; + }, + entry, rightNs_->payloadType_); } -template void JoinedSelector::readValuesFromPreResult(VariantArray &, KeyValueType, int, std::string_view) const; -template void JoinedSelector::readValuesFromPreResult(VariantArray &, KeyValueType, int, std::string_view) const; +VariantArray JoinedSelector::readValuesFromPreResult(const QueryJoinEntry &entry) const { + return readValuesOfRightNsFrom( + preResult_->values, + [this](const ItemRef &item) -> std::optional { + if (item.Value().IsFree()) { + return std::nullopt; + } + return ConstPayload{preResult_->values.payloadType, item.Value()}; + }, + entry, preResult_->values.payloadType); +} void JoinedSelector::AppendSelectIteratorOfJoinIndexData(SelectIteratorContainer &iterators, int *maxIterations, unsigned sortId, const SelectFunction::Ptr &selectFnc, const RdxContext &rdxCtx) { @@ -184,39 +188,25 @@ void JoinedSelector::AppendSelectIteratorOfJoinIndexData(SelectIteratorContainer return; } unsigned optimized = 0; - assertrx(preResult_->dataMode != JoinPreResult::ModeValues || itemQuery_.entries.Size() == joinQuery_.joinEntries_.size()); + assertrx_throw(preResult_->dataMode != JoinPreResult::ModeValues || itemQuery_.entries.Size() == joinQuery_.joinEntries_.size()); for (size_t i = 0; i < joinQuery_.joinEntries_.size(); ++i) { const QueryJoinEntry &joinEntry = joinQuery_.joinEntries_[i]; - if (joinEntry.op_ != OpAnd || (joinEntry.condition_ != CondEq && joinEntry.condition_ != CondSet) || - (i + 1 < joinQuery_.joinEntries_.size() && joinQuery_.joinEntries_[i + 1].op_ == OpOr) || - joinEntry.idxNo == IndexValueType::SetByJsonPath) { + if (!joinEntry.IsLeftFieldIndexed() || joinEntry.Operation() != OpAnd || + (joinEntry.Condition() != CondEq && joinEntry.Condition() != CondSet) || + (i + 1 < joinQuery_.joinEntries_.size() && joinQuery_.joinEntries_[i + 1].Operation() == OpOr)) { continue; } - const auto &leftIndex = leftNs_->indexes_[joinEntry.idxNo]; + const auto &leftIndex = leftNs_->indexes_[joinEntry.LeftIdxNo()]; assertrx(!IsFullText(leftIndex->Type())); if (leftIndex->Opts().IsSparse()) continue; VariantArray values; if (preResult_->dataMode == JoinPreResult::ModeIdSet) { - int rightIdxNo = IndexValueType::NotSet; - if (rightNs_->getIndexByNameOrJsonPath(joinEntry.joinIndex_, rightIdxNo) && - !rightNs_->indexes_[rightIdxNo]->Opts().IsSparse()) { - readValuesFromRightNs(values, leftIndex->SelectKeyType(), rightIdxNo, joinEntry.joinIndex_); - } else { - readValuesFromRightNs(values, leftIndex->SelectKeyType(), rightIdxNo, joinEntry.joinIndex_); - } + values = readValuesFromRightNs(joinEntry); } else { - assertrx(itemQuery_.entries.HoldsOrReferTo(i)); - const QueryEntry &qe = itemQuery_.entries.Get(i); - assertrx(qe.index == joinEntry.joinIndex_); - const int rightIdxNo = qe.idxNo; - if (rightIdxNo == IndexValueType::SetByJsonPath) { - readValuesFromPreResult(values, leftIndex->SelectKeyType(), rightIdxNo, joinEntry.joinIndex_); - } else { - readValuesFromPreResult(values, leftIndex->SelectKeyType(), rightIdxNo, joinEntry.joinIndex_); - } + values = readValuesFromPreResult(joinEntry); } - auto ctx = selectFnc ? selectFnc->CreateCtx(joinEntry.idxNo) : BaseFunctionCtx::Ptr{}; + auto ctx = selectFnc ? selectFnc->CreateCtx(joinEntry.LeftIdxNo()) : BaseFunctionCtx::Ptr{}; assertrx(!ctx || ctx->type != BaseFunctionCtx::kFtCtx); if (leftIndex->Opts().GetCollateMode() == CollateUTF8) { @@ -230,9 +220,9 @@ void JoinedSelector::AppendSelectIteratorOfJoinIndexData(SelectIteratorContainer bool was = false; for (SelectKeyResult &res : leftIndex->SelectKey(values, CondSet, sortId, opts, ctx, rdxCtx)) { if (!res.comparators_.empty()) continue; - SelectIterator selIter{res, false, joinEntry.index_, - (joinEntry.idxNo < 0 ? IteratorFieldKind::NonIndexed : IteratorFieldKind::Indexed), false}; - selIter.Bind(leftNs_->payloadType_, joinEntry.idxNo); + SelectIterator selIter{res, false, joinEntry.LeftFieldName(), + (joinEntry.LeftIdxNo() < 0 ? IteratorFieldKind::NonIndexed : IteratorFieldKind::Indexed), false}; + selIter.Bind(leftNs_->payloadType_, joinEntry.LeftIdxNo()); const int curIterations = selIter.GetMaxIterations(); if (curIterations && curIterations < *maxIterations) *maxIterations = curIterations; iterators.Append(OpAnd, std::move(selIter)); diff --git a/cpp_src/core/nsselecter/joinedselector.h b/cpp_src/core/nsselecter/joinedselector.h index 0fb63758d..531a1c25e 100644 --- a/cpp_src/core/nsselecter/joinedselector.h +++ b/cpp_src/core/nsselecter/joinedselector.h @@ -88,7 +88,13 @@ class JoinedSelector { joinedSelectorsCount_(joinedSelectorsCount), rdxCtx_(rdxCtx), optimized_(false), - inTransaction_{inTransaction} {} + inTransaction_{inTransaction} { +#ifndef NDEBUG + for (const auto &jqe : joinQuery_.joinEntries_) { + assertrx_throw(jqe.FieldsHaveBeenSet()); + } +#endif + } JoinedSelector(JoinedSelector &&) = default; JoinedSelector &operator=(JoinedSelector &&) = delete; @@ -98,7 +104,7 @@ class JoinedSelector { bool Process(IdType, int nsId, ConstPayload, bool match); JoinType Type() const noexcept { return joinType_; } void SetType(JoinType type) noexcept { joinType_ = type; } - const std::string &RightNsName() const noexcept { return itemQuery_._namespace; } + const std::string &RightNsName() const noexcept { return itemQuery_.NsName(); } const JoinedQuery &JoinQuery() const noexcept { return joinQuery_; } int Called() const noexcept { return called_; } int Matched() const noexcept { return matched_; } @@ -109,10 +115,11 @@ class JoinedSelector { const NamespaceImpl::Ptr &RightNs() const noexcept { return rightNs_; } private: - template - void readValuesFromRightNs(VariantArray &values, KeyValueType leftIndexType, int rightIdxNo, std::string_view rightIndex) const; - template - void readValuesFromPreResult(VariantArray &values, KeyValueType leftIndexType, int rightIdxNo, std::string_view rightIndex) const; + [[nodiscard]] VariantArray readValuesFromRightNs(const QueryJoinEntry &) const; + [[nodiscard]] VariantArray readValuesFromPreResult(const QueryJoinEntry &) const; + template + [[nodiscard]] VariantArray readValuesOfRightNsFrom(const Cont &from, const Fn &createPayload, const QueryJoinEntry &, + const PayloadType &) const; void selectFromRightNs(QueryResults &joinItemR, const Query &, bool &found, bool &matchedAtLeastOnce); void selectFromPreResultValues(QueryResults &joinItemR, const Query &, bool &found, bool &matchedAtLeastOnce) const; @@ -134,7 +141,4 @@ class JoinedSelector { }; using JoinedSelectors = std::vector; -extern template void JoinedSelector::readValuesFromPreResult(VariantArray &, KeyValueType, int, std::string_view) const; -extern template void JoinedSelector::readValuesFromPreResult(VariantArray &, KeyValueType, int, std::string_view) const; - } // namespace reindexer diff --git a/cpp_src/core/nsselecter/joinedselectormock.h b/cpp_src/core/nsselecter/joinedselectormock.h index 5db3a49e6..e5cc58ea3 100644 --- a/cpp_src/core/nsselecter/joinedselectormock.h +++ b/cpp_src/core/nsselecter/joinedselectormock.h @@ -17,7 +17,7 @@ class JoinedSelectorMock { public: JoinedSelectorMock(JoinType jt, reindexer::JoinedQuery q) : query_{std::move(q)}, qr_{}, joinType_{jt} {} const reindexer::JoinedQuery& JoinQuery() const noexcept { return query_; } - const std::string& RightNsName() const noexcept { return query_._namespace; } + const std::string& RightNsName() const noexcept { return query_.NsName(); } reindexer::QueryResults& QueryResults() noexcept { return qr_; } const reindexer::QueryResults& QueryResults() const noexcept { return qr_; } JoinType Type() const noexcept { return joinType_; } diff --git a/cpp_src/core/nsselecter/nsselecter.cc b/cpp_src/core/nsselecter/nsselecter.cc index 0e7bf7650..48444789e 100644 --- a/cpp_src/core/nsselecter/nsselecter.cc +++ b/cpp_src/core/nsselecter/nsselecter.cc @@ -1,6 +1,5 @@ #include "nsselecter.h" -#include "core/cjson/jsonbuilder.h" #include "core/namespace/namespaceimpl.h" #include "core/queryresults/joinresults.h" #include "crashqueryreporter.h" @@ -48,10 +47,10 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte bool containAggCount = containSomeAggCount(AggCount); bool containAggCountCached = containAggCount ? false : containSomeAggCount(AggCountCached); - bool needCalcTotal = aggregationQueryRef.calcTotal == ModeAccurateTotal || containAggCount; + bool needCalcTotal = aggregationQueryRef.CalcTotal() == ModeAccurateTotal || containAggCount; QueryCacheKey ckey; - if (aggregationQueryRef.calcTotal == ModeCachedTotal || containAggCountCached) { + if (aggregationQueryRef.CalcTotal() == ModeCachedTotal || containAggCountCached) { ckey = QueryCacheKey{ctx.query}; auto cached = ns_->queryTotalCountCache_->Get(ckey); @@ -362,10 +361,10 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte } } // Put count/count_cached to aggretions - if (aggregationQueryRef.calcTotal != ModeNoTotal || containAggCount || containAggCountCached) { + if (aggregationQueryRef.CalcTotal() != ModeNoTotal || containAggCount || containAggCountCached) { AggregationResult ret; ret.fields = {"*"}; - ret.type = (aggregationQueryRef.calcTotal == ModeAccurateTotal || containAggCount) ? AggCount : AggCountCached; + ret.type = (aggregationQueryRef.CalcTotal() == ModeAccurateTotal || containAggCount) ? AggCount : AggCountCached; if (ctx.isMergeQuerySubQuery()) { assertrx_throw(!result.aggregationResults.empty()); auto &agg = result.aggregationResults.back(); @@ -783,7 +782,7 @@ It NsSelecter::applyForcedSortImpl(NamespaceImpl &ns, It begin, It end, const It // implementation for composite indexes const auto &payloadType = ns.payloadType_; const FieldsSet &fields = ns.indexes_[idx]->Fields(); - unordered_payload_map sortMap(0, payloadType, fields); + unordered_payload_map sortMap(0, PayloadType{payloadType}, FieldsSet{fields}); ForcedMapInserter inserter{sortMap}; for (auto value : forcedSortOrder) { value.convert(fieldType, &payloadType, &fields); @@ -1323,7 +1322,7 @@ void NsSelecter::prepareSortJoinedIndex(size_t nsIdx, std::string_view column, i .FieldByName(std::string{column}, index); if (index == IndexValueType::SetByJsonPath) { skipSortingEntry |= !validateField( - strictMode, column, js.joinQuery_._namespace, + strictMode, column, js.joinQuery_.NsName(), js.preResult_->dataMode == JoinPreResult::ModeValues ? js.preResult_->values.tagsMatcher : js.rightNs_->tagsMatcher_); } } @@ -1565,11 +1564,11 @@ size_t NsSelecter::calculateNormalCost(const QueryEntries &qentries, SelectCtx & [&costCalculator](const JoinQueryEntry &) { costCalculator.MarkInapposite(); }, [&costCalculator](const BetweenFieldsQueryEntry &) { costCalculator.MarkInapposite(); }, [&](const QueryEntry &qe) { - if (qe.idxNo < 0) { + if (!qe.IsFieldIndexed()) { costCalculator.MarkInapposite(); return; } - if (qe.idxNo == ctx.sortingContext.uncommitedIndex) { + if (qe.IndexNo() == ctx.sortingContext.uncommitedIndex) { if (sortIndexSearchState == SortIndexNotFound) { const bool isExpectingIdSet = qentries.GetOperation(i) == OpAnd && (next == sz || qentries.GetOperation(next) != OpOr); @@ -1591,7 +1590,7 @@ size_t NsSelecter::calculateNormalCost(const QueryEntries &qentries, SelectCtx & return; } - auto &index = ns_->indexes_[qe.idxNo]; + auto &index = ns_->indexes_[qe.IndexNo()]; if (IsFullText(index->Type())) { costCalculator.MarkInapposite(); return; @@ -1604,8 +1603,8 @@ size_t NsSelecter::calculateNormalCost(const QueryEntries &qentries, SelectCtx & opts.inTransaction = ctx.inTransaction; try { - SelectKeyResults reslts = index->SelectKey(qe.values, qe.condition, 0, opts, nullptr, rdxCtx); - costCalculator.Add(reslts, qe.idxNo == ctx.sortingContext.uncommitedIndex); + SelectKeyResults reslts = index->SelectKey(qe.Values(), qe.Condition(), 0, opts, nullptr, rdxCtx); + costCalculator.Add(reslts, qe.IndexNo() == ctx.sortingContext.uncommitedIndex); } catch (const Error &) { costCalculator.MarkInapposite(); } @@ -1632,7 +1631,7 @@ size_t NsSelecter::calculateOptimizedCost(size_t costNormal, const QueryEntries [&costCalculator](const JoinQueryEntry &) { costCalculator.MarkInapposite(); }, [&costCalculator](const BetweenFieldsQueryEntry &) { costCalculator.MarkInapposite(); }, [&](const QueryEntry &qe) { - if (qe.idxNo < 0 || qe.idxNo != ctx.sortingContext.uncommitedIndex) { + if (!qe.IsFieldIndexed() || qe.IndexNo() != ctx.sortingContext.uncommitedIndex) { costCalculator.MarkInapposite(); return; } @@ -1645,7 +1644,7 @@ size_t NsSelecter::calculateOptimizedCost(size_t costNormal, const QueryEntries opts.inTransaction = ctx.inTransaction; try { - SelectKeyResults reslts = ns_->indexes_[qe.idxNo]->SelectKey(qe.values, qe.condition, 0, opts, nullptr, rdxCtx); + SelectKeyResults reslts = ns_->indexes_[qe.IndexNo()]->SelectKey(qe.Values(), qe.Condition(), 0, opts, nullptr, rdxCtx); costCalculator.Add(reslts); } catch (const Error &) { costCalculator.MarkInapposite(); @@ -1662,7 +1661,7 @@ bool NsSelecter::isSortOptimizatonEffective(const QueryEntries &qentries, Select } if (qentries.Size() == 1 && qentries.HoldsOrReferTo(0)) { const auto &qe = qentries.Get(0); - if (qe.idxNo == ctx.sortingContext.uncommitedIndex) { + if (qe.IndexNo() == ctx.sortingContext.uncommitedIndex) { return SelectIteratorContainer::IsExpectingOrderedResults(qe); } } @@ -1696,8 +1695,8 @@ bool NsSelecter::isSortOptimizatonEffective(const QueryEntries &qentries, Select // TODO: It's possible to evaluate this multiplier, based on the query conditions, but the only way to avoid corner cases is to // allow user to hint this optimization. const size_t limitMultiplier = std::max(size_t(20), size_t(totalItemsCount / expectedMaxIterationsNormal) * 4); - const auto offset = ctx.query.HasOffset() ? ctx.query.start : 1; - costOptimized = limitMultiplier * (ctx.query.count + offset); + const auto offset = ctx.query.HasOffset() ? ctx.query.Offset() : 1; + costOptimized = limitMultiplier * (ctx.query.Limit() + offset); } return costOptimized <= costNormal; } diff --git a/cpp_src/core/nsselecter/querypreprocessor.cc b/cpp_src/core/nsselecter/querypreprocessor.cc index e7abaf49e..6070eb478 100644 --- a/cpp_src/core/nsselecter/querypreprocessor.cc +++ b/cpp_src/core/nsselecter/querypreprocessor.cc @@ -1,7 +1,6 @@ #include "querypreprocessor.h" #include "core/index/index.h" -#include "core/index/indextext/indextext.h" #include "core/namespace/namespaceimpl.h" #include "core/nsselecter/joinedselector.h" #include "core/nsselecter/selectiteratorcontainer.h" @@ -9,7 +8,6 @@ #include "core/payload/fieldsset.h" #include "core/query/dsl/dslencoder.h" #include "core/query/queryentry.h" -#include "estl/overloaded.h" #include "nsselecter.h" #include "qresexplainholder.h" #include "substitutionhelpers.h" @@ -22,25 +20,23 @@ QueryPreprocessor::QueryPreprocessor(QueryEntries &&queries, NamespaceImpl *ns, query_{ctx.query}, strictMode_(ctx.inTransaction ? StrictModeNone : ((query_.strictMode == StrictModeNotSet) ? ns_.config_.strictMode : query_.strictMode)), - start_(query_.start), - count_(query_.count), + start_(query_.Offset()), + count_(query_.Limit()), forcedSortOrder_(!query_.forcedSortOrder_.empty()), reqMatchedOnce_(ctx.reqMatchedOnceFlag) { if (forcedSortOrder_ && (start_ > QueryEntry::kDefaultOffset || count_ < QueryEntry::kDefaultLimit)) { - assertrx(!query_.sortingEntries_.empty()); + assertrx_throw(!query_.sortingEntries_.empty()); static const std::vector emptyJoinedSelectors; const auto &sEntry = query_.sortingEntries_[0]; if (SortExpression::Parse(sEntry.expression, emptyJoinedSelectors).ByIndexField()) { - QueryEntry qe; - qe.values.reserve(query_.forcedSortOrder_.size()); - for (const auto &v : query_.forcedSortOrder_) qe.values.push_back(v); - qe.condition = query_.forcedSortOrder_.size() == 1 ? CondEq : CondSet; - qe.index = sEntry.expression; - if (!ns_.getIndexByNameOrJsonPath(qe.index, qe.idxNo)) { - qe.idxNo = IndexValueType::SetByJsonPath; - } + VariantArray values; + values.reserve(query_.forcedSortOrder_.size()); + for (const auto &v : query_.forcedSortOrder_) values.push_back(v); desc_ = sEntry.desc; - Append(desc_ ? OpNot : OpAnd, std::move(qe)); + QueryField fld{sEntry.expression}; + SetQueryField(fld, ns_); + Append(desc_ ? OpNot : OpAnd, std::move(fld), query_.forcedSortOrder_.size() == 1 ? CondEq : CondSet, + std::move(values)); queryEntryAddedByForcedSortOptimization_ = true; } } @@ -58,9 +54,8 @@ void QueryPreprocessor::ExcludeFtQuery(const RdxContext &rdxCtx) { if (queryEntryAddedByForcedSortOptimization_ || Size() <= 1) return; for (auto it = begin(), next = it, endIt = end(); it != endIt; it = next) { ++next; - if (it->HoldsOrReferTo() && it->Value().idxNo != IndexValueType::SetByJsonPath) { - const auto indexNo = it->Value().idxNo; - auto &index = ns_.indexes_[indexNo]; + if (it->HoldsOrReferTo() && it->Value().IsFieldIndexed()) { + auto &index = ns_.indexes_[it->Value().IndexNo()]; if (!IsFastFullText(index->Type())) continue; if (it->operation != OpAnd || (next != endIt && next->operation == OpOr) || !index->EnablePreselectBeforeFt()) break; ftPreselect_ = index->FtPreselect(rdxCtx); @@ -80,16 +75,16 @@ bool QueryPreprocessor::NeedNextEvaluation(unsigned start, unsigned count, bool if (evaluationsCount_++) return false; if (queryEntryAddedByForcedSortOptimization_) { container_.back().operation = desc_ ? OpAnd : OpNot; - assertrx(start <= start_); + assertrx_throw(start <= start_); start_ = start; - assertrx(count <= count_); + assertrx_throw(count <= count_); count_ = count; return count_ || (reqMatchedOnce_ && !matchedAtLeastOnce); } else if (ftEntry_) { if (!matchedAtLeastOnce) return false; qresHolder.BackupContainer(); - start_ = query_.start; - count_ = query_.count; + start_ = query_.Offset(); + count_ = query_.Limit(); forcedSortOrder_ = !query_.forcedSortOrder_.empty(); clear(); Append(OpAnd, std::move(*ftEntry_)); @@ -101,19 +96,19 @@ bool QueryPreprocessor::NeedNextEvaluation(unsigned start, unsigned count, bool return false; } -void QueryPreprocessor::checkStrictMode(const std::string &index, int idxNo) const { - if (idxNo != IndexValueType::SetByJsonPath) return; +void QueryPreprocessor::checkStrictMode(const QueryField &field) const { + if (field.IsFieldIndexed()) return; switch (strictMode_) { case StrictModeIndexes: - throw Error(errParams, + throw Error(errQueryExec, "Current query strict mode allows filtering by indexes only. There are no indexes with name '%s' in namespace '%s'", - index, ns_.name_); + field.FieldName(), ns_.name_); case StrictModeNames: - if (ns_.tagsMatcher_.path2tag(index).empty()) { - throw Error(errParams, + if (field.HaveEmptyField()) { + throw Error(errQueryExec, "Current query strict mode allows filtering by existing fields only. There are no fields with name '%s' in " "namespace '%s'", - index, ns_.name_); + field.FieldName(), ns_.name_); } case StrictModeNotSet: case StrictModeNone: @@ -151,7 +146,7 @@ bool QueryPreprocessor::removeBrackets() { bool QueryPreprocessor::canRemoveBracket(size_t i) const { if (Size(i) < 2) { - throw Error{errParams, "Bracket cannot be empty"}; + throw Error{errQueryExec, "Bracket cannot be empty"}; } const size_t next = Next(i); const OpType op = GetOperation(i); @@ -162,7 +157,7 @@ bool QueryPreprocessor::canRemoveBracket(size_t i) const { size_t QueryPreprocessor::removeBrackets(size_t begin, size_t end) { if (begin != end && GetOperation(begin) == OpOr) { - throw Error{errParams, "First condition cannot be with operation OR"}; + throw Error{errQueryExec, "OR operator in first condition or after left join"}; } size_t deleted = 0; for (size_t i = begin; i < end - deleted; i = Next(i)) { @@ -183,31 +178,23 @@ void QueryPreprocessor::InitIndexNumbers() { ExecuteAppropriateForEach( Skip{}, [this](QueryEntry &entry) { - if (entry.idxNo == IndexValueType::NotSet) { - if (!ns_.getIndexByNameOrJsonPath(entry.index, entry.idxNo)) { - entry.idxNo = IndexValueType::SetByJsonPath; - } + if (!entry.FieldsHaveBeenSet()) { + SetQueryField(entry.FieldData(), ns_); } - checkStrictMode(entry.index, entry.idxNo); + checkStrictMode(entry.FieldData()); }, [this](BetweenFieldsQueryEntry &entry) { - if (entry.firstIdxNo == IndexValueType::NotSet) { - if (!ns_.getIndexByNameOrJsonPath(entry.firstIndex, entry.firstIdxNo)) { - entry.firstIdxNo = IndexValueType::SetByJsonPath; - } - } - checkStrictMode(entry.firstIndex, entry.firstIdxNo); - if (entry.secondIdxNo == IndexValueType::NotSet) { - if (!ns_.getIndexByNameOrJsonPath(entry.secondIndex, entry.secondIdxNo)) { - entry.secondIdxNo = IndexValueType::SetByJsonPath; - } + if (!entry.FieldsHaveBeenSet()) { + SetQueryField(entry.LeftFieldData(), ns_); + SetQueryField(entry.RightFieldData(), ns_); } - checkStrictMode(entry.secondIndex, entry.secondIdxNo); + checkStrictMode(entry.LeftFieldData()); + checkStrictMode(entry.RightFieldData()); }); } size_t QueryPreprocessor::lookupQueryIndexes(uint16_t dst, uint16_t srcBegin, uint16_t srcEnd) { - assertrx(dst <= srcBegin); + assertrx_throw(dst <= srcBegin); h_vector iidx(kMaxIndexes, uint16_t(0)); size_t merged = 0; for (size_t src = srcBegin, nextSrc; src < srcEnd; src = nextSrc) { @@ -221,17 +208,16 @@ size_t QueryPreprocessor::lookupQueryIndexes(uint16_t dst, uint16_t srcBegin, ui return true; }, [&](QueryEntry &entry) { - const bool isIndexField = (entry.idxNo >= 0); - if (isIndexField) { + if (entry.IsFieldIndexed()) { // try merge entries with AND opetator if ((GetOperation(src) == OpAnd) && (nextSrc >= srcEnd || GetOperation(nextSrc) != OpOr)) { - if (size_t(entry.idxNo) >= iidx.size()) { + if (size_t(entry.IndexNo()) >= iidx.size()) { const auto oldSize = iidx.size(); - iidx.resize(size_t(entry.idxNo) + 1); + iidx.resize(entry.IndexNo() + 1); std::fill(iidx.begin() + oldSize, iidx.begin() + iidx.size(), 0); } - auto &iidxRef = iidx[entry.idxNo]; - if (iidxRef > 0 && !ns_.indexes_[entry.idxNo]->Opts().IsArray()) { + auto &iidxRef = iidx[entry.IndexNo()]; + if (iidxRef > 0 && !ns_.indexes_[entry.IndexNo()]->Opts().IsArray()) { if (mergeQueryEntries(iidxRef - 1, src)) { ++merged; return false; @@ -265,9 +251,9 @@ size_t QueryPreprocessor::lookupQueryIndexes(uint16_t dst, uint16_t srcBegin, ui void QueryPreprocessor::CheckUniqueFtQuery() const { bool found = false; ExecuteAppropriateForEach(Skip{}, [&](const QueryEntry &qe) { - if (qe.idxNo != IndexValueType::SetByJsonPath && IsFullText(ns_.indexes_[qe.idxNo]->Type())) { + if (qe.IsFieldIndexed() && IsFullText(ns_.indexes_[qe.IndexNo()]->Type())) { if (found) { - throw Error{errParams, "Query cannot contain more than one full text condition"}; + throw Error{errQueryExec, "Query cannot contain more than one full text condition"}; } else { found = true; } @@ -277,8 +263,8 @@ void QueryPreprocessor::CheckUniqueFtQuery() const { bool QueryPreprocessor::ContainsFullTextIndexes() const { for (auto it = cbegin().PlainIterator(), end = cend().PlainIterator(); it != end; ++it) { - if (it->HoldsOrReferTo() && it->Value().idxNo != IndexValueType::SetByJsonPath && - IsFullText(ns_.indexes_[it->Value().idxNo]->Type())) { + if (it->HoldsOrReferTo() && it->Value().IsFieldIndexed() && + IsFullText(ns_.indexes_[it->Value().IndexNo()]->Type())) { return true; } } @@ -321,11 +307,14 @@ static void createCompositeKeyValues(const h_vector } } -static void createCompositeKeyValues(const h_vector, 4> &values, const PayloadType &plType, - VariantArray &ret) { +static VariantArray createCompositeKeyValues(const h_vector, 4> &values, const PayloadType &plType, + uint32_t resultSetSize) { PayloadValue d(plType.TotalSize()); Payload pl(plType, d); + VariantArray ret; + ret.reserve(resultSetSize); createCompositeKeyValues(values, plType, pl, ret, 0); + return ret; } size_t QueryPreprocessor::substituteCompositeIndexes(const size_t from, const size_t to) { @@ -350,15 +339,16 @@ size_t QueryPreprocessor::substituteCompositeIndexes(const size_t from, const si continue; } auto &qe = Get(cur); - if ((qe.condition != CondEq && qe.condition != CondSet) || qe.idxNo >= ns_.payloadType_.NumFields() || qe.idxNo < 0) { + if ((qe.Condition() != CondEq && qe.Condition() != CondSet) || !qe.IsFieldIndexed() || + qe.IndexNo() >= ns_.payloadType_.NumFields()) { continue; } - const std::vector *found = getCompositeIndex(qe.idxNo); + const std::vector *found = getCompositeIndex(qe.IndexNo()); if (!found || found->empty()) { continue; } - searcher.Add(qe.idxNo, *found, cur); + searcher.Add(qe.IndexNo(), *found, cur); } EntriesRanges deleteRanges; @@ -371,14 +361,13 @@ size_t QueryPreprocessor::substituteCompositeIndexes(const size_t from, const si uint32_t maxSetSize = 0; for (auto i : res.entries) { auto &qe = Get(i); - if rx_unlikely (!res.fields.contains(qe.idxNo)) { + if rx_unlikely (!res.fields.contains(qe.IndexNo())) { throw Error(errLogic, "Error during composite index's fields substitution (this should not happen)"); } - - maxSetSize = std::max(maxSetSize, qe.values.size()); - resultSetSize = (resultSetSize == 0) ? qe.values.size() : (resultSetSize * qe.values.size()); + maxSetSize = std::max(maxSetSize, qe.Values().size()); + resultSetSize = (resultSetSize == 0) ? qe.Values().size() : (resultSetSize * qe.Values().size()); } - static const CompositeValuesCountLimits kCompositeSetLimits; + constexpr static CompositeValuesCountLimits kCompositeSetLimits; if (resultSetSize != maxSetSize) { // Do not perform substitution if result set size becoms larger than initial indexes set size // and this size is greater than limit @@ -390,22 +379,17 @@ size_t QueryPreprocessor::substituteCompositeIndexes(const size_t from, const si } for (auto i : res.entries) { auto &qe = Get(i); - const auto idxKeyType = ns_.indexes_[qe.idxNo]->KeyType(); - for (auto &v : qe.values) { - v.convert(idxKeyType); - } - values.emplace_back(qe.idxNo, std::move(qe.values)); + qe.ConvertValuesToFieldType(); + const int idxNo = qe.IndexNo(); + values.emplace_back(idxNo, std::move(qe).Values()); } { - QueryEntry ce(CondSet, ns_.indexes_[res.idx]->Name(), res.idx); - ce.values.reserve(resultSetSize); - createCompositeKeyValues(values, ns_.payloadType_, ce.values); - if (ce.values.size() == 1) { - ce.condition = CondEq; - } + VariantArray qValues = createCompositeKeyValues(values, ns_.payloadType_, resultSetSize); const auto first = res.entries.front(); SetOperation(OpAnd, first); - container_[first].SetValue(std::move(ce)); + QueryField fld{ns_.indexes_[res.idx]->Name()}; + setQueryIndex(fld, res.idx, ns_); + container_[first].Emplace(std::move(fld), qValues.size() == 1 ? CondEq : CondSet, std::move(qValues)); } deleteRanges.Add(span(res.entries.data() + 1, res.entries.size() - 1)); resIdx = searcher.RemoveUsedAndGetNext(resIdx); @@ -417,29 +401,14 @@ size_t QueryPreprocessor::substituteCompositeIndexes(const size_t from, const si return deleted; } -void QueryPreprocessor::convertWhereValues(QueryEntry *qe) const { - const FieldsSet *fields = nullptr; - KeyValueType keyType{KeyValueType::Undefined{}}; - const bool isIndexField = (qe->idxNo != IndexValueType::SetByJsonPath); - if (isIndexField) { - keyType = ns_.indexes_[qe->idxNo]->SelectKeyType(); - fields = &ns_.indexes_[qe->idxNo]->Fields(); - } - if (!keyType.Is()) { - if (qe->condition != CondDWithin) { - for (auto &key : qe->values) { - key.convert(keyType, &ns_.payloadType_, fields); - } - } - } -} +void QueryPreprocessor::convertWhereValues(QueryEntry &qe) const { qe.ConvertValuesToFieldType(ns_.payloadType_); } void QueryPreprocessor::convertWhereValues(QueryEntries::iterator begin, QueryEntries::iterator end) const { for (auto it = begin; it != end; ++it) { it->InvokeAppropriate( Skip{}, [this, &it](const QueryEntriesBracket &) { convertWhereValues(it.begin(), it.end()); }, - [this](QueryEntry &qe) { convertWhereValues(&qe); }); + [this](QueryEntry &qe) { convertWhereValues(qe); }); } } @@ -481,12 +450,12 @@ void QueryPreprocessor::findMaxIndex(QueryEntries::const_iterator begin, QueryEn return FoundIndexInfo(); }, [this](const QueryEntry &entry) -> FoundIndexInfo { - if (entry.idxNo != IndexValueType::SetByJsonPath && !entry.distinct) { - const auto idxPtr = ns_.indexes_[entry.idxNo].get(); + if (entry.IsFieldIndexed() && !entry.Distinct()) { + const auto idxPtr = ns_.indexes_[entry.IndexNo()].get(); if (idxPtr->IsOrdered() && !idxPtr->Opts().IsArray()) { - if (IsOrderedCondition(entry.condition)) { + if (IsOrderedCondition(entry.Condition())) { return FoundIndexInfo{idxPtr, FoundIndexInfo::ConditionType::Compatible}; - } else if (entry.condition == CondAny || entry.values.size() > 1) { + } else if (entry.Condition() == CondAny || entry.Values().size() > 1) { return FoundIndexInfo{idxPtr, FoundIndexInfo::ConditionType::Incompatible}; } } @@ -510,42 +479,42 @@ void QueryPreprocessor::findMaxIndex(QueryEntries::const_iterator begin, QueryEn bool QueryPreprocessor::mergeQueryEntries(size_t lhs, size_t rhs) { QueryEntry *lqe = &Get(lhs); QueryEntry &rqe = Get(rhs); - if ((lqe->condition == CondEq || lqe->condition == CondSet) && (rqe.condition == CondEq || rqe.condition == CondSet)) { + if ((lqe->Condition() == CondEq || lqe->Condition() == CondSet) && (rqe.Condition() == CondEq || rqe.Condition() == CondSet)) { // intersect 2 queryentries on the same index - if rx_unlikely (lqe->values.empty()) { + if rx_unlikely (lqe->Values().empty()) { return true; } if (container_[lhs].IsRef()) { container_[lhs].SetValue(const_cast(*lqe)); lqe = &Get(lhs); } + const bool distinct = lqe->Distinct() || rqe.Distinct(); VariantArray setValues; - if (rx_likely(!rqe.values.empty())) { - convertWhereValues(lqe); - convertWhereValues(&rqe); - VariantArray *first = &lqe->values; - VariantArray *second = &rqe.values; - if (lqe->values.size() > rqe.values.size()) { - std::swap(first, second); - } - setValues.reserve(first->size()); + if (rx_likely(!rqe.Values().empty())) { + convertWhereValues(*lqe); + convertWhereValues(rqe); + auto &&[first, second] = lqe->Values().size() < rqe.Values().size() + ? std::make_pair(std::move(*lqe).Values(), std::move(rqe).Values()) + : std::make_pair(std::move(rqe).Values(), std::move(*lqe).Values()); + + setValues.reserve(first.size()); constexpr size_t kMinArraySizeToUseHashSet = 250; - if (second->size() < kMinArraySizeToUseHashSet) { + if (second.size() < kMinArraySizeToUseHashSet) { // Intersect via binary search + sort for small vectors - std::sort(first->begin(), first->end()); - for (auto &&v : *second) { - if (std::binary_search(first->begin(), first->end(), v)) { + std::sort(first.begin(), first.end()); + for (auto &&v : second) { + if (std::binary_search(first.begin(), first.end(), v)) { setValues.emplace_back(std::move(v)); } } } else { // Intersect via hash_set for large vectors reindexer::fast_hash_set set; - set.reserve(first->size() * 2); - for (auto &&v : *first) { + set.reserve(first.size() * 2); + for (auto &&v : first) { set.emplace(std::move(v)); } - for (auto &&v : *second) { + for (auto &&v : second) { if (set.erase(v)) { setValues.emplace_back(std::move(v)); } @@ -553,27 +522,26 @@ bool QueryPreprocessor::mergeQueryEntries(size_t lhs, size_t rhs) { } } - lqe->values = std::move(setValues); - lqe->condition = (lqe->values.size() == 1) ? CondEq : CondSet; - lqe->distinct |= rqe.distinct; + lqe->SetCondAndValues(CondSet, std::move(setValues)); + lqe->Distinct(distinct); return true; - } else if (rqe.condition == CondAny) { - if (!lqe->distinct && rqe.distinct) { + } else if (rqe.Condition() == CondAny) { + if (!lqe->Distinct() && rqe.Distinct()) { if (container_[lhs].IsRef()) { container_[lhs].SetValue(const_cast(*lqe)); lqe = &Get(lhs); } - lqe->distinct = true; + lqe->Distinct(true); } return true; - } else if (lqe->condition == CondAny) { - const bool distinct = lqe->distinct || rqe.distinct; + } else if (lqe->Condition() == CondAny) { + const bool distinct = lqe->Distinct() || rqe.Distinct(); if (container_[rhs].IsRef()) { container_[lhs].SetValue(const_cast(rqe)); } else { container_[lhs].SetValue(std::move(rqe)); } - Get(lhs).distinct = distinct; + Get(lhs).Distinct(distinct); return true; } @@ -584,19 +552,16 @@ void QueryPreprocessor::AddDistinctEntries(const h_vector &aggreg bool wasAdded = false; for (auto &ag : aggregators) { if (ag.Type() != AggDistinct) continue; - QueryEntry qe; - assertrx(ag.Names().size() == 1); - qe.index = ag.Names()[0]; - qe.condition = CondAny; - qe.distinct = true; - Append(wasAdded ? OpOr : OpAnd, std::move(qe)); + assertrx_throw(ag.Names().size() == 1); + Append(wasAdded ? OpOr : OpAnd, ag.Names()[0], QueryEntry::DistinctTag{}); wasAdded = true; } } -void QueryPreprocessor::fillQueryEntryFromOnCondition(QueryEntry &queryEntry, std::string &explainStr, AggType &oAggType, - NamespaceImpl &rightNs, Query joinQuery, std::string joinIndex, CondType condition, - KeyValueType valuesType, const RdxContext &rdxCtx) { +std::pair QueryPreprocessor::queryValuesFromOnCondition(std::string &explainStr, AggType &oAggType, + NamespaceImpl &rightNs, Query joinQuery, + const QueryJoinEntry &joinEntry, CondType condition, + const RdxContext &rdxCtx) { size_t limit; const auto &rNsCfg = rightNs.Config(); if (rNsCfg.maxPreselectSize == 0) { @@ -608,25 +573,25 @@ void QueryPreprocessor::fillQueryEntryFromOnCondition(QueryEntry &queryEntry, st std::min(std::max(rNsCfg.minPreselectSize, rightNs.ItemsCount() * rNsCfg.maxPreselectPart), rNsCfg.maxPreselectSize); } joinQuery.explain_ = query_.explain_; - joinQuery.count = limit + 2; - joinQuery.start = 0; + joinQuery.Limit(limit + 2); + joinQuery.Offset(QueryEntry::kDefaultOffset); joinQuery.sortingEntries_.clear(); joinQuery.forcedSortOrder_.clear(); joinQuery.aggregations_.clear(); switch (condition) { case CondEq: case CondSet: - joinQuery.Distinct(std::move(joinIndex)); + joinQuery.Distinct(joinEntry.RightFieldName()); oAggType = AggType::AggDistinct; break; case CondLt: case CondLe: - joinQuery.Aggregate(AggMax, {std::move(joinIndex)}); + joinQuery.Aggregate(AggMax, {joinEntry.RightFieldName()}); oAggType = AggType::AggMax; break; case CondGt: case CondGe: - joinQuery.Aggregate(AggMin, {std::move(joinIndex)}); + joinQuery.Aggregate(AggMin, {joinEntry.RightFieldName()}); oAggType = AggType::AggMin; break; case CondAny: @@ -635,104 +600,87 @@ void QueryPreprocessor::fillQueryEntryFromOnCondition(QueryEntry &queryEntry, st case CondEmpty: case CondLike: case CondDWithin: - throw Error(errParams, "Unsupported condition in ON statment: %s", CondTypeToStr(condition)); + throw Error(errQueryExec, "Unsupported condition in ON statment: %s", CondTypeToStr(condition)); } SelectCtx ctx{joinQuery, nullptr}; QueryResults qr; rightNs.Select(qr, ctx, rdxCtx); - if (qr.Count() > limit) return; - assertrx(qr.aggregationResults.size() == 1); + if (qr.Count() > limit) return {CondAny, {}}; + assertrx_throw(qr.aggregationResults.size() == 1); + auto &aggRes = qr.aggregationResults[0]; explainStr = qr.explainResults; switch (condition) { case CondEq: case CondSet: { - assertrx(qr.aggregationResults[0].type == AggDistinct); - queryEntry.values.reserve(qr.aggregationResults[0].distincts.size()); - assertrx(qr.aggregationResults[0].distinctsFields.size() == 1); - const auto field = qr.aggregationResults[0].distinctsFields[0]; - for (Variant &distValue : qr.aggregationResults[0].distincts) { + assertrx_throw(aggRes.type == AggDistinct); + VariantArray values; + values.reserve(aggRes.distincts.size()); + for (Variant &distValue : aggRes.distincts) { if (distValue.Type().Is()) { - ConstPayload pl(qr.aggregationResults[0].payloadType, distValue.operator const PayloadValue &()); - VariantArray v; - if (field == IndexValueType::SetByJsonPath) { - assertrx(qr.aggregationResults[0].distinctsFields.getTagsPathsLength() == 1); - pl.GetByJsonPath(qr.aggregationResults[0].distinctsFields.getTagsPath(0), v, valuesType); - } else { - pl.Get(field, v); - } - assertrx(v.size() == 1); - queryEntry.values.emplace_back(std::move(v[0])); + ConstPayload pl(aggRes.payloadType, distValue.operator const PayloadValue &()); + values.emplace_back(pl.GetComposite(aggRes.distinctsFields, joinEntry.RightCompositeFieldsTypes())); } else { - queryEntry.values.emplace_back(std::move(distValue)); + values.emplace_back(std::move(distValue)); } } - queryEntry.condition = (queryEntry.values.size() == 1) ? CondEq : CondSet; - break; + return {CondSet, std::move(values)}; } case CondLt: case CondLe: case CondGt: case CondGe: - if (auto value = qr.aggregationResults[0].GetValue()) { - queryEntry.condition = condition; - queryEntry.values.emplace_back(*value); + if (auto value = aggRes.GetValue()) { + return {condition, {Variant{*value}}}; + } else { + return {CondAny, {}}; } - break; case CondAny: case CondRange: case CondAllSet: case CondEmpty: case CondLike: case CondDWithin: - throw Error(errParams, "Unsupported condition in ON statment: %s", CondTypeToStr(condition)); + default: + throw Error(errQueryExec, "Unsupported condition in ON statment: %s", CondTypeToStr(condition)); } } -template -void QueryPreprocessor::fillQueryEntryFromOnCondition(QueryEntry &queryEntry, std::string_view joinIndex, CondType condition, - const JoinedSelector &joinedSelector, KeyValueType valuesType, const int rightIdxNo, - const CollateOpts &collate) { - JoinPreResult::Values &values = joinedSelector.preResult_->values; +std::pair QueryPreprocessor::queryValuesFromOnCondition(CondType condition, const QueryJoinEntry &joinEntry, + const JoinedSelector &joinedSelector, + const CollateOpts &collate) { switch (condition) { case CondEq: - case CondSet: { - joinedSelector.readValuesFromPreResult(queryEntry.values, valuesType, rightIdxNo, joinIndex); - queryEntry.condition = (queryEntry.values.size() == 1) ? CondEq : CondSet; - return; - } + case CondSet: + return {CondSet, joinedSelector.readValuesFromPreResult(joinEntry)}; case CondLt: case CondLe: case CondGt: case CondGe: { - queryEntry.condition = condition; - VariantArray buffer; + const JoinPreResult::Values &values = joinedSelector.preResult_->values; + VariantArray buffer, keyValues; for (const ItemRef &item : values) { - buffer.clear(); - assertrx(!item.Value().IsFree()); + assertrx_throw(!item.Value().IsFree()); const ConstPayload pl{values.payloadType, item.Value()}; - if constexpr (byJsonPath) { - pl.GetByJsonPath(joinIndex, values.tagsMatcher, buffer, valuesType); - } else { - pl.Get(rightIdxNo, buffer); - } + pl.GetByFieldsSet(joinEntry.RightFields(), buffer, joinEntry.RightFieldType(), joinEntry.RightCompositeFieldsTypes()); for (Variant &v : buffer) { - if (queryEntry.values.empty()) { - queryEntry.values.emplace_back(std::move(v)); + if (keyValues.empty()) { + keyValues.emplace_back(std::move(v)); } else { - const auto cmp = queryEntry.values[0].Compare(v, collate); + const auto cmp = keyValues[0].Compare(v, collate); if (condition == CondLt || condition == CondLe) { if (cmp < 0) { - queryEntry.values[0] = std::move(v); + keyValues[0] = std::move(v); } } else { if (cmp > 0) { - queryEntry.values[0] = std::move(v); + keyValues[0] = std::move(v); } } } } } + return {condition, std::move(keyValues)}; } break; case CondAny: case CondRange: @@ -740,7 +688,8 @@ void QueryPreprocessor::fillQueryEntryFromOnCondition(QueryEntry &queryEntry, st case CondEmpty: case CondLike: case CondDWithin: - throw Error(errParams, "Unsupported condition in ON statment: %s", CondTypeToStr(condition)); + default: + throw Error(errQueryExec, "Unsupported condition in ON statment: %s", CondTypeToStr(condition)); } } @@ -782,8 +731,7 @@ size_t QueryPreprocessor::injectConditionsFromJoins(size_t from, size_t to, Join assertrx_throw(to <= container_.size()); }, [&](const JoinQueryEntry &jqe) { - assertrx(js.size() > jqe.joinIndex); - + assertrx_throw(js.size() > jqe.joinIndex); JoinedSelector &joinedSelector = js[jqe.joinIndex]; const bool byValues = joinedSelector.PreResult() && joinedSelector.PreResult()->dataMode == JoinPreResult::ModeValues; @@ -803,14 +751,13 @@ size_t QueryPreprocessor::injectConditionsFromJoins(size_t from, size_t to, Join return; } } - const auto &joinEntries = joinedSelector.joinQuery_.joinEntries_; // LeftJoin-s shall not be in QueryEntries container_ by construction - assertrx(joinedSelector.Type() == InnerJoin || joinedSelector.Type() == OrInnerJoin); + assertrx_throw(joinedSelector.Type() == InnerJoin || joinedSelector.Type() == OrInnerJoin); // Checking if we have anything to inject into main Where clause bool foundANDOrOR = false; for (const auto &je : joinEntries) { - if (je.op_ != OpNot) { + if (je.Operation() != OpNot) { foundANDOrOR = true; break; } @@ -822,7 +769,7 @@ size_t QueryPreprocessor::injectConditionsFromJoins(size_t from, size_t to, Join OpType op = GetOperation(cur); if (joinedSelector.Type() == OrInnerJoin) { - if (op == OpNot) throw Error(errParams, "OR INNER JOIN with operation NOT"); + if (op == OpNot) throw Error(errQueryExec, "OR INNER JOIN with operation NOT"); op = OpOr; joinedSelector.SetType(InnerJoin); } @@ -832,6 +779,8 @@ size_t QueryPreprocessor::injectConditionsFromJoins(size_t from, size_t to, Join // !!!Warning jqe reference will be invalidated after EncloseInBracket EncloseInBracket(cur, cur + 1, op); ++cur; + ++to; + ++injectedCount; explainJoinOn.ReserveOnEntries(joinEntries.size()); @@ -840,12 +789,10 @@ size_t QueryPreprocessor::injectConditionsFromJoins(size_t from, size_t to, Join size_t orChainLength = 0; for (size_t i = 0, s = joinEntries.size(); i < s; ++i) { const QueryJoinEntry &joinEntry = joinEntries[i]; - auto explainEntry = explainJoinOn.AppendOnEntryExplain(); explainEntry.InitialCondition(joinEntry, joinedSelector); - - CondType condition = joinEntry.condition_; - OpType operation = joinEntry.op_; + CondType condition = joinEntry.Condition(); + OpType operation = joinEntry.Operation(); switch (operation) { case OpNot: orChainLength = 0; @@ -873,7 +820,7 @@ size_t QueryPreprocessor::injectConditionsFromJoins(size_t from, size_t to, Join case CondEmpty: case CondLike: case CondDWithin: - throw Error(errParams, "Unsupported condition in ON statment: %s", CondTypeToStr(condition)); + throw Error(errQueryExec, "Unsupported condition in ON statment: %s", CondTypeToStr(condition)); } operation = OpAnd; break; @@ -888,67 +835,67 @@ size_t QueryPreprocessor::injectConditionsFromJoins(size_t from, size_t to, Join orChainLength = 0; break; } - - QueryEntry newEntry; - newEntry.index = joinEntry.index_; - newEntry.idxNo = IndexValueType::SetByJsonPath; - KeyValueType valuesType = KeyValueType::Undefined{}; - CollateOpts collate; - if (ns_.getIndexByNameOrJsonPath(newEntry.index, newEntry.idxNo)) { - const Index &index = *ns_.indexes_[newEntry.idxNo]; - valuesType = index.SelectKeyType(); - collate = index.Opts().collateOpts_; - } - + CondType queryCondition{CondAny}; + VariantArray values; if (byValues) { - assertrx(joinedSelector.itemQuery_.entries.HoldsOrReferTo(i)); - const QueryEntry &qe = joinedSelector.itemQuery_.entries.Get(i); - assertrx(qe.index == joinEntry.joinIndex_); - const int rightIdxNo = qe.idxNo; - if (rightIdxNo == IndexValueType::SetByJsonPath) { - fillQueryEntryFromOnCondition(newEntry, joinEntry.joinIndex_, condition, joinedSelector, valuesType, - rightIdxNo, collate); - } else { - fillQueryEntryFromOnCondition(newEntry, joinEntry.joinIndex_, condition, joinedSelector, valuesType, - rightIdxNo, collate); + assertrx_throw(joinedSelector.itemQuery_.entries.HoldsOrReferTo(i)); + assertrx_throw(joinedSelector.itemQuery_.entries.Get(i).FieldName() == joinEntry.RightFieldName()); + CollateOpts collate; + if (joinEntry.IsLeftFieldIndexed()) { + collate = ns_.indexes_[joinEntry.LeftIdxNo()]->Opts().collateOpts_; } + std::tie(queryCondition, values) = queryValuesFromOnCondition(condition, joinEntry, joinedSelector, collate); } else { bool skip = false; switch (condition) { + case CondAny: + case CondEmpty: + case CondLike: + case CondDWithin: + explainEntry.Skipped("Skipped due to unsupperted on condition"sv); + skip = true; + break; + case CondRange: case CondLt: case CondLe: case CondGt: - case CondGe: { - const QueryEntry &qe = joinedSelector.itemQuery_.entries.Get(i); - skip = qe.idxNo != IndexValueType::SetByJsonPath && joinedSelector.RightNs()->indexes_[qe.idxNo]->IsUuid(); - if (skip) { - explainEntry.Skipped("Skipped due to condition Lt|Le|Gt|Ge with UUID index field."sv); - } + case CondGe: + joinedSelector.itemQuery_.entries.Get(i).FieldType().EvaluateOneOf( + [&skip, + &explainEntry](OneOf) noexcept { + skip = true; + explainEntry.Skipped( + "Skipped due to condition Lt|Le|Gt|Ge|Range with not indexed or not numeric field."sv); + }, + [](OneOf) noexcept { + }); break; - } case CondEq: case CondSet: case CondAllSet: - case CondAny: - case CondEmpty: - case CondRange: - case CondLike: - case CondDWithin: + joinedSelector.itemQuery_.entries.Get(i).FieldType().EvaluateOneOf( + [&skip, &explainEntry](OneOf) noexcept { + skip = true; + explainEntry.Skipped("Skipped due to condition Eq|Set|AllSet with composite index."sv); + }, + [](OneOf) noexcept {}); break; } if (!skip) { std::string explainSelect; AggType selectAggType; - fillQueryEntryFromOnCondition(newEntry, explainSelect, selectAggType, *joinedSelector.RightNs(), - joinedSelector.JoinQuery(), joinEntry.joinIndex_, condition, valuesType, rdxCtx); + std::tie(queryCondition, values) = + queryValuesFromOnCondition(explainSelect, selectAggType, *joinedSelector.RightNs(), + joinedSelector.JoinQuery(), joinEntry, condition, rdxCtx); explainEntry.ExplainSelect(std::move(explainSelect), selectAggType); } } - - if (!newEntry.values.empty()) { - explainEntry.Succeed(newEntry); - - Insert(cur, operation, std::move(newEntry)); + if (!values.empty()) { + Insert(cur, operation, QueryEntry{QueryField(joinEntry.LeftFieldData()), queryCondition, std::move(values)}); + explainEntry.Succeed(Get(cur)); ++cur; ++count; prevIsSkipped = false; @@ -956,6 +903,7 @@ size_t QueryPreprocessor::injectConditionsFromJoins(size_t from, size_t to, Join explainEntry.Skipped("Skipped as cannot obtain values from right namespace."sv); if (operation == OpOr) { Erase(cur - orChainLength, cur); + cur -= orChainLength; count -= orChainLength; // Marking On-injections as fail for removed entries. explainJoinOn.FailOnEntriesAsOrChain(orChainLength); @@ -971,8 +919,10 @@ size_t QueryPreprocessor::injectConditionsFromJoins(size_t from, size_t to, Join [this, cur, count, &js](WrSerializer &ser) { briefDump(cur - count, Next(cur - count), js, ser); }); ++cur; - injectedCount += count + 2; - to += count + 2; + injectedCount += count + 1; + to += count + 1; + } else { + explainJoinOn.Skipped("Skipped as there are no injected conditions"); } }); } @@ -1020,7 +970,7 @@ class JoinOnExplainEnabled { explainEntry_.succeed = true; explainEntry_.reason = ""; explainEntry_.newCond = newEntry.DumpBrief(); - explainEntry_.valuesCount = newEntry.values.size(); + explainEntry_.valuesCount = newEntry.Values().size(); } void Skipped(std::string_view reason) noexcept { @@ -1090,4 +1040,34 @@ class JoinOnExplainEnabled { time_point_t startTime_; }; +void QueryPreprocessor::setQueryIndex(QueryField &qField, int idxNo, const NamespaceImpl &ns) { + const auto &idx = *ns.indexes_[idxNo]; + std::vector compositeFieldsTypes; + if (idxNo >= ns.indexes_.firstCompositePos()) { +#ifndef NDEBUG + const bool ftIdx = IsFullText(idx.Type()); +#endif + for (const auto f : ns.indexes_[idxNo]->Fields()) { + if (f == IndexValueType::SetByJsonPath) { + // not indexed fields allowed only in ft composite indexes + assertrx_throw(ftIdx); + compositeFieldsTypes.push_back(KeyValueType::String{}); + } else { + assertrx_throw(f <= ns.indexes_.firstCompositePos()); + compositeFieldsTypes.push_back(ns.indexes_[f]->SelectKeyType()); + } + } + } + qField.SetIndexData(idxNo, FieldsSet(idx.Fields()), idx.KeyType(), idx.SelectKeyType(), std::move(compositeFieldsTypes)); +} + +void QueryPreprocessor::SetQueryField(QueryField &qField, const NamespaceImpl &ns) { + int idxNo = IndexValueType::SetByJsonPath; + if (ns.getIndexByNameOrJsonPath(qField.FieldName(), idxNo)) { + setQueryIndex(qField, idxNo, ns); + } else { + qField.SetField({ns.tagsMatcher_.path2tag(qField.FieldName())}); + } +} + } // namespace reindexer diff --git a/cpp_src/core/nsselecter/querypreprocessor.h b/cpp_src/core/nsselecter/querypreprocessor.h index 907636251..ba79d4c10 100644 --- a/cpp_src/core/nsselecter/querypreprocessor.h +++ b/cpp_src/core/nsselecter/querypreprocessor.h @@ -65,6 +65,7 @@ class QueryPreprocessor : private QueryEntries { return std::move(*ftPreselect_); } bool IsFtPreselected() const noexcept { return ftPreselect_ && !ftEntry_; } + static void SetQueryField(QueryField &, const NamespaceImpl &); private: struct FoundIndexInfo { @@ -78,6 +79,7 @@ class QueryPreprocessor : private QueryEntries { uint64_t isFitForSortOptimization : 1; }; + static void setQueryIndex(QueryField &, int idxNo, const NamespaceImpl &); [[nodiscard]] SortingEntries detectOptimalSortOrder() const; bool forcedStage() const noexcept { return evaluationsCount_ == (desc_ ? 1 : 0); } size_t lookupQueryIndexes(uint16_t dst, uint16_t srcBegin, uint16_t srcEnd); @@ -85,7 +87,7 @@ class QueryPreprocessor : private QueryEntries { bool mergeQueryEntries(size_t lhs, size_t rhs); const std::vector *getCompositeIndex(int field) const; void convertWhereValues(QueryEntries::iterator begin, QueryEntries::iterator end) const; - void convertWhereValues(QueryEntry *) const; + void convertWhereValues(QueryEntry &) const; [[nodiscard]] const Index *findMaxIndex(QueryEntries::const_iterator begin, QueryEntries::const_iterator end) const; void findMaxIndex(QueryEntries::const_iterator begin, QueryEntries::const_iterator end, h_vector &foundIndexes) const; @@ -94,12 +96,13 @@ class QueryPreprocessor : private QueryEntries { */ template size_t injectConditionsFromJoins(size_t from, size_t to, JoinedSelectors &, OnConditionInjections &, const RdxContext &); - void fillQueryEntryFromOnCondition(QueryEntry &, std::string &outExplainStr, AggType &, NamespaceImpl &rightNs, Query joinQuery, - std::string joinIndex, CondType condition, KeyValueType, const RdxContext &); - template - void fillQueryEntryFromOnCondition(QueryEntry &, std::string_view joinIndex, CondType condition, const JoinedSelector &, KeyValueType, - int rightIdxNo, const CollateOpts &); - void checkStrictMode(const std::string &index, int idxNo) const; + [[nodiscard]] std::pair queryValuesFromOnCondition(std::string &outExplainStr, AggType &, + NamespaceImpl &rightNs, Query joinQuery, + const QueryJoinEntry &, CondType condition, + const RdxContext &); + [[nodiscard]] std::pair queryValuesFromOnCondition(CondType condition, const QueryJoinEntry &, + const JoinedSelector &, const CollateOpts &); + void checkStrictMode(const QueryField &) const; bool removeBrackets(); size_t removeBrackets(size_t begin, size_t end); bool canRemoveBracket(size_t i) const; diff --git a/cpp_src/core/nsselecter/selectiteratorcontainer.cc b/cpp_src/core/nsselecter/selectiteratorcontainer.cc index a926d6fb9..f34f73137 100644 --- a/cpp_src/core/nsselecter/selectiteratorcontainer.cc +++ b/cpp_src/core/nsselecter/selectiteratorcontainer.cc @@ -188,46 +188,15 @@ void SelectIteratorContainer::SetExpectMaxIterations(int expectedIterations) { SelectKeyResults SelectIteratorContainer::processQueryEntry(const QueryEntry &qe, const NamespaceImpl &ns, StrictMode strictMode) { SelectKeyResults selectResults; - FieldsSet fields; - TagsPath tagsPath = ns.tagsMatcher_.path2tag(qe.index); - - // TODO: it may be necessary to remove or change this switch after QueryEntry refactoring - switch (qe.condition) { - case CondAny: - case CondEmpty: - case CondAllSet: - case CondEq: - case CondSet: - break; - case CondRange: - case CondDWithin: - if (qe.values.size() != 2) { - throw Error(errParams, "For condition %s required exactly 2 arguments, but provided %d", CondTypeToStr(qe.condition), - qe.values.size()); - } - break; - case CondLt: - case CondLe: - case CondGt: - case CondGe: - case CondLike: - if (qe.values.size() != 1) { - throw Error(errParams, "For condition %s required exactly 1 argument, but provided %d", CondTypeToStr(qe.condition), - qe.values.size()); - } - break; - } - - if (!tagsPath.empty()) { + if (!qe.HaveEmptyField()) { SelectKeyResult comparisonResult; - fields.push_back(tagsPath); - comparisonResult.comparators_.emplace_back(qe.condition, KeyValueType::Null{}, qe.values, false, qe.distinct, ns.payloadType_, - fields, nullptr, CollateOpts()); + comparisonResult.comparators_.emplace_back(qe.Condition(), KeyValueType::Null{}, qe.Values(), false, qe.Distinct(), ns.payloadType_, + qe.Fields(), nullptr, CollateOpts()); selectResults.emplace_back(std::move(comparisonResult)); } else if (strictMode == StrictModeNone) { SelectKeyResult res; // Ignore non-index/non-existing fields - if (qe.condition == CondEmpty) { + if (qe.Condition() == CondEmpty) { res.emplace_back(SingleSelectKeyResult(IdType(0), IdType(ns.items_.size()))); } else { res.emplace_back(SingleSelectKeyResult(IdType(0), IdType(0))); @@ -237,33 +206,30 @@ SelectKeyResults SelectIteratorContainer::processQueryEntry(const QueryEntry &qe throw Error( errParams, "Current query strict mode allows filtering by existing fields only. There are no fields with name '%s' in namespace '%s'", - qe.index, ns.name_); + qe.FieldName(), ns.name_); } return selectResults; } template -void SelectIteratorContainer::processField(FieldsComparator &fc, std::string_view field, int idxNo, const NamespaceImpl &ns) const { - const bool nonIndexField = (idxNo == IndexValueType::SetByJsonPath); - if (nonIndexField) { - TagsPath tagsPath = ns.tagsMatcher_.path2tag(field); - if (tagsPath.empty()) { - throw Error{errQueryExec, "Only existing fields can be compared. There are no fields with name '%s' in namespace '%s'", field, - ns.name_}; - } +void SelectIteratorContainer::processField(FieldsComparator &fc, const QueryField &field, const NamespaceImpl &ns) const { + if (field.IsFieldIndexed()) { + auto &index = ns.indexes_[field.IndexNo()]; if constexpr (left) { - fc.SetLeftField(tagsPath); + fc.SetCollateOpts(index->Opts().collateOpts_); + fc.SetLeftField(field.Fields(), field.FieldType(), index->Opts().IsArray()); } else { - fc.SetRightField(tagsPath); + fc.SetRightField(field.Fields(), field.FieldType(), index->Opts().IsArray()); } + } else if (field.HaveEmptyField()) { + throw Error{errQueryExec, "Only existing fields can be compared. There are no fields with name '%s' in namespace '%s'", + field.FieldName(), ns.name_}; } else { - auto &index = ns.indexes_[idxNo]; if constexpr (left) { - fc.SetCollateOpts(index->Opts().collateOpts_); - fc.SetLeftField(index->Fields(), index->KeyType(), index->Opts().IsArray()); + fc.SetLeftField(field.Fields()); } else { - fc.SetRightField(index->Fields(), index->KeyType(), index->Opts().IsArray()); + fc.SetRightField(field.Fields()); } } } @@ -272,7 +238,7 @@ SelectKeyResults SelectIteratorContainer::processQueryEntry(const QueryEntry &qe unsigned sortId, bool isQueryFt, SelectFunction::Ptr &selectFnc, bool &isIndexFt, bool &isIndexSparse, FtCtx::Ptr &ftCtx, QueryPreprocessor &qPreproc, const RdxContext &rdxCtx) { - auto &index = ns.indexes_[qe.idxNo]; + auto &index = ns.indexes_[qe.IndexNo()]; isIndexFt = IsFullText(index->Type()); isIndexSparse = index->Opts().IsSparse(); @@ -289,31 +255,31 @@ SelectKeyResults SelectIteratorContainer::processQueryEntry(const QueryEntry &qe opts.forceComparator = 1; } } - if (qe.distinct) { + if (qe.Distinct()) { opts.distinct = 1; } opts.maxIterations = GetMaxIterations(); opts.indexesNotOptimized = !ctx_->sortingContext.enableSortOrders; opts.inTransaction = ctx_->inTransaction; - auto ctx = selectFnc ? selectFnc->CreateCtx(qe.idxNo) : BaseFunctionCtx::Ptr{}; + auto ctx = selectFnc ? selectFnc->CreateCtx(qe.IndexNo()) : BaseFunctionCtx::Ptr{}; if (ctx && ctx->type == BaseFunctionCtx::kFtCtx) ftCtx = reindexer::reinterpret_pointer_cast(ctx); if (index->Opts().GetCollateMode() == CollateUTF8 || isIndexFt) { - for (auto &key : qe.values) key.EnsureUTF8(); + for (auto &key : qe.Values()) key.EnsureUTF8(); } PerfStatCalculatorMT calc(index->GetSelectPerfCounter(), ns.enablePerfCounters_); if (qPreproc.IsFtPreselected()) { - return index->SelectKey(qe.values, qe.condition, opts, ctx, qPreproc.MoveFtPreselect(), rdxCtx); + return index->SelectKey(qe.Values(), qe.Condition(), opts, ctx, qPreproc.MoveFtPreselect(), rdxCtx); } else { - return index->SelectKey(qe.values, qe.condition, sortId, opts, ctx, rdxCtx); + return index->SelectKey(qe.Values(), qe.Condition(), sortId, opts, ctx, rdxCtx); } } void SelectIteratorContainer::processJoinEntry(const JoinQueryEntry &jqe, OpType op) { auto &js = (*ctx_->joinedSelectors)[jqe.joinIndex]; if (js.JoinQuery().joinEntries_.empty()) throw Error(errQueryExec, "Join without ON conditions"); - if (js.JoinQuery().joinEntries_[0].op_ == OpOr) throw Error(errQueryExec, "The first ON condition cannot have OR operation"); + if (js.JoinQuery().joinEntries_[0].Operation() == OpOr) throw Error(errQueryExec, "The first ON condition cannot have OR operation"); if (js.Type() != InnerJoin && js.Type() != OrInnerJoin) throw Error(errLogic, "Not INNER JOIN in QueryEntry"); if (js.Type() == OrInnerJoin) { if (op == OpNot) throw Error(errQueryExec, "NOT operator with or_inner_join"); @@ -327,7 +293,7 @@ void SelectIteratorContainer::processJoinEntry(const JoinQueryEntry &jqe, OpType } void SelectIteratorContainer::processQueryEntryResults(SelectKeyResults &selectResults, OpType op, const NamespaceImpl &ns, - const QueryEntry &qe, bool isIndexFt, bool isIndexSparse, bool nonIndexField, + const QueryEntry &qe, bool isIndexFt, bool isIndexSparse, std::optional nextOp) { if (selectResults.empty()) { if (op == OpAnd) { @@ -340,19 +306,19 @@ void SelectIteratorContainer::processQueryEntryResults(SelectKeyResults &selectR case OpOr: { const iterator last = lastAppendedOrClosed(); if (last == this->end()) { - throw Error(errQueryExec, "OR operator in first condition or after left join "); + throw Error(errQueryExec, "OR operator in first condition or after left join"); } if (last->HoldsOrReferTo() && !last->Value().distinct && last->operation != OpNot) { if (last->IsRef()) { last->SetValue(last->Value()); } SelectIterator &it = last->Value(); - if (nonIndexField || isIndexSparse) { + if (!qe.IsFieldIndexed() || isIndexSparse) { it.Append(res); } else { - it.AppendAndBind(res, ns.payloadType_, qe.idxNo); + it.AppendAndBind(res, ns.payloadType_, qe.IndexNo()); } - it.name += " or " + qe.index; + it.name += " or " + qe.FieldName(); break; } } @@ -360,16 +326,16 @@ void SelectIteratorContainer::processQueryEntryResults(SelectKeyResults &selectR case OpNot: case OpAnd: // Iterator Field Kind: Query entry results. Field known. - Append(op, SelectIterator(res, qe.distinct, qe.index, - qe.idxNo < 0 ? IteratorFieldKind::NonIndexed : IteratorFieldKind::Indexed, isIndexFt)); - if (!nonIndexField && !isIndexSparse) { + Append(op, res, qe.Distinct(), qe.FieldName(), + qe.IndexNo() < 0 ? IteratorFieldKind::NonIndexed : IteratorFieldKind::Indexed, isIndexFt); + if (qe.IsFieldIndexed() && !isIndexSparse) { // last appended is always a SelectIterator const auto lastAppendedIt = lastAppendedOrClosed(); if (lastAppendedIt->IsRef()) { lastAppendedIt->SetValue(lastAppendedIt->Value()); } SelectIterator &lastAppended = lastAppendedIt->Value(); - lastAppended.Bind(ns.payloadType_, qe.idxNo); + lastAppended.Bind(ns.payloadType_, qe.IndexNo()); lastAppended.SetNotOperationFlag(op == OpNot); const auto maxIterations = lastAppended.GetMaxIterations(); const int cur = op == OpNot ? ns.items_.size() - maxIterations : maxIterations; @@ -393,25 +359,23 @@ void SelectIteratorContainer::processEqualPositions(const std::vector(eqPos.queryEntriesPositions[0])}; - if (firstQe.condition == CondEmpty || (firstQe.condition == CondSet && firstQe.values.empty())) { + if (firstQe.Condition() == CondEmpty || (firstQe.Condition() == CondSet && firstQe.Values().empty())) { throw Error(errLogic, "Condition IN(with empty parameter list), IS NULL, IS EMPTY not allowed for equal position!"); } - const KeyValueType type = firstQe.values.size() ? firstQe.values[0].Type() : KeyValueType::Null{}; - Comparator cmp(firstQe.condition, type, firstQe.values, true, firstQe.distinct, ns.payloadType_, FieldsSet({firstQe.idxNo})); + const KeyValueType type = firstQe.Values().size() ? firstQe.Values()[0].Type() : KeyValueType::Null{}; + Comparator cmp{firstQe.Condition(), type, firstQe.Values(), true, firstQe.Distinct(), ns.payloadType_, firstQe.Fields()}; for (size_t i = 0; i < eqPos.queryEntriesPositions.size(); ++i) { const QueryEntry &qe = queries.Get(eqPos.queryEntriesPositions[i]); - if (qe.condition == CondEmpty || (qe.condition == CondSet && qe.values.empty())) { + if (qe.Condition() == CondEmpty || (qe.Condition() == CondSet && qe.Values().empty())) { throw Error(errLogic, "Condition IN(with empty parameter list), IS NULL, IS EMPTY not allowed for equal position!"); } - if (qe.idxNo == IndexValueType::SetByJsonPath) { - cmp.BindEqualPosition(ns.tagsMatcher_.path2tag(qe.index), qe.values, qe.condition); - } else if (ns.indexes_[qe.idxNo]->Opts().IsSparse()) { - const TagsPath &tp = ns.indexes_[qe.idxNo]->Fields().getTagsPath(0); - cmp.BindEqualPosition(tp, qe.values, qe.condition); + assertrx_throw(qe.Fields().size() == 1); + if (qe.Fields()[0] == IndexValueType::SetByJsonPath) { + cmp.BindEqualPosition(qe.Fields().getFieldsPath(0), qe.Values(), qe.Condition()); } else { - cmp.BindEqualPosition(qe.idxNo, qe.values, qe.condition); + cmp.BindEqualPosition(qe.Fields()[0], qe.Values(), qe.Condition()); } } @@ -450,33 +414,33 @@ std::vector SelectIteratorContainer::pr queries.InvokeAppropriate( j, Skip{}, [&](const QueryEntry &eq) { - if (foundFields.find(eq.index) != foundFields.end()) { + if (foundFields.find(eq.FieldName()) != foundFields.end()) { throw Error(errParams, "Equal position field '%s' found twice in enclosing bracket; equal position fields: [%s]", - eq.index, getEpFieldsStr()); + eq.FieldName(), getEpFieldsStr()); } - const auto it = epFields.find(eq.index); + const auto it = epFields.find(eq.FieldName()); if (it == epFields.end()) return; if (queries.GetOperation(j) != OpAnd || (next < end && queries.GetOperation(next) == OpOr)) { throw Error(errParams, "Only AND operation allowed for equal position; equal position field with not AND operation: '%s'; " "equal position fields: [%s]", - eq.index, getEpFieldsStr()); + eq.FieldName(), getEpFieldsStr()); } result[i].queryEntriesPositions.push_back(j); foundFields.insert(epFields.extract(it)); }, [&](const BetweenFieldsQueryEntry &eq) { // TODO equal positions for BetweenFieldsQueryEntry #1092 - if (epFields.find(eq.firstIndex) != epFields.end()) { + if (epFields.find(eq.LeftFieldName()) != epFields.end()) { throw Error( errParams, "Equal positions for conditions between fields are not supported; field: '%s'; equal position fields: [%s]", - eq.firstIndex, getEpFieldsStr()); + eq.LeftFieldName(), getEpFieldsStr()); } - if (epFields.find(eq.secondIndex) != epFields.end()) { + if (epFields.find(eq.RightFieldName()) != epFields.end()) { throw Error( errParams, "Equal positions for conditions between fields are not supported; field: '%s'; equal position fields: [%s]", - eq.secondIndex, getEpFieldsStr()); + eq.RightFieldName(), getEpFieldsStr()); } }); } @@ -517,28 +481,16 @@ bool SelectIteratorContainer::prepareIteratorsForSelectLoop(QueryPreprocessor &q return contFT; }, [&](const QueryEntry &qe) { - const bool isFT = qe.idxNo != IndexValueType::SetByJsonPath && IsFullText(ns.indexes_[qe.idxNo]->Type()); + const bool isFT = qe.IsFieldIndexed() && IsFullText(ns.indexes_[qe.IndexNo()]->Type()); if (isFT && (op == OpOr || (next < end && queries.GetOperation(next) == OpOr))) { throw Error(errLogic, "OR operation is not allowed with fulltext index"); } SelectKeyResults selectResults; bool isIndexFt = false, isIndexSparse = false; - const bool nonIndexField = (qe.idxNo == IndexValueType::SetByJsonPath); - - if (nonIndexField) { - auto strictMode = ns.config_.strictMode; - if (ctx_) { - if (ctx_->inTransaction) { - strictMode = StrictModeNone; - } else if (ctx_->query.strictMode != StrictModeNotSet) { - strictMode = ctx_->query.strictMode; - } - } - selectResults = processQueryEntry(qe, ns, strictMode); - } else { - bool enableSortIndexOptimize = (ctx_->sortingContext.uncommitedIndex == qe.idxNo) && !sortIndexFound && - (op == OpAnd) && !qe.distinct && (begin == 0) && + if (qe.IsFieldIndexed()) { + bool enableSortIndexOptimize = (ctx_->sortingContext.uncommitedIndex == qe.IndexNo()) && !sortIndexFound && + (op == OpAnd) && !qe.Distinct() && (begin == 0) && (next == end || queries.GetOperation(next) != OpOr); if (enableSortIndexOptimize) { if (!IsExpectingOrderedResults(qe)) { @@ -549,12 +501,22 @@ bool SelectIteratorContainer::prepareIteratorsForSelectLoop(QueryPreprocessor &q } selectResults = processQueryEntry(qe, enableSortIndexOptimize, ns, sortId, isQueryFt, selectFnc, isIndexFt, isIndexSparse, ftCtx, qPreproc, rdxCtx); + } else { + auto strictMode = ns.config_.strictMode; + if (ctx_) { + if (ctx_->inTransaction) { + strictMode = StrictModeNone; + } else if (ctx_->query.strictMode != StrictModeNotSet) { + strictMode = ctx_->query.strictMode; + } + } + selectResults = processQueryEntry(qe, ns, strictMode); } std::optional nextOp; if (next != end) { nextOp = queries.GetOperation(next); } - processQueryEntryResults(selectResults, op, ns, qe, isIndexFt, isIndexSparse, nonIndexField, nextOp); + processQueryEntryResults(selectResults, op, ns, qe, isIndexFt, isIndexSparse, nextOp); if (op != OpOr) { for (auto &ep : equalPositions) { const auto lastPosition = ep.queryEntriesPositions.back(); @@ -574,9 +536,9 @@ bool SelectIteratorContainer::prepareIteratorsForSelectLoop(QueryPreprocessor &q return false; }, [&](const BetweenFieldsQueryEntry &qe) { - FieldsComparator fc{qe.firstIndex, qe.Condition(), qe.secondIndex, ns.payloadType_}; - processField(fc, qe.firstIndex, qe.firstIdxNo, ns); - processField(fc, qe.secondIndex, qe.secondIdxNo, ns); + FieldsComparator fc{qe.LeftFieldName(), qe.Condition(), qe.RightFieldName(), ns.payloadType_}; + processField(fc, qe.LeftFieldData(), ns); + processField(fc, qe.RightFieldData(), ns); Append(op, std::move(fc)); return false; }, @@ -759,11 +721,11 @@ void JoinSelectIterator::Dump(WrSerializer &ser, const std::vector - void processField(FieldsComparator &, std::string_view field, int idxNo, const NamespaceImpl &ns) const; + void processField(FieldsComparator &, const QueryField &, const NamespaceImpl &) const; void processJoinEntry(const JoinQueryEntry &, OpType); void processQueryEntryResults(SelectKeyResults &selectResults, OpType, const NamespaceImpl &ns, const QueryEntry &qe, bool isIndexFt, - bool isIndexSparse, bool nonIndexField, std::optional nextOp); + bool isIndexSparse, std::optional nextOp); struct EqualPositions { h_vector queryEntriesPositions; size_t positionToInsertIterator = 0; diff --git a/cpp_src/core/payload/fieldsset.h b/cpp_src/core/payload/fieldsset.h index 87da7eefe..98c0062ae 100644 --- a/cpp_src/core/payload/fieldsset.h +++ b/cpp_src/core/payload/fieldsset.h @@ -75,18 +75,8 @@ class FieldsSet : protected base_fields_set { } } - void push_back(const TagsPath &tagsPath) { - if (!contains(tagsPath)) { - base_fields_set::push_back(IndexValueType::SetByJsonPath); - tagsPaths_.emplace_back(tagsPath); - } - } - void push_back(TagsPath &&tagsPath) { - if (!contains(tagsPath)) { - base_fields_set::push_back(IndexValueType::SetByJsonPath); - tagsPaths_.emplace_back(std::move(tagsPath)); - } - } + void push_back(const TagsPath &tagsPath) { pushBack(tagsPath); } + void push_back(TagsPath &&tagsPath) { pushBack(std::move(tagsPath)); } void push_front(TagsPath &&tagsPath) { if (!contains(tagsPath)) { base_fields_set::insert(begin(), IndexValueType::SetByJsonPath); @@ -94,18 +84,10 @@ class FieldsSet : protected base_fields_set { } } - void push_back(const IndexedTagsPath &tagsPath) { - if (!contains(tagsPath)) { - base_fields_set::push_back(IndexValueType::SetByJsonPath); - tagsPaths_.emplace_back(tagsPath); - } - } - void push_back(IndexedTagsPath &&tagsPath) { - if (!contains(tagsPath)) { - base_fields_set::push_back(IndexValueType::SetByJsonPath); - tagsPaths_.emplace_back(std::move(tagsPath)); - } - } + void push_back(const IndexedTagsPath &tagsPath) { pushBack(tagsPath); } + void push_back(IndexedTagsPath &&tagsPath) { pushBack(std::move(tagsPath)); } + void push_back(const FieldsPath &fieldPath) { pushBack(fieldPath); } + void push_back(FieldsPath &&fieldPath) { pushBack(std::move(fieldPath)); } void push_back(int f) { if (f < 0) return; @@ -146,31 +128,34 @@ class FieldsSet : protected base_fields_set { bool contains(const IndexesFieldsSet &f) const noexcept { return (mask_ & f.mask()) == f.mask(); } bool contains(const TagsPath &tagsPath) const noexcept { for (const FieldsPath &path : tagsPaths_) { - if (path.index() == 0) { - if (std::get(path) == tagsPath) return true; - } else { - if (std::get(path).Compare(tagsPath)) return true; + if (std::visit(overloaded{[&tagsPath](const TagsPath &path) { return path == tagsPath; }, + [&tagsPath](const IndexedTagsPath &path) { return path.Compare(tagsPath); }}, + path)) { + return true; } } return false; } bool contains(const IndexedTagsPath &tagsPath) const noexcept { for (const FieldsPath &path : tagsPaths_) { - if (path.index() == 1) { - if (std::get(path) == tagsPath) return true; - } else { - if (tagsPath.Compare(std::get(path))) return true; + if (std::visit(overloaded{[&tagsPath](const TagsPath &path) { return tagsPath.Compare(path); }, + [&tagsPath](const IndexedTagsPath &path) { return path == tagsPath; }}, + path)) { + return true; } } return false; } + bool contains(const FieldsPath &fieldsPath) const noexcept { + return std::visit([&](const auto &fp) { return contains(fp); }, fieldsPath); + } bool match(const TagsPath &tagsPath) const noexcept { if (tagsPaths_.empty()) return true; - for (auto &flt : tagsPaths_) { - if (flt.index() == 0) { - if (comparePaths(tagsPath, std::get(flt))) return true; - } else { - if (comparePaths(std::get(flt), tagsPath)) return true; + for (auto &path : tagsPaths_) { + if (std::visit(overloaded{[&tagsPath, this](const TagsPath &path) { return comparePaths(tagsPath, path); }, + [&tagsPath, this](const IndexedTagsPath &path) { return comparePaths(path, tagsPath); }}, + path)) { + return true; } } return false; @@ -178,11 +163,11 @@ class FieldsSet : protected base_fields_set { template bool match(const IndexedTagsPathImpl &tagsPath) const noexcept { if (tagsPaths_.empty()) return true; - for (auto &flt : tagsPaths_) { - if (flt.index() == 1) { - if (comparePaths(tagsPath, std::get(flt))) return true; - } else { - if (comparePaths(tagsPath, std::get(flt))) return true; + for (auto &path : tagsPaths_) { + if (std::visit(overloaded{[&tagsPath, this](const TagsPath &path) { return comparePaths(tagsPath, path); }, + [&tagsPath, this](const IndexedTagsPath &path) { return comparePaths(tagsPath, path); }}, + path)) { + return true; } } return false; @@ -199,12 +184,15 @@ class FieldsSet : protected base_fields_set { const h_vector &getJsonPaths() const noexcept { return jsonPaths_; } bool isTagsPathIndexed(size_t idx) const noexcept { assertrx(idx < tagsPaths_.size()); - return (tagsPaths_[idx].index() == 1); + return std::visit(overloaded{[](const TagsPath &) { return false; }, [](const IndexedTagsPath &) { return true; }}, + tagsPaths_[idx]); } const TagsPath &getTagsPath(size_t idx) const & { return std::get(tagsPaths_[idx]); } const TagsPath &getTagsPath(size_t idx) const && = delete; const IndexedTagsPath &getIndexedTagsPath(size_t idx) const & { return std::get(tagsPaths_[idx]); } const IndexedTagsPath &getIndexedTagsPath(size_t idx) const && = delete; + const FieldsPath &getFieldsPath(size_t idx) const & { return tagsPaths_[idx]; } + const FieldsPath &getFieldsPath(size_t idx) const && = delete; const std::string &getJsonPath(size_t idx) const &noexcept { return jsonPaths_[idx]; } const std::string &getJsonPath(size_t idx) const && = delete; @@ -235,7 +223,14 @@ class FieldsSet : protected base_fields_set { os << "]}"; } -protected: +private: + template + void pushBack(F &&fieldPath) { + if (!contains(fieldPath)) { + base_fields_set::push_back(IndexValueType::SetByJsonPath); + tagsPaths_.emplace_back(std::forward(fieldPath)); + } + } template bool comparePaths(const TPath1 &lhs, const TPath2 &rhs) const noexcept { unsigned i = 0, count = std::min(lhs.size(), rhs.size()); diff --git a/cpp_src/core/payload/payloadiface.cc b/cpp_src/core/payload/payloadiface.cc index 110d0b80f..64b50daec 100644 --- a/cpp_src/core/payload/payloadiface.cc +++ b/cpp_src/core/payload/payloadiface.cc @@ -5,7 +5,6 @@ #include "core/keyvalue/p_string.h" #include "core/keyvalue/variant.h" #include "core/namespace/stringsholder.h" -#include "itoa/itoa.h" #include "payloadiface.h" #include "payloadvalue.h" @@ -77,29 +76,77 @@ void PayloadIface::GetByJsonPath(std::string_view jsonPath, TagsMatcher &tags } template -void PayloadIface::GetByJsonPath(const TagsPath &jsonPath, VariantArray &krefs, KeyValueType expectedType) const { - ConstPayload pl(t_, *v_); - FieldsSet filter({jsonPath}); - BaseEncoder encoder(nullptr, &filter); +template +void PayloadIface::getByJsonPath(const P &path, VariantArray &krefs, KeyValueType expectedType) const { krefs.clear(); - if (!jsonPath.empty()) { - FieldsExtractor extractor(&krefs, expectedType, jsonPath.size()); - encoder.Encode(pl, extractor); + if (path.empty()) { + return; } + const FieldsSet filter{{path}}; + ConstPayload pl(t_, *v_); + BaseEncoder encoder(nullptr, &filter); + FieldsExtractor extractor(&krefs, expectedType, path.size(), &filter); + encoder.Encode(pl, extractor); +} + +template +void PayloadIface::GetByJsonPath(const TagsPath &tagsPath, VariantArray &krefs, KeyValueType expectedType) const { + getByJsonPath(tagsPath, krefs, expectedType); } template void PayloadIface::GetByJsonPath(const IndexedTagsPath &tagsPath, VariantArray &krefs, KeyValueType expectedType) const { - ConstPayload pl(t_, *v_); - FieldsSet filter({tagsPath}); - BaseEncoder encoder(nullptr, &filter); - krefs.Clear(); - if (!tagsPath.empty()) { - FieldsExtractor extractor(&krefs, expectedType, tagsPath.size(), &filter); - encoder.Encode(pl, extractor); + getByJsonPath(tagsPath, krefs, expectedType); +} + +template +void PayloadIface::GetByFieldsSet(const FieldsSet &fields, VariantArray &kvs, KeyValueType expectedType, + const std::vector &expectedCompositeTypes) const { + if (expectedType.Is()) { + kvs.Clear(); + kvs.emplace_back(GetComposite(fields, expectedCompositeTypes)); + } else { + assertrx_throw(fields.size() == 1); + if (fields[0] == IndexValueType::SetByJsonPath) { + assertrx_throw(fields.getTagsPathsLength() == 1); + if (fields.isTagsPathIndexed(0)) { + getByJsonPath(fields.getIndexedTagsPath(0), kvs, expectedType); + } else { + getByJsonPath(fields.getTagsPath(0), kvs, expectedType); + } + } else { + Get(fields[0], kvs); + } } } +template +Variant PayloadIface::GetComposite(const FieldsSet &fields, const std::vector &expectedTypes) const { + thread_local VariantArray buffer; + buffer.clear(); + assertrx_throw(fields.size() == expectedTypes.size()); + size_t jsonFieldIdx{0}; + [[maybe_unused]] const size_t maxJsonFieldIdx{fields.getTagsPathsLength()}; + VariantArray buf; + for (size_t i = 0, s = fields.size(); i < s; ++i) { + buf.clear(); + if (fields[i] == IndexValueType::SetByJsonPath) { + assertrx_throw(jsonFieldIdx < maxJsonFieldIdx); + if (fields.isTagsPathIndexed(jsonFieldIdx)) { + getByJsonPath(fields.getIndexedTagsPath(jsonFieldIdx), buf, expectedTypes[i]); + } else { + getByJsonPath(fields.getTagsPath(jsonFieldIdx), buf, expectedTypes[i]); + } + ++jsonFieldIdx; + } else { + Get(fields[i], buf); + } + assertrx_throw(buf.size() == 1); + buffer.emplace_back(std::move(buf[0])); + } + return Variant{buffer}; +} + template VariantArray PayloadIface::GetIndexedArrayData(const IndexedTagsPath &tagsPath, int field, int &offset, int &size) const { if (tagsPath.empty()) { @@ -199,8 +246,13 @@ void PayloadIface::SerializeFields(WrSerializer &ser, const FieldsSet &fields for (int field : fields) { if (field == IndexValueType::SetByJsonPath) { assertrx(tagPathIdx < fields.getTagsPathsLength()); - const TagsPath &tagsPath = fields.getTagsPath(tagPathIdx); - GetByJsonPath(tagsPath, varr, KeyValueType::Undefined{}); + if (fields.isTagsPathIndexed(tagPathIdx)) { + const IndexedTagsPath &tagsPath = fields.getIndexedTagsPath(tagPathIdx); + GetByJsonPath(tagsPath, varr, KeyValueType::Undefined{}); + } else { + const TagsPath &tagsPath = fields.getTagsPath(tagPathIdx); + GetByJsonPath(tagsPath, varr, KeyValueType::Undefined{}); + } if (varr.empty()) { throw Error(errParams, "PK serializing error: field [%s] cannot not be empty", fields.getJsonPath(tagPathIdx)); } @@ -281,8 +333,13 @@ size_t PayloadIface::GetHash(const FieldsSet &fields) const { ret ^= Field(field).Hash(); } else { assertrx(tagPathIdx < fields.getTagsPathsLength()); - const TagsPath &tagsPath = fields.getTagsPath(tagPathIdx++); - GetByJsonPath(tagsPath, keys1, KeyValueType::Undefined{}); + if (fields.isTagsPathIndexed(tagPathIdx)) { + const IndexedTagsPath &tagsPath = fields.getIndexedTagsPath(tagPathIdx++); + GetByJsonPath(tagsPath, keys1, KeyValueType::Undefined{}); + } else { + const TagsPath &tagsPath = fields.getTagsPath(tagPathIdx++); + GetByJsonPath(tagsPath, keys1, KeyValueType::Undefined{}); + } ret ^= keys1.Hash(); } } @@ -333,9 +390,15 @@ bool PayloadIface::IsEQ(const T &other, const FieldsSet &fields) const { if (!Field(field).IsEQ(o.Field(field))) return false; } } else { - const TagsPath &tagsPath = fields.getTagsPath(tagPathIdx++); - GetByJsonPath(tagsPath, keys1, KeyValueType::Undefined{}); - o.GetByJsonPath(tagsPath, keys2, KeyValueType::Undefined{}); + if (fields.isTagsPathIndexed(tagPathIdx)) { + const IndexedTagsPath &tagsPath = fields.getIndexedTagsPath(tagPathIdx++); + GetByJsonPath(tagsPath, keys1, KeyValueType::Undefined{}); + o.GetByJsonPath(tagsPath, keys2, KeyValueType::Undefined{}); + } else { + const TagsPath &tagsPath = fields.getTagsPath(tagPathIdx++); + GetByJsonPath(tagsPath, keys1, KeyValueType::Undefined{}); + o.GetByJsonPath(tagsPath, keys2, KeyValueType::Undefined{}); + } if (keys1 != keys2) { return false; } @@ -355,35 +418,44 @@ int PayloadIface::Compare(const T &other, const FieldsSet &fields, size_t &fi bool commonOpts = (collateOpts.size() == 1); for (size_t i = 0; i < fields.size(); ++i) { - int cmpRes = 0; const auto field(fields[i]); const CollateOpts *opts(commonOpts ? collateOpts[0] : collateOpts[i]); if (field != IndexValueType::SetByJsonPath) { - cmpRes = Field(field).Get().Compare(o.Field(field).Get(), opts ? *opts : CollateOpts()); + int cmpRes = Field(field).Get().Compare(o.Field(field).Get(), opts ? *opts : CollateOpts()); + if (cmpRes) { + firstDifferentFieldIdx = i; + return cmpRes; + } } else { assertrx(tagPathIdx < fields.getTagsPathsLength()); - const TagsPath &tagsPath = fields.getTagsPath(tagPathIdx++); - GetByJsonPath(tagsPath, krefs1, KeyValueType::Undefined{}); - o.GetByJsonPath(tagsPath, krefs2, KeyValueType::Undefined{}); + if (fields.isTagsPathIndexed(tagPathIdx)) { + const IndexedTagsPath &tagsPath = fields.getIndexedTagsPath(tagPathIdx++); + GetByJsonPath(tagsPath, krefs1, KeyValueType::Undefined{}); + o.GetByJsonPath(tagsPath, krefs2, KeyValueType::Undefined{}); + } else { + const TagsPath &tagsPath = fields.getTagsPath(tagPathIdx++); + GetByJsonPath(tagsPath, krefs1, KeyValueType::Undefined{}); + o.GetByJsonPath(tagsPath, krefs2, KeyValueType::Undefined{}); + } size_t length = std::min(krefs1.size(), krefs2.size()); for (size_t j = 0; j < length; ++j) { - cmpRes = krefs1[j].RelaxCompare(krefs2[j], opts ? *opts : CollateOpts()); - if (cmpRes) break; - } - if (cmpRes == 0) { - if (krefs1.size() < krefs2.size()) { - cmpRes = -1; - } else if (krefs1.size() > krefs2.size()) { - cmpRes = 1; + int cmpRes = krefs1[j].RelaxCompare(krefs2[j], opts ? *opts : CollateOpts()); + if (cmpRes) { + firstDifferentFieldIdx = i; + return cmpRes; } } - } - - firstDifferentFieldIdx = i; - if (cmpRes > 0) return 1; - if (cmpRes < 0) return -1; + if (krefs1.size() < krefs2.size()) { + firstDifferentFieldIdx = i; + return -1; + } + if (krefs1.size() > krefs2.size()) { + firstDifferentFieldIdx = i; + return 1; + } + } } return 0; } diff --git a/cpp_src/core/payload/payloadiface.h b/cpp_src/core/payload/payloadiface.h index 39569e4c3..1781e1ce9 100644 --- a/cpp_src/core/payload/payloadiface.h +++ b/cpp_src/core/payload/payloadiface.h @@ -112,6 +112,9 @@ class PayloadIface { void GetByJsonPath(std::string_view jsonPath, TagsMatcher &tagsMatcher, VariantArray &, KeyValueType expectedType) const; void GetByJsonPath(const TagsPath &jsonPath, VariantArray &, KeyValueType expectedType) const; void GetByJsonPath(const IndexedTagsPath &jsonPath, VariantArray &, KeyValueType expectedType) const; + void GetByFieldsSet(const FieldsSet &, VariantArray &, KeyValueType expectedType, + const std::vector &expectedCompositeTypes) const; + [[nodiscard]] Variant GetComposite(const FieldsSet &, const std::vector &expectedTypes) const; VariantArray GetIndexedArrayData(const IndexedTagsPath &jsonPath, int field, int &offset, int &size) const; // Get fields count @@ -167,6 +170,8 @@ class PayloadIface { T CopyWithRemovedFields(PayloadType t); template void copyOrMoveStrings(int field, StrHolder &dest, bool copy); + template + void getByJsonPath(const P &path, VariantArray &, KeyValueType expectedType) const; template ::value>::type * = nullptr> void setArray(int field, const VariantArray &keys, bool append); diff --git a/cpp_src/core/payload/payloadvalue.cc b/cpp_src/core/payload/payloadvalue.cc index 7e438b0cd..d1b07a365 100644 --- a/cpp_src/core/payload/payloadvalue.cc +++ b/cpp_src/core/payload/payloadvalue.cc @@ -1,27 +1,19 @@ #include "payloadvalue.h" -#include +#include #include "core/keyvalue/p_string.h" -#include "string.h" -#include "tools/errors.h" + namespace reindexer { PayloadValue::PayloadValue(size_t size, const uint8_t *ptr, size_t cap) : p_(nullptr) { p_ = alloc((cap != 0) ? cap : size); - if (ptr) + if (ptr) { memcpy(Ptr(), ptr, size); - else + } else { memset(Ptr(), 0, size); -} - -PayloadValue::PayloadValue(const PayloadValue &other) noexcept : p_(other.p_) { - if (p_) { - header()->refcount.fetch_add(1, std::memory_order_relaxed); } } -PayloadValue::~PayloadValue() { release(); } - uint8_t *PayloadValue::alloc(size_t cap) { auto pn = reinterpret_cast(operator new(cap + sizeof(dataHeader))); dataHeader *nheader = reinterpret_cast(pn); @@ -44,7 +36,7 @@ void PayloadValue::release() noexcept { void PayloadValue::Clone(size_t size) { // If we have exclusive data - just up lsn - if (p_ && header()->refcount.load() == 1) { + if (p_ && header()->refcount.load(std::memory_order_acquire) == 1) { return; } assertrx(size || p_); @@ -64,7 +56,7 @@ void PayloadValue::Clone(size_t size) { void PayloadValue::Resize(size_t oldSize, size_t newSize) { assertrx(p_); - assertrx(header()->refcount.load() == 1); + assertrx(header()->refcount.load(std::memory_order_acquire) == 1); if (newSize <= header()->cap) return; diff --git a/cpp_src/core/payload/payloadvalue.h b/cpp_src/core/payload/payloadvalue.h index 96a53b0b1..b94ba1198 100644 --- a/cpp_src/core/payload/payloadvalue.h +++ b/cpp_src/core/payload/payloadvalue.h @@ -14,18 +14,22 @@ class PayloadValue { struct dataHeader { dataHeader() noexcept : refcount(1), cap(0), lsn(-1) {} - ~dataHeader() { assertrx(refcount.load() == 0); } + ~dataHeader() { assertrx(refcount.load(std::memory_order_acquire) == 0); } refcounter refcount; unsigned cap; int64_t lsn; }; PayloadValue() noexcept : p_(nullptr) {} - PayloadValue(const PayloadValue &) noexcept; + PayloadValue(const PayloadValue &other) noexcept : p_(other.p_) { + if (p_) { + header()->refcount.fetch_add(1, std::memory_order_relaxed); + } + } // Alloc payload store with size, and copy data from another array PayloadValue(size_t size, const uint8_t *ptr = nullptr, size_t cap = 0); - ~PayloadValue(); - PayloadValue &operator=(const PayloadValue &other) { + ~PayloadValue() { release(); } + PayloadValue &operator=(const PayloadValue &other) noexcept { if (&other != this) { release(); p_ = other.p_; @@ -52,8 +56,8 @@ class PayloadValue { uint8_t *Ptr() const noexcept { return p_ + sizeof(dataHeader); } void SetLSN(int64_t lsn) { header()->lsn = lsn; } int64_t GetLSN() const { return p_ ? header()->lsn : 0; } - bool IsFree() const { return bool(p_ == nullptr); } - void Free() { release(); } + bool IsFree() const noexcept { return bool(p_ == nullptr); } + void Free() noexcept { release(); } size_t GetCapacity() const noexcept { return header()->cap; } const uint8_t *get() const noexcept { return p_; } diff --git a/cpp_src/core/query/dsl/dslencoder.cc b/cpp_src/core/query/dsl/dslencoder.cc index 3561d5b24..b8e92f6e3 100644 --- a/cpp_src/core/query/dsl/dslencoder.cc +++ b/cpp_src/core/query/dsl/dslencoder.cc @@ -114,10 +114,10 @@ void encodeAggregationFunctions(const Query& query, JsonBuilder& builder) { } void encodeJoinEntry(const QueryJoinEntry& joinEntry, JsonBuilder& builder) { - builder.Put("left_field", joinEntry.index_); - builder.Put("right_field", joinEntry.joinIndex_); - builder.Put("cond", get(cond_map, joinEntry.condition_)); - builder.Put("op", get(op_map, joinEntry.op_)); + builder.Put("left_field", joinEntry.LeftFieldName()); + builder.Put("right_field", joinEntry.RightFieldName()); + builder.Put("cond", get(cond_map, joinEntry.Condition())); + builder.Put("op", get(op_map, joinEntry.Operation())); } void encodeSingleJoinQuery(const JoinedQuery& joinQuery, JsonBuilder& builder) { @@ -125,9 +125,9 @@ void encodeSingleJoinQuery(const JoinedQuery& joinQuery, JsonBuilder& builder) { auto node = builder.Object("join_query"sv); node.Put("type", get(join_types, joinQuery.joinType)); - node.Put("namespace", joinQuery._namespace); - node.Put("limit", joinQuery.count); - node.Put("offset", joinQuery.start); + node.Put("namespace", joinQuery.NsName()); + node.Put("limit", joinQuery.Limit()); + node.Put("offset", joinQuery.Offset()); encodeFilters(joinQuery, node); encodeSorting(joinQuery.sortingEntries_, node); @@ -147,18 +147,18 @@ void encodeSingleJoinQuery(const JoinedQuery& joinQuery, JsonBuilder& builder) { } void encodeFilter(const QueryEntry& qentry, JsonBuilder& builder) { - if (qentry.distinct) return; - builder.Put("cond", get(cond_map, CondType(qentry.condition))); - builder.Put("field", qentry.index); + if (qentry.Distinct()) return; + builder.Put("cond", get(cond_map, CondType(qentry.Condition()))); + builder.Put("field", qentry.FieldName()); - if (qentry.values.empty()) return; - if (qentry.values.size() > 1 || qentry.values[0].Type().Is()) { + if (qentry.Values().empty()) return; + if (qentry.Values().size() > 1 || qentry.Values()[0].Type().Is()) { auto arrNode = builder.Array("value"); - for (const Variant& kv : qentry.values) { + for (const Variant& kv : qentry.Values()) { arrNode.Put(nullptr, kv); } } else { - builder.Put("value", qentry.values[0]); + builder.Put("value", qentry.Values()[0]); } } @@ -201,10 +201,10 @@ void encodeUpdateFields(const Query& query, JsonBuilder& builder) { void toDsl(const Query& query, JsonBuilder& builder) { switch (query.Type()) { case QueryType::QuerySelect: { - builder.Put("namespace", query._namespace); - builder.Put("limit", query.count); - builder.Put("offset", query.start); - builder.Put("req_total", get(reqtotal_values, query.calcTotal)); + builder.Put("namespace", query.NsName()); + builder.Put("limit", query.Limit()); + builder.Put("offset", query.Offset()); + builder.Put("req_total", get(reqtotal_values, query.CalcTotal())); builder.Put("explain", query.explain_); builder.Put("type", "select"); auto strictMode = strictModeToString(query.strictMode); @@ -222,7 +222,7 @@ void toDsl(const Query& query, JsonBuilder& builder) { break; } case QueryType::QueryUpdate: { - builder.Put("namespace", query._namespace); + builder.Put("namespace", query.NsName()); builder.Put("explain", query.explain_); builder.Put("type", "update"); encodeFilters(query, builder); @@ -244,14 +244,14 @@ void toDsl(const Query& query, JsonBuilder& builder) { break; } case QueryType::QueryDelete: { - builder.Put("namespace", query._namespace); + builder.Put("namespace", query.NsName()); builder.Put("explain", query.explain_); builder.Put("type", "delete"); encodeFilters(query, builder); break; } case QueryType::QueryTruncate: { - builder.Put("namespace", query._namespace); + builder.Put("namespace", query.NsName()); builder.Put("type", "truncate"); break; } @@ -283,7 +283,7 @@ void QueryEntries::toDsl(const_iterator it, const_iterator to, const Query& pare dsl::encodeEqualPositions(bracket.equalPositions, arrNode); }, [&node](const QueryEntry& qe) { - if (qe.distinct) return; + if (qe.Distinct()) return; dsl::encodeFilter(qe, node); }, [&node, &parentQuery](const JoinQueryEntry& jqe) { @@ -292,8 +292,8 @@ void QueryEntries::toDsl(const_iterator it, const_iterator to, const Query& pare }, [&node](const BetweenFieldsQueryEntry& qe) { node.Put("cond", dsl::get(dsl::cond_map, CondType(qe.Condition()))); - node.Put("first_field", qe.firstIndex); - node.Put("second_field", qe.secondIndex); + node.Put("first_field", qe.LeftFieldName()); + node.Put("second_field", qe.RightFieldName()); }); } } diff --git a/cpp_src/core/query/dsl/dslparser.cc b/cpp_src/core/query/dsl/dslparser.cc index f5fbfc709..13bac0c61 100644 --- a/cpp_src/core/query/dsl/dslparser.cc +++ b/cpp_src/core/query/dsl/dslparser.cc @@ -1,6 +1,5 @@ #include "dslparser.h" #include "core/cjson/jschemachecker.h" -#include "core/cjson/jsonbuilder.h" #include "core/query/query.h" #include "estl/fast_hash_map.h" #include "gason/gason.h" @@ -324,34 +323,6 @@ static void parseFilter(const JsonValue& filter, Query& q, std::vector(joined_entry_map, name, "join_query.on"sv)) { case JoinEntry::LeftField: checkJsonValueType(value, name, JSON_STRING); - qjoinEntry.index_ = std::string(value.toString()); + leftField = std::string(value.toString()); break; case JoinEntry::RightField: checkJsonValueType(value, name, JSON_STRING); - qjoinEntry.joinIndex_ = std::string(value.toString()); + rightField = std::string(value.toString()); break; case JoinEntry::Cond: checkJsonValueType(value, name, JSON_STRING); - qjoinEntry.condition_ = get(cond_map, value.toString(), "condition enum"sv); + cond = get(cond_map, value.toString(), "condition enum"sv); break; case JoinEntry::Op: checkJsonValueType(value, name, JSON_STRING); - qjoinEntry.op_ = get(op_map, value.toString(), "operation enum"sv); + op = get(op_map, value.toString(), "operation enum"sv); break; } } - qjoin.joinEntries_.emplace_back(qjoinEntry); + qjoin.joinEntries_.emplace_back(op, cond, std::move(leftField), std::move(rightField)); } } @@ -419,7 +392,7 @@ void parseSingleJoinQuery(const JsonValue& join, Query& query) { break; case JoinRoot::Namespace: checkJsonValueType(value, name, JSON_STRING); - qjoin._namespace = std::string(value.toString()); + qjoin.SetNsName(value.toString()); break; case JoinRoot::Filters: checkJsonValueType(value, name, JSON_ARRAY); @@ -430,11 +403,11 @@ void parseSingleJoinQuery(const JsonValue& join, Query& query) { break; case JoinRoot::Limit: checkJsonValueType(value, name, JSON_NUMBER, JSON_DOUBLE); - qjoin.count = static_cast(value.toNumber()); + qjoin.Limit(static_cast(value.toNumber())); break; case JoinRoot::Offset: checkJsonValueType(value, name, JSON_NUMBER, JSON_DOUBLE); - qjoin.start = static_cast(value.toNumber()); + qjoin.Offset(static_cast(value.toNumber())); break; case JoinRoot::On: parseJoinedEntries(value, qjoin); @@ -604,17 +577,17 @@ void parse(const JsonValue& root, Query& q) { switch (get(root_map, name, "root"sv)) { case Root::Namespace: checkJsonValueType(v, name, JSON_STRING); - q._namespace = std::string(v.toString()); + q.SetNsName(v.toString()); break; case Root::Limit: checkJsonValueType(v, name, JSON_NUMBER, JSON_DOUBLE); - q.count = static_cast(v.toNumber()); + q.Limit(static_cast(v.toNumber())); break; case Root::Offset: checkJsonValueType(v, name, JSON_NUMBER, JSON_DOUBLE); - q.start = static_cast(v.toNumber()); + q.Offset(static_cast(v.toNumber())); break; case Root::Filters: @@ -643,7 +616,7 @@ void parse(const JsonValue& root, Query& q) { break; case Root::ReqTotal: checkJsonValueType(v, name, JSON_STRING); - q.calcTotal = get(reqtotal_values, v.toString(), "req_total enum"sv); + q.CalcTotal(get(reqtotal_values, v.toString(), "req_total enum"sv)); break; case Root::Aggregations: checkJsonValueType(v, name, JSON_ARRAY); diff --git a/cpp_src/core/query/query.cc b/cpp_src/core/query/query.cc index 2e7b1da3e..2e2e9afa9 100644 --- a/cpp_src/core/query/query.cc +++ b/cpp_src/core/query/query.cc @@ -13,18 +13,15 @@ using namespace std::string_view_literals; const std::string_view kLsnIndexName = "#lsn"sv; const std::string_view kSlaveVersionIndexName = "#slave_version"sv; -Query::Query(const std::string &__namespace, unsigned _start, unsigned _count, CalcTotalMode _calcTotal) - : _namespace(__namespace), start(_start), count(_count), calcTotal(_calcTotal) {} - bool Query::operator==(const Query &obj) const { if (entries != obj.entries) return false; if (aggregations_ != obj.aggregations_) return false; - if (_namespace != obj._namespace) return false; + if (NsName() != obj.NsName()) return false; if (sortingEntries_ != obj.sortingEntries_) return false; - if (calcTotal != obj.calcTotal) return false; - if (start != obj.start) return false; - if (count != obj.count) return false; + if (CalcTotal() != obj.CalcTotal()) return false; + if (Offset() != obj.Offset()) return false; + if (Limit() != obj.Limit()) return false; if (debugLevel != obj.debugLevel) return false; if (strictMode != obj.strictMode) return false; if (forcedSortOrder_.size() != obj.forcedSortOrder_.size()) return false; @@ -52,17 +49,6 @@ Error Query::FromJSON(const std::string &dsl) { return dsl::Parse(dsl, *this); } std::string Query::GetJSON() const { return dsl::toDsl(*this); } -Query &Query::SetObject(std::string field, VariantArray value, bool hasExpressions) & { - for (auto &it : value) { - if (!it.Type().Is()) { - throw Error(errLogic, "Unexpected variant type in SetObject: %s. Expecting KeyValueType::String with JSON-content", - it.Type().Name()); - } - } - updateFields_.emplace_back(std::move(field), std::move(value), FieldModeSetJson, hasExpressions); - return *this; -} - WrSerializer &Query::GetSQL(WrSerializer &ser, bool stripArgs) const { return SQLEncoder(*this).GetSQL(ser, stripArgs); } std::string Query::GetSQL(bool stripArgs) const { @@ -77,12 +63,12 @@ void Query::deserialize(Serializer &ser, bool &hasJoinConditions) { int qtype = ser.GetVarUint(); switch (qtype) { case QueryCondition: { - QueryEntry qe; - qe.index = std::string(ser.GetVString()); - OpType op = OpType(ser.GetVarUint()); - qe.condition = CondType(ser.GetVarUint()); + const auto fieldName = ser.GetVString(); + const OpType op = OpType(ser.GetVarUint()); + const CondType condition = CondType(ser.GetVarUint()); int cnt = ser.GetVarUint(); - if (qe.condition == CondDWithin) { + VariantArray values; + if (condition == CondDWithin) { if (cnt != 3) { throw Error(errParseBin, "Expected point and distance for DWithin"); } @@ -90,14 +76,14 @@ void Query::deserialize(Serializer &ser, bool &hasJoinConditions) { point.reserve(2); point.emplace_back(ser.GetVariant().EnsureHold()); point.emplace_back(ser.GetVariant().EnsureHold()); - qe.values.reserve(2); - qe.values.emplace_back(std::move(point)); - qe.values.emplace_back(ser.GetVariant().EnsureHold()); + values.reserve(2); + values.emplace_back(std::move(point)); + values.emplace_back(ser.GetVariant().EnsureHold()); } else { - qe.values.reserve(cnt); - while (cnt--) qe.values.emplace_back(ser.GetVariant().EnsureHold()); + values.reserve(cnt); + while (cnt--) values.emplace_back(ser.GetVariant().EnsureHold()); } - entries.Append(op, std::move(qe)); + entries.Append(op, std::string{fieldName}, condition, std::move(values)); break; } case QueryBetweenFieldsCondition: { @@ -105,12 +91,12 @@ void Query::deserialize(Serializer &ser, bool &hasJoinConditions) { std::string firstField{ser.GetVString()}; CondType condition = static_cast(ser.GetVarUint()); std::string secondField{ser.GetVString()}; - entries.Append(op, BetweenFieldsQueryEntry{std::move(firstField), condition, std::move(secondField)}); + entries.Append(op, std::move(firstField), condition, std::move(secondField)); break; } case QueryAlwaysFalseCondition: { const OpType op = OpType(ser.GetVarUint()); - entries.Append(op, AlwaysFalse{}); + entries.Append(op); break; } case QueryJoinCondition: { @@ -155,12 +141,9 @@ void Query::deserialize(Serializer &ser, bool &hasJoinConditions) { break; } case QueryDistinct: { - QueryEntry qe; - qe.index = std::string(ser.GetVString()); - if (!qe.index.empty()) { - qe.distinct = true; - qe.condition = CondAny; - entries.Append(OpAnd, std::move(qe)); + const auto fieldName = ser.GetVString(); + if (!fieldName.empty()) { + entries.Append(OpAnd, std::string{fieldName}, QueryEntry::DistinctTag{}); } break; } @@ -180,12 +163,12 @@ void Query::deserialize(Serializer &ser, bool &hasJoinConditions) { break; } case QueryJoinOn: { - QueryJoinEntry qje; - qje.op_ = OpType(ser.GetVarUint()); - qje.condition_ = CondType(ser.GetVarUint()); - qje.index_ = std::string(ser.GetVString()); - qje.joinIndex_ = std::string(ser.GetVString()); - reinterpret_cast(this)->joinEntries_.push_back(std::move(qje)); + const OpType op = static_cast(ser.GetVarUint()); + const CondType condition = static_cast(ser.GetVarUint()); + std::string leftFieldName{ser.GetVString()}; + std::string rightFieldName{ser.GetVString()}; + reinterpret_cast(this)->joinEntries_.emplace_back(op, condition, std::move(leftFieldName), + std::move(rightFieldName)); break; } case QueryDebugLevel: @@ -195,13 +178,13 @@ void Query::deserialize(Serializer &ser, bool &hasJoinConditions) { strictMode = StrictMode(ser.GetVarUint()); break; case QueryLimit: - count = ser.GetVarUint(); + count_ = ser.GetVarUint(); break; case QueryOffset: - start = ser.GetVarUint(); + start_ = ser.GetVarUint(); break; case QueryReqTotal: - calcTotal = CalcTotalMode(ser.GetVarUint()); + calcTotal_ = CalcTotalMode(ser.GetVarUint()); break; case QuerySelectFilter: selectFilter_.push_back(std::string(ser.GetVString())); @@ -291,7 +274,7 @@ void Query::deserialize(Serializer &ser, bool &hasJoinConditions) { } void Query::Serialize(WrSerializer &ser, uint8_t mode) const { - ser.PutVString(_namespace); + ser.PutVString(NsName()); entries.Serialize(ser); for (const auto &agg : aggregations_) { @@ -328,10 +311,10 @@ void Query::Serialize(WrSerializer &ser, uint8_t mode) const { if (mode & WithJoinEntries) { for (const auto &qje : reinterpret_cast(this)->joinEntries_) { ser.PutVarUint(QueryJoinOn); - ser.PutVarUint(qje.op_); - ser.PutVarUint(qje.condition_); - ser.PutVString(qje.index_); - ser.PutVString(qje.joinIndex_); + ser.PutVarUint(qje.Operation()); + ser.PutVarUint(qje.Condition()); + ser.PutVString(qje.LeftFieldName()); + ser.PutVString(qje.RightFieldName()); } } @@ -364,17 +347,17 @@ void Query::Serialize(WrSerializer &ser, uint8_t mode) const { if (!(mode & SkipLimitOffset)) { if (HasLimit()) { ser.PutVarUint(QueryLimit); - ser.PutVarUint(count); + ser.PutVarUint(Limit()); } if (HasOffset()) { ser.PutVarUint(QueryOffset); - ser.PutVarUint(start); + ser.PutVarUint(Offset()); } } - if (calcTotal != ModeNoTotal) { + if (CalcTotal() != ModeNoTotal) { ser.PutVarUint(QueryReqTotal); - ser.PutVarUint(calcTotal); + ser.PutVarUint(CalcTotal()); } for (const auto &sf : selectFilter_) { @@ -426,7 +409,7 @@ void Query::Serialize(WrSerializer &ser, uint8_t mode) const { } void Query::Deserialize(Serializer &ser) { - _namespace = std::string(ser.GetVString()); + namespace_ = std::string(ser.GetVString()); bool hasJoinConditions = false; deserialize(ser, hasJoinConditions); @@ -445,37 +428,27 @@ void Query::Deserialize(Serializer &ser) { Query &q = nested ? mergeQueries_.back() : *this; if (joinType != JoinType::LeftJoin && !hasJoinConditions) { const size_t joinIdx = joinQueries_.size(); - entries.Append((joinType == JoinType::OrInnerJoin) ? OpOr : OpAnd, JoinQueryEntry{joinIdx}); + entries.Append((joinType == JoinType::OrInnerJoin) ? OpOr : OpAnd, joinIdx); } q.joinQueries_.emplace_back(std::move(q1)); } } } -Query &Query::Join(JoinType joinType, const std::string &index, const std::string &joinIndex, CondType cond, OpType op, Query &&qr) & { - QueryJoinEntry joinEntry; - joinEntry.op_ = op; - joinEntry.condition_ = cond; - joinEntry.index_ = index; - joinEntry.joinIndex_ = joinIndex; +Query &Query::Join(JoinType joinType, std::string leftField, std::string rightField, CondType cond, OpType op, Query &&qr) & { auto &jq = joinQueries_.emplace_back(joinType, std::move(qr)); - jq.joinEntries_.emplace_back(std::move(joinEntry)); + jq.joinEntries_.emplace_back(op, cond, std::move(leftField), std::move(rightField)); if (joinType != JoinType::LeftJoin) { - entries.Append((joinType == JoinType::InnerJoin) ? OpType::OpAnd : OpType::OpOr, JoinQueryEntry(joinQueries_.size() - 1)); + entries.Append((joinType == JoinType::InnerJoin) ? OpType::OpAnd : OpType::OpOr, joinQueries_.size() - 1); } return *this; } -Query &Query::Join(JoinType joinType, const std::string &index, const std::string &joinIndex, CondType cond, OpType op, const Query &qr) & { - QueryJoinEntry joinEntry; - joinEntry.op_ = op; - joinEntry.condition_ = cond; - joinEntry.index_ = index; - joinEntry.joinIndex_ = joinIndex; +Query &Query::Join(JoinType joinType, std::string leftField, std::string rightField, CondType cond, OpType op, const Query &qr) & { joinQueries_.emplace_back(joinType, qr); - joinQueries_.back().joinEntries_.emplace_back(std::move(joinEntry)); + joinQueries_.back().joinEntries_.emplace_back(op, cond, std::move(leftField), std::move(rightField)); if (joinType != JoinType::LeftJoin) { - entries.Append((joinType == JoinType::InnerJoin) ? OpType::OpAnd : OpType::OpOr, JoinQueryEntry(joinQueries_.size() - 1)); + entries.Append((joinType == JoinType::InnerJoin) ? OpType::OpAnd : OpType::OpOr, joinQueries_.size() - 1); } return *this; } @@ -483,7 +456,7 @@ Query &Query::Join(JoinType joinType, const std::string &index, const std::strin Query::OnHelper Query::Join(JoinType joinType, Query &&q) & { joinQueries_.emplace_back(joinType, std::move(q)); if (joinType != JoinType::LeftJoin) { - entries.Append((joinType == JoinType::InnerJoin) ? OpType::OpAnd : OpType::OpOr, JoinQueryEntry(joinQueries_.size() - 1)); + entries.Append((joinType == JoinType::InnerJoin) ? OpType::OpAnd : OpType::OpOr, joinQueries_.size() - 1); } return {*this, joinQueries_.back()}; } @@ -491,7 +464,7 @@ Query::OnHelper Query::Join(JoinType joinType, Query &&q) & { Query::OnHelper Query::Join(JoinType joinType, const Query &q) & { joinQueries_.emplace_back(joinType, q); if (joinType != JoinType::LeftJoin) { - entries.Append((joinType == JoinType::InnerJoin) ? OpType::OpAnd : OpType::OpOr, JoinQueryEntry(joinQueries_.size() - 1)); + entries.Append((joinType == JoinType::InnerJoin) ? OpType::OpAnd : OpType::OpOr, joinQueries_.size() - 1); } return {*this, joinQueries_.back()}; } @@ -499,7 +472,7 @@ Query::OnHelper Query::Join(JoinType joinType, const Query &q) & { Query::OnHelperR Query::Join(JoinType joinType, Query &&q) && { joinQueries_.emplace_back(joinType, std::move(q)); if (joinType != JoinType::LeftJoin) { - entries.Append((joinType == JoinType::InnerJoin) ? OpType::OpAnd : OpType::OpOr, JoinQueryEntry(joinQueries_.size() - 1)); + entries.Append((joinType == JoinType::InnerJoin) ? OpType::OpAnd : OpType::OpOr, joinQueries_.size() - 1); } return {std::move(*this), joinQueries_.back()}; } @@ -507,7 +480,7 @@ Query::OnHelperR Query::Join(JoinType joinType, Query &&q) && { Query::OnHelperR Query::Join(JoinType joinType, const Query &q) && { joinQueries_.emplace_back(joinType, q); if (joinType != JoinType::LeftJoin) { - entries.Append((joinType == JoinType::InnerJoin) ? OpType::OpAnd : OpType::OpOr, JoinQueryEntry(joinQueries_.size() - 1)); + entries.Append((joinType == JoinType::InnerJoin) ? OpType::OpAnd : OpType::OpOr, joinQueries_.size() - 1); } return {std::move(*this), joinQueries_.back()}; } @@ -548,11 +521,11 @@ void Query::WalkNested(bool withSelf, bool withMerged, const std::function(0) && kLsnIndexName == entries.Get(0).index) { + if (entries.Size() == 1 && entries.HoldsOrReferTo(0) && kLsnIndexName == entries.Get(0).FieldName()) { return true; } else if (entries.Size() == 2 && entries.HoldsOrReferTo(0) && entries.HoldsOrReferTo(1)) { - const auto &index0 = entries.Get(0).index; - const auto &index1 = entries.Get(1).index; + const auto &index0 = entries.Get(0).FieldName(); + const auto &index1 = entries.Get(1).FieldName(); return (kLsnIndexName == index0 && kSlaveVersionIndexName == index1) || (kLsnIndexName == index1 && kSlaveVersionIndexName == index0); } diff --git a/cpp_src/core/query/query.h b/cpp_src/core/query/query.h index 00b2b7203..f944b3a3e 100644 --- a/cpp_src/core/query/query.h +++ b/cpp_src/core/query/query.h @@ -71,8 +71,10 @@ class Query { /// @param start - number of the first row to get from selected set. Analog to sql OFFSET Offset. /// @param count - number of rows to get from result set. Analog to sql LIMIT RowsCount. /// @param calcTotal - calculation mode. - explicit Query(const std::string &nsName, unsigned start = QueryEntry::kDefaultOffset, unsigned count = QueryEntry::kDefaultLimit, - CalcTotalMode calcTotal = ModeNoTotal); + template + explicit Query(T nsName, unsigned start = QueryEntry::kDefaultOffset, unsigned count = QueryEntry::kDefaultLimit, + CalcTotalMode calcTotal = ModeNoTotal) + : namespace_(std::forward(nsName)), start_(start), count_(count), calcTotal_(calcTotal) {} /// Creates an empty object. Query() = default; @@ -112,79 +114,75 @@ class Query { Query &&Explain(bool on = true) && { return std::move(Explain(on)); } /// Adds a condition with a single value. Analog to sql Where clause. - /// @param idx - index used in condition clause. + /// @param field - field used in condition clause. /// @param cond - type of condition. /// @param val - value of index to be compared with. /// @return Query object ready to be executed. - template - Query &Where(const std::string &idx, CondType cond, Input val) & { - return Where(idx, cond, {val}); + template + Query &Where(Str &&field, CondType cond, Input val) & { + return Where(std::forward(field), cond, {std::forward(val)}); } - template - Query &&Where(const std::string &idx, CondType cond, Input val) && { - return std::move(Where(idx, cond, {val})); + template + Query &&Where(Str &&field, CondType cond, Input val) && { + return std::move(Where(std::forward(field), cond, {std::move(val)})); } /// Adds a condition with several values. Analog to sql Where clause. - /// @param idx - index used in condition clause. + /// @param field - field used in condition clause. /// @param cond - type of condition. /// @param l - list of index values to be compared with. /// @return Query object ready to be executed. - template - Query &Where(const std::string &idx, CondType cond, std::initializer_list l) & { - QueryEntry qe; - qe.condition = cond; - qe.index = idx; - for (auto it = l.begin(); it != l.end(); it++) qe.values.push_back(Variant(*it)); - entries.Append(nextOp_, std::move(qe)); + template + Query &Where(Str &&field, CondType cond, std::initializer_list l) & { + VariantArray values; + values.reserve(l.size()); + for (auto it = l.begin(); it != l.end(); it++) values.emplace_back(*it); + entries.Append(nextOp_, std::forward(field), cond, std::move(values)); nextOp_ = OpAnd; return *this; } - template - Query &&Where(const std::string &idx, CondType cond, std::initializer_list l) && { - return std::move(Where(idx, cond, std::move(l))); + template + Query &&Where(Str &&field, CondType cond, std::initializer_list l) && { + return std::move(Where(std::forward(field), cond, std::move(l))); } /// Adds a condition with several values. Analog to sql Where clause. - /// @param idx - index used in condition clause. + /// @param field - field used in condition clause. /// @param cond - type of condition. /// @param l - vector of index values to be compared with. /// @return Query object ready to be executed. - template - Query &Where(const std::string &idx, CondType cond, const std::vector &l) & { - QueryEntry qe; - qe.condition = cond; - qe.index = idx; - qe.values.reserve(l.size()); - for (auto it = l.begin(); it != l.end(); it++) qe.values.push_back(Variant(*it)); - entries.Append(nextOp_, std::move(qe)); + template + Query &Where(Str &&field, CondType cond, const std::vector &l) & { + VariantArray values; + values.reserve(l.size()); + for (auto it = l.begin(); it != l.end(); it++) values.emplace_back(*it); + entries.Append(nextOp_, std::forward(field), cond, std::move(values)); nextOp_ = OpAnd; return *this; } - template - Query &&Where(const std::string &idx, CondType cond, const std::vector &l) && { - return std::move(Where(idx, cond, l)); + template + Query &&Where(Str &&field, CondType cond, const std::vector &l) && { + return std::move(Where(std::forward(field), cond, l)); } /// Adds a condition with several values. Analog to sql Where clause. - /// @param idx - index used in condition clause. + /// @param field - field used in condition clause. /// @param cond - type of condition. /// @param l - vector of index values to be compared with. /// @return Query object ready to be executed. - Query &Where(const std::string &idx, CondType cond, const VariantArray &l) & { - QueryEntry qe; - qe.condition = cond; - qe.index = idx; - qe.values.reserve(l.size()); - for (auto it = l.begin(); it != l.end(); it++) qe.values.push_back(Variant(*it)); - entries.Append(nextOp_, std::move(qe)); + template + Query &Where(Str &&field, CondType cond, VariantArray l) & { + entries.Append(nextOp_, std::forward(field), cond, std::move(l)); nextOp_ = OpAnd; return *this; } - Query &&Where(const std::string &idx, CondType cond, const VariantArray &l) && { return std::move(Where(idx, cond, l)); } + template + Query &&Where(Str &&field, CondType cond, VariantArray l) && { + return std::move(Where(std::forward(field), cond, std::move(l))); + } /// Adds a condition with several values to a composite index. - /// @param idx - index name. + /// @param idx - composite index name. /// @param cond - type of condition. /// @param l - list of values to be compared according to the order /// of indexes in composite index name. @@ -194,172 +192,192 @@ class Query { /// in case of CondRange) belongs to "bookid" and l[0][1] (and l[1][1] in case of CondRange) /// belongs to "price" indexes. /// @return Query object ready to be executed. - Query &WhereComposite(const std::string &idx, CondType cond, std::initializer_list l) & { - QueryEntry qe; - qe.condition = cond; - qe.index = idx; - qe.values.reserve(l.size()); + template + Query &WhereComposite(Str &&idx, CondType cond, std::initializer_list l) & { + VariantArray values; + values.reserve(l.size()); for (auto it = l.begin(); it != l.end(); it++) { - qe.values.push_back(Variant(*it)); + values.emplace_back(*it); } - entries.Append(nextOp_, std::move(qe)); + entries.Append(nextOp_, std::forward(idx), cond, std::move(values)); nextOp_ = OpAnd; return *this; } - Query &&WhereComposite(const std::string &idx, CondType cond, std::initializer_list l) && { - return std::move(WhereComposite(idx, cond, l)); + template + Query &&WhereComposite(Str &&idx, CondType cond, std::initializer_list l) && { + return std::move(WhereComposite(std::forward(idx), cond, std::move(l))); } - Query &WhereComposite(const std::string &idx, CondType cond, const std::vector &v) & { - QueryEntry qe; - qe.condition = cond; - qe.index = idx; - qe.values.reserve(v.size()); + template + Query &WhereComposite(Str &&idx, CondType cond, const std::vector &v) & { + VariantArray values; + values.reserve(v.size()); for (auto it = v.begin(); it != v.end(); it++) { - qe.values.push_back(Variant(*it)); + values.emplace_back(*it); } - entries.Append(nextOp_, std::move(qe)); + entries.Append(nextOp_, std::forward(idx), cond, std::move(values)); nextOp_ = OpAnd; return *this; } - Query &&WhereComposite(const std::string &idx, CondType cond, const std::vector &v) && { - return std::move(WhereComposite(idx, cond, v)); + template + Query &&WhereComposite(Str &&idx, CondType cond, const std::vector &v) && { + return std::move(WhereComposite(std::forward(idx), cond, v)); } - Query &WhereBetweenFields(std::string firstIdx, CondType cond, std::string secondIdx) & { - entries.Append(nextOp_, BetweenFieldsQueryEntry{std::move(firstIdx), cond, std::move(secondIdx)}); + template + Query &WhereBetweenFields(Str1 &&firstIdx, CondType cond, Str2 &&secondIdx) & { + entries.Append(nextOp_, std::forward(firstIdx), cond, std::forward(secondIdx)); nextOp_ = OpAnd; return *this; } - Query &&WhereBetweenFields(std::string firstIdx, CondType cond, std::string secondIdx) && { - return std::move(WhereBetweenFields(std::move(firstIdx), cond, std::move(secondIdx))); + template + Query &&WhereBetweenFields(Str1 &&firstIdx, CondType cond, Str2 &&secondIdx) && { + return std::move(WhereBetweenFields(std::forward(firstIdx), cond, std::forward(secondIdx))); } - Query &DWithin(const std::string &idx, Point p, double distance) & { - QueryEntry qe; - qe.condition = CondDWithin; - qe.index = idx; - qe.values.reserve(2); - qe.values.emplace_back(p); - qe.values.emplace_back(distance); - entries.Append(nextOp_, std::move(qe)); + template + Query &DWithin(Str &&field, Point p, double distance) & { + entries.Append(nextOp_, std::forward(field), CondDWithin, VariantArray::Create(p, distance)); nextOp_ = OpAnd; return *this; } - Query &&DWithin(const std::string &idx, Point p, double distance) && { return std::move(DWithin(idx, p, distance)); } + template + Query &&DWithin(Str &&field, Point p, double distance) && { + return std::move(DWithin(std::forward(field), p, distance)); + } /// Sets a new value for a field. /// @param field - field name. /// @param value - new value. /// @param hasExpressions - true: value has expresions in it - template - Query &Set(std::string field, ValueType value, bool hasExpressions = false) & { - return Set(std::move(field), {value}, hasExpressions); + template + Query &Set(Str &&field, ValueType value, bool hasExpressions = false) & { + return Set(std::forward(field), {value}, hasExpressions); } - template - Query &&Set(std::string field, ValueType value, bool hasExpressions = false) && { - return std::move(Set(std::move(field), std::move(value), hasExpressions)); + template + Query &&Set(Str &&field, ValueType value, bool hasExpressions = false) && { + return std::move(Set(std::forward(field), std::move(value), hasExpressions)); } /// Sets a new value for a field. /// @param field - field name. /// @param l - new value. /// @param hasExpressions - true: value has expresions in it - template - Query &Set(std::string field, std::initializer_list l, bool hasExpressions = false) & { + template + Query &Set(Str &&field, std::initializer_list l, bool hasExpressions = false) & { VariantArray value; value.reserve(l.size()); for (auto it = l.begin(); it != l.end(); it++) value.emplace_back(*it); - return Set(std::move(field), std::move(value), hasExpressions); + return Set(std::forward(field), std::move(value), hasExpressions); } - template - Query &&Set(std::string field, std::initializer_list l, bool hasExpressions = false) && { - return std::move(Set(std::move(field), std::move(l), hasExpressions)); + template + Query &&Set(Str &&field, std::initializer_list l, bool hasExpressions = false) && { + return std::move(Set(std::forward(field), std::move(l), hasExpressions)); } /// Sets a new value for a field. /// @param field - field name. /// @param l - new value. /// @param hasExpressions - true: value has expresions in it - template - Query &Set(std::string field, const std::vector &l, bool hasExpressions = false) & { + template + Query &Set(Str &&field, const std::vector &l, bool hasExpressions = false) & { VariantArray value; value.reserve(l.size()); for (auto it = l.begin(); it != l.end(); it++) value.emplace_back(*it); - return Set(std::move(field), std::move(value.MarkArray()), hasExpressions); + return Set(std::forward(field), std::move(value.MarkArray()), hasExpressions); } - template - Query &&Set(std::string field, const std::vector &l, bool hasExpressions = false) && { - return std::move(Set(std::move(field), l, hasExpressions)); + template + Query &&Set(Str &&field, const std::vector &l, bool hasExpressions = false) && { + return std::move(Set(std::forward(field), l, hasExpressions)); } /// Sets a new value for a field. /// @param field - field name. /// @param value - new value. /// @param hasExpressions - true: value has expresions in it - Query &Set(std::string field, VariantArray value, bool hasExpressions = false) & { - updateFields_.emplace_back(std::move(field), std::move(value), FieldModeSet, hasExpressions); + template + Query &Set(Str &&field, VariantArray value, bool hasExpressions = false) & { + updateFields_.emplace_back(std::forward(field), std::move(value), FieldModeSet, hasExpressions); return *this; } - Query &&Set(std::string field, VariantArray value, bool hasExpressions = false) && { - return std::move(Set(std::move(field), std::move(value), hasExpressions)); + template + Query &&Set(Str &&field, VariantArray value, bool hasExpressions = false) && { + return std::move(Set(std::forward(field), std::move(value), hasExpressions)); } /// Sets a value for a field as an object. /// @param field - field name. /// @param value - new value. /// @param hasExpressions - true: value has expresions in it - template - Query &SetObject(std::string field, ValueType value, bool hasExpressions = false) & { - return SetObject(std::move(field), {value}, hasExpressions); + template + Query &SetObject(Str &&field, ValueType value, bool hasExpressions = false) & { + return SetObject(std::forward(field), {value}, hasExpressions); } - template - Query &&SetObject(std::string field, ValueType value, bool hasExpressions = false) && { - return std::move(SetObject(std::move(field), std::move(value), hasExpressions)); + template + Query &&SetObject(Str &&field, ValueType value, bool hasExpressions = false) && { + return std::move(SetObject(std::forward(field), std::move(value), hasExpressions)); } /// Sets a new value for a field as an object. /// @param field - field name. /// @param l - new value. /// @param hasExpressions - true: value has expresions in it - template - Query &SetObject(std::string field, std::initializer_list l, bool hasExpressions = false) & { + template + Query &SetObject(Str &&field, std::initializer_list l, bool hasExpressions = false) & { VariantArray value; value.reserve(l.size()); - for (auto it = l.begin(); it != l.end(); it++) value.emplace_back(Variant(*it)); - return SetObject(std::move(field), std::move(value), hasExpressions); + for (auto it = l.begin(); it != l.end(); it++) value.emplace_back(*it); + return SetObject(std::forward(field), std::move(value), hasExpressions); } - template - Query &&SetObject(std::string field, std::initializer_list l, bool hasExpressions = false) && { - return std::move(SetObject(std::move(field), std::move(l), hasExpressions)); + template + Query &&SetObject(Str &&field, std::initializer_list l, bool hasExpressions = false) && { + return std::move(SetObject(std::forward(field), std::move(l), hasExpressions)); } /// Sets a new value for a field as an object. /// @param field - field name. /// @param l - new value. /// @param hasExpressions - true: value has expresions in it - template - Query &SetObject(std::string field, const std::vector &l, bool hasExpressions = false) & { + template + Query &SetObject(Str &&field, const std::vector &l, bool hasExpressions = false) & { VariantArray value; value.reserve(l.size()); for (auto it = l.begin(); it != l.end(); it++) value.emplace_back(Variant(*it)); - return SetObject(std::move(field), std::move(value.MarkArray()), hasExpressions); + return SetObject(std::forward(field), std::move(value.MarkArray()), hasExpressions); } - template - Query &&SetObject(std::string field, const std::vector &l, bool hasExpressions = false) && { - return std::move(SetObject(std::move(field), l, hasExpressions)); + template + Query &&SetObject(Str &&field, const std::vector &l, bool hasExpressions = false) && { + return std::move(SetObject(std::forward(field), l, hasExpressions)); } /// Sets a value for a field as an object. /// @param field - field name. /// @param value - new value. /// @param hasExpressions - true: value has expresions in it - Query &SetObject(std::string field, VariantArray value, bool hasExpressions = false) &; - Query &&SetObject(std::string field, VariantArray value, bool hasExpressions = false) && { - return std::move(SetObject(std::move(field), std::move(value), hasExpressions)); + template + Query &SetObject(Str &&field, VariantArray value, bool hasExpressions = false) & { + for (auto &it : value) { + if (!it.Type().Is()) { + throw Error(errLogic, "Unexpected variant type in SetObject: %s. Expecting KeyValueType::String with JSON-content", + it.Type().Name()); + } + } + updateFields_.emplace_back(std::forward(field), std::move(value), FieldModeSetJson, hasExpressions); + return *this; + } + template + Query &&SetObject(Str &&field, VariantArray value, bool hasExpressions = false) && { + return std::move(SetObject(std::forward(field), std::move(value), hasExpressions)); } /// Drops a value for a field. /// @param field - field name. - Query &Drop(std::string field) & { - updateFields_.emplace_back(std::move(field), VariantArray(), FieldModeDrop); + template + Query &Drop(Str &&field) & { + updateFields_.emplace_back(std::forward(field), VariantArray(), FieldModeDrop); return *this; } - Query &&Drop(std::string field) && { return std::move(Drop(std::move(field))); } + template + Query &&Drop(Str &&field) && { + return std::move(Drop(std::forward(field))); + } /// Add sql-function to query. /// @param function - function declaration. - void AddFunction(std::string function) { selectFunctions_.emplace_back(std::move(function)); } + template + void AddFunction(Str &&function) { + selectFunctions_.emplace_back(std::forward(function)); + } /// Adds equal position fields to arrays queries. /// @param equalPosition - list of fields with equal array index position. @@ -387,19 +405,21 @@ class Query { /// Joins namespace with another namespace. Analog to sql JOIN. /// @param joinType - type of Join (Inner, Left or OrInner). - /// @param index - name of the field in the namespace of this Query object. - /// @param joinIndex - name of the field in the namespace of qr Query object. + /// @param leftField - name of the field in the namespace of this Query object. + /// @param rightField - name of the field in the namespace of qr Query object. /// @param cond - condition type (Eq, Leq, Geq, etc). /// @param op - operation type (and, or, not). /// @param qr - query of the namespace that is going to be joined with this one. /// @return Query object ready to be executed. - Query &Join(JoinType joinType, const std::string &index, const std::string &joinIndex, CondType cond, OpType op, Query &&qr) &; - Query &&Join(JoinType joinType, const std::string &index, const std::string &joinIndex, CondType cond, OpType op, Query &&qr) && { - return std::move(Join(joinType, index, joinIndex, cond, op, std::move(qr))); + Query &Join(JoinType joinType, std::string leftField, std::string rightField, CondType cond, OpType op, Query &&qr) &; + template + Query &&Join(JoinType joinType, StrL &&leftField, StrR &&rightField, CondType cond, OpType op, Query &&qr) && { + return std::move(Join(joinType, std::forward(leftField), std::forward(rightField), cond, op, std::move(qr))); } - Query &Join(JoinType joinType, const std::string &index, const std::string &joinIndex, CondType cond, OpType op, const Query &qr) &; - Query &&Join(JoinType joinType, const std::string &index, const std::string &joinIndex, CondType cond, OpType op, const Query &qr) && { - return std::move(Join(joinType, index, joinIndex, cond, op, qr)); + Query &Join(JoinType joinType, std::string leftField, std::string rightField, CondType cond, OpType op, const Query &qr) &; + template + Query &&Join(JoinType joinType, StrL &&leftField, StrR &&rightField, CondType cond, OpType op, const Query &qr) && { + return std::move(Join(joinType, std::forward(leftField), std::forward(rightField), cond, op, qr)); } OnHelper Join(JoinType joinType, Query &&q) &; @@ -409,60 +429,72 @@ class Query { /// @public /// Inner Join of this namespace with another one. - /// @param index - name of the field in the namespace of this Query object. - /// @param joinIndex - name of the field in the namespace of qr Query object. + /// @param leftField - name of the field in the namespace of this Query object. + /// @param rightField - name of the field in the namespace of qr Query object. /// @param cond - condition type (Eq, Leq, Geq, etc). /// @param qr - query of the namespace that is going to be joined with this one. /// @return Query object ready to be executed. - Query &InnerJoin(const std::string &index, const std::string &joinIndex, CondType cond, Query &&qr) & { // -V1071 - return Join(JoinType::InnerJoin, index, joinIndex, cond, OpAnd, std::move(qr)); + template + Query &InnerJoin(StrL &&leftField, StrR &&rightField, CondType cond, Query &&qr) & { // -V1071 + return Join(JoinType::InnerJoin, std::forward(leftField), std::forward(rightField), cond, OpAnd, std::move(qr)); } - Query &&InnerJoin(const std::string &index, const std::string &joinIndex, CondType cond, Query &&qr) && { - return std::move(InnerJoin(index, joinIndex, cond, std::move(qr))); + template + Query &&InnerJoin(StrL &&leftField, StrR &&rightField, CondType cond, Query &&qr) && { + return std::move(InnerJoin(std::forward(leftField), std::forward(rightField), cond, std::move(qr))); } - Query &InnerJoin(const std::string &index, const std::string &joinIndex, CondType cond, const Query &qr) & { - return Join(JoinType::InnerJoin, index, joinIndex, cond, OpAnd, qr); + template + Query &InnerJoin(StrL &&leftField, StrR &&rightField, CondType cond, const Query &qr) & { + return Join(JoinType::InnerJoin, std::forward(leftField), std::forward(rightField), cond, OpAnd, qr); } - Query &&InnerJoin(const std::string &index, const std::string &joinIndex, CondType cond, const Query &qr) && { - return std::move(InnerJoin(index, joinIndex, cond, qr)); + template + Query &&InnerJoin(StrL &&leftField, StrR &&rightField, CondType cond, const Query &qr) && { + return std::move(InnerJoin(std::forward(leftField), std::forward(rightField), cond, qr)); } /// Left Join of this namespace with another one. - /// @param index - name of the field in the namespace of this Query object. - /// @param joinIndex - name of the field in the namespace of qr Query object. + /// @param leftField - name of the field in the namespace of this Query object. + /// @param rightField - name of the field in the namespace of qr Query object. /// @param cond - condition type (Eq, Leq, Geq, etc). /// @param qr - query of the namespace that is going to be joined with this one. /// @return Query object ready to be executed. - Query &LeftJoin(const std::string &index, const std::string &joinIndex, CondType cond, Query &&qr) & { - return Join(JoinType::LeftJoin, index, joinIndex, cond, OpAnd, std::move(qr)); + template + Query &LeftJoin(StrL &&leftField, StrR &&rightField, CondType cond, Query &&qr) & { + return Join(JoinType::LeftJoin, std::forward(leftField), std::forward(rightField), cond, OpAnd, std::move(qr)); } - Query &&LeftJoin(const std::string &index, const std::string &joinIndex, CondType cond, Query &&qr) && { - return std::move(LeftJoin(index, joinIndex, cond, std::move(qr))); + template + Query &&LeftJoin(StrL &&leftField, StrR &&rightField, CondType cond, Query &&qr) && { + return std::move(LeftJoin(std::forward(leftField), std::forward(rightField), cond, std::move(qr))); } - Query &LeftJoin(const std::string &index, const std::string &joinIndex, CondType cond, const Query &qr) & { - return Join(JoinType::LeftJoin, index, joinIndex, cond, OpAnd, qr); + template + Query &LeftJoin(StrL &&leftField, StrR &&rightField, CondType cond, const Query &qr) & { + return Join(JoinType::LeftJoin, std::forward(leftField), std::forward(rightField), cond, OpAnd, qr); } - Query &&LeftJoin(const std::string &index, const std::string &joinIndex, CondType cond, const Query &qr) && { - return std::move(LeftJoin(index, joinIndex, cond, qr)); + template + Query &&LeftJoin(StrL &&leftField, StrR &&rightField, CondType cond, const Query &qr) && { + return std::move(LeftJoin(std::forward(leftField), std::forward(rightField), cond, qr)); } /// OrInnerJoin of this namespace with another one. - /// @param index - name of the field in the namespace of this Query object. - /// @param joinIndex - name of the field in the namespace of qr Query object. + /// @param leftField - name of the field in the namespace of this Query object. + /// @param rightField - name of the field in the namespace of qr Query object. /// @param cond - condition type (Eq, Leq, Geq, etc). /// @param qr - query of the namespace that is going to be joined with this one. /// @return a reference to a query object ready to be executed. - Query &OrInnerJoin(const std::string &index, const std::string &joinIndex, CondType cond, Query &&qr) & { - return Join(JoinType::OrInnerJoin, index, joinIndex, cond, OpAnd, std::move(qr)); + template + Query &OrInnerJoin(StrL &&leftField, StrR &&rightField, CondType cond, Query &&qr) & { + return Join(JoinType::OrInnerJoin, std::forward(leftField), std::forward(rightField), cond, OpAnd, std::move(qr)); } - Query &&OrInnerJoin(const std::string &index, const std::string &joinIndex, CondType cond, Query &&qr) && { - return std::move(OrInnerJoin(index, joinIndex, cond, std::move(qr))); + template + Query &&OrInnerJoin(StrL &&leftField, StrR &&rightField, CondType cond, Query &&qr) && { + return std::move(OrInnerJoin(std::forward(leftField), std::forward(rightField), cond, std::move(qr))); } - Query &OrInnerJoin(const std::string &index, const std::string &joinIndex, CondType cond, const Query &qr) & { - return Join(JoinType::OrInnerJoin, index, joinIndex, cond, OpAnd, qr); + template + Query &OrInnerJoin(StrL &&leftField, StrR &&rightField, CondType cond, const Query &qr) & { + return Join(JoinType::OrInnerJoin, std::forward(leftField), std::forward(rightField), cond, OpAnd, qr); } - Query &&OrInnerJoin(const std::string &index, const std::string &joinIndex, CondType cond, const Query &qr) && { - return std::move(OrInnerJoin(index, joinIndex, cond, qr)); + template + Query &&OrInnerJoin(StrL &&leftField, StrR &&rightField, CondType cond, const Query &qr) && { + return std::move(OrInnerJoin(std::forward(leftField), std::forward(rightField), cond, qr)); } Query &Merge(const Query &q) &; Query &&Merge(const Query &q) && { return std::move(Merge(q)); } @@ -491,11 +523,15 @@ class Query { /// @param sort - sorting column name. /// @param desc - is sorting direction descending or ascending. /// @return Query object. - Query &Sort(std::string sort, bool desc) & { // -V1071 - if (sort.length()) sortingEntries_.emplace_back(std::move(sort), desc); + template + Query &Sort(Str &&sort, bool desc) & { // -V1071 + if (!strEmpty(sort)) sortingEntries_.emplace_back(std::forward(sort), desc); return *this; } - Query &&Sort(std::string sort, bool desc) && { return std::move(Sort(std::move(sort), desc)); } + template + Query &&Sort(Str &&sort, bool desc) && { + return std::move(Sort(std::forward(sort), desc)); + } /// Performs sorting by ST_Distance() expressions for geometry index. Sorting function will use distance between field and target point. /// @param field - field's name. This field must contain Point. @@ -519,17 +555,17 @@ class Query { /// @param desc - is sorting direction descending or ascending. /// @param forcedSortOrder - list of values for forced sort order. /// @return Query object. - template - Query &Sort(std::string sort, bool desc, std::initializer_list forcedSortOrder) & { + template + Query &Sort(Str &&sort, bool desc, std::initializer_list forcedSortOrder) & { if (!sortingEntries_.empty() && !std::empty(forcedSortOrder)) throw Error(errParams, "Forced sort order is allowed for the first sorting entry only"); - sortingEntries_.emplace_back(std::move(sort), desc); + sortingEntries_.emplace_back(std::forward(sort), desc); for (const T &v : forcedSortOrder) forcedSortOrder_.emplace_back(v); return *this; } - template - Query &&Sort(std::string sort, bool desc, std::initializer_list forcedSortOrder) && { - return std::move(Sort(std::move(sort), desc, std::move(forcedSortOrder))); + template + Query &&Sort(Str &&sort, bool desc, std::initializer_list forcedSortOrder) && { + return std::move(Sort(std::forward(sort), desc, std::move(forcedSortOrder))); } /// Performs sorting by certain column. Analog to sql ORDER BY. @@ -537,28 +573,32 @@ class Query { /// @param desc - is sorting direction descending or ascending. /// @param forcedSortOrder - list of values for forced sort order. /// @return Query object. - template - Query &Sort(std::string sort, bool desc, const T &forcedSortOrder) & { + template + Query &Sort(Str &&sort, bool desc, const T &forcedSortOrder) & { if (!sortingEntries_.empty() && !forcedSortOrder.empty()) throw Error(errParams, "Forced sort order is allowed for the first sorting entry only"); - sortingEntries_.emplace_back(std::move(sort), desc); + sortingEntries_.emplace_back(std::forward(sort), desc); for (const auto &v : forcedSortOrder) forcedSortOrder_.emplace_back(v); return *this; } - template - Query &&Sort(std::string sort, bool desc, const T &forcedSortOrder) && { - return std::move(Sort(std::move(sort), desc, forcedSortOrder)); + template + Query &&Sort(Str &&sort, bool desc, const T &forcedSortOrder) && { + return std::move(Sort(std::forward(sort), desc, forcedSortOrder)); } /// Performs distinct for a certain index. /// @param indexName - name of index for distict operation. - Query &Distinct(std::string indexName) & { - if (indexName.length()) { - aggregations_.emplace_back(AggDistinct, h_vector{std::move(indexName)}); + template + Query &Distinct(Str &&indexName) & { + if (!strEmpty(indexName)) { + aggregations_.emplace_back(AggDistinct, h_vector{std::forward(indexName)}); } return *this; } - Query &&Distinct(std::string indexName) && { return std::move(Distinct(std::move(indexName))); } + template + Query &&Distinct(Str &&indexName) && { + return std::move(Distinct(std::forward(indexName))); + } /// Sets list of columns in this namespace to be finally selected. /// @param l - list of columns to be selected. @@ -636,7 +676,7 @@ class Query { /// @param limit - number of rows to get from result set. /// @return Query object. Query &Limit(unsigned limit) &noexcept { - count = limit; + count_ = limit; return *this; } Query &&Limit(unsigned limit) &&noexcept { return std::move(Limit(limit)); } @@ -646,7 +686,7 @@ class Query { /// @param offset - index of the first row to get from result set. /// @return Query object. Query &Offset(unsigned offset) &noexcept { - start = offset; + start_ = offset; return *this; } Query &&Offset(unsigned offset) &&noexcept { return std::move(Offset(offset)); } @@ -654,7 +694,7 @@ class Query { /// Set the total count calculation mode to Accurate /// @return Query object Query &ReqTotal() &noexcept { - calcTotal = ModeAccurateTotal; + calcTotal_ = ModeAccurateTotal; return *this; } Query &&ReqTotal() &&noexcept { return std::move(ReqTotal()); } @@ -663,7 +703,7 @@ class Query { /// It will be use LRUCache for total count result /// @return Query object Query &CachedTotal() &noexcept { - calcTotal = ModeCachedTotal; + calcTotal_ = ModeCachedTotal; return *this; } Query &&CachedTotal() &&noexcept { return std::move(CachedTotal()); } @@ -696,38 +736,47 @@ class Query { void WalkNested(bool withSelf, bool withMerged, const std::function &visitor) const; - bool HasLimit() const noexcept { return count != QueryEntry::kDefaultLimit; } - bool HasOffset() const noexcept { return start != QueryEntry::kDefaultOffset; } + bool HasLimit() const noexcept { return count_ != QueryEntry::kDefaultLimit; } + bool HasOffset() const noexcept { return start_ != QueryEntry::kDefaultOffset; } bool IsWALQuery() const noexcept; const std::vector &UpdateFields() const noexcept { return updateFields_; } QueryType Type() const noexcept { return type_; } - -protected: - void deserialize(Serializer &ser, bool &hasJoinConditions); - -public: - std::string _namespace; /// Name of the namespace. - unsigned start = QueryEntry::kDefaultOffset; /// First row index from result set. - unsigned count = QueryEntry::kDefaultLimit; /// Number of rows from result set. - int debugLevel = 0; /// Debug level. - StrictMode strictMode = StrictModeNotSet; /// Strict mode. - bool explain_ = false; /// Explain query if true - CalcTotalMode calcTotal = ModeNoTotal; /// Calculation mode. - QueryType type_ = QuerySelect; /// Query type - OpType nextOp_ = OpAnd; /// Next operation constant. - SortingEntries sortingEntries_; /// Sorting data. - std::vector forcedSortOrder_; /// Keys that always go first - before any ordered values. - std::vector joinQueries_; /// List of queries for join. - std::vector mergeQueries_; /// List of merge queries. - h_vector selectFilter_; /// List of columns in a final result set. - std::vector selectFunctions_; /// List of sql functions + const std::string &NsName() const &noexcept { return namespace_; } + template + void SetNsName(T &&nsName) &noexcept { + namespace_ = std::forward(nsName); + } + unsigned Limit() const noexcept { return count_; } + unsigned Offset() const noexcept { return start_; } + CalcTotalMode CalcTotal() const noexcept { return calcTotal_; } + void CalcTotal(CalcTotalMode calcTotal) noexcept { calcTotal_ = calcTotal; } + + int debugLevel = 0; /// Debug level. + StrictMode strictMode = StrictModeNotSet; /// Strict mode. + bool explain_ = false; /// Explain query if true + QueryType type_ = QuerySelect; /// Query type + OpType nextOp_ = OpAnd; /// Next operation constant. + SortingEntries sortingEntries_; /// Sorting data. + std::vector forcedSortOrder_; /// Keys that always go first - before any ordered values. + std::vector joinQueries_; /// List of queries for join. + std::vector mergeQueries_; /// List of merge queries. + h_vector selectFilter_; /// List of columns in a final result set. + std::vector selectFunctions_; /// List of sql functions QueryEntries entries; std::vector aggregations_; + auto NsName() const && = delete; + private: - std::vector updateFields_; /// List of fields (and values) for update. + void deserialize(Serializer &ser, bool &hasJoinConditions); + + std::string namespace_; /// Name of the namespace. + unsigned start_ = QueryEntry::kDefaultOffset; /// First row index from result set. + unsigned count_ = QueryEntry::kDefaultLimit; /// Number of rows from result set. + CalcTotalMode calcTotal_ = ModeNoTotal; /// Calculation mode. + std::vector updateFields_; /// List of fields (and values) for update. bool withRank_ = false; friend class SQLParser; }; diff --git a/cpp_src/core/query/queryentry.cc b/cpp_src/core/query/queryentry.cc index a7cc7f4e3..d7148dc49 100644 --- a/cpp_src/core/query/queryentry.cc +++ b/cpp_src/core/query/queryentry.cc @@ -8,7 +8,6 @@ #include "query.h" #include "tools/serializer.h" #include "tools/string_regexp_functions.h" -#include "tools/stringstools.h" namespace reindexer { @@ -21,11 +20,11 @@ std::string JoinQueryEntry::Dump(const std::vector &joinedSelectors) const { ser << '('; for (const auto &jqe : q.joinEntries_) { if (&jqe != &q.joinEntries_.front()) { - ser << ' ' << jqe.op_ << ' '; + ser << ' ' << jqe.Operation() << ' '; } else { - assertrx(jqe.op_ == OpAnd); + assertrx(jqe.Operation() == OpAnd); } - ser << q._namespace << '.' << jqe.joinIndex_ << ' ' << InvertJoinCondition(jqe.condition_) << ' ' << jqe.index_; + ser << q.NsName() << '.' << jqe.RightFieldName() << ' ' << InvertJoinCondition(jqe.Condition()) << ' ' << jqe.LeftFieldName(); } ser << ')'; return std::string{ser.Slice()}; @@ -41,30 +40,133 @@ std::string JoinQueryEntry::DumpOnCondition(const std::vector &joinedSelecto ser << js.Type() << " ON ("; for (const auto &jqe : q.joinEntries_) { if (&jqe != &q.joinEntries_.front()) { - ser << ' ' << jqe.op_ << ' '; + ser << ' ' << jqe.Operation() << ' '; } - ser << q._namespace << '.' << jqe.joinIndex_ << ' ' << InvertJoinCondition(jqe.condition_) << ' ' << jqe.index_; + ser << q.NsName() << '.' << jqe.RightFieldName() << ' ' << InvertJoinCondition(jqe.Condition()) << ' ' << jqe.LeftFieldName(); } ser << ')'; return std::string{ser.Slice()}; } template std::string JoinQueryEntry::DumpOnCondition(const JoinedSelectors &) const; -bool QueryEntry::operator==(const QueryEntry &obj) const { - return condition == obj.condition && index == obj.index && idxNo == obj.idxNo && distinct == obj.distinct && - values.RelaxCompare(obj.values) == 0; +bool QueryField::operator==(const QueryField &other) const noexcept { + if (fieldName_ != other.fieldName_ || idxNo_ != other.idxNo_ || fieldsSet_ != other.fieldsSet_ || + !fieldType_.IsSame(other.fieldType_) || !selectType_.IsSame(other.selectType_) || + compositeFieldsTypes_.size() != other.compositeFieldsTypes_.size()) { + return false; + } + for (size_t i = 0, s = compositeFieldsTypes_.size(); i < s; ++i) { + if (!compositeFieldsTypes_[i].IsSame(other.compositeFieldsTypes_[i])) { + return false; + } + } + return true; +} + +void QueryField::SetField(FieldsSet &&fields) & { + assertrx_throw(fields.size() == 1); + assertrx_throw(fields[0] == IndexValueType::SetByJsonPath); + assertrx_throw(idxNo_ == IndexValueType::NotSet); + idxNo_ = IndexValueType::SetByJsonPath; + fieldsSet_ = std::move(fields); +} + +static void checkIndexData([[maybe_unused]] int idxNo, [[maybe_unused]] const FieldsSet &fields, KeyValueType fieldType, + [[maybe_unused]] const std::vector &compositeFieldsTypes) { + assertrx_throw(idxNo >= 0); + if (fieldType.Is()) { + assertrx_throw(fields.size() == compositeFieldsTypes.size()); + } else { + assertrx_throw(fields.size() == 1); + assertrx_throw(compositeFieldsTypes.empty()); + } +} + +void QueryField::SetIndexData(int idxNo, FieldsSet &&fields, KeyValueType fieldType, KeyValueType selectType, + std::vector &&compositeFieldsTypes) & { + checkIndexData(idxNo, fields, fieldType, compositeFieldsTypes); + idxNo_ = idxNo; + fieldsSet_ = std::move(fields); + fieldType_ = fieldType; + selectType_ = selectType; + compositeFieldsTypes_ = std::move(compositeFieldsTypes); +} + +bool QueryField::HaveEmptyField() const noexcept { + size_t tagsNo = 0; + for (auto f : Fields()) { + if (f == IndexValueType::SetByJsonPath) { + if (Fields().getTagsPath(tagsNo).empty()) { + return true; + } + ++tagsNo; + } + } + return Fields().empty(); +} + +bool QueryEntry::operator==(const QueryEntry &other) const noexcept { + return QueryField::operator==(other) && condition_ == other.condition_ && distinct_ == other.distinct_ && + values_.RelaxCompare(other.values_) == 0; } +template +void QueryEntry::verify(CondType cond, const VariantArray &values) { + if constexpr (flags & kIgnoreEmptyValues) { + if (values.empty()) { + return; + } + } + const auto checkArgsCount = [&](size_t argsCountReq) { + if (values.size() != argsCountReq) { + throw Error{errLogic, "Condition %s must have exact %d argument, but %d arguments was provided", CondTypeToStr(cond), + argsCountReq, values.size()}; + } + }; + switch (cond) { + case CondEq: + case CondSet: + case CondAllSet: + break; + case CondAny: + case CondEmpty: + if (!values.empty() && !(values.size() == 1 && values[0].Type().Is())) { + throw Error{errLogic, "Condition %s must have no argument or single null argument, but %d not null arguments was provided", + CondTypeToStr(cond), values.size()}; + } + break; + case CondGe: + case CondGt: + case CondLt: + case CondLe: + checkArgsCount(1); + break; + case CondLike: + checkArgsCount(1); + if (!values[0].Type().Is()) { + throw Error{errLogic, "Condition %s must have string argument, but %d argument was provided", CondTypeToStr(cond), + values[0].Type().Name()}; + } + break; + case CondRange: + case CondDWithin: + checkArgsCount(2); + break; + } +} +template void QueryEntry::verify<0u>(CondType, const VariantArray &); +template void QueryEntry::verify(CondType, const VariantArray &); + std::string QueryEntry::Dump() const { WrSerializer ser; - if (distinct) { - ser << "Distinct index: " << index; + if (Distinct()) { + ser << "Distinct index: " << FieldName(); } else { - ser << index << ' ' << condition << ' '; - const bool severalValues = (values.size() > 1); + ser << FieldName() << ' ' << condition_ << ' '; + const bool severalValues = (Values().size() > 1); if (severalValues) ser << '('; - for (auto &v : values) { - if (&v != &*values.begin()) ser << ','; + for (auto &v : Values()) { + if (&v != &*Values().begin()) ser << ','; ser << '\'' << v.As() << '\''; } if (severalValues) ser << ')'; @@ -75,18 +177,18 @@ std::string QueryEntry::Dump() const { std::string QueryEntry::DumpBrief() const { WrSerializer ser; { - ser << index << ' ' << condition << ' '; - const bool severalValues = (values.size() > 1); + ser << FieldName() << ' ' << Condition() << ' '; + const bool severalValues = (Values().size() > 1); if (severalValues) { ser << "(...)"; } else { - ser << '\'' << values.front().As() << '\''; + ser << '\'' << Values().front().As() << '\''; } } return std::string(ser.Slice()); } -AggregateEntry::AggregateEntry(AggType type, h_vector fields, SortingEntries sort, unsigned limit, unsigned offset) +AggregateEntry::AggregateEntry(AggType type, h_vector &&fields, SortingEntries &&sort, unsigned limit, unsigned offset) : type_(type), fields_(std::move(fields)), sortingEntries_{std::move(sort)}, limit_(limit), offset_(offset) { switch (type_) { case AggFacet: @@ -134,7 +236,7 @@ AggregateEntry::AggregateEntry(AggType type, h_vector fields, So } } -void AggregateEntry::AddSortingEntry(SortingEntry sorting) { +void AggregateEntry::AddSortingEntry(SortingEntry &&sorting) { if (type_ != AggFacet) { throw Error(errQueryExec, "Sort is not available for aggregation %s", AggTypeToStr(type_)); } @@ -155,21 +257,13 @@ void AggregateEntry::SetOffset(unsigned o) { offset_ = o; } -BetweenFieldsQueryEntry::BetweenFieldsQueryEntry(std::string fstIdx, CondType cond, std::string sndIdx) - : firstIndex{std::move(fstIdx)}, secondIndex{std::move(sndIdx)}, condition_{cond} { - if (condition_ == CondAny || condition_ == CondEmpty || condition_ == CondDWithin) { - throw Error{errLogic, "Condition '%s' is inapplicable between two fields", std::string{CondTypeToStr(condition_)}}; - } -} - bool BetweenFieldsQueryEntry::operator==(const BetweenFieldsQueryEntry &other) const noexcept { - return firstIdxNo == other.firstIdxNo && secondIdxNo == other.secondIdxNo && Condition() == other.Condition() && - firstIndex == other.firstIndex && secondIndex == other.secondIndex; + return leftField_ == other.leftField_ && rightField_ == other.rightField_ && Condition() == other.Condition(); } std::string BetweenFieldsQueryEntry::Dump() const { WrSerializer ser; - ser << firstIndex << ' ' << Condition() << ' ' << secondIndex; + ser << LeftFieldName() << ' ' << Condition() << ' ' << RightFieldName(); return std::string{ser.Slice()}; } @@ -184,30 +278,31 @@ void QueryEntries::serialize(const_iterator it, const_iterator to, WrSerializer ser.PutVarUint(QueryCloseBracket); }, [&ser, op](const QueryEntry &entry) { - entry.distinct ? ser.PutVarUint(QueryDistinct) : ser.PutVarUint(QueryCondition); - ser.PutVString(entry.index); - if (entry.distinct) return; + entry.Distinct() ? ser.PutVarUint(QueryDistinct) : ser.PutVarUint(QueryCondition); + ser.PutVString(entry.FieldName()); + if (entry.Distinct()) return; ser.PutVarUint(op); - ser.PutVarUint(entry.condition); - if (entry.condition == CondDWithin) { - if (entry.values.size() != 2) { - throw Error(errLogic, "Condition DWithin must have exact 2 value, but %d values was provided", entry.values.size()); + ser.PutVarUint(entry.Condition()); + if (entry.Condition() == CondDWithin) { + if (entry.Values().size() != 2) { + throw Error(errLogic, "Condition DWithin must have exact 2 value, but %d values was provided", + entry.Values().size()); } ser.PutVarUint(3); - if (entry.values[0].Type().Is()) { - const Point point = static_cast(entry.values[0]); + if (entry.Values()[0].Type().Is()) { + const Point point = static_cast(entry.Values()[0]); ser.PutDouble(point.X()); ser.PutDouble(point.Y()); - ser.PutVariant(entry.values[1]); + ser.PutVariant(entry.Values()[1]); } else { - const Point point = static_cast(entry.values[1]); + const Point point = static_cast(entry.Values()[1]); ser.PutDouble(point.X()); ser.PutDouble(point.Y()); - ser.PutVariant(entry.values[0]); + ser.PutVariant(entry.Values()[0]); } } else { - ser.PutVarUint(entry.values.size()); - for (auto &kv : entry.values) ser.PutVariant(kv); + ser.PutVarUint(entry.Values().size()); + for (auto &kv : entry.Values()) ser.PutVariant(kv); } }, [&ser, op](const JoinQueryEntry &jqe) { @@ -218,9 +313,9 @@ void QueryEntries::serialize(const_iterator it, const_iterator to, WrSerializer [&ser, op](const BetweenFieldsQueryEntry &entry) { ser.PutVarUint(QueryBetweenFieldsCondition); ser.PutVarUint(op); - ser.PutVString(entry.firstIndex); + ser.PutVString(entry.LeftFieldName()); ser.PutVarUint(entry.Condition()); - ser.PutVString(entry.secondIndex); + ser.PutVString(entry.RightFieldName()); }, [&ser, op](const AlwaysFalse &) { ser.PutVarUint(QueryAlwaysFalseCondition); @@ -233,13 +328,8 @@ bool UpdateEntry::operator==(const UpdateEntry &obj) const noexcept { return isExpression_ == obj.isExpression_ && column_ == obj.column_ && mode_ == obj.mode_ && values_ == obj.values_; } -bool QueryJoinEntry::operator==(const QueryJoinEntry &obj) const noexcept { - if (op_ != obj.op_) return false; - if (static_cast(condition_) != obj.condition_) return false; - if (index_ != obj.index_) return false; - if (joinIndex_ != obj.joinIndex_) return false; - if (idxNo != obj.idxNo) return false; - return true; +bool QueryJoinEntry::operator==(const QueryJoinEntry &other) const noexcept { + return op_ == other.op_ && condition_ == other.condition_ && leftField_ == other.leftField_ && rightField_ == other.rightField_; } bool AggregateEntry::operator==(const AggregateEntry &obj) const noexcept { @@ -254,7 +344,7 @@ bool SortingEntry::operator==(const SortingEntry &obj) const noexcept { return true; } -bool QueryEntries::checkIfSatisfyConditions(const_iterator begin, const_iterator end, const ConstPayload &pl, TagsMatcher &tagsMatcher) { +bool QueryEntries::checkIfSatisfyConditions(const_iterator begin, const_iterator end, const ConstPayload &pl) { assertrx(begin != end && begin->operation != OpOr); bool result = true; for (auto it = begin; it != end; ++it) { @@ -264,40 +354,26 @@ bool QueryEntries::checkIfSatisfyConditions(const_iterator begin, const_iterator break; } const bool lastResult = it->InvokeAppropriate( - [&it, &pl, &tagsMatcher](const QueryEntriesBracket &) { - return checkIfSatisfyConditions(it.cbegin(), it.cend(), pl, tagsMatcher); - }, - [&pl, &tagsMatcher](const QueryEntry &qe) { return checkIfSatisfyCondition(qe, pl, tagsMatcher); }, - [&pl, &tagsMatcher](const BetweenFieldsQueryEntry &qe) { return checkIfSatisfyCondition(qe, pl, tagsMatcher); }, + [&it, &pl](const QueryEntriesBracket &) { return checkIfSatisfyConditions(it.cbegin(), it.cend(), pl); }, + [&pl](const QueryEntry &qe) { return checkIfSatisfyCondition(qe, pl); }, + [&pl](const BetweenFieldsQueryEntry &qe) { return checkIfSatisfyCondition(qe, pl); }, [](const JoinQueryEntry &) -> bool { abort(); }, [](const AlwaysFalse &) { return false; }); result = (lastResult != (it->operation == OpNot)); } return result; } -bool QueryEntries::checkIfSatisfyCondition(const QueryEntry &qEntry, const ConstPayload &pl, TagsMatcher &tagsMatcher) { +bool QueryEntries::checkIfSatisfyCondition(const QueryEntry &qEntry, const ConstPayload &pl) { VariantArray values; - if (qEntry.idxNo == IndexValueType::SetByJsonPath) { - pl.GetByJsonPath(qEntry.index, tagsMatcher, values, KeyValueType::Undefined{}); - } else { - pl.Get(qEntry.idxNo, values); - } - return checkIfSatisfyCondition(values, qEntry.condition, qEntry.values); + pl.GetByFieldsSet(qEntry.Fields(), values, qEntry.FieldType(), qEntry.CompositeFieldsTypes()); + return checkIfSatisfyCondition(values, qEntry.Condition(), qEntry.Values()); } -bool QueryEntries::checkIfSatisfyCondition(const BetweenFieldsQueryEntry &qEntry, const ConstPayload &pl, TagsMatcher &tagsMatcher) { +bool QueryEntries::checkIfSatisfyCondition(const BetweenFieldsQueryEntry &qEntry, const ConstPayload &pl) { VariantArray lValues; - if (qEntry.firstIdxNo == IndexValueType::SetByJsonPath) { - pl.GetByJsonPath(qEntry.firstIndex, tagsMatcher, lValues, KeyValueType::Undefined{}); - } else { - pl.Get(qEntry.firstIdxNo, lValues); - } + pl.GetByFieldsSet(qEntry.LeftFields(), lValues, qEntry.LeftFieldType(), qEntry.LeftCompositeFieldsTypes()); VariantArray rValues; - if (qEntry.secondIdxNo == IndexValueType::SetByJsonPath) { - pl.GetByJsonPath(qEntry.secondIndex, tagsMatcher, rValues, KeyValueType::Undefined{}); - } else { - pl.Get(qEntry.secondIdxNo, rValues); - } + pl.GetByFieldsSet(qEntry.RightFields(), rValues, qEntry.RightFieldType(), qEntry.RightCompositeFieldsTypes()); return checkIfSatisfyCondition(lValues, qEntry.Condition(), rValues); } @@ -350,18 +426,11 @@ bool QueryEntries::checkIfSatisfyCondition(const VariantArray &lValues, CondType return false; } case CondType::CondRange: - if (rValues.size() != 2) throw Error(errParams, "For ranged query reuqired 2 arguments, but provided %d", rValues.size()); for (const auto &v : lValues) { if (v.RelaxCompare(rValues[0]) < 0 || v.RelaxCompare(rValues[1]) > 0) return false; } return true; case CondType::CondLike: - if (rValues.size() != 1) { - throw Error(errLogic, "Condition LIKE must have exact 1 value, but %d values was provided", rValues.size()); - } - if (!rValues[0].Type().Is()) { - throw Error(errLogic, "Condition LIKE must have value of string type, but %s value was provided", rValues[0].Type().Name()); - } for (const auto &v : lValues) { if (!v.Type().Is()) { throw Error(errLogic, "Condition LIKE must be applied to data of string type, but %s was provided", v.Type().Name()); @@ -370,9 +439,6 @@ bool QueryEntries::checkIfSatisfyCondition(const VariantArray &lValues, CondType } return false; case CondType::CondDWithin: { - if (rValues.size() != 2) { - throw Error(errLogic, "Condition DWithin must have exact 2 value, but %d values was provided", rValues.size()); - } Point point; double distance; if (rValues[0].Type().Is()) { @@ -390,4 +456,56 @@ bool QueryEntries::checkIfSatisfyCondition(const VariantArray &lValues, CondType return false; } +template +std::string QueryJoinEntry::DumpCondition(const JS &joinedSelector, bool needOp) const { + WrSerializer ser; + const auto &q = joinedSelector.JoinQuery(); + if (needOp) { + ser << ' ' << op_ << ' '; + } + ser << q.NsName() << '.' << RightFieldName() << ' ' << InvertJoinCondition(condition_) << ' ' << LeftFieldName(); + return std::string{ser.Slice()}; +} +template std::string QueryJoinEntry::DumpCondition(const JoinedSelector &, bool) const; + +void QueryEntries::dumpEqualPositions(size_t level, WrSerializer &ser, const EqualPositions_t &equalPositions) { + for (const auto &eq : equalPositions) { + for (size_t i = 0; i < level; ++i) { + ser << " "; + } + ser << "equal_poisition("; + for (size_t i = 0, s = eq.size(); i < s; ++i) { + if (i != 0) ser << ", "; + ser << eq[i]; + } + ser << ")\n"; + } +} + +template +void QueryEntries::dump(size_t level, const_iterator begin, const_iterator end, const std::vector &joinedSelectors, WrSerializer &ser) { + for (const_iterator it = begin; it != end; ++it) { + for (size_t i = 0; i < level; ++i) { + ser << " "; + } + if (it != begin || it->operation != OpAnd) { + ser << it->operation << ' '; + } + it->InvokeAppropriate( + [&](const QueryEntriesBracket &b) { + ser << "(\n"; + dump(level + 1, it.cbegin(), it.cend(), joinedSelectors, ser); + dumpEqualPositions(level + 1, ser, b.equalPositions); + for (size_t i = 0; i < level; ++i) { + ser << " "; + } + ser << ")\n"; + }, + [&ser](const QueryEntry &qe) { ser << qe.Dump() << '\n'; }, + [&joinedSelectors, &ser](const JoinQueryEntry &jqe) { ser << jqe.Dump(joinedSelectors) << '\n'; }, + [&ser](const BetweenFieldsQueryEntry &qe) { ser << qe.Dump() << '\n'; }, + [&ser](const AlwaysFalse &) { ser << "AlwaysFalse" << 'n'; }); + } +} + } // namespace reindexer diff --git a/cpp_src/core/query/queryentry.h b/cpp_src/core/query/queryentry.h index d2b53ebf8..6d00faf35 100644 --- a/cpp_src/core/query/queryentry.h +++ b/cpp_src/core/query/queryentry.h @@ -5,10 +5,11 @@ #include #include "core/expressiontree.h" #include "core/keyvalue/variant.h" +#include "core/payload/fieldsset.h" #include "core/type_consts.h" -#include "core/type_consts_helpers.h" #include "estl/h_vector.h" #include "tools/serializer.h" +#include "tools/verifying_updater.h" namespace reindexer { @@ -31,44 +32,181 @@ struct JoinQueryEntry { std::string DumpOnCondition(const std::vector &joinedSelectors) const; }; -struct QueryEntry { +class QueryField { +public: + template + explicit QueryField(Str &&fieldName) noexcept : fieldName_{std::forward(fieldName)} {} + QueryField(std::string &&fieldName, int idxNo, FieldsSet fields, KeyValueType fieldType, + std::vector &&compositeFieldsTypes); + QueryField(QueryField &&) noexcept = default; + QueryField(const QueryField &) = default; + QueryField &operator=(QueryField &&) noexcept = default; + + [[nodiscard]] bool operator==(const QueryField &) const noexcept; + [[nodiscard]] bool operator!=(const QueryField &other) const noexcept { return !operator==(other); } + + [[nodiscard]] int IndexNo() const noexcept { return idxNo_; } + [[nodiscard]] bool IsFieldIndexed() const noexcept { return idxNo_ >= 0; } + [[nodiscard]] bool FieldsHaveBeenSet() const noexcept { return idxNo_ != IndexValueType::NotSet; } + [[nodiscard]] const FieldsSet &Fields() const &noexcept { return fieldsSet_; } + [[nodiscard]] const std::string &FieldName() const &noexcept { return fieldName_; } + [[nodiscard]] KeyValueType FieldType() const noexcept { return fieldType_; } + [[nodiscard]] KeyValueType SelectType() const noexcept { return selectType_; } + [[nodiscard]] const std::vector &CompositeFieldsTypes() const &noexcept { return compositeFieldsTypes_; } + [[nodiscard]] bool HaveEmptyField() const noexcept; + void SetField(FieldsSet &&fields) &; + void SetIndexData(int idxNo, FieldsSet &&fields, KeyValueType fieldType, KeyValueType selectType, + std::vector &&compositeFieldsTypes) &; + + QueryField &operator=(const QueryField &) = delete; + auto Fields() const && = delete; + auto FieldName() const && = delete; + auto CompositeFieldsTypes() const && = delete; + +private: + std::string fieldName_; + int idxNo_{IndexValueType::NotSet}; + FieldsSet fieldsSet_; + KeyValueType fieldType_{KeyValueType::Undefined{}}; + KeyValueType selectType_{KeyValueType::Undefined{}}; + std::vector compositeFieldsTypes_; +}; + +class QueryEntry : private QueryField { + static constexpr unsigned kIgnoreEmptyValues = 1u; + +public: + struct DistinctTag {}; + struct IgnoreEmptyValues {}; static constexpr unsigned kDefaultLimit = UINT_MAX; static constexpr unsigned kDefaultOffset = 0; - QueryEntry(std::string idx, CondType cond, VariantArray v) : index{std::move(idx)}, condition{cond}, values(std::move(v)) {} - QueryEntry(CondType cond, std::string idx, int idxN, bool dist = false) - : index(std::move(idx)), idxNo(idxN), condition(cond), distinct(dist) {} - QueryEntry() = default; + template + QueryEntry(Str &&fieldName, CondType cond, VariantArray &&v) + : QueryField{std::forward(fieldName)}, values_{std::move(v)}, condition_{cond} { + Verify(); + } + template + QueryEntry(Str &&fieldName, DistinctTag) : QueryField{std::forward(fieldName)}, condition_{CondAny}, distinct_{true} { + Verify(); + } + QueryEntry(QueryField &&field, CondType cond, VariantArray &&v) + : QueryField{std::move(field)}, values_{std::move(v)}, condition_{cond} { + Verify(); + } + QueryEntry(QueryField &&field, CondType cond, IgnoreEmptyValues) : QueryField{std::move(field)}, condition_{cond} { + verifyIgnoringEmptyValues(); + } + [[nodiscard]] CondType Condition() const noexcept { return condition_; } + [[nodiscard]] const VariantArray &Values() const &noexcept { return values_; } + [[nodiscard]] VariantArray &&Values() &&noexcept { return std::move(values_); } + [[nodiscard]] auto UpdatableValues(IgnoreEmptyValues) &noexcept { + return VerifyingUpdater{*this}; + } + [[nodiscard]] bool Distinct() const noexcept { return distinct_; } + void Distinct(bool d) noexcept { distinct_ = d; } + using QueryField::IndexNo; + using QueryField::IsFieldIndexed; + using QueryField::FieldsHaveBeenSet; + using QueryField::Fields; + using QueryField::FieldName; + using QueryField::FieldType; + using QueryField::SelectType; + using QueryField::CompositeFieldsTypes; + using QueryField::SetField; + using QueryField::SetIndexData; + using QueryField::HaveEmptyField; + void SetCondAndValues(CondType cond, VariantArray &&values) { + verify(cond, values); + condition_ = cond; + values_ = std::move(values); + } - bool operator==(const QueryEntry &) const; - bool operator!=(const QueryEntry &other) const { return !operator==(other); } + const QueryField &FieldData() const &noexcept { return static_cast(*this); } + QueryField &FieldData() &noexcept { return static_cast(*this); } + void ConvertValuesToFieldType() & { + for (Variant &v : values_) { + v.convert(SelectType()); + } + } + void ConvertValuesToFieldType(const PayloadType &pt) & { + if (SelectType().Is() || Condition() == CondDWithin) { + return; + } + for (Variant &v : values_) { + v.convert(SelectType(), &pt, &Fields()); + } + } + void Verify() const { verify(condition_, values_); } - std::string index; - int idxNo = IndexValueType::NotSet; - CondType condition = CondType::CondAny; - bool distinct = false; - VariantArray values; + [[nodiscard]] bool operator==(const QueryEntry &) const noexcept; + [[nodiscard]] bool operator!=(const QueryEntry &other) const noexcept { return !operator==(other); } - std::string Dump() const; - std::string DumpBrief() const; + [[nodiscard]] std::string Dump() const; + [[nodiscard]] std::string DumpBrief() const; + + auto Values() const && = delete; + auto FieldData() const && = delete; + +private: + template + static void verify(CondType, const VariantArray &); + void verifyIgnoringEmptyValues() const { verify(condition_, values_); } + + VariantArray values_; + CondType condition_; + bool distinct_{false}; }; +extern template void QueryEntry::verify<0u>(CondType, const VariantArray &); +extern template void QueryEntry::verify(CondType, const VariantArray &); class BetweenFieldsQueryEntry { public: - BetweenFieldsQueryEntry(std::string fstIdx, CondType cond, std::string sndIdx); - - bool operator==(const BetweenFieldsQueryEntry &) const noexcept; - bool operator!=(const BetweenFieldsQueryEntry &other) const noexcept { return !operator==(other); } - - std::string firstIndex; - std::string secondIndex; - int firstIdxNo = IndexValueType::NotSet; - int secondIdxNo = IndexValueType::NotSet; + template + BetweenFieldsQueryEntry(StrL &&fstIdx, CondType cond, StrR &&sndIdx) + : leftField_{std::forward(fstIdx)}, rightField_{std::forward(sndIdx)}, condition_{cond} { + if (condition_ == CondAny || condition_ == CondEmpty || condition_ == CondDWithin) { + throw Error{errLogic, "Condition '%s' is inapplicable between two fields", std::string{CondTypeToStr(condition_)}}; + } + } - CondType Condition() const noexcept { return condition_; } - std::string Dump() const; + [[nodiscard]] bool operator==(const BetweenFieldsQueryEntry &) const noexcept; + [[nodiscard]] bool operator!=(const BetweenFieldsQueryEntry &other) const noexcept { return !operator==(other); } + + [[nodiscard]] CondType Condition() const noexcept { return condition_; } + [[nodiscard]] int LeftIdxNo() const noexcept { return leftField_.IndexNo(); } + [[nodiscard]] int RightIdxNo() const noexcept { return rightField_.IndexNo(); } + [[nodiscard]] const std::string &LeftFieldName() const &noexcept { return leftField_.FieldName(); } + [[nodiscard]] const std::string &RightFieldName() const &noexcept { return rightField_.FieldName(); } + [[nodiscard]] const FieldsSet &LeftFields() const &noexcept { return leftField_.Fields(); } + [[nodiscard]] const FieldsSet &RightFields() const &noexcept { return rightField_.Fields(); } + [[nodiscard]] KeyValueType LeftFieldType() const noexcept { return leftField_.FieldType(); } + [[nodiscard]] KeyValueType RightFieldType() const noexcept { return rightField_.FieldType(); } + [[nodiscard]] const std::vector &LeftCompositeFieldsTypes() const &noexcept { return leftField_.CompositeFieldsTypes(); } + [[nodiscard]] const std::vector &RightCompositeFieldsTypes() const &noexcept { + return rightField_.CompositeFieldsTypes(); + } + [[nodiscard]] const QueryField &LeftFieldData() const &noexcept { return leftField_; } + [[nodiscard]] QueryField &LeftFieldData() &noexcept { return leftField_; } + [[nodiscard]] const QueryField &RightFieldData() const &noexcept { return rightField_; } + [[nodiscard]] QueryField &RightFieldData() &noexcept { return rightField_; } + [[nodiscard]] bool FieldsHaveBeenSet() const noexcept { return leftField_.FieldsHaveBeenSet() && rightField_.FieldsHaveBeenSet(); } + [[nodiscard]] bool IsLeftFieldIndexed() const noexcept { return leftField_.IsFieldIndexed(); } + [[nodiscard]] bool IsRightFieldIndexed() const noexcept { return rightField_.IsFieldIndexed(); } + [[nodiscard]] std::string Dump() const; + + auto LeftFieldName() const && = delete; + auto RightFieldName() const && = delete; + auto LeftFields() const && = delete; + auto RightFields() const && = delete; + auto LeftCompositeFieldsTypes() const && = delete; + auto RightCompositeFieldsTypes() const && = delete; + auto LeftFieldData() const && = delete; + auto RightFieldData() const && = delete; private: + QueryField leftField_; + QueryField rightField_; CondType condition_; }; @@ -103,9 +241,7 @@ class QueryEntries void ToDsl(const Query &parentQuery, JsonBuilder &builder) const { return toDsl(cbegin(), cend(), parentQuery, builder); } void WriteSQLWhere(const Query &parentQuery, WrSerializer &, bool stripArgs) const; void Serialize(WrSerializer &ser) const { serialize(cbegin(), cend(), ser); } - bool CheckIfSatisfyConditions(const ConstPayload &pl, TagsMatcher &tm) const { - return checkIfSatisfyConditions(cbegin(), cend(), pl, tm); - } + bool CheckIfSatisfyConditions(const ConstPayload &pl) const { return checkIfSatisfyConditions(cbegin(), cend(), pl); } template std::string Dump(const std::vector &joinedSelectors) const { WrSerializer ser; @@ -120,57 +256,22 @@ class QueryEntries static void toDsl(const_iterator it, const_iterator to, const Query &parentQuery, JsonBuilder &); static void writeSQL(const Query &parentQuery, const_iterator from, const_iterator to, WrSerializer &, bool stripArgs); static void serialize(const_iterator it, const_iterator to, WrSerializer &); - static bool checkIfSatisfyConditions(const_iterator begin, const_iterator end, const ConstPayload &, TagsMatcher &); - static bool checkIfSatisfyCondition(const QueryEntry &, const ConstPayload &, TagsMatcher &); - static bool checkIfSatisfyCondition(const BetweenFieldsQueryEntry &, const ConstPayload &, TagsMatcher &); + static bool checkIfSatisfyConditions(const_iterator begin, const_iterator end, const ConstPayload &); + static bool checkIfSatisfyCondition(const QueryEntry &, const ConstPayload &); + static bool checkIfSatisfyCondition(const BetweenFieldsQueryEntry &, const ConstPayload &); static bool checkIfSatisfyCondition(const VariantArray &lValues, CondType, const VariantArray &rValues); protected: - static void dumpEqualPositions(size_t level, WrSerializer &ser, const EqualPositions_t &equalPositions) { - for (const auto &eq : equalPositions) { - for (size_t i = 0; i < level; ++i) { - ser << " "; - } - ser << "equal_poisition("; - for (size_t i = 0, s = eq.size(); i < s; ++i) { - if (i != 0) ser << ", "; - ser << eq[i]; - } - ser << ")\n"; - } - } - + static void dumpEqualPositions(size_t level, WrSerializer &, const EqualPositions_t &); template - static void dump(size_t level, const_iterator begin, const_iterator end, const std::vector &joinedSelectors, WrSerializer &ser) { - for (const_iterator it = begin; it != end; ++it) { - for (size_t i = 0; i < level; ++i) { - ser << " "; - } - if (it != begin || it->operation != OpAnd) { - ser << it->operation << ' '; - } - it->InvokeAppropriate( - [&](const QueryEntriesBracket &b) { - ser << "(\n"; - dump(level + 1, it.cbegin(), it.cend(), joinedSelectors, ser); - dumpEqualPositions(level + 1, ser, b.equalPositions); - for (size_t i = 0; i < level; ++i) { - ser << " "; - } - ser << ")\n"; - }, - [&ser](const QueryEntry &qe) { ser << qe.Dump() << '\n'; }, - [&joinedSelectors, &ser](const JoinQueryEntry &jqe) { ser << jqe.Dump(joinedSelectors) << '\n'; }, - [&ser](const BetweenFieldsQueryEntry &qe) { ser << qe.Dump() << '\n'; }, - [&ser](const AlwaysFalse &) { ser << "AlwaysFalse" << 'n'; }); - } - } + static void dump(size_t level, const_iterator begin, const_iterator end, const std::vector &joinedSelectors, WrSerializer &); }; class UpdateEntry { public: - UpdateEntry(std::string c, VariantArray v, FieldModifyMode m = FieldModeSet, bool e = false) - : column_(std::move(c)), values_(std::move(v)), mode_(m), isExpression_(e) { + template + UpdateEntry(Str &&c, VariantArray &&v, FieldModifyMode m = FieldModeSet, bool e = false) + : column_(std::forward(c)), values_(std::move(v)), mode_(m), isExpression_(e) { if (column_.empty()) { throw Error{errParams, "Empty update column name"}; } @@ -192,36 +293,71 @@ class UpdateEntry { bool isExpression_ = false; }; -struct QueryJoinEntry { - QueryJoinEntry() = default; - QueryJoinEntry(OpType op, CondType cond, std::string idx, std::string jIdx) - : op_{op}, condition_{cond}, index_{std::move(idx)}, joinIndex_{std::move(jIdx)} {} - bool operator==(const QueryJoinEntry &) const noexcept; - bool operator!=(const QueryJoinEntry &qje) const noexcept { return !operator==(qje); } - OpType op_ = OpAnd; - CondType condition_ = CondEq; ///< Condition applied to expression: index_ COND joinIndex_ - std::string index_; ///< main ns index field name - std::string joinIndex_; ///< joining ns index field name - int idxNo = -1; ///< index_ field Index number in main ns - bool reverseNamespacesOrder = false; ///< controls SQL encoding order - ///< false: mainNs.index Condition joinNs.joinIndex - ///< true: joinNs.joinIndex Invert(Condition) mainNs.index +class QueryJoinEntry { +public: + QueryJoinEntry(OpType op, CondType cond, std::string &&leftFld, std::string &&rightFld, bool reverseNs = false) noexcept + : leftField_{std::move(leftFld)}, rightField_{std::move(rightFld)}, op_{op}, condition_{cond}, reverseNamespacesOrder_{reverseNs} {} + [[nodiscard]] bool operator==(const QueryJoinEntry &) const noexcept; + [[nodiscard]] bool operator!=(const QueryJoinEntry &other) const noexcept { return !operator==(other); } + [[nodiscard]] bool IsLeftFieldIndexed() const noexcept { return leftField_.IsFieldIndexed(); } + [[nodiscard]] bool IsRightFieldIndexed() const noexcept { return rightField_.IsFieldIndexed(); } + [[nodiscard]] int LeftIdxNo() const noexcept { return leftField_.IndexNo(); } + [[nodiscard]] int RightIdxNo() const noexcept { return rightField_.IndexNo(); } + [[nodiscard]] const FieldsSet &LeftFields() const &noexcept { return leftField_.Fields(); } + [[nodiscard]] const FieldsSet &RightFields() const &noexcept { return rightField_.Fields(); } + [[nodiscard]] KeyValueType LeftFieldType() const noexcept { return leftField_.FieldType(); } + [[nodiscard]] KeyValueType RightFieldType() const noexcept { return rightField_.FieldType(); } + [[nodiscard]] const std::vector &LeftCompositeFieldsTypes() const &noexcept { return leftField_.CompositeFieldsTypes(); } + [[nodiscard]] const std::vector &RightCompositeFieldsTypes() const &noexcept { + return rightField_.CompositeFieldsTypes(); + } + [[nodiscard]] OpType Operation() const noexcept { return op_; } + [[nodiscard]] CondType Condition() const noexcept { return condition_; } + [[nodiscard]] const std::string &LeftFieldName() const &noexcept { return leftField_.FieldName(); } + [[nodiscard]] const std::string &RightFieldName() const &noexcept { return rightField_.FieldName(); } + [[nodiscard]] bool ReverseNamespacesOrder() const noexcept { return reverseNamespacesOrder_; } + [[nodiscard]] const QueryField &LeftFieldData() const &noexcept { return leftField_; } + [[nodiscard]] QueryField &LeftFieldData() &noexcept { return leftField_; } + [[nodiscard]] const QueryField &RightFieldData() const &noexcept { return rightField_; } + [[nodiscard]] QueryField &RightFieldData() &noexcept { return rightField_; } + void SetLeftIndexData(int idxNo, FieldsSet &&fields, KeyValueType fieldType, KeyValueType selectType, + std::vector &&compositeFieldsTypes) & { + leftField_.SetIndexData(idxNo, std::move(fields), fieldType, selectType, std::move(compositeFieldsTypes)); + } + void SetRightIndexData(int idxNo, FieldsSet &&fields, KeyValueType fieldType, KeyValueType selectType, + std::vector &&compositeFieldsTypes) & { + rightField_.SetIndexData(idxNo, std::move(fields), fieldType, selectType, std::move(compositeFieldsTypes)); + } + void SetLeftField(FieldsSet &&fields) & { leftField_.SetField(std::move(fields)); } + void SetRightField(FieldsSet &&fields) & { rightField_.SetField(std::move(fields)); } + [[nodiscard]] bool FieldsHaveBeenSet() const noexcept { return leftField_.FieldsHaveBeenSet() && rightField_.FieldsHaveBeenSet(); } template - std::string DumpCondition(const JS &joinedSelector, bool needOp = false) const { - WrSerializer ser; - const auto &q = joinedSelector.JoinQuery(); - if (needOp) { - ser << ' ' << op_ << ' '; - } - ser << q._namespace << '.' << joinIndex_ << ' ' << InvertJoinCondition(condition_) << ' ' << index_; - return std::string{ser.Slice()}; - } + std::string DumpCondition(const JS &joinedSelector, bool needOp = false) const; + + auto LeftFields() const && = delete; + auto RightFields() const && = delete; + auto LeftCompositeFieldsTypes() const && = delete; + auto RightCompositeFieldsTypes() const && = delete; + auto LeftFieldName() const && = delete; + auto RightFieldName() const && = delete; + auto LeftFieldData() const && = delete; + auto RightFieldData() const && = delete; + +private: + QueryField leftField_; + QueryField rightField_; + const OpType op_; + const CondType condition_; + const bool reverseNamespacesOrder_; ///< controls SQL encoding order + ///< false: mainNs.index Condition joinNs.joinIndex + ///< true: joinNs.joinIndex Invert(Condition) mainNs.index }; struct SortingEntry { SortingEntry() noexcept = default; - SortingEntry(std::string e, bool d) noexcept : expression(std::move(e)), desc(d) {} + template + SortingEntry(Str &&e, bool d) noexcept : expression(std::forward(e)), desc(d) {} bool operator==(const SortingEntry &) const noexcept; bool operator!=(const SortingEntry &se) const noexcept { return !operator==(se); } std::string expression; @@ -233,7 +369,7 @@ struct SortingEntries : public h_vector {}; class AggregateEntry { public: - AggregateEntry(AggType type, h_vector fields, SortingEntries sort = {}, unsigned limit = QueryEntry::kDefaultLimit, + AggregateEntry(AggType type, h_vector &&fields, SortingEntries &&sort = {}, unsigned limit = QueryEntry::kDefaultLimit, unsigned offset = QueryEntry::kDefaultOffset); [[nodiscard]] bool operator==(const AggregateEntry &) const noexcept; [[nodiscard]] bool operator!=(const AggregateEntry &ae) const noexcept { return !operator==(ae); } @@ -242,7 +378,7 @@ class AggregateEntry { [[nodiscard]] const SortingEntries &Sorting() const noexcept { return sortingEntries_; } [[nodiscard]] unsigned Limit() const noexcept { return limit_; } [[nodiscard]] unsigned Offset() const noexcept { return offset_; } - void AddSortingEntry(SortingEntry); + void AddSortingEntry(SortingEntry &&); void SetLimit(unsigned); void SetOffset(unsigned); diff --git a/cpp_src/core/query/sql/sqlencoder.cc b/cpp_src/core/query/sql/sqlencoder.cc index aa82c1110..3234225cd 100644 --- a/cpp_src/core/query/sql/sqlencoder.cc +++ b/cpp_src/core/query/sql/sqlencoder.cc @@ -1,7 +1,7 @@ #include "core/query/sql/sqlencoder.h" #include "core/keyvalue/geometry.h" -#include "core/nsselecter/sortexpression.h" +#include "core/keyvalue/p_string.h" #include "core/queryresults/aggregationresult.h" #include "core/type_consts_helpers.h" #include "tools/serializer.h" @@ -16,7 +16,7 @@ static void indexToSql(const std::string &index, WrSerializer &ser) { } } -static WrSerializer &stringToSql(const std::string &str, WrSerializer &ser) { +static WrSerializer &stringToSql(std::string_view str, WrSerializer &ser) { ser << '\''; for (auto c : str) { switch (c) { @@ -55,8 +55,8 @@ void SQLEncoder::DumpSingleJoinQuery(size_t idx, WrSerializer &ser, bool stripAr assertrx(idx < query_.joinQueries_.size()); const auto &jq = query_.joinQueries_[idx]; ser << ' ' << jq.joinType; - if (jq.entries.Empty() && jq.count == QueryEntry::kDefaultLimit && jq.sortingEntries_.empty()) { - ser << ' ' << jq._namespace << " ON "; + if (jq.entries.Empty() && !jq.HasLimit() && jq.sortingEntries_.empty()) { + ser << ' ' << jq.NsName() << " ON "; } else { ser << " ("; jq.GetSQL(ser, stripArgs); @@ -65,13 +65,13 @@ void SQLEncoder::DumpSingleJoinQuery(size_t idx, WrSerializer &ser, bool stripAr if (jq.joinEntries_.size() != 1) ser << "("; for (auto &e : jq.joinEntries_) { if (&e != &*jq.joinEntries_.begin()) { - ser << ' ' << e.op_ << ' '; + ser << ' ' << e.Operation() << ' '; } - if (e.reverseNamespacesOrder) { - ser << jq._namespace << '.' << e.joinIndex_ << ' ' << InvertJoinCondition(e.condition_) << ' ' << query_._namespace << '.' - << e.index_; + if (e.ReverseNamespacesOrder()) { + ser << jq.NsName() << '.' << e.RightFieldName() << ' ' << InvertJoinCondition(e.Condition()) << ' ' << query_.NsName() << '.' + << e.LeftFieldName(); } else { - ser << query_._namespace << '.' << e.index_ << ' ' << e.condition_ << ' ' << jq._namespace << '.' << e.joinIndex_; + ser << query_.NsName() << '.' << e.LeftFieldName() << ' ' << e.Condition() << ' ' << jq.NsName() << '.' << e.RightFieldName(); } } if (jq.joinEntries_.size() != 1) ser << ')'; @@ -171,10 +171,10 @@ WrSerializer &SQLEncoder::GetSQL(WrSerializer &ser, bool stripArgs) const { distinctIndex = query_.aggregations_[0].Fields()[0]; } if (query_.selectFilter_.empty()) { - if (query_.count != 0 || query_.calcTotal == ModeNoTotal) { + if (query_.Limit() != 0 || query_.CalcTotal() == ModeNoTotal) { if (needComma) ser << ", "; ser << '*'; - if (query_.calcTotal != ModeNoTotal) { + if (query_.CalcTotal() != ModeNoTotal) { needComma = true; } } @@ -190,19 +190,19 @@ WrSerializer &SQLEncoder::GetSQL(WrSerializer &ser, bool stripArgs) const { } } } - if (query_.calcTotal != ModeNoTotal) { + if (query_.CalcTotal() != ModeNoTotal) { if (needComma) ser << ", "; - if (query_.calcTotal == ModeAccurateTotal) ser << "COUNT(*)"; - if (query_.calcTotal == ModeCachedTotal) ser << "COUNT_CACHED(*)"; + if (query_.CalcTotal() == ModeAccurateTotal) ser << "COUNT(*)"; + if (query_.CalcTotal() == ModeCachedTotal) ser << "COUNT_CACHED(*)"; } - ser << " FROM " << query_._namespace; + ser << " FROM " << query_.NsName(); } break; case QueryDelete: - ser << "DELETE FROM " << query_._namespace; + ser << "DELETE FROM " << query_.NsName(); break; case QueryUpdate: { if (query_.UpdateFields().empty()) break; - ser << "UPDATE " << query_._namespace; + ser << "UPDATE " << query_.NsName(); FieldModifyMode mode = query_.UpdateFields().front().Mode(); bool isUpdate = (mode == FieldModeSet || mode == FieldModeSetJson); if (isUpdate) { @@ -219,11 +219,19 @@ WrSerializer &SQLEncoder::GetSQL(WrSerializer &ser, bool stripArgs) const { if (isArray) ser << '['; for (const Variant &v : field.Values()) { if (&v != &*field.Values().begin()) ser << ','; - if (v.Type().Is() && !field.IsExpression() && (mode != FieldModeSetJson)) { - stringToSql(v.As(), ser); - } else { - ser << v.As(); - } + v.Type().EvaluateOneOf(overloaded{ + [&](KeyValueType::String) { + if (!field.IsExpression() && mode != FieldModeSetJson) { + stringToSql(v.As(), ser); + } else { + ser << v.As(); + } + }, + [&](KeyValueType::Uuid) { ser << '\'' << v.As() << '\''; }, + [&](OneOf) { + ser << v.As(); + }}); } if (isArray) ser << "]"; } @@ -231,7 +239,7 @@ WrSerializer &SQLEncoder::GetSQL(WrSerializer &ser, bool stripArgs) const { break; } case QueryTruncate: - ser << "TRUNCATE " << query_._namespace; + ser << "TRUNCATE " << query_.NsName(); break; default: throw Error(errParams, "Not implemented"); @@ -242,8 +250,8 @@ WrSerializer &SQLEncoder::GetSQL(WrSerializer &ser, bool stripArgs) const { dumpMerged(ser, stripArgs); dumpOrderBy(ser, stripArgs); - if (query_.start != QueryEntry::kDefaultOffset && !stripArgs) ser << " OFFSET " << query_.start; - if (query_.count != QueryEntry::kDefaultLimit && !stripArgs) ser << " LIMIT " << query_.count; + if (query_.HasOffset() && !stripArgs) ser << " OFFSET " << query_.Offset(); + if (query_.HasLimit() && !stripArgs) ser << " LIMIT " << query_.Limit(); return ser; } @@ -274,41 +282,43 @@ void SQLEncoder::dumpWhereEntries(QueryEntries::const_iterator from, QueryEntrie if (encodedEntries) { ser << opNames[op] << ' '; } - if (entry.condition == CondDWithin) { + if (entry.Condition() == CondDWithin) { ser << "ST_DWithin("; - indexToSql(entry.index, ser); + indexToSql(entry.FieldName(), ser); if (stripArgs) { ser << ", ?, ?)"; } else { - assertrx(entry.values.size() == 2); + assertrx(entry.Values().size() == 2); Point point; double distance; - if (entry.values[0].Type().Is()) { - point = static_cast(entry.values[0]); - distance = entry.values[1].As(); + if (entry.Values()[0].Type().Is()) { + point = static_cast(entry.Values()[0]); + distance = entry.Values()[1].As(); } else { - point = static_cast(entry.values[1]); - distance = entry.values[0].As(); + point = static_cast(entry.Values()[1]); + distance = entry.Values()[0].As(); } ser << ", ST_GeomFromText('POINT(" << point.X() << ' ' << point.Y() << ")'), " << distance << ')'; } } else { - indexToSql(entry.index, ser); - ser << ' ' << entry.condition << ' '; - if (entry.condition == CondEmpty || entry.condition == CondAny) { + indexToSql(entry.FieldName(), ser); + ser << ' ' << entry.Condition() << ' '; + if (entry.Condition() == CondEmpty || entry.Condition() == CondAny) { } else if (stripArgs) { ser << '?'; } else { - if (entry.values.size() != 1) ser << '('; - for (auto &v : entry.values) { - if (&v != &entry.values[0]) ser << ','; - if (v.Type().Is()) { - stringToSql(v.As(), ser); - } else { - ser << v.As(); - } + if (entry.Values().size() != 1) ser << '('; + for (auto &v : entry.Values()) { + if (&v != &entry.Values()[0]) ser << ','; + v.Type().EvaluateOneOf(overloaded{ + [&](KeyValueType::String) { stringToSql(v.As(), ser); }, + [&](KeyValueType::Uuid) { ser << '\'' << v.As() << '\''; }, + [&](OneOf) { + ser << v.As(); + }}); } - if (entry.values.size() != 1) ser << ")"; + if (entry.Values().size() != 1) ser << ")"; } } }, @@ -319,9 +329,9 @@ void SQLEncoder::dumpWhereEntries(QueryEntries::const_iterator from, QueryEntrie SQLEncoder(query_).DumpSingleJoinQuery(jqe.joinIndex, ser, stripArgs); }, [&ser](const BetweenFieldsQueryEntry &entry) { - indexToSql(entry.firstIndex, ser); + indexToSql(entry.LeftFieldName(), ser); ser << ' ' << entry.Condition() << ' '; - indexToSql(entry.secondIndex, ser); + indexToSql(entry.RightFieldName(), ser); }); ++encodedEntries; } diff --git a/cpp_src/core/query/sql/sqlencoder.h b/cpp_src/core/query/sql/sqlencoder.h index c1b217609..f43e0a6c0 100644 --- a/cpp_src/core/query/sql/sqlencoder.h +++ b/cpp_src/core/query/sql/sqlencoder.h @@ -2,7 +2,6 @@ #include #include "core/query/query.h" -#include "core/type_consts.h" /// @namespace reindexer /// The base namespace diff --git a/cpp_src/core/query/sql/sqlparser.cc b/cpp_src/core/query/sql/sqlparser.cc index 8549d03ae..1cb96cfd1 100644 --- a/cpp_src/core/query/sql/sqlparser.cc +++ b/cpp_src/core/query/sql/sqlparser.cc @@ -6,6 +6,7 @@ #include "core/queryresults/aggregationresult.h" #include "core/type_consts_helpers.h" #include "sqltokentype.h" +#include "tools/stringstools.h" #include "vendor/double-conversion/double-conversion.h" #include "vendor/gason/gason.h" @@ -91,16 +92,16 @@ int SQLParser::selectParse(tokenizer &parser) { parser.next_token(); tok = peekSqlToken(parser, SingleSelectFieldSqlToken); if (name.text() == "count"sv) { - query_.calcTotal = ModeAccurateTotal; + query_.CalcTotal(ModeAccurateTotal); if (!wasSelectFilter) { - query_.count = 0; + query_.Limit(0); } tok = parser.next_token(); if (tok.text() != "*") throw Error(errParseSQL, "Expected '*', but found '%s' in query, %s", tok.text(), parser.where()); } else if (name.text() == "count_cached"sv) { - query_.calcTotal = ModeCachedTotal; + query_.CalcTotal(ModeCachedTotal); if (!wasSelectFilter) { - query_.count = 0; + query_.Limit(0); } tok = parser.next_token(); if (tok.text() != "*"sv) throw Error(errParseSQL, "Expected '*', but found '%s' in query, %s", tok.text(), parser.where()); @@ -168,13 +169,13 @@ int SQLParser::selectParse(tokenizer &parser) { throw Error(errConflict, kAggregationWithSelectFieldsMsgError); } query_.selectFilter_.emplace_back(nameWithCase.text()); - query_.count = QueryEntry::kDefaultLimit; + query_.Limit(QueryEntry::kDefaultLimit); wasSelectFilter = true; } else if (name.text() == "*"sv) { if (!query_.CanAddSelectFilter()) { throw Error(errConflict, kAggregationWithSelectFieldsMsgError); } - query_.count = QueryEntry::kDefaultLimit; + query_.Limit(QueryEntry::kDefaultLimit); wasSelectFilter = true; query_.selectFilter_.clear(); } @@ -187,8 +188,8 @@ int SQLParser::selectParse(tokenizer &parser) { throw Error(errParams, "Expected 'FROM', but found '%s' in query, %s", tok.text(), parser.where()); peekSqlToken(parser, NamespaceSqlToken); - query_._namespace = std::string(parser.next_token().text()); - ctx_.updateLinkedNs(query_._namespace); + query_.SetNsName(parser.next_token().text()); + ctx_.updateLinkedNs(query_.NsName()); while (!parser.end()) { tok = peekSqlToken(parser, SelectConditionsStart); @@ -200,17 +201,17 @@ int SQLParser::selectParse(tokenizer &parser) { tok = parser.next_token(); if (tok.type != TokenNumber) throw Error(errParseSQL, "Expected number, but found '%s' in query, %s", tok.text(), parser.where()); - query_.count = stoi(tok.text()); + query_.Limit(stoi(tok.text())); } else if (tok.text() == "offset"sv) { parser.next_token(); tok = parser.next_token(); if (tok.type != TokenNumber) throw Error(errParseSQL, "Expected number, but found '%s' in query, %s", tok.text(), parser.where()); - query_.start = stoi(tok.text()); + query_.Offset(stoi(tok.text())); } else if (tok.text() == "order"sv) { parser.next_token(); parseOrderBy(parser, query_.sortingEntries_, query_.forcedSortOrder_); - ctx_.updateLinkedNs(query_._namespace); + ctx_.updateLinkedNs(query_.NsName()); } else if (tok.text() == "join"sv) { parser.next_token(); parseJoin(JoinType::LeftJoin, parser); @@ -402,8 +403,8 @@ int SQLParser::deleteParse(tokenizer &parser) { throw Error(errParams, "Expected 'FROM', but found '%s' in query, %s", tok.text(), parser.where()); peekSqlToken(parser, NamespaceSqlToken); - query_._namespace = std::string(parser.next_token().text()); - ctx_.updateLinkedNs(query_._namespace); + query_.SetNsName(parser.next_token().text()); + ctx_.updateLinkedNs(query_.NsName()); while (!parser.end()) { tok = peekSqlToken(parser, DeleteConditionsStart); @@ -415,17 +416,17 @@ int SQLParser::deleteParse(tokenizer &parser) { tok = parser.next_token(); if (tok.type != TokenNumber) throw Error(errParseSQL, "Expected number, but found '%s' in query, %s", tok.text(), parser.where()); - query_.count = stoi(tok.text()); + query_.Limit(stoi(tok.text())); } else if (tok.text() == "offset"sv) { parser.next_token(); tok = parser.next_token(); if (tok.type != TokenNumber) throw Error(errParseSQL, "Expected number, but found '%s' in query, %s", tok.text(), parser.where()); - query_.start = stoi(tok.text()); + query_.Offset(stoi(tok.text())); } else if (tok.text() == "order"sv) { parser.next_token(); parseOrderBy(parser, query_.sortingEntries_, query_.forcedSortOrder_); - ctx_.updateLinkedNs(query_._namespace); + ctx_.updateLinkedNs(query_.NsName()); } else break; } @@ -491,7 +492,7 @@ UpdateEntry SQLParser::parseUpdateField(tokenizer &parser) { if (tok.type != TokenName) { throw Error(errParseSQL, "Expected field name but found '%s' in query %s", tok.text(), parser.where()); } - UpdateEntry updateField{{tok.text().data(), tok.text().length()}, {}}; + UpdateEntry updateField{tok.text(), {}}; parser.next_token(); tok = parser.next_token(); @@ -545,8 +546,8 @@ int SQLParser::updateParse(tokenizer &parser) { parser.next_token(); token tok = peekSqlToken(parser, NamespaceSqlToken); - query_._namespace = std::string(tok.text()); - ctx_.updateLinkedNs(query_._namespace); + query_.SetNsName(tok.text()); + ctx_.updateLinkedNs(query_.NsName()); parser.next_token(); tok = peekSqlToken(parser, UpdateOptionsSqlToken); @@ -586,8 +587,8 @@ int SQLParser::updateParse(tokenizer &parser) { int SQLParser::truncateParse(tokenizer &parser) { parser.next_token(); token tok = peekSqlToken(parser, NamespaceSqlToken); - query_._namespace = std::string(tok.text()); - ctx_.updateLinkedNs(query_._namespace); + query_.SetNsName(tok.text()); + ctx_.updateLinkedNs(query_.NsName()); parser.next_token(); return 0; } @@ -650,7 +651,7 @@ int SQLParser::parseWhere(tokenizer &parser) { nextOp = OpAnd; } else { // Index name - const std::string index{tok.text()}; + std::string index{tok.text()}; // Operator CondType condition; @@ -673,13 +674,13 @@ int SQLParser::parseWhere(tokenizer &parser) { if (ctx_.autocompleteMode) peekSqlToken(parser, WhereFieldValueSqlToken, false); tok = parser.next_token(); if (iequals(tok.text(), "null"sv) || iequals(tok.text(), "empty"sv)) { - query_.entries.Append(nextOp, QueryEntry{index, CondEmpty, {}}); + query_.entries.Append(nextOp, QueryEntry{std::move(index), CondEmpty, {}}); } else if (iequals(tok.text(), "not"sv)) { tok = peekSqlToken(parser, WhereFieldNegateValueSqlToken, false); if (!iequals(tok.text(), "null"sv) && !iequals(tok.text(), "empty"sv)) { throw Error(errParseSQL, "Expected NULL, but found '%s' in query, %s", tok.text(), parser.where()); } - query_.entries.Append(nextOp, QueryEntry{index, CondAny, {}}); + query_.entries.Append(nextOp, QueryEntry{std::move(index), CondAny, {}}); tok = parser.next_token(false); } else if (tok.text() == "("sv) { VariantArray values; @@ -692,12 +693,12 @@ int SQLParser::parseWhere(tokenizer &parser) { if (tok.text() != ","sv) throw Error(errParseSQL, "Expected ')' or ',', but found '%s' in query, %s", tok.text(), parser.where()); } - query_.entries.Append(nextOp, QueryEntry{index, condition, std::move(values)}); + query_.entries.Append(nextOp, QueryEntry{std::move(index), condition, std::move(values)}); } else if (tok.type != TokenName || toLower(tok.text()) == "true" || toLower(tok.text()) == "false") { - query_.entries.Append(nextOp, QueryEntry{index, condition, {token2kv(tok, parser, true)}}); + query_.entries.Append(nextOp, QueryEntry{std::move(index), condition, {token2kv(tok, parser, true)}}); // Second field } else { - query_.entries.Append(nextOp, BetweenFieldsQueryEntry{index, condition, std::string{tok.text()}}); + query_.entries.Append(nextOp, BetweenFieldsQueryEntry{std::move(index), condition, std::string{tok.text()}}); } nextOp = OpAnd; } @@ -766,7 +767,7 @@ void SQLParser::parseEqualPositions(tokenizer &parser, std::vectorHoldsOrReferTo() && nameWithCase.text() == it->Value().index) { + if (it->HoldsOrReferTo() && nameWithCase.text() == it->Value().FieldName()) { validField = true; break; } @@ -933,11 +934,11 @@ void SQLParser::parseJoin(JoinType type, tokenizer &parser) { throw Error(errParseSQL, "Expected ')', but found %s, %s", tok.text(), parser.where()); } } else { - jquery._namespace = std::string(tok.text()); - ctx_.updateLinkedNs(jquery._namespace); + jquery.SetNsName(tok.text()); + ctx_.updateLinkedNs(jquery.NsName()); } jquery.joinType = type; - jparser.parseJoinEntries(parser, query_._namespace, jquery); + jparser.parseJoinEntries(parser, query_.NsName(), jquery); if (type != JoinType::LeftJoin) { query_.entries.Append((type == JoinType::InnerJoin) ? OpAnd : OpOr, JoinQueryEntry(query_.joinQueries_.size())); @@ -1023,28 +1024,25 @@ void SQLParser::parseJoinEntries(tokenizer &parser, const std::string &mainNs, J return; } - QueryJoinEntry je; - std::string ns1 = mainNs, ns2 = jquery._namespace; - std::string idx1 = parseJoinedFieldName(parser, ns1); - je.condition_ = getCondType(parser.next_token().text()); - std::string idx2 = parseJoinedFieldName(parser, ns2); - - if (ns1 == mainNs && ns2 == jquery._namespace) { - je.index_ = std::move(idx1); - je.joinIndex_ = std::move(idx2); - } else if (ns2 == mainNs && ns1 == jquery._namespace) { - je.index_ = std::move(idx2); - je.joinIndex_ = std::move(idx1); - je.condition_ = InvertJoinCondition(je.condition_); - je.reverseNamespacesOrder = true; - } else { - throw Error(errParseSQL, "Unexpected tables with ON statement: ('%s' and '%s') but expected ('%s' and '%s'), %s", ns1, ns2, - mainNs, jquery._namespace, parser.where()); + std::string ns1 = mainNs, ns2 = jquery.NsName(); + std::string fld1 = parseJoinedFieldName(parser, ns1); + CondType condition = getCondType(parser.next_token().text()); + std::string fld2 = parseJoinedFieldName(parser, ns2); + bool reverseNamespacesOrder{false}; + + if (ns1 != mainNs || ns2 != jquery.NsName()) { + if (ns2 == mainNs && ns1 == jquery.NsName()) { + std::swap(fld1, fld2); + condition = InvertJoinCondition(condition); + reverseNamespacesOrder = true; + } else { + throw Error(errParseSQL, "Unexpected tables with ON statement: ('%s' and '%s') but expected ('%s' and '%s'), %s", ns1, ns2, + mainNs, jquery.NsName(), parser.where()); + } } - je.op_ = jquery.nextOp_; + jquery.joinEntries_.emplace_back(jquery.nextOp_, condition, std::move(fld1), std::move(fld2), reverseNamespacesOrder); jquery.nextOp_ = OpAnd; - jquery.joinEntries_.emplace_back(std::move(je)); if (!braces) { return; } diff --git a/cpp_src/core/query/sql/sqlsuggester.h b/cpp_src/core/query/sql/sqlsuggester.h index f0e0feebf..e0d55ec29 100644 --- a/cpp_src/core/query/sql/sqlsuggester.h +++ b/cpp_src/core/query/sql/sqlsuggester.h @@ -2,9 +2,7 @@ #include #include "core/schema.h" -#include "estl/fast_hash_map.h" #include "sqlparser.h" -#include "tools/stringstools.h" namespace reindexer { diff --git a/cpp_src/core/queryresults/queryresults.cc b/cpp_src/core/queryresults/queryresults.cc index 1ba73eaec..a17df13d5 100644 --- a/cpp_src/core/queryresults/queryresults.cc +++ b/cpp_src/core/queryresults/queryresults.cc @@ -427,18 +427,6 @@ Item QueryResults::Iterator::GetItem(bool enableHold) { return item; } -QueryResults::Iterator &QueryResults::Iterator::operator++() { - idx_++; - return *this; -} -QueryResults::Iterator &QueryResults::Iterator::operator+(int val) { - idx_ += val; - return *this; -} - -bool QueryResults::Iterator::operator!=(const Iterator &other) const { return idx_ != other.idx_; } -bool QueryResults::Iterator::operator==(const Iterator &other) const { return idx_ == other.idx_; } - void QueryResults::AddItem(Item &item, bool withData, bool enableHold) { auto ritem = item.impl_; if (item.GetID() != -1) { diff --git a/cpp_src/core/queryresults/queryresults.h b/cpp_src/core/queryresults/queryresults.h index e3e4d11f1..4c06d9d5f 100644 --- a/cpp_src/core/queryresults/queryresults.h +++ b/cpp_src/core/queryresults/queryresults.h @@ -52,10 +52,12 @@ class QueryResults { void AddItem(Item &item, bool withData = false, bool enableHold = true); std::string Dump() const; void Erase(ItemRefVector::iterator begin, ItemRefVector::iterator end); - size_t Count() const { return items_.size(); } - size_t TotalCount() const { return totalCount; } - const std::string &GetExplainResults() const { return explainResults; } - const std::vector &GetAggregationResults() const { return aggregationResults; } + size_t Count() const noexcept { return items_.size(); } + size_t TotalCount() const noexcept { return totalCount; } + const std::string &GetExplainResults() const &noexcept { return explainResults; } + const std::string &GetExplainResults() const && = delete; + const std::vector &GetAggregationResults() const &noexcept { return aggregationResults; } + const std::vector &GetAggregationResults() const && = delete; void Clear(); h_vector GetNamespaces() const; bool IsCacheEnabled() const { return !nonCacheableData; } @@ -77,12 +79,19 @@ class QueryResults { int64_t GetLSN() const { return qr_->items_[idx_].Value().GetLSN(); } bool IsRaw() const; std::string_view GetRaw() const; - Iterator &operator++(); - Iterator &operator+(int delta); - const Error &Status() const noexcept { return err_; } - bool operator!=(const Iterator &) const; - bool operator==(const Iterator &) const; - Iterator &operator*() { return *this; } + Iterator &operator++() noexcept { + idx_++; + return *this; + } + Iterator &operator+(int delta) noexcept { + idx_ += delta; + return *this; + } + + Error Status() const noexcept { return err_; } + bool operator==(const Iterator &other) const noexcept { return idx_ == other.idx_; } + bool operator!=(const Iterator &other) const noexcept { return !operator==(other); } + Iterator &operator*() noexcept { return *this; } const QueryResults *qr_; int idx_; diff --git a/cpp_src/core/queryresults/tableviewbuilder.h b/cpp_src/core/queryresults/tableviewbuilder.h index d9d092127..52fa3247a 100644 --- a/cpp_src/core/queryresults/tableviewbuilder.h +++ b/cpp_src/core/queryresults/tableviewbuilder.h @@ -2,15 +2,13 @@ #include #include +#include #include #include #include #include -#include "tools/errors.h" #include "tools/terminalutils.h" -#include - namespace reindexer { struct ColumnData { diff --git a/cpp_src/core/rdxcontext.h b/cpp_src/core/rdxcontext.h index c79b1eaed..ac7cfce47 100644 --- a/cpp_src/core/rdxcontext.h +++ b/cpp_src/core/rdxcontext.h @@ -18,22 +18,24 @@ struct IRdxCancelContext { virtual ~IRdxCancelContext() = default; }; +constexpr std::string_view kDefaultTimeoutError = "Context timeout"; +constexpr std::string_view kDefaultCancelError = "Context was canceled"; + template -void ThrowOnCancel(const Context& ctx, std::string_view errMsg = std::string_view()) { // TODO may be ""sv +void ThrowOnCancel(const Context& ctx, std::string_view errMsg = std::string_view()) { if (!ctx.isCancelable()) return; - auto cancel = ctx.checkCancel(); + const auto cancel = ctx.checkCancel(); switch (cancel) { case CancelType::Explicit: - throw Error(errCanceled, errMsg); + throw Error(errCanceled, errMsg.empty() ? kDefaultCancelError : errMsg); case CancelType::Timeout: - throw Error(errTimeout, errMsg); + throw Error(errTimeout, errMsg.empty() ? kDefaultTimeoutError : errMsg); case CancelType::None: return; - default: - assertrx(false); - throw Error(errCanceled, errMsg); } + assertrx(false); + throw Error(errCanceled, errMsg.empty() ? kDefaultCancelError : errMsg); } class RdxDeadlineContext : public IRdxCancelContext { @@ -156,7 +158,7 @@ class InternalRdxContext { std::move(user), connectionId); } InternalRdxContext WithContextParams(milliseconds timeout, std::string_view activityTracer, std::string&& user, - int connectionId) const { + int connectionId) const { return activityTracer.empty() ? InternalRdxContext(cmpl_, RdxDeadlineContext(timeout, deadlineCtx_.parent()), activityTracer_, user_, connectionId_) : InternalRdxContext(cmpl_, RdxDeadlineContext(timeout, deadlineCtx_.parent()), diff --git a/cpp_src/core/reindexerimpl.cc b/cpp_src/core/reindexerimpl.cc index b34c8778b..65ec563e5 100644 --- a/cpp_src/core/reindexerimpl.cc +++ b/cpp_src/core/reindexerimpl.cc @@ -4,17 +4,16 @@ #include #include #include "cjson/jsonbuilder.h" -#include "core/cjson/jsondecoder.h" #include "core/cjson/protobufschemabuilder.h" #include "core/iclientsstats.h" #include "core/index/index.h" #include "core/itemimpl.h" #include "core/nsselecter/crashqueryreporter.h" +#include "core/nsselecter/querypreprocessor.h" #include "core/query/sql/sqlsuggester.h" #include "core/selectfunc/selectfunc.h" #include "core/type_consts_helpers.h" #include "defnsconfigs.h" -#include "estl/contexted_locks.h" #include "estl/defines.h" #include "queryresults/joinresults.h" #include "replicator/replicator.h" @@ -244,7 +243,7 @@ Error ReindexerImpl::Connect(const std::string& dsn, ConnectOpts opts) { idx = nsIdx.fetch_add(1, std::memory_order_relaxed)) { auto& de = foundNs[idx]; if (de.isDir && validateObjectName(de.name, true)) { - if (de.name[0] == '@') { + if (de.name[0] == kTmpNsPrefix) { const std::string tmpPath = fs::JoinPath(storagePath_, de.name); logPrintf(LogWarning, "Dropping tmp namespace '%s'", de.name); if (fs::RmDirAll(tmpPath) < 0) { @@ -336,7 +335,7 @@ Error ReindexerImpl::addNamespace(const NamespaceDef& nsDef, const RdxContext& r Namespace::Ptr ns; try { { - ULock lock(mtx_, &rdxCtx); + SLock lock(mtx_, &rdxCtx); if (namespaces_.find(nsDef.name) != namespaces_.end()) { return Error(errParams, "Namespace '%s' already exists", nsDef.name); } @@ -361,7 +360,9 @@ Error ReindexerImpl::addNamespace(const NamespaceDef& nsDef, const RdxContext& r namespaces_.insert({nsDef.name, ns}); } if (!nsDef.isTemporary) observers_.OnWALUpdate(LSNPair(), nsDef.name, WALRecord(WalNamespaceAdd)); - for (auto& indexDef : nsDef.indexes) ns->AddIndex(indexDef, rdxCtx); + for (auto& indexDef : nsDef.indexes) { + ns->AddIndex(indexDef, rdxCtx); + } ns->SetSchema(nsDef.schemaJson, rdxCtx); if (nsDef.storage.IsSlaveMode()) ns->setSlaveMode(rdxCtx); @@ -427,7 +428,6 @@ Error ReindexerImpl::closeNamespace(std::string_view nsName, const RdxContext& c try { ULock lock(mtx_, &ctx); auto nsIt = namespaces_.find(nsName); - if (nsIt == namespaces_.end()) { return Error(errNotFound, "Namespace '%s' does not exist", nsName); } @@ -521,7 +521,9 @@ Error ReindexerImpl::renameNamespace(std::string_view srcNsName, const std::stri const InternalRdxContext& ctx) { Namespace::Ptr dstNs, srcNs; try { - if (dstNsName == srcNsName.data()) return errOK; + if (dstNsName == srcNsName.data()) { + return {}; + } if (isSystemNamespaceNameStrict(srcNsName)) { return Error(errParams, "Can't rename system namespace (%s)", srcNsName); } @@ -536,6 +538,22 @@ Error ReindexerImpl::renameNamespace(std::string_view srcNsName, const std::stri const auto rdxCtx = ctx.CreateRdxContext( ctx.NeedTraceActivity() ? (ser << "RENAME " << srcNsName << " to " << dstNsName).Slice() : ""sv, activities_); + { + // Perform namespace flushes to minimize chances of the flush under lock + SLock lock(mtx_, &rdxCtx); + auto srcIt = namespaces_.find(srcNsName); + srcNs = (srcIt != namespaces_.end()) ? srcIt->second : Namespace::Ptr(); + lock.unlock(); + + if (srcNs) { + auto err = srcNs->awaitMainNs(rdxCtx)->FlushStorage(rdxCtx); + if (!err.ok()) { + return Error(err.code(), "Unable to flush storage before rename: %s", err.what()); + } + srcNs.reset(); + } + } + ULock lock(mtx_, &rdxCtx); auto srcIt = namespaces_.find(srcNsName); if (srcIt == namespaces_.end()) { @@ -546,8 +564,7 @@ Error ReindexerImpl::renameNamespace(std::string_view srcNsName, const std::stri auto replState = srcNs->GetReplState(rdxCtx); - if (fromReplication || !replState.slaveMode) // rename from replicator forced temporary ns - { + if (fromReplication || !replState.slaveMode) { // rename from replicator forced temporary ns auto dstIt = namespaces_.find(dstNsName); auto needWalUpdate = !srcNs->GetDefinition(rdxCtx).isTemporary; if (dstIt != namespaces_.end()) { @@ -557,18 +574,16 @@ Error ReindexerImpl::renameNamespace(std::string_view srcNsName, const std::stri } else { srcNs->Rename(dstNsName, storagePath_, rdxCtx); } - if (needWalUpdate) observers_.OnWALUpdate(LSNPair(), srcNsName, WALRecord(WalNamespaceRename, dstNsName)); - - auto srcNamespace = srcIt->second; namespaces_.erase(srcIt); - namespaces_[dstNsName] = std::move(srcNamespace); + namespaces_[dstNsName] = std::move(srcNs); + if (needWalUpdate) observers_.OnWALUpdate(LSNPair(), srcNsName, WALRecord(WalNamespaceRename, dstNsName)); } else { return Error(errLogic, "Can't rename namespace in slave mode '%s'", srcNsName); } } catch (const Error& err) { return err; } - return errOK; + return {}; } template @@ -678,7 +693,7 @@ Error ReindexerImpl::Update(const Query& q, QueryResults& result, const Internal try { WrSerializer ser; const auto rdxCtx = ctx.CreateRdxContext(ctx.NeedTraceActivity() ? q.GetSQL(ser).Slice() : ""sv, activities_, result); - auto ns = getNamespace(q._namespace, rdxCtx); + auto ns = getNamespace(q.NsName(), rdxCtx); ns->Update(q, result, rdxCtx); if (ns->IsSystem(rdxCtx)) { const std::string kNsName = ns->GetName(rdxCtx); @@ -788,7 +803,7 @@ Error ReindexerImpl::Delete(std::string_view nsName, Item& item, QueryResults& q Error ReindexerImpl::Delete(const Query& q, QueryResults& result, const InternalRdxContext& ctx) { const auto makeCtxStr = [&q](WrSerializer& ser) -> WrSerializer& { return q.GetSQL(ser); }; - const std::string_view nsName = q._namespace; + const std::string_view nsName = q.NsName(); APPLY_NS_FUNCTION2(false, Delete, q, result); } @@ -808,7 +823,7 @@ Error ReindexerImpl::Select(std::string_view query, QueryResults& result, const err = Update(q, result, ctx); break; case QueryTruncate: - err = TruncateNamespace(q._namespace, ctx); + err = TruncateNamespace(q.NsName(), ctx); break; default: err = Error(errParams, "Error unsupported query type %d", q.type_); @@ -840,7 +855,7 @@ Error ReindexerImpl::Select(const Query& q, QueryResults& result, const Internal const auto rdxCtx = ctx.CreateRdxContext(ctx.NeedTraceActivity() ? nonNormalizedSQL.Slice() : "", activities_, result); NsLocker locks(rdxCtx); - auto mainNsWrp = getNamespace(q._namespace, rdxCtx); + auto mainNsWrp = getNamespace(q.NsName(), rdxCtx); auto mainNs = q.IsWALQuery() ? mainNsWrp->awaitMainNs(rdxCtx) : mainNsWrp->getMainNs(); const auto queriesPerfStatsEnabled = configProvider_.QueriesPerfStatsEnabled(); @@ -862,7 +877,7 @@ Error ReindexerImpl::Select(const Query& q, QueryResults& result, const Internal } : std::function{}; - const bool isSystemNsRequest = isSystemNamespaceNameFast(q._namespace); + const bool isSystemNsRequest = isSystemNamespaceNameFast(q.NsName()); QueryStatCalculator statCalculator( std::move(hitter), std::chrono::microseconds(queriesThresholdUS), queriesPerfStatsEnabled || configProvider_.GetSelectLoggingParams().thresholdUs >= 0, @@ -871,13 +886,13 @@ Error ReindexerImpl::Select(const Query& q, QueryResults& result, const Internal StatsLocker::StatsLockT statsSelectLck; if (isSystemNsRequest) { - statsSelectLck = syncSystemNamespaces(q._namespace, detectFilterNsNames(q), rdxCtx); + statsSelectLck = syncSystemNamespaces(q.NsName(), detectFilterNsNames(q), rdxCtx); } // Lookup and lock namespaces_ mainNs->updateSelectTime(); locks.Add(mainNs); q.WalkNested(false, true, [this, &locks, &rdxCtx](const Query& q) { - auto nsWrp = getNamespace(q._namespace, rdxCtx); + auto nsWrp = getNamespace(q.NsName(), rdxCtx); auto ns = q.IsWALQuery() ? nsWrp->awaitMainNs(rdxCtx) : nsWrp->getMainNs(); ns->updateSelectTime(); locks.Add(ns); @@ -955,26 +970,26 @@ bool ReindexerImpl::isPreResultValuesModeOptimizationAvailable(const Query& jIte jItemQ.entries.ExecuteAppropriateForEach( Skip{}, [&jns, &result](const QueryEntry& qe) { - if (qe.idxNo >= 0) { - assertrx(jns->indexes_.size() > static_cast(qe.idxNo)); - const IndexType indexType = jns->indexes_[qe.idxNo]->Type(); + if (qe.IsFieldIndexed()) { + assertrx(jns->indexes_.size() > static_cast(qe.IndexNo())); + const IndexType indexType = jns->indexes_[qe.IndexNo()]->Type(); if (IsComposite(indexType) || IsFullText(indexType)) result = false; } }, [&jns, &result](const BetweenFieldsQueryEntry& qe) { - if (qe.firstIdxNo >= 0) { - assertrx(jns->indexes_.size() > static_cast(qe.firstIdxNo)); - const IndexType indexType = jns->indexes_[qe.firstIdxNo]->Type(); + if (qe.IsLeftFieldIndexed()) { + assertrx(jns->indexes_.size() > static_cast(qe.LeftIdxNo())); + const IndexType indexType = jns->indexes_[qe.LeftIdxNo()]->Type(); if (IsComposite(indexType) || IsFullText(indexType)) result = false; } - if (qe.secondIdxNo >= 0) { - assertrx(jns->indexes_.size() > static_cast(qe.secondIdxNo)); - if (IsComposite(jns->indexes_[qe.secondIdxNo]->Type())) result = false; + if (qe.IsRightFieldIndexed()) { + assertrx(jns->indexes_.size() > static_cast(qe.RightIdxNo())); + if (IsComposite(jns->indexes_[qe.RightIdxNo()]->Type())) result = false; } }); if (!result) return false; for (const auto& se : mainQ.sortingEntries_) { - if (byJoinedField(se.expression, jItemQ._namespace)) return false; // TODO maybe allow #1410 + if (byJoinedField(se.expression, jItemQ.NsName())) return false; // TODO maybe allow #1410 } return true; } @@ -984,14 +999,14 @@ JoinedSelectors ReindexerImpl::prepareJoinedSelectors(const Query& q, QueryResul std::vector& queryResultsContexts, const RdxContext& rdxCtx) { JoinedSelectors joinedSelectors; if (q.joinQueries_.empty()) return joinedSelectors; - auto ns = locks.Get(q._namespace); + auto ns = locks.Get(q.NsName()); assertrx(ns); // For each joined queries uint32_t joinedSelectorsCount = uint32_t(q.joinQueries_.size()); for (auto& jq : q.joinQueries_) { - if rx_unlikely (isSystemNamespaceNameFast(jq._namespace)) { - throw Error(errParams, "Queries to system namespaces ('%s') are not supported inside JOIN statement", jq._namespace); + if rx_unlikely (isSystemNamespaceNameFast(jq.NsName())) { + throw Error(errParams, "Queries to system namespaces ('%s') are not supported inside JOIN statement", jq.NsName()); } if (rx_unlikely(!jq.joinQueries_.empty())) { throw Error(errParams, "JOINs nested into the other JOINs are not supported"); @@ -1001,13 +1016,13 @@ JoinedSelectors ReindexerImpl::prepareJoinedSelectors(const Query& q, QueryResul } // Get common results from joined namespaces_ - auto jns = locks.Get(jq._namespace); + auto jns = locks.Get(jq.NsName()); assertrx(jns); // Do join for each item in main result - Query jItemQ(jq._namespace); + Query jItemQ(jq.NsName()); jItemQ.explain_ = q.explain_; - jItemQ.Debug(jq.debugLevel).Limit(jq.count); + jItemQ.Debug(jq.debugLevel).Limit(jq.Limit()); jItemQ.Strict(q.strictMode); for (size_t i = 0; i < jq.sortingEntries_.size(); ++i) { jItemQ.Sort(jq.sortingEntries_[i].expression, jq.sortingEntries_[i].desc); @@ -1017,18 +1032,13 @@ JoinedSelectors ReindexerImpl::prepareJoinedSelectors(const Query& q, QueryResul // Construct join conditions for (auto& je : jq.joinEntries_) { - int joinIdx = IndexValueType::NotSet; - if (!jns->getIndexByName(je.joinIndex_, joinIdx)) { - joinIdx = IndexValueType::SetByJsonPath; - } - QueryEntry qe(InvertJoinCondition(je.condition_), je.joinIndex_, joinIdx); - if (!ns->getIndexByName(je.index_, const_cast(je).idxNo)) { - const_cast(je).idxNo = IndexValueType::SetByJsonPath; - } - jItemQ.entries.Append(je.op_, std::move(qe)); + QueryPreprocessor::SetQueryField(const_cast(je).LeftFieldData(), *ns); + QueryPreprocessor::SetQueryField(const_cast(je).RightFieldData(), *jns); + jItemQ.entries.Append(je.Operation(), QueryField(je.RightFieldData()), InvertJoinCondition(je.Condition()), + QueryEntry::IgnoreEmptyValues{}); } - Query jjq(jq); + Query jjq(static_cast(jq)); JoinPreResult::Ptr preResult = std::make_shared(); uint32_t joinedFieldIdx = uint32_t(joinedSelectors.size()); JoinCacheRes joinRes; @@ -1058,24 +1068,6 @@ JoinedSelectors ReindexerImpl::prepareJoinedSelectors(const Query& q, QueryResul result.AddNamespace(jns, true); if (preResult->dataMode == JoinPreResult::ModeValues) { - jItemQ.entries.ExecuteAppropriateForEach( - Skip{}, - [&jns](QueryEntry& qe) { - if (qe.idxNo != IndexValueType::SetByJsonPath) { - assertrx(qe.idxNo >= 0 && static_cast(qe.idxNo) < jns->indexes_.size()); - if (jns->indexes_[qe.idxNo]->Opts().IsSparse()) qe.idxNo = IndexValueType::SetByJsonPath; - } - }, - [&jns](BetweenFieldsQueryEntry& qe) { - if (qe.firstIdxNo != IndexValueType::SetByJsonPath) { - assertrx(qe.firstIdxNo >= 0 && static_cast(qe.firstIdxNo) < jns->indexes_.size()); - if (jns->indexes_[qe.firstIdxNo]->Opts().IsSparse()) qe.firstIdxNo = IndexValueType::SetByJsonPath; - } - if (qe.secondIdxNo != IndexValueType::SetByJsonPath) { - assertrx(qe.secondIdxNo >= 0 && static_cast(qe.secondIdxNo) < jns->indexes_.size()); - if (jns->indexes_[qe.secondIdxNo]->Opts().IsSparse()) qe.secondIdxNo = IndexValueType::SetByJsonPath; - } - }); preResult->values.PreselectAllowed(static_cast(jns->Config().maxPreselectSize) >= preResult->values.size()); if (!preResult->values.Locked()) preResult->values.Lock(); // If not from cache locks.Delete(jns); @@ -1091,9 +1083,9 @@ JoinedSelectors ReindexerImpl::prepareJoinedSelectors(const Query& q, QueryResul template void ReindexerImpl::doSelect(const Query& q, QueryResults& result, NsLocker& locks, SelectFunctionsHolder& func, const RdxContext& ctx, QueryStatCalculator& queryStatCalculator) { - auto ns = locks.Get(q._namespace); + auto ns = locks.Get(q.NsName()); if rx_unlikely (!ns) { - throw Error(errParams, "Namespace '%s' does not exist", q._namespace); + throw Error(errParams, "Namespace '%s' does not exist", q.NsName()); } std::vector joinQueryResultsContexts; bool thereAreJoins = !q.joinQueries_.empty(); @@ -1173,13 +1165,13 @@ void ReindexerImpl::doSelect(const Query& q, QueryResults& result, NsLocker& throw Error(errParams, "Limit and offset are not supported for aggregations '%s'", AggTypeToStr(errType)); } for (auto& mq : q.mergeQueries_) { - if rx_unlikely (isSystemNamespaceNameFast(mq._namespace)) { - throw Error(errParams, "Queries to system namespaces ('%s') are not supported inside MERGE statement", mq._namespace); + if rx_unlikely (isSystemNamespaceNameFast(mq.NsName())) { + throw Error(errParams, "Queries to system namespaces ('%s') are not supported inside MERGE statement", mq.NsName()); } if rx_unlikely (!mq.sortingEntries_.empty()) { throw Error(errParams, "Sorting in inner merge query is not allowed"); } - if rx_unlikely (!mq.aggregations_.empty() || mq.calcTotal != ModeNoTotal) { + if rx_unlikely (!mq.aggregations_.empty() || mq.CalcTotal() != ModeNoTotal) { throw Error(errParams, "Aggregations in inner merge query is not allowed"); } if rx_unlikely (mq.HasLimit() || mq.HasOffset()) { @@ -1189,7 +1181,7 @@ void ReindexerImpl::doSelect(const Query& q, QueryResults& result, NsLocker& throw Error(errParams, "MERGEs nested into the MERGEs are not supported"); } - auto mns = locks.Get(mq._namespace); + auto mns = locks.Get(mq.NsName()); assertrx(mns); SelectCtx mctx(mq, &q); mctx.nsid = ++counter; @@ -1204,16 +1196,16 @@ void ReindexerImpl::doSelect(const Query& q, QueryResults& result, NsLocker& result.AddNamespace(mns, true); } ItemRefVector& itemRefVec = result.Items(); - if (static_cast(q.start) >= itemRefVec.size()) { + if (q.Offset() >= itemRefVec.size()) { result.Erase(itemRefVec.begin(), itemRefVec.end()); return; } std::sort(itemRefVec.begin(), itemRefVec.end(), ItemRefLess()); - if (q.start > QueryEntry::kDefaultOffset) { - result.Erase(itemRefVec.begin(), itemRefVec.begin() + q.start); + if (q.HasOffset()) { + result.Erase(itemRefVec.begin(), itemRefVec.begin() + q.Offset()); } - if (itemRefVec.size() > q.count) { - result.Erase(itemRefVec.begin() + q.count, itemRefVec.end()); + if (itemRefVec.size() > q.Limit()) { + result.Erase(itemRefVec.begin() + q.Limit(), itemRefVec.end()); } } // Adding context to QueryResults @@ -1234,7 +1226,6 @@ Error ReindexerImpl::Commit(std::string_view /*_namespace*/) { Namespace::Ptr ReindexerImpl::getNamespace(std::string_view nsName, const RdxContext& ctx) { SLock lock(mtx_, &ctx); auto nsIt = namespaces_.find(nsName); - if (nsIt == namespaces_.end()) { throw Error(errParams, "Namespace '%s' does not exist", nsName); } @@ -1645,7 +1636,7 @@ ReindexerImpl::FilterNsNamesT ReindexerImpl::detectFilterNsNames(const Query& q) } if (entries.HoldsOrReferTo(i)) { auto& qe = entries.Get(i); - if (qe.index == kNsNameField) { + if (qe.FieldName() == kNsNameField) { if (op == OpNot) { return std::nullopt; } @@ -1653,15 +1644,15 @@ ReindexerImpl::FilterNsNamesT ReindexerImpl::detectFilterNsNames(const Query& q) [i](const BracketRange& br) noexcept { return i >= br.begin && i < br.end; }) != notBrackets.end()) { return std::nullopt; } - if (qe.condition != CondSet && qe.condition != CondEq) { + if (qe.Condition() != CondSet && qe.Condition() != CondEq) { return std::nullopt; } if (res.has_value()) { return std::nullopt; } res.emplace(); - res->reserve(qe.values.size()); - for (auto& v : qe.values) { + res->reserve(qe.Values().size()); + for (auto& v : qe.Values()) { if (!v.Type().Is()) { return std::nullopt; } @@ -1670,7 +1661,7 @@ ReindexerImpl::FilterNsNamesT ReindexerImpl::detectFilterNsNames(const Query& q) } } else if (entries.HoldsOrReferTo(i)) { auto& qe = entries.Get(i); - if (qe.firstIndex == kNsNameField || qe.secondIndex == kNsNameField) { + if (qe.LeftFieldName() == kNsNameField || qe.RightFieldName() == kNsNameField) { return std::nullopt; } } else if (op == OpNot && entries.IsSubTree(i)) { diff --git a/cpp_src/core/storage/leveldbstorage.cc b/cpp_src/core/storage/leveldbstorage.cc index 00520a2fb..090acd124 100644 --- a/cpp_src/core/storage/leveldbstorage.cc +++ b/cpp_src/core/storage/leveldbstorage.cc @@ -6,6 +6,7 @@ #include #include #include "tools/assertrx.h" +#include "tools/fsops.h" namespace reindexer { namespace datastorage { @@ -144,7 +145,11 @@ void LevelDbStorage::doDestroy(const std::string& path) { db_.reset(); leveldb::Status status = leveldb::DestroyDB(path.c_str(), options); if (!status.ok()) { - printf("Cannot destroy DB: %s, %s\n", path.c_str(), status.ToString().c_str()); + fprintf(stderr, "Cannot destroy LevelDB's storage: %s, %s. Trying to remove files by the backup mechanism...\n", path.c_str(), + status.ToString().c_str()); + if (fs::RmDirAll(path) != 0) { + fprintf(stderr, "Unable to remove LevelDB's storage: %s, %s", path.c_str(), strerror(errno)); + } } } diff --git a/cpp_src/core/storage/rocksdbstorage.cc b/cpp_src/core/storage/rocksdbstorage.cc index 4d20d8ef7..9535275b9 100644 --- a/cpp_src/core/storage/rocksdbstorage.cc +++ b/cpp_src/core/storage/rocksdbstorage.cc @@ -6,6 +6,7 @@ #include #include #include +#include "tools/fsops.h" namespace reindexer { namespace datastorage { @@ -143,8 +144,10 @@ void RocksDbStorage::doDestroy(const std::string& path) { options.create_if_missing = true; db_.reset(); rocksdb::Status status = rocksdb::DestroyDB(path.c_str(), options); - if (!status.ok()) { - printf("Cannot destroy DB: %s, %s\n", path.c_str(), status.ToString().c_str()); + fprintf(stderr, "Cannot destroy RocksDB's storage: %s, %s. Trying to remove files by the backup mechanism...\n", path.c_str(), + status.ToString().c_str()); + if (fs::RmDirAll(path) != 0) { + fprintf(stderr, "Unable to remove RocksDB's storage: %s, %s", path.c_str(), strerror(errno)); } } diff --git a/cpp_src/core/txstats.h b/cpp_src/core/txstats.h index f30d7e27a..30e364909 100644 --- a/cpp_src/core/txstats.h +++ b/cpp_src/core/txstats.h @@ -1,9 +1,9 @@ #pragma once #include "core/transaction.h" +#include "core/transactionimpl.h" #include "namespace/namespacestat.h" #include "perfstatcounter.h" -#include "tools/stringstools.h" namespace reindexer { diff --git a/cpp_src/core/type_consts_helpers.cc b/cpp_src/core/type_consts_helpers.cc index 18fd15e1c..60db10409 100644 --- a/cpp_src/core/type_consts_helpers.cc +++ b/cpp_src/core/type_consts_helpers.cc @@ -110,6 +110,8 @@ namespace reindexer { return "unknown"sv; } +} // namespace reindexer + [[nodiscard]] std::string_view JoinTypeName(JoinType type) { using namespace std::string_view_literals; @@ -126,5 +128,3 @@ namespace reindexer { assertrx(false); return "unknown"sv; } - -} // namespace reindexer diff --git a/cpp_src/core/type_consts_helpers.h b/cpp_src/core/type_consts_helpers.h index 2fc5e36df..d4d32897e 100644 --- a/cpp_src/core/type_consts_helpers.h +++ b/cpp_src/core/type_consts_helpers.h @@ -11,6 +11,18 @@ namespace reindexer { [[nodiscard]] std::string_view TagTypeToStr(TagType); [[nodiscard]] std::string_view AggTypeToStr(AggType t) noexcept; +constexpr bool IsComposite(IndexType type) noexcept { + return type == IndexCompositeBTree || type == IndexCompositeFastFT || type == IndexCompositeFuzzyFT || type == IndexCompositeHash; +} + +constexpr bool IsFullText(IndexType type) noexcept { + return type == IndexFastFT || type == IndexFuzzyFT || type == IndexCompositeFastFT || type == IndexCompositeFuzzyFT; +} + +constexpr bool IsFastFullText(IndexType type) noexcept { return type == IndexFastFT || type == IndexCompositeFastFT; } + +} // namespace reindexer + /// Get readable Join Type /// @param type - join type /// @returns string with join type name @@ -132,15 +144,3 @@ T& operator<<(T& os, CollateMode m) { } std::abort(); } - -constexpr bool IsComposite(IndexType type) noexcept { - return type == IndexCompositeBTree || type == IndexCompositeFastFT || type == IndexCompositeFuzzyFT || type == IndexCompositeHash; -} - -constexpr bool IsFullText(IndexType type) noexcept { - return type == IndexFastFT || type == IndexFuzzyFT || type == IndexCompositeFastFT || type == IndexCompositeFuzzyFT; -} - -constexpr bool IsFastFullText(IndexType type) noexcept { return type == IndexFastFT || type == IndexCompositeFastFT; } - -} // namespace reindexer diff --git a/cpp_src/gtests/bench/fixtures/api_tv_composite.cc b/cpp_src/gtests/bench/fixtures/api_tv_composite.cc index fdeceac96..dc131e080 100644 --- a/cpp_src/gtests/bench/fixtures/api_tv_composite.cc +++ b/cpp_src/gtests/bench/fixtures/api_tv_composite.cc @@ -299,13 +299,14 @@ void ApiTvComposite::RangeHashCompositeIntStr(benchmark::State& state) { } void ApiTvComposite::RangeTreeIntSortByHashInt(State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); auto idRange = id_seq_->GetRandomIdRange(id_seq_->Count() * 0.02); - q.Where("id", CondRange, {idRange.first, idRange.second}).Sort("age", false).Limit(20); + q.Where("id", CondRange, {idRange.first, idRange.second}).Sort("age"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -314,13 +315,14 @@ void ApiTvComposite::RangeTreeIntSortByHashInt(State& state) { } void ApiTvComposite::RangeTreeIntSortByTreeInt(State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); auto idRange = id_seq_->GetRandomIdRange(id_seq_->Count() * 0.02); - q.Where("id", CondRange, {idRange.first, idRange.second}).Sort("year", false).Limit(20); + q.Where("id"sv, CondRange, {idRange.first, idRange.second}).Sort("year"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -329,13 +331,14 @@ void ApiTvComposite::RangeTreeIntSortByTreeInt(State& state) { } void ApiTvComposite::RangeTreeStrSortByHashInt(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); auto idRange = id_seq_->GetRandomIdRange(id_seq_->Count() * 0.02); - q.Where("id", CondRange, {std::to_string(idRange.first), std::to_string(idRange.second)}).Sort("age", false).Limit(20); + q.Where("id"sv, CondRange, {std::to_string(idRange.first), std::to_string(idRange.second)}).Sort("age"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -344,13 +347,14 @@ void ApiTvComposite::RangeTreeStrSortByHashInt(benchmark::State& state) { } void ApiTvComposite::RangeTreeStrSortByTreeInt(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); auto idRange = id_seq_->GetRandomIdRange(id_seq_->Count() * 0.02); - q.Where("id", CondRange, {std::to_string(idRange.first), std::to_string(idRange.second)}).Sort("year", false).Limit(20); + q.Where("id"sv, CondRange, {std::to_string(idRange.first), std::to_string(idRange.second)}).Sort("year"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -359,6 +363,7 @@ void ApiTvComposite::RangeTreeStrSortByTreeInt(benchmark::State& state) { } void ApiTvComposite::RangeTreeDoubleSortByTreeInt(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); @@ -366,7 +371,7 @@ void ApiTvComposite::RangeTreeDoubleSortByTreeInt(benchmark::State& state) { auto leftRate = random(0.0, 4.99); auto rightRate = random(5.0, 10.0); - q.Where("rate", CondRange, {leftRate, rightRate}).Sort("year", false).Limit(20); + q.Where("rate"sv, CondRange, {leftRate, rightRate}).Sort("year"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -375,6 +380,7 @@ void ApiTvComposite::RangeTreeDoubleSortByTreeInt(benchmark::State& state) { } void ApiTvComposite::RangeTreeDoubleSortByHashInt(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); @@ -382,7 +388,7 @@ void ApiTvComposite::RangeTreeDoubleSortByHashInt(benchmark::State& state) { auto leftRate = random(0.0, 4.99); auto rightRate = random(5.0, 10.0); - q.Where("rate", CondRange, {leftRate, rightRate}).Sort("age", false).Limit(20); + q.Where("rate"sv, CondRange, {leftRate, rightRate}).Sort("age"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -391,13 +397,14 @@ void ApiTvComposite::RangeTreeDoubleSortByHashInt(benchmark::State& state) { } void ApiTvComposite::RangeTreeStrSortByHashStrCollateASCII(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); auto idRange = id_seq_->GetRandomIdRange(id_seq_->Count() * 0.02); - q.Where("id", CondRange, {std::to_string(idRange.first), std::to_string(idRange.second)}).Sort("location", false).Limit(20); + q.Where("id"sv, CondRange, {std::to_string(idRange.first), std::to_string(idRange.second)}).Sort("location"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -406,13 +413,14 @@ void ApiTvComposite::RangeTreeStrSortByHashStrCollateASCII(benchmark::State& sta } void ApiTvComposite::RangeTreeStrSortByHashStrCollateUTF8(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); auto idRange = id_seq_->GetRandomIdRange(id_seq_->Count() * 0.02); - q.Where("id", CondRange, {std::to_string(idRange.first), std::to_string(idRange.second)}).Sort("name", false).Limit(20); + q.Where("id"sv, CondRange, {std::to_string(idRange.first), std::to_string(idRange.second)}).Sort("name"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -421,11 +429,12 @@ void ApiTvComposite::RangeTreeStrSortByHashStrCollateUTF8(benchmark::State& stat } void ApiTvComposite::SortByHashInt(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); - q.Sort("id", false).Limit(20); + q.Sort("id"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -434,11 +443,12 @@ void ApiTvComposite::SortByHashInt(benchmark::State& state) { } void ApiTvComposite::ForcedSortByHashInt(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); - q.Sort("id", false, {10, 20, 30, 40, 50}).Limit(20); + q.Sort("id"sv, false, {10, 20, 30, 40, 50}).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -447,11 +457,12 @@ void ApiTvComposite::ForcedSortByHashInt(benchmark::State& state) { } void ApiTvComposite::ForcedSortWithSecondCondition(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); - q.Sort("id", false, {10, 20, 30, 40, 50}).Sort("location", false).Limit(20); + q.Sort("id"sv, false, {10, 20, 30, 40, 50}).Sort("location"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -474,11 +485,12 @@ void ApiTvComposite::Query2CondIdSetComposite(benchmark::State& state) { } void ApiTvComposite::SortByHashStrCollateASCII(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); - q.Sort("location", false).Limit(20); + q.Sort("location"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -487,11 +499,12 @@ void ApiTvComposite::SortByHashStrCollateASCII(benchmark::State& state) { } void ApiTvComposite::SortByHashStrCollateUTF8(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); - q.Sort("name", false).Limit(20); + q.Sort("name"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -500,11 +513,12 @@ void ApiTvComposite::SortByHashStrCollateUTF8(benchmark::State& state) { } void ApiTvComposite::SortByHashCompositeIntInt(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); - q.Sort("id+start_time", false).Limit(20); + q.Sort("id+start_time"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -513,11 +527,12 @@ void ApiTvComposite::SortByHashCompositeIntInt(benchmark::State& state) { } void ApiTvComposite::SortByHashCompositeIntStr(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); - q.Sort("id+genre", false).Limit(20); + q.Sort("id+genre"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -526,11 +541,12 @@ void ApiTvComposite::SortByHashCompositeIntStr(benchmark::State& state) { } void ApiTvComposite::SortByTreeCompositeIntInt(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); - q.Sort("id+year", false).Limit(20); + q.Sort("id+year"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); @@ -539,11 +555,12 @@ void ApiTvComposite::SortByTreeCompositeIntInt(benchmark::State& state) { } void ApiTvComposite::SortByTreeCompositeIntStrCollateUTF8(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); - q.Sort("id+name", false).Limit(20); + q.Sort("id+name"sv, false).Limit(20); QueryResults qres; auto err = db_->Select(q, qres); diff --git a/cpp_src/gtests/bench/fixtures/ft_fixture.cc b/cpp_src/gtests/bench/fixtures/ft_fixture.cc index b02143761..955ee9d66 100644 --- a/cpp_src/gtests/bench/fixtures/ft_fixture.cc +++ b/cpp_src/gtests/bench/fixtures/ft_fixture.cc @@ -7,6 +7,7 @@ #include "core/cjson/jsonbuilder.h" #include "core/ft/config/ftfastconfig.h" +#include "tools/errors.h" #include "tools/stringstools.h" #include @@ -241,18 +242,33 @@ reindexer::Item FullText::MakeItem(benchmark::State&) { void FullText::BuildInsertSteps(State& state) { AllocsTracker allocsTracker(state, printFlags); - db_->DropNamespace(nsdef_.name); + auto err = db_->DropNamespace(nsdef_.name); + if (!err.ok()) { + state.SkipWithError(err.what().c_str()); + assertf(err.ok(), "%s", err.what()); + } id_seq_->Reset(); - auto err = BaseFixture::Initialize(); + err = BaseFixture::Initialize(); + if (!err.ok()) { + state.SkipWithError(err.what().c_str()); + assertf(err.ok(), "%s", err.what()); + } size_t i = 0; size_t mem = 0; + assert(!words_.empty()); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) auto item = MakeSpecialItem(); - if (!item.Status().ok()) state.SkipWithError(item.Status().what().c_str()); + if (!item.Status().ok()) { + state.SkipWithError(item.Status().what().c_str()); + assertf(item.Status().ok(), "%s", item.Status().what()); + } err = db_->Insert(nsdef_.name, item); - if (!err.ok()) state.SkipWithError(err.what().c_str()); + if (!err.ok()) { + state.SkipWithError(err.what().c_str()); + assertf(err.ok(), "%s", err.what()); + } if (i % 12000 == 0) { Query q(nsdef_.name); @@ -279,7 +295,10 @@ void FullText::Insert(State& state) { AllocsTracker allocsTracker(state, printFlags); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) auto item = MakeItem(state); - if (!item.Status().ok()) state.SkipWithError(item.Status().what().c_str()); + if (!item.Status().ok()) { + state.SkipWithError(item.Status().what().c_str()); + continue; + } auto err = db_->Insert(nsdef_.name, item); if (!err.ok()) state.SkipWithError(err.what().c_str()); @@ -290,10 +309,11 @@ void FullText::Insert(State& state) { } void FullText::BuildCommonIndexes(benchmark::State& state) { + using namespace std::string_view_literals; AllocsTracker allocsTracker(state, printFlags); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) Query q(nsdef_.name); - q.Where("year", CondRange, {2010, 2016}).Limit(20).Sort("year", false); + q.Where("year"sv, CondRange, {2010, 2016}).Limit(20).Sort("year"sv, false); std::this_thread::sleep_for(std::chrono::milliseconds(2000)); QueryResults qres; @@ -325,7 +345,10 @@ void FullText::BuildAndInsertLowWordsDiversityNs(State& state) { item["description1"] = d1; item["description2"] = d2; - if (!item.Status().ok()) state.SkipWithError(item.Status().what().c_str()); + if (!item.Status().ok()) { + state.SkipWithError(item.Status().what().c_str()); + continue; + } auto err = db_->Insert(lowWordsDiversityNsDef_.name, item); if (!err.ok()) state.SkipWithError(err.what().c_str()); @@ -908,7 +931,13 @@ void FullText::InitForAlternatingUpdatesAndSelects(State& state) { ftCfg.optimization = opt; ftIndexOpts.config = ftCfg.GetJson({}); AllocsTracker allocsTracker(state, printFlags); - db_->DropNamespace(alternatingNs_); + auto err = db_->DropNamespace(alternatingNs_); + if (!err.ok()) { + if (err.code() != errNotFound || err.what() != "Namespace '" + alternatingNs_ + "' does not exist") { + state.SkipWithError(err.what().c_str()); + assertf(err.ok(), "%s", err.what()); + } + } for (auto _ : state) { // NOLINT(*deadcode.DeadStores) NamespaceDef nsDef{alternatingNs_}; nsDef.AddIndex("id", "hash", "int", IndexOpts().PK()) @@ -918,7 +947,10 @@ void FullText::InitForAlternatingUpdatesAndSelects(State& state) { .AddIndex("search_comp", {"search1", "search2"}, "text", "composite", ftIndexOpts) .AddIndex("search_comp_not_index_fields", {"field1", "field2"}, "text", "composite", ftIndexOpts); auto err = db_->AddNamespace(nsDef); - if (!err.ok()) state.SkipWithError(err.what().c_str()); + if (!err.ok()) { + state.SkipWithError(err.what().c_str()); + assertf(err.ok(), "%s", err.what()); + } values_.clear(); values_.reserve(kNsSize); reindexer::WrSerializer ser; @@ -934,18 +966,25 @@ void FullText::InitForAlternatingUpdatesAndSelects(State& state) { bld.Put("rand", rand()); bld.End(); auto item = db_->NewItem(alternatingNs_); - if (!item.Status().ok()) state.SkipWithError(item.Status().what().c_str()); + if (!item.Status().ok()) { + state.SkipWithError(item.Status().what().c_str()); + continue; + } err = item.FromJSON(ser.Slice()); - if (!err.ok()) state.SkipWithError(err.what().c_str()); + if (!err.ok()) { + state.SkipWithError(err.what().c_str()); + continue; + } err = db_->Insert(alternatingNs_, item); if (!err.ok()) state.SkipWithError(err.what().c_str()); } } - auto err = db_->Commit(alternatingNs_); + err = db_->Commit(alternatingNs_); if (!err.ok()) state.SkipWithError(err.what().c_str()); // Init index build + assert(!values_.empty()); Query q = Query(alternatingNs_) .Where("search1", CondEq, @@ -970,6 +1009,7 @@ void FullText::InitForAlternatingUpdatesAndSelects(State& state) { void FullText::updateAlternatingNs(reindexer::WrSerializer& ser, benchmark::State& state) { using namespace std::string_literals; + assert(!values_.empty()); const int i = randomGenerator_(randomEngine_, std::uniform_int_distribution::param_type{0, int(values_.size() - 1)}); ser.Reset(); reindexer::JsonBuilder bld(ser); @@ -982,9 +1022,15 @@ void FullText::updateAlternatingNs(reindexer::WrSerializer& ser, benchmark::Stat bld.End(); auto item = db_->NewItem(alternatingNs_); item.Unsafe(false); - if (!item.Status().ok()) state.SkipWithError(item.Status().what().c_str()); + if (!item.Status().ok()) { + state.SkipWithError(item.Status().what().c_str()); + return; + } auto err = item.FromJSON(ser.Slice()); - if (!err.ok()) state.SkipWithError(err.what().c_str()); + if (!err.ok()) { + state.SkipWithError(err.what().c_str()); + return; + } err = db_->Update(alternatingNs_, item); if (!err.ok()) state.SkipWithError(err.what().c_str()); @@ -997,6 +1043,7 @@ void FullText::updateAlternatingNs(reindexer::WrSerializer& ser, benchmark::Stat } void FullText::AlternatingUpdatesAndSelects(benchmark::State& state) { + assert(!values_.empty()); reindexer::WrSerializer ser; AllocsTracker allocsTracker(state, printFlags); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) @@ -1015,6 +1062,7 @@ void FullText::AlternatingUpdatesAndSelects(benchmark::State& state) { } void FullText::AlternatingUpdatesAndSelectsByComposite(benchmark::State& state) { + assert(!values_.empty()); reindexer::WrSerializer ser; AllocsTracker allocsTracker(state, printFlags); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) @@ -1030,6 +1078,7 @@ void FullText::AlternatingUpdatesAndSelectsByComposite(benchmark::State& state) } void FullText::AlternatingUpdatesAndSelectsByCompositeByNotIndexFields(benchmark::State& state) { + assert(!values_.empty()); reindexer::WrSerializer ser; AllocsTracker allocsTracker(state, printFlags); for (auto _ : state) { // NOLINT(*deadcode.DeadStores) diff --git a/cpp_src/gtests/bench/fixtures/ft_fixture.h b/cpp_src/gtests/bench/fixtures/ft_fixture.h index 309c0de75..13f69eb43 100644 --- a/cpp_src/gtests/bench/fixtures/ft_fixture.h +++ b/cpp_src/gtests/bench/fixtures/ft_fixture.h @@ -200,7 +200,7 @@ class FullText : private BaseFixture { void updateAlternatingNs(reindexer::WrSerializer&, benchmark::State&); reindexer::Error readDictFile(const std::string& fileName, std::vector& words); - const char* alternatingNs_ = "FtAlternatingUpdatesAndSelects"; + const std::string alternatingNs_ = "FtAlternatingUpdatesAndSelects"; size_t raw_data_sz_ = 0; std::mt19937 randomEngine_{1}; diff --git a/cpp_src/gtests/bench/ft_bench.cc b/cpp_src/gtests/bench/ft_bench.cc index 2358cd617..05b81d1de 100644 --- a/cpp_src/gtests/bench/ft_bench.cc +++ b/cpp_src/gtests/bench/ft_bench.cc @@ -30,11 +30,12 @@ int main(int argc, char** argv) { } shared_ptr DB = std::make_shared(); - DB->Connect("builtin://" + kStoragePath); + auto err = DB->Connect("builtin://" + kStoragePath); + if (!err.ok()) return err.code(); FullText ft(DB.get(), "fulltext", kItemsInBenchDataset); - auto err = ft.Initialize(); + err = ft.Initialize(); if (!err.ok()) return err.code(); ::benchmark::Initialize(&argc, argv); diff --git a/cpp_src/gtests/bench/reindexer_bench.cc b/cpp_src/gtests/bench/reindexer_bench.cc index 2053b0d1f..216a8d42a 100644 --- a/cpp_src/gtests/bench/reindexer_bench.cc +++ b/cpp_src/gtests/bench/reindexer_bench.cc @@ -39,7 +39,8 @@ int main(int argc, char** argv) { } shared_ptr DB = std::make_shared(); - DB->Connect("builtin://" + kStoragePath); + auto err = DB->Connect("builtin://" + kStoragePath); + if (!err.ok()) return err.code(); JoinItems joinItems(DB.get(), 500); ApiTvSimple apiTvSimple(DB.get(), "ApiTvSimple", kItemsInBenchDataset); @@ -48,7 +49,7 @@ int main(int argc, char** argv) { Geometry geometry(DB.get(), "Geometry", kItemsInBenchDataset); Aggregation aggregation(DB.get(), "Aggregation", kItemsInBenchDataset); - auto err = apiTvSimple.Initialize(); + err = apiTvSimple.Initialize(); if (!err.ok()) return err.code(); err = apiTvSimpleComparators.Initialize(); diff --git a/cpp_src/gtests/tests/API/base_tests.cc b/cpp_src/gtests/tests/API/base_tests.cc index a466ceb28..b6e72f672 100644 --- a/cpp_src/gtests/tests/API/base_tests.cc +++ b/cpp_src/gtests/tests/API/base_tests.cc @@ -89,11 +89,14 @@ TEST_F(ReindexerApi, RenameNamespace) { auto getRowsInJSON = [&](const std::string& namespaceName, std::vector& resStrings) { QueryResults result; - rt.reindexer->Select(Query(namespaceName), result); + auto err = rt.reindexer->Select(Query(namespaceName), result); + ASSERT_TRUE(err.ok()) << err.what(); resStrings.clear(); for (auto it = result.begin(); it != result.end(); ++it) { + ASSERT_TRUE(it.Status().ok()) << it.Status().what(); reindexer::WrSerializer sr; - it.GetJSON(sr, false); + err = it.GetJSON(sr, false); + ASSERT_TRUE(err.ok()) << err.what(); std::string_view sv = sr.Slice(); resStrings.emplace_back(sv.data(), sv.size()); } @@ -246,8 +249,7 @@ TEST_F(ReindexerApi, DistinctCompositeIndex) { EXPECT_TRUE(err.ok()) << err.what(); } - Query q; - q._namespace = default_namespace; + Query q{default_namespace}; q.Distinct("v1+v2"); { QueryResults qr; @@ -1333,15 +1335,15 @@ TEST_F(ReindexerApi, DistinctQueriesEncodingTest) { std::string dsl = q1.GetJSON(); Query q2; q2.FromJSON(dsl); - EXPECT_TRUE(q1 == q2); + EXPECT_EQ(q1, q2) << "q1: " << q1.GetSQL() << "\nq2: " << q2.GetSQL(); Query q3{Query(default_namespace).Distinct("name").Distinct("city").Where("id", CondGt, static_cast(10))}; std::string sql2 = q3.GetSQL(); Query q4; q4.FromSQL(sql2); - EXPECT_TRUE(q3 == q4); - EXPECT_TRUE(sql2 == q4.GetSQL()); + EXPECT_EQ(q3, q4) << "q3: " << q3.GetSQL() << "\nq4: " << q4.GetSQL(); + EXPECT_EQ(sql2, q4.GetSQL()); } TEST_F(ReindexerApi, ContextCancelingTest) { diff --git a/cpp_src/gtests/tests/fixtures/ft_api.cc b/cpp_src/gtests/tests/fixtures/ft_api.cc index d4668d665..9f5c9ac81 100644 --- a/cpp_src/gtests/tests/fixtures/ft_api.cc +++ b/cpp_src/gtests/tests/fixtures/ft_api.cc @@ -61,7 +61,10 @@ reindexer::Error FTApi::SetFTConfig(const reindexer::FtFastConfig& ftCfg, const fieldsMap.emplace(fields[i], i); } std::vector nses; - rt.reindexer->EnumNamespaces(nses, reindexer::EnumNamespacesOpts().WithFilter(ns)); + auto err = rt.reindexer->EnumNamespaces(nses, reindexer::EnumNamespacesOpts().WithFilter(ns)); + if (!err.ok()) { + return err; + } const auto it = std::find_if(nses[0].indexes.begin(), nses[0].indexes.end(), [&index](const reindexer::IndexDef& idef) { return idef.name_ == index; }); it->opts_.SetConfig(ftCfg.GetJson(fieldsMap)); @@ -174,11 +177,11 @@ reindexer::QueryResults FTApi::SimpleSelect3(std::string word) { return res; } -void FTApi::Delete(int id) { +reindexer::Error FTApi::Delete(int id) { reindexer::Item item = rt.NewItem("nm1"); item["id"] = id; - this->rt.reindexer->Delete("nm1", item); + return this->rt.reindexer->Delete("nm1", item); } reindexer::QueryResults FTApi::SimpleCompositeSelect(std::string word) { diff --git a/cpp_src/gtests/tests/fixtures/ft_api.h b/cpp_src/gtests/tests/fixtures/ft_api.h index 5e56bf8b3..542630dec 100644 --- a/cpp_src/gtests/tests/fixtures/ft_api.h +++ b/cpp_src/gtests/tests/fixtures/ft_api.h @@ -32,7 +32,7 @@ class FTApi : public ::testing::TestWithParam& indexes) const { + const FieldType fldType = std::visit( + reindexer::overloaded{[](const Child& c) noexcept { return c.type; }, [](const Children&) noexcept { return FieldType::Struct; }}, + content_); IndexOpts opts; - const bool pk = rnd.PkIndex(isPk); + const bool pk = rnd.PkIndex(isPk_); opts.PK(pk); - opts.Array(rnd.RndArrayField(isArray)); - opts.Sparse(rnd.SparseIndex(pk)); + opts.Array(rnd.RndArrayField(isArray_) == IsArray::Yes); + opts.Sparse(rnd.RndSparseIndex(isSparse_)); opts.Dense(rnd.DenseIndex()); opts.RTreeType(static_cast(rnd.RndInt(IndexOpts::Linear, IndexOpts::RStar))); - FieldType fldType = std::visit( - reindexer::overloaded{[](const Child& c) noexcept { return c.type; }, [](const Children&) noexcept { return FieldType::Struct; }}, - content); std::string fieldType = rnd.IndexFieldType(fldType); - std::string indexType = rnd.RndIndexType(fldType, isPk); + std::string indexType{ToText(rnd.RndIndexType(type_))}; reindexer::JsonPaths jsonPaths; std::visit(reindexer::overloaded{[&](const Child& c) { jsonPaths.push_back(scheme.GetJsonPath(c.fieldPath)); }, [&](const Children& c) { + jsonPaths.reserve(c.size()); for (const auto& child : c) { + if (rnd.RndBool(0.5)) { + std::vector scalarIndexes; + scalarIndexes.reserve(indexes.size()); + for (size_t i = 0, s = indexes.size(); i < s; ++i) { + if (const auto* c = std::get_if(&indexes[i].content_); + c && c->fieldPath == child.fieldPath) { + scalarIndexes.push_back(i); + } + } + if (!scalarIndexes.empty()) { + jsonPaths.push_back(indexes[rnd.RndWhich(scalarIndexes)].name_); + continue; + } + } jsonPaths.push_back(scheme.GetJsonPath(child.fieldPath)); } }}, - content); - return {name, std::move(jsonPaths), std::move(indexType), std::move(fieldType), std::move(opts), rnd.ExpiredIndex()}; + content_); + return {name_, std::move(jsonPaths), std::move(indexType), std::move(fieldType), std::move(opts), rnd.ExpiredIndex()}; } void Index::Dump(std::ostream& os, const NsScheme& scheme, size_t offset) const { for (size_t i = 0; i < offset; ++i) os << " "; os << "{\n"; for (size_t i = 0; i <= offset; ++i) os << " "; - os << "name: " << name << '\n'; + os << "name: " << name_ << '\n'; + for (size_t i = 0; i <= offset; ++i) os << " "; + os << "type: " << type_ << '\n'; for (size_t i = 0; i <= offset; ++i) os << " "; - os << "pk: " << (isPk ? "true" : "false") << '\n'; + os << "pk: " << std::boolalpha << isPk_ << '\n'; for (size_t i = 0; i <= offset; ++i) os << " "; - os << "array: " << (isArray ? "true" : "false") << '\n'; + os << "array: " << std::boolalpha << IsArray() << '\n'; + for (size_t i = 0; i <= offset; ++i) os << " "; + os << "sparse: " << std::boolalpha << (IsSparse() == IsSparse::Yes) << '\n'; for (size_t i = 0; i <= offset; ++i) os << " "; std::visit(reindexer::overloaded{[&](const Child& child) { + os << "composite: false\n"; + for (size_t i = 0; i <= offset; ++i) os << " "; os << "field: {\n"; for (size_t i = 0; i < offset + 2; ++i) os << " "; os << "type: " << child.type << '\n'; @@ -48,6 +72,8 @@ void Index::Dump(std::ostream& os, const NsScheme& scheme, size_t offset) const os << "}\n"; }, [&](const Children& children) { + os << "composite: true\n"; + for (size_t i = 0; i <= offset; ++i) os << " "; os << "fields: [\n"; for (const auto& c : children) { for (size_t i = 0; i < offset + 2; ++i) os << " "; @@ -62,7 +88,7 @@ void Index::Dump(std::ostream& os, const NsScheme& scheme, size_t offset) const for (size_t i = 0; i <= offset; ++i) os << " "; os << "]\n"; }}, - content); + content_); } } // namespace fuzzing diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/index.h b/cpp_src/gtests/tests/fixtures/fuzzing/index.h index 1daeb9266..80407211c 100644 --- a/cpp_src/gtests/tests/fixtures/fuzzing/index.h +++ b/cpp_src/gtests/tests/fixtures/fuzzing/index.h @@ -1,7 +1,9 @@ #pragma once -#include "ns_scheme.h" -#include "random_generator.h" +#include +#include +#include +#include "types.h" namespace reindexer { struct IndexDef; @@ -9,20 +11,43 @@ struct IndexDef; namespace fuzzing { -struct Index { - reindexer::IndexDef IndexDef(RandomGenerator&, const NsScheme&) const; +class RandomGenerator; +class NsScheme; +class Index { +public: struct Child { FieldType type; FieldPath fieldPath; }; using Children = std::vector; - std::string name; - std::variant content; - bool isPk{false}; - bool isArray{false}; + Index(std::string name, IndexType type, IsArray isArray, IsSparse isSparse, Children content) noexcept + : name_{std::move(name)}, type_{type}, content_{std::move(content)}, isArray_{isArray}, isSparse_{isSparse} {} + Index(std::string name, IndexType type, IsArray isArray, IsSparse isSparse, Child content) noexcept + : name_{std::move(name)}, type_{type}, content_{std::move(content)}, isArray_{isArray}, isSparse_{isSparse} {} + + const std::string& Name() const& noexcept { return name_; } + const std::string& Name() const&& = delete; + IndexType Type() const noexcept { return type_; } + const auto& Content() const& noexcept { return content_; } + const auto& Content() const&& = delete; + bool IsPk() const noexcept { return isPk_; } + void SetPk() noexcept { isPk_ = true; } + bool IsArray() const noexcept { return isArray_ == IsArray::Yes; } + auto IsSparse() const noexcept { return isSparse_; } + + reindexer::IndexDef IndexDef(RandomGenerator&, const NsScheme&, const std::vector&) const; + void Dump(std::ostream&, const NsScheme&, size_t offset) const; + +private: + std::string name_; + IndexType type_; + std::variant content_; + bool isPk_{false}; + enum IsArray isArray_ { IsArray::No }; + enum IsSparse isSparse_ { IsSparse::No }; }; } // namespace fuzzing diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/ns.cc b/cpp_src/gtests/tests/fixtures/fuzzing/ns.cc index 93d5f16fc..5a520d4a0 100644 --- a/cpp_src/gtests/tests/fixtures/fuzzing/ns.cc +++ b/cpp_src/gtests/tests/fixtures/fuzzing/ns.cc @@ -1,5 +1,8 @@ #include "ns.h" #include +#include "estl/overloaded.h" +#include "index.h" +#include "tools/assertrx.h" struct FieldPathHash { size_t operator()(const fuzzing::FieldPath& fp) const noexcept { @@ -30,47 +33,74 @@ static bool availablePkFieldType(FieldType ft) { } } -Ns::Ns(std::string name, std::ostream& os, RandomGenerator::ErrFactorType errorFactor) - : name_{std::move(name)}, rndGen_{os, errorFactor}, scheme_{name_, rndGen_} { - std::unordered_set generatedNames; +static bool availablePkIndexType(IndexType it) { + switch (it) { + case IndexType::Store: + case IndexType::FastFT: + case IndexType::FuzzyFT: + case IndexType::RTree: + return false; + case IndexType::Hash: + case IndexType::Tree: + case IndexType::Ttl: + return true; + default: + assertrx(false); + std::abort(); + } +} + +Ns::Ns(std::string name, RandomGenerator::ErrFactorType errorFactor) + : name_{std::move(name)}, rndGen_{errorFactor}, scheme_{name_, rndGen_} { + std::unordered_set usedIndexNames; std::unordered_set usedPaths; constexpr static size_t kMaxTries = 10; const size_t idxCount = rndGen_.IndexesCount(); const bool withErr = rndGen_.RndErr(); indexes_.reserve(idxCount); + std::vector scalarIndexes; + scalarIndexes.reserve(idxCount); for (size_t i = 0; i < idxCount; ++i) { const bool uniqueName = rndGen_.UniqueName(); - if (rndGen_.CompositeIndex()) { - bool fail = false; - Index index{uniqueName ? rndGen_.IndexName(generatedNames) : std::string{}, Index::Children{}}; - auto& children = std::get(index.content); - const size_t size = rndGen_.CompositeIndexSize(); - children.reserve(size); - for (size_t i = 0; i < size; ++i) { - auto fldPath = rndGen_.RndScalarField(scheme_); - FieldType fldType; - if (scheme_.IsStruct(fldPath)) { - if (!rndGen_.RndErr()) { - fail = true; - break; - } - fldType = rndGen_.RndFieldType(); + if (rndGen_.CompositeIndex(scalarIndexes.size())) { + bool array = false; + bool containsUuid = false; + std::string name; + Index::Children children; + const auto fields = rndGen_.RndFieldsForCompositeIndex(scalarIndexes); + children.reserve(fields.size()); + for (size_t f : fields) { + Index::Child fieldData; + if (f < indexes_.size()) { + const auto& idx = indexes_[f]; + fieldData = std::get(idx.Content()); + array |= idx.IsArray(); } else { - fldType = scheme_.GetFieldType(fldPath); - } - if (!uniqueName) { - if (!index.name.empty()) index.name += '+'; - if (fldPath.empty()) { - index.name += rndGen_.FieldName(generatedNames); + fieldData.fieldPath = rndGen_.RndScalarField(scheme_); + if (scheme_.IsStruct(fieldData.fieldPath)) { + fieldData.type = rndGen_.RndFieldType(); } else { - index.name += scheme_.GetJsonPath(fldPath); + fieldData.type = scheme_.GetFieldType(fieldData.fieldPath); } } - children.emplace_back(Index::Child{fldType, std::move(fldPath)}); + if (!uniqueName) { + if (!name.empty()) name += '+'; + name += scheme_.GetJsonPath(fieldData.fieldPath); + } + containsUuid |= fieldData.type == FieldType::Uuid; + children.emplace_back(std::move(fieldData)); + } + const auto indexType = + containsUuid ? rndGen_.RndIndexType({FieldType::Struct, FieldType::Uuid}) : rndGen_.RndIndexType({FieldType::Struct}); + if (uniqueName) { + name = rndGen_.IndexName(usedIndexNames); + } else if (!usedIndexNames.insert(name).second) { + name = rndGen_.IndexName(usedIndexNames); + usedIndexNames.insert(name); } - if (fail) continue; - index.isArray = rndGen_.RndArrayField(false); - indexes_.emplace_back(std::move(index)); + + indexes_.emplace_back(std::move(name), indexType, rndGen_.RndArrayField(array ? IsArray::Yes : IsArray::No), IsSparse::No, + std::move(children)); } else { FieldPath fldPath; size_t tryCounts = 0; @@ -82,47 +112,73 @@ Ns::Ns(std::string name, std::ostream& os, RandomGenerator::ErrFactorType errorF if (scheme_.IsStruct(fldPath)) { if (!rndGen_.RndErr()) continue; const auto fldType = rndGen_.RndFieldType(); - indexes_.emplace_back(Index{rndGen_.IndexName(generatedNames), Index::Child{fldType, std::move(fldPath)}}); + indexes_.emplace_back(rndGen_.IndexName(usedIndexNames), rndGen_.RndIndexType({fldType}), + rndGen_.RndBool(0.5) ? IsArray::Yes : IsArray::No, + rndGen_.RndBool(0.5) ? IsSparse::Yes : IsSparse::No, Index::Child{fldType, std::move(fldPath)}); } else { const auto fldType = scheme_.GetFieldType(fldPath); - const bool isArray = scheme_.IsArray(fldPath); - std::string idxName = uniqueName ? rndGen_.IndexName(generatedNames) : scheme_.GetJsonPath(fldPath); - indexes_.emplace_back(Index{std::move(idxName), Index::Child{fldType, std::move(fldPath)}}); - indexes_.back().isArray = rndGen_.RndArrayField(isArray); + const auto isArray = scheme_.IsArray(fldPath); + std::string idxName; + if (uniqueName) { + idxName = rndGen_.IndexName(usedIndexNames); + } else { + idxName = scheme_.GetJsonPath(fldPath); + if (!usedIndexNames.insert(idxName).second) { + idxName = rndGen_.IndexName(usedIndexNames); + usedIndexNames.insert(idxName); + } + } + indexes_.emplace_back(std::move(idxName), rndGen_.RndIndexType({fldType}), rndGen_.RndArrayField(isArray), + rndGen_.RndSparseIndex(fldType), Index::Child{fldType, std::move(fldPath)}); + } + if (const auto& idx = indexes_.back(); + !idx.IsArray() && idx.IsSparse() == IsSparse::No && + std::get(idx.Content()).type != FieldType::Point) { // TODO remove point check after #1352 + scalarIndexes.push_back(indexes_.size() - 1); } } } + if (rndGen_.RndErr()) { + // Do not set PK index + return; + } std::vector ii; for (size_t i = 0, s = indexes_.size(); i < s; ++i) { const auto& idx = indexes_[i]; - if (!idx.isArray && - (std::holds_alternative(idx.content) || availablePkFieldType(std::get(idx.content).type))) { + if (!idx.IsArray() && idx.IsSparse() == IsSparse::No && availablePkIndexType(idx.Type()) && + (std::holds_alternative(idx.Content()) || availablePkFieldType(std::get(idx.Content()).type))) { ii.push_back(i); } } if (ii.empty()) { - if (!rndGen_.RndErr()) { - auto path = scheme_.AddRndPkField(rndGen_); - const auto fldType = scheme_.GetFieldType(path); - std::string name = rndGen_.UniqueName() ? rndGen_.IndexName(generatedNames) : scheme_.GetJsonPath(path); - indexes_.emplace_back(Index{std::move(name), Index::Child{fldType, std::move(path)}}); - indexes_.back().isArray = false; - indexes_.back().isPk = true; + auto path = scheme_.AddRndPkField(rndGen_); + const auto fldType = scheme_.GetFieldType(path); + std::string name; + if (rndGen_.UniqueName()) { + name = rndGen_.IndexName(usedIndexNames); + } else { + name = scheme_.GetJsonPath(path); + if (!usedIndexNames.insert(name).second) { + name = rndGen_.IndexName(usedIndexNames); + usedIndexNames.insert(name); + } } + indexes_.emplace_back(std::move(name), rndGen_.RndPkIndexType({fldType}), IsArray::No, IsSparse::No, + Index::Child{fldType, std::move(path)}); + indexes_.back().SetPk(); } else { - indexes_[rndGen_.RndWhich(ii)].isPk = true; + indexes_[rndGen_.RndWhich(ii)].SetPk(); } } -void Ns::AddIndex(Index& index, bool isSparse) { - if (isSparse) return; - std::visit(reindexer::overloaded{[&](const Index::Child& c) { scheme_.AddIndex(c.fieldPath, isSparse); }, +void Ns::AddIndexToScheme(const Index& index, size_t indexNumber) { + std::visit(reindexer::overloaded{[&](const Index::Child& c) { scheme_.AddIndex(c.fieldPath, indexNumber, index.IsSparse()); }, [&](const Index::Children& c) { for (const auto& child : c) { - scheme_.AddIndex(child.fieldPath, isSparse); + scheme_.AddIndex(child.fieldPath, indexNumber, index.IsSparse()); } }}, - index.content); + index.Content()); } void Ns::Dump(std::ostream& os) const { diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/ns.h b/cpp_src/gtests/tests/fixtures/fuzzing/ns.h index 53baf5a3e..6c93abb46 100644 --- a/cpp_src/gtests/tests/fixtures/fuzzing/ns.h +++ b/cpp_src/gtests/tests/fixtures/fuzzing/ns.h @@ -1,21 +1,23 @@ #pragma once -#include "index.h" #include "ns_scheme.h" #include "random_generator.h" namespace fuzzing { +class Index; + class Ns { public: - Ns(std::string name, std::ostream&, RandomGenerator::ErrFactorType errorFactor); - std::vector& GetIndexes() noexcept { return indexes_; } + Ns(std::string name, RandomGenerator::ErrFactorType errorFactor); + const std::vector& GetIndexes() const& noexcept { return indexes_; } + std::vector& GetIndexes() & noexcept { return indexes_; } + const std::vector& GetIndexes() const&& = delete; const std::string& GetName() const noexcept { return name_; } const NsScheme& GetScheme() const noexcept { return scheme_; } RandomGenerator& GetRandomGenerator() noexcept { return rndGen_; } - void AddIndex(Index&, bool isSparse); - void NewItem(reindexer::WrSerializer& ser) { scheme_.NewItem(ser, rndGen_); } - const std::vector& GetIndexes() const noexcept { return indexes_; } + void AddIndexToScheme(const Index&, size_t indexNumber); + void NewItem(reindexer::WrSerializer& ser) { scheme_.NewItem(ser, rndGen_, indexes_); } void Dump(std::ostream&) const; private: diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/ns_scheme.cc b/cpp_src/gtests/tests/fixtures/fuzzing/ns_scheme.cc index 8fb4e9e52..8c027dbc6 100644 --- a/cpp_src/gtests/tests/fixtures/fuzzing/ns_scheme.cc +++ b/cpp_src/gtests/tests/fixtures/fuzzing/ns_scheme.cc @@ -1,10 +1,12 @@ #include "ns_scheme.h" #include "core/cjson/jsonbuilder.h" +#include "index.h" +#include "random_generator.h" #include "tools/serializer.h" namespace fuzzing { -void NsScheme::NewItem(reindexer::WrSerializer& ser, RandomGenerator& rnd) { +void NsScheme::NewItem(reindexer::WrSerializer& ser, RandomGenerator& rnd, const std::vector& indexes) { ser.Reset(); if (rnd.RndErr()) { enum Err : uint8_t { Zero, Random, END = Random }; @@ -23,10 +25,195 @@ void NsScheme::NewItem(reindexer::WrSerializer& ser, RandomGenerator& rnd) { } } reindexer::JsonBuilder builder{ser}; - toJson(builder, std::get(ns_.content), rnd); + toJson(builder, std::get(ns_.content), rnd, indexes); } -void NsScheme::rndValueToJson(reindexer::JsonBuilder& builder, FieldType ft, std::string_view name, RandomGenerator& rnd) { +bool NsScheme::IsStruct(const FieldPath& path) const noexcept { + if (path.empty()) return true; + const Node::Children& ref = findLastContainer(path); + assertrx(ref.size() > path.back()); + return std::holds_alternative(ref[path.back()].content); +} + +bool NsScheme::IsPoint(const FieldPath& path) const noexcept { + if (path.empty()) return false; + const Node::Children& ref = findLastContainer(path); + assertrx(ref.size() > path.back()); + return !std::holds_alternative(ref[path.back()].content) && + std::get(ref[path.back()].content).type == FieldType::Point; +} + +bool NsScheme::isTtl(const std::vector& idxNumbers, const std::vector& indexes) noexcept { + for (size_t idx : idxNumbers) { + assertrx(idx < indexes.size()); + if (indexes[idx].Type() == IndexType::Ttl) { + return true; + } + } + return false; +} + +bool NsScheme::IsTtl(const FieldPath& path, const std::vector& indexes) const noexcept { + if (path.empty()) return false; + const Node::Children& ref = findLastContainer(path); + assertrx(ref.size() > path.back()); + if (std::holds_alternative(ref[path.back()].content)) { + return false; + } + return isTtl(std::get(ref[path.back()].content).indexes, indexes); +} + +size_t NsScheme::FieldsCount(const FieldPath& path) const noexcept { + if (path.empty()) { + return std::get(ns_.content).size(); + } + const Node::Children& ref = findLastContainer(path); + assertrx(ref.size() > path.back()); + return std::visit(reindexer::overloaded{[](const Node::Child&) noexcept -> size_t { + assertrx(false); + return 0; + }, + [](const Node::Children& c) noexcept { return c.size(); }}, + ref[path.back()].content); +} + +IsArray NsScheme::IsArray(const FieldPath& path) const noexcept { + if (path.empty()) return ns_.array; + const Node::Children* ptr = &std::get(ns_.content); + for (size_t i = 0, s = path.size() - 1; i < s; ++i) { + assertrx(ptr->size() > path[i]); + const auto& idx = (*ptr)[path[i]]; + if (idx.array == IsArray::Yes) return IsArray::Yes; + std::visit( + reindexer::overloaded{[&ptr](const Node::Children& c) noexcept { ptr = &c; }, [](const Node::Child&) noexcept { assertrx(0); }}, + idx.content); + } + assertrx(ptr->size() > path.back()); + return (*ptr)[path.back()].array; +} + +FieldType NsScheme::GetFieldType(const FieldPath& path) const noexcept { + assertrx(!path.empty()); + const Node::Children& ref = findLastContainer(path); + assertrx(ref.size() > path.back()); + return std::visit(reindexer::overloaded{[](const Node::Child& c) noexcept { return c.type; }, + [](const Node::Children&) noexcept { return FieldType::Struct; }}, + ref[path.back()].content); +} + +void NsScheme::SetFieldType(const FieldPath& path, FieldType ft) noexcept { + assertrx(!path.empty()); + Node::Children& ref = findLastContainer(path); + assertrx(ref.size() > path.back()); + return std::visit(reindexer::overloaded{[ft](Node::Child& c) noexcept { c.type = ft; }, [](Node::Children&) noexcept { assertrx(0); }}, + ref[path.back()].content); +} + +std::string NsScheme::GetJsonPath(const FieldPath& path) const noexcept { + if (path.empty()) return {}; + std::string res; + const Node::Children* ptr = &std::get(ns_.content); + for (size_t i = 0, s = path.size() - 1; i < s; ++i) { + assertrx(ptr->size() > path[i]); + const auto& idx = (*ptr)[path[i]]; + res += idx.name; + std::visit( + reindexer::overloaded{[&ptr](const Node::Children& c) noexcept { ptr = &c; }, [](const Node::Child&) noexcept { assertrx(0); }}, + idx.content); + res += '.'; + } + assertrx(ptr->size() > path.back()); + res += (*ptr)[path.back()].name; + return res; +} + +void NsScheme::AddIndex(const FieldPath& path, size_t index, IsSparse isSparse) { + assertrx(!path.empty()); + if (isSparse == IsSparse::No) { + ns_.sparse = IsSparse::No; + } + Node::Children* ptr = &std::get(ns_.content); + for (size_t i = 0, s = path.size() - 1; i < s; ++i) { + assertrx(ptr->size() > path[i]); + if (isSparse == IsSparse::No) { + (*ptr)[path[i]].sparse = IsSparse::No; + } + std::visit(reindexer::overloaded{[&ptr](Node::Children& c) noexcept { ptr = &c; }, [](Node::Child&) noexcept { assertrx(0); }}, + (*ptr)[path[i]].content); + } + assertrx(ptr->size() > path.back()); + addIndex((*ptr)[path.back()], index, isSparse); +} + +FieldPath NsScheme::AddRndPkField(RandomGenerator& rnd) { + auto& children = std::get(ns_.content); + children.emplace_back(Node{rnd.FieldName(generatedNames_), Node::Child{rnd.RndPkIndexFieldType()}}); + children.back().array = IsArray::No; + children.back().sparse = IsSparse::No; + return {children.size() - 1}; +} + +void NsScheme::addIndex(Node& node, size_t index, IsSparse isSparse) { + if (isSparse == IsSparse::No) { + node.sparse = IsSparse::No; + } + std::visit(reindexer::overloaded{[index](Node::Child& c) noexcept { c.indexes.push_back(index); }, + [](Node::Children&) noexcept { assertrx(0); }}, + node.content); +} + +void NsScheme::fillChildren(Node::Children& children, RandomGenerator& rnd, unsigned level, bool& canBeArray, bool& canBeSparse) { + const size_t fieldsCount = rnd.FieldsCount(level == 0); + children.reserve(fieldsCount); + for (size_t i = 0; i < fieldsCount; ++i) { + auto fName = rnd.FieldName(generatedNames_); + const auto type = rnd.RndFieldType(level); + if (type == FieldType::Struct) { + children.emplace_back(Node{std::move(fName), Node::Children{}}); + fillChildren(std::get(children.back().content), rnd, level + 1, canBeArray, canBeSparse); + if (canBeArray || rnd.RndErr()) { + children.back().array = rnd.RndArrayField(); + } + if (!canBeSparse && !rnd.RndErr()) { + children.back().sparse = IsSparse::No; + } + } else { + children.emplace_back(Node{std::move(fName), Node::Child{type}}); + if (type == FieldType::Point) { + canBeSparse = false; + canBeArray = false; + children.back().sparse = IsSparse::No; + } + if (canBeArray || rnd.RndErr()) { + children.back().array = rnd.RndArrayField(); + } + } + } +} + +const NsScheme::Node::Children& NsScheme::findLastContainer(const FieldPath& path) const noexcept { + const Node::Children* ptr = &std::get(ns_.content); + for (size_t i = 0, s = path.size() - 1; i < s; ++i) { + assertrx(ptr->size() > path[i]); + std::visit( + reindexer::overloaded{[&ptr](const Node::Children& c) noexcept { ptr = &c; }, [](const Node::Child&) noexcept { assertrx(0); }}, + (*ptr)[path[i]].content); + } + return *ptr; +} + +NsScheme::Node::Children& NsScheme::findLastContainer(const FieldPath& path) noexcept { + Node::Children* ptr = &std::get(ns_.content); + for (size_t i = 0, s = path.size() - 1; i < s; ++i) { + assertrx(ptr->size() > path[i]); + std::visit(reindexer::overloaded{[&ptr](Node::Children& c) noexcept { ptr = &c; }, [](Node::Child&) noexcept { assertrx(0); }}, + (*ptr)[path[i]].content); + } + return *ptr; +} + +void NsScheme::rndValueToJson(reindexer::JsonBuilder& builder, FieldType ft, std::string_view name, const std::vector& idxNumbers, + const std::vector& indexes, RandomGenerator& rnd) { switch (ft) { case FieldType::Bool: builder.Put(name, rnd.RndBool(0.5)); @@ -35,7 +222,11 @@ void NsScheme::rndValueToJson(reindexer::JsonBuilder& builder, FieldType ft, std builder.Put(name, rnd.RndIntValue()); break; case FieldType::Int64: - builder.Put(name, rnd.RndInt64Value()); + if (isTtl(idxNumbers, indexes)) { + builder.Put(name, rnd.RndTtlValue()); + } else { + builder.Put(name, rnd.RndInt64Value()); + } break; case FieldType::Double: builder.Put(name, rnd.RndDoubleValue()); @@ -54,41 +245,44 @@ void NsScheme::rndValueToJson(reindexer::JsonBuilder& builder, FieldType ft, std Node::Children children; fillChildren(children, rnd, 2, canBeArray, canBeSparse); auto obj = builder.Object(name); - toJson(obj, children, rnd); + toJson(obj, children, rnd, indexes); } break; default: assertrx(0); } } -void NsScheme::toJson(reindexer::JsonBuilder& builder, const Node::Children& children, RandomGenerator& rnd) { +void NsScheme::toJson(reindexer::JsonBuilder& builder, const Node::Children& children, RandomGenerator& rnd, + const std::vector& indexes) { for (const Node& n : children) { if (!rnd.NeedThisNode(n.sparse)) continue; - if (rnd.RndArrayField(n.array)) { + if (rnd.RndArrayField(n.array) == IsArray::Yes) { auto arr = builder.Array(n.name); const size_t arrSize = rnd.ArraySize(); for (size_t i = 0; i < arrSize; ++i) { if (rnd.RndErr()) { - rndValueToJson(arr, rnd.RndFieldType(), {}, rnd); + rndValueToJson(arr, rnd.RndFieldType(), {}, {}, indexes, rnd); } else { - std::visit(reindexer::overloaded{[&](const Node::Child& c) { rndValueToJson(arr, c.type, {}, rnd); }, - [&](const Node::Children& c) { - auto obj = arr.Object(); - toJson(obj, c, rnd); - }}, - n.content); + std::visit( + reindexer::overloaded{[&](const Node::Child& c) { rndValueToJson(arr, c.type, {}, c.indexes, indexes, rnd); }, + [&](const Node::Children& c) { + auto obj = arr.Object(); + toJson(obj, c, rnd, indexes); + }}, + n.content); } } } else { if (rnd.RndErr()) { - rndValueToJson(builder, rnd.RndFieldType(), n.name, rnd); + rndValueToJson(builder, rnd.RndFieldType(), n.name, {}, indexes, rnd); } else { - std::visit(reindexer::overloaded{[&](const Node::Child& c) { rndValueToJson(builder, c.type, n.name, rnd); }, - [&](const Node::Children& c) { - auto obj = builder.Object(n.name); - toJson(obj, c, rnd); - }}, - n.content); + std::visit( + reindexer::overloaded{[&](const Node::Child& c) { rndValueToJson(builder, c.type, n.name, c.indexes, indexes, rnd); }, + [&](const Node::Children& c) { + auto obj = builder.Object(n.name); + toJson(obj, c, rnd, indexes); + }}, + n.content); } } } @@ -100,9 +294,9 @@ void NsScheme::Node::Dump(std::ostream& os, size_t offset) const { for (size_t i = 0; i <= offset; ++i) os << " "; os << "name: " << name << '\n'; for (size_t i = 0; i <= offset; ++i) os << " "; - os << "sparse: " << (sparse ? "true" : "false") << '\n'; + os << "sparse: " << std::boolalpha << (sparse == IsSparse::Yes) << '\n'; for (size_t i = 0; i <= offset; ++i) os << " "; - os << "array: " << (array ? "true" : "false") << '\n'; + os << "array: " << std::boolalpha << (array == IsArray::Yes) << '\n'; std::visit(reindexer::overloaded{[&](const Child& child) { for (size_t i = 0; i <= offset; ++i) os << " "; os << "type: " << child.type << '\n'; diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/ns_scheme.h b/cpp_src/gtests/tests/fixtures/fuzzing/ns_scheme.h index 675173c36..a63a76e38 100644 --- a/cpp_src/gtests/tests/fixtures/fuzzing/ns_scheme.h +++ b/cpp_src/gtests/tests/fixtures/fuzzing/ns_scheme.h @@ -1,10 +1,10 @@ #pragma once #include +#include #include #include -#include "estl/overloaded.h" -#include "random_generator.h" +#include "types.h" namespace reindexer { @@ -15,17 +15,22 @@ class JsonBuilder; namespace fuzzing { +class RandomGenerator; +class Index; + class NsScheme { struct Node { using Children = std::vector; struct Child { + Child(FieldType t) noexcept : type{t} {} FieldType type; + std::vector indexes; }; std::string name; std::variant content; - bool sparse{true}; - bool array{false}; + IsSparse sparse{IsSparse::Yes}; + IsArray array{IsArray::No}; void Dump(std::ostream&, size_t offset) const; }; @@ -34,164 +39,29 @@ class NsScheme { bool canBeArray = true, canBeSparse = true; fillChildren(std::get(ns_.content), rnd, 0, canBeArray, canBeSparse); } - size_t FieldsCount(const FieldPath& path) const noexcept { - if (path.empty()) { - return std::get(ns_.content).size(); - } - const Node::Children& ref = findLastContainer(path); - assertrx(ref.size() > path.back()); - return std::visit(reindexer::overloaded{[](const Node::Child&) noexcept -> size_t { - assertrx(false); - return 0; - }, - [](const Node::Children& c) noexcept { return c.size(); }}, - ref[path.back()].content); - } - bool IsStruct(const FieldPath& path) const noexcept { - if (path.empty()) return true; - const Node::Children& ref = findLastContainer(path); - assertrx(ref.size() > path.back()); - return std::holds_alternative(ref[path.back()].content); - } - bool IsPoint(const FieldPath& path) const noexcept { - if (path.empty()) return false; - const Node::Children& ref = findLastContainer(path); - assertrx(ref.size() > path.back()); - return !std::holds_alternative(ref[path.back()].content) && - std::get(ref[path.back()].content).type == FieldType::Point; - } - bool IsArray(const FieldPath& path) const noexcept { - if (path.empty()) return ns_.array; - const Node::Children* ptr = &std::get(ns_.content); - for (size_t i = 0, s = path.size() - 1; i < s; ++i) { - assertrx(ptr->size() > path[i]); - const auto& idx = (*ptr)[path[i]]; - if (idx.array) return true; - std::visit(reindexer::overloaded{[&ptr](const Node::Children& c) noexcept { ptr = &c; }, - [](const Node::Child&) noexcept { assert(0); }}, - idx.content); - } - assertrx(ptr->size() > path.back()); - return (*ptr)[path.back()].array; - } - FieldType GetFieldType(const FieldPath& path) const noexcept { - assertrx(!path.empty()); - const Node::Children& ref = findLastContainer(path); - assertrx(ref.size() > path.back()); - return std::visit(reindexer::overloaded{[](const Node::Child& c) noexcept { return c.type; }, - [](const Node::Children&) noexcept { return FieldType::Struct; }}, - ref[path.back()].content); - } - void SetFieldType(const FieldPath& path, FieldType ft) noexcept { - assertrx(!path.empty()); - Node::Children& ref = findLastContainer(path); - assertrx(ref.size() > path.back()); - return std::visit( - reindexer::overloaded{[ft](Node::Child& c) noexcept { c.type = ft; }, [](Node::Children&) noexcept { assert(0); }}, - ref[path.back()].content); - } - std::string GetJsonPath(const FieldPath& path) const noexcept { - if (path.empty()) return {}; - std::string res; - const Node::Children* ptr = &std::get(ns_.content); - for (size_t i = 0, s = path.size() - 1; i < s; ++i) { - assertrx(ptr->size() > path[i]); - const auto& idx = (*ptr)[path[i]]; - res += idx.name; - std::visit(reindexer::overloaded{[&ptr](const Node::Children& c) noexcept { ptr = &c; }, - [](const Node::Child&) noexcept { assert(0); }}, - idx.content); - res += '.'; - } - assertrx(ptr->size() > path.back()); - res += (*ptr)[path.back()].name; - return res; - } - void AddIndex(const FieldPath& path, bool isSparse) { - if (path.empty()) return; - if (!isSparse) ns_.sparse = false; - Node::Children* ptr = &std::get(ns_.content); - for (size_t i = 0, s = path.size() - 1; i < s; ++i) { - assertrx(ptr->size() > path[i]); - if (!isSparse) { - (*ptr)[path[i]].sparse = false; - } - std::visit(reindexer::overloaded{[&ptr](Node::Children& c) noexcept { ptr = &c; }, [](Node::Child&) noexcept { assert(0); }}, - (*ptr)[path[i]].content); - } - assertrx(ptr->size() > path.back()); - mark((*ptr)[path.back()], isSparse); - } - void NewItem(reindexer::WrSerializer&, RandomGenerator&); + size_t FieldsCount(const FieldPath&) const noexcept; + bool IsStruct(const FieldPath&) const noexcept; + bool IsPoint(const FieldPath&) const noexcept; + bool IsTtl(const FieldPath&, const std::vector&) const noexcept; + enum IsArray IsArray(const FieldPath&) const noexcept; + FieldType GetFieldType(const FieldPath&) const noexcept; + void SetFieldType(const FieldPath&, FieldType) noexcept; + std::string GetJsonPath(const FieldPath&) const noexcept; + void AddIndex(const FieldPath&, size_t index, IsSparse); + void NewItem(reindexer::WrSerializer&, RandomGenerator&, const std::vector&); void Dump(std::ostream& os, size_t offset) const { ns_.Dump(os, offset); } - FieldPath AddRndPkField(RandomGenerator& rnd) { - auto& children = std::get(ns_.content); - children.emplace_back(Node{rnd.FieldName(generatedNames_), Node::Child{rnd.RndPkIndexFieldType()}}); - children.back().array = false; - children.back().sparse = false; - return {children.size() - 1}; - } + FieldPath AddRndPkField(RandomGenerator&); private: - static void mark(Node& node, bool isSparse) { - if (!isSparse) { - node.sparse = false; - } - std::visit(reindexer::overloaded{[](Node::Child&) noexcept {}, - [isSparse](Node::Children& c) noexcept { - for (Node& n : c) mark(n, isSparse); - }}, - node.content); - } - void fillChildren(Node::Children& children, RandomGenerator& rnd, unsigned level, bool& canBeArray, bool& canBeSparse) { - const size_t fieldsCount = rnd.FieldsCount(level == 0); - children.reserve(fieldsCount); - for (size_t i = 0; i < fieldsCount; ++i) { - auto fName = rnd.FieldName(generatedNames_); - const auto type = rnd.RndFieldType(level); - if (type == FieldType::Struct) { - children.emplace_back(Node{std::move(fName), Node::Children{}}); - fillChildren(std::get(children.back().content), rnd, level + 1, canBeArray, canBeSparse); - if (canBeArray || rnd.RndErr()) { - children.back().array = rnd.RndArrayField(); - } - if (!canBeSparse && !rnd.RndErr()) { - children.back().sparse = false; - } - } else { - children.emplace_back(Node{std::move(fName), Node::Child{type}}); - if (type == FieldType::Point) { - canBeSparse = false; - canBeArray = false; - children.back().sparse = false; - } - if (canBeArray || rnd.RndErr()) { - children.back().array = rnd.RndArrayField(); - } - } - } - } - const Node::Children& findLastContainer(const FieldPath& path) const noexcept { - const Node::Children* ptr = &std::get(ns_.content); - for (size_t i = 0, s = path.size() - 1; i < s; ++i) { - assertrx(ptr->size() > path[i]); - std::visit(reindexer::overloaded{[&ptr](const Node::Children& c) noexcept { ptr = &c; }, - [](const Node::Child&) noexcept { assert(0); }}, - (*ptr)[path[i]].content); - } - return *ptr; - } - Node::Children& findLastContainer(const FieldPath& path) noexcept { - Node::Children* ptr = &std::get(ns_.content); - for (size_t i = 0, s = path.size() - 1; i < s; ++i) { - assertrx(ptr->size() > path[i]); - std::visit(reindexer::overloaded{[&ptr](Node::Children& c) noexcept { ptr = &c; }, [](Node::Child&) noexcept { assert(0); }}, - (*ptr)[path[i]].content); - } - return *ptr; - } - void toJson(reindexer::JsonBuilder&, const Node::Children&, RandomGenerator&); - void rndValueToJson(reindexer::JsonBuilder&, FieldType, std::string_view name, RandomGenerator&); + static void addIndex(Node&, size_t index, IsSparse); + void fillChildren(Node::Children&, RandomGenerator&, unsigned level, bool& canBeArray, bool& canBeSparse); + const Node::Children& findLastContainer(const FieldPath&) const noexcept; + Node::Children& findLastContainer(const FieldPath&) noexcept; + void toJson(reindexer::JsonBuilder&, const Node::Children&, RandomGenerator&, const std::vector&); + void rndValueToJson(reindexer::JsonBuilder&, FieldType, std::string_view name, const std::vector& idxNumbers, + const std::vector&, RandomGenerator&); + static bool isTtl(const std::vector& idxNumbers, const std::vector&) noexcept; + Node ns_; std::unordered_set generatedNames_; }; diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/query_generator.cc b/cpp_src/gtests/tests/fixtures/fuzzing/query_generator.cc index 520c3abe7..da0f1d081 100644 --- a/cpp_src/gtests/tests/fixtures/fuzzing/query_generator.cc +++ b/cpp_src/gtests/tests/fixtures/fuzzing/query_generator.cc @@ -1,12 +1,13 @@ #include "query_generator.h" #include "core/query/query.h" +#include "index.h" +#include "ns.h" namespace fuzzing { reindexer::Query QueryGenerator::operator()() { if (namespaces_.empty() || rndGen_.RndErr()) { - std::unordered_set generatedNames; - return reindexer::Query{rndGen_.NsName(generatedNames)}; + return reindexer::Query{rndGen_.GenerateNsName()}; } const auto& ns = rndGen_.RndWhich(namespaces_); reindexer::Query query{ns.GetName()}; @@ -15,21 +16,24 @@ reindexer::Query QueryGenerator::operator()() { case Index: if (const auto& indexes = ns.GetIndexes(); !indexes.empty()) { const auto& idx = rndGen_.RndWhich(indexes); - std::visit(reindexer::overloaded{[&](const Index::Child& c) { rndGen_.RndWhere(query, idx.name, {c.type}); }, + std::visit(reindexer::overloaded{[&](const Index::Child& c) { rndGen_.RndWhere(query, idx.Name(), c.type, idx.Type()); }, [&](const Index::Children& c) { std::vector types; types.reserve(c.size()); for (const auto& child : c) types.push_back(child.type); - rndGen_.RndWhere(query, idx.name, types); + rndGen_.RndWhereComposite(query, idx.Name(), std::move(types), idx.Type()); }}, - idx.content); + idx.Content()); } break; case Field: { const auto path = rndGen_.RndField(ns.GetScheme()); const FieldType type = ns.GetScheme().GetFieldType(path); - if (type != FieldType::Struct) { - rndGen_.RndWhere(query, ns.GetScheme().GetJsonPath(path), {type}); + if (type == FieldType::Struct) { // TODO object find + } else { + const std::optional indexType = + ns.GetScheme().IsTtl(path, ns.GetIndexes()) ? IndexType::Ttl : std::optional{}; + rndGen_.RndWhere(query, ns.GetScheme().GetJsonPath(path), type, indexType); } } break; case Empty: diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/query_generator.h b/cpp_src/gtests/tests/fixtures/fuzzing/query_generator.h index 2724b23c5..c2644492b 100644 --- a/cpp_src/gtests/tests/fixtures/fuzzing/query_generator.h +++ b/cpp_src/gtests/tests/fixtures/fuzzing/query_generator.h @@ -1,6 +1,6 @@ #pragma once -#include "ns.h" +#include "random_generator.h" namespace reindexer { @@ -10,10 +10,11 @@ class Query; namespace fuzzing { +class Ns; + class QueryGenerator { public: - QueryGenerator(const std::vector& nss, std::ostream& os, RandomGenerator::ErrFactorType errorFactor) - : namespaces_{nss}, rndGen_{os, errorFactor} {} + QueryGenerator(const std::vector& nss, RandomGenerator::ErrFactorType errorFactor) : namespaces_{nss}, rndGen_{errorFactor} {} reindexer::Query operator()(); private: diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/random_generator.cc b/cpp_src/gtests/tests/fixtures/fuzzing/random_generator.cc index 25e0f640e..e207a6667 100644 --- a/cpp_src/gtests/tests/fixtures/fuzzing/random_generator.cc +++ b/cpp_src/gtests/tests/fixtures/fuzzing/random_generator.cc @@ -1,17 +1,97 @@ #include "random_generator.h" +#include +#include #include +#include +#include "core/payload/fieldsset.h" #include "core/query/query.h" +#include "index.h" #include "ns_scheme.h" namespace fuzzing { -RandomGenerator::RandomGenerator(std::ostream& os, ErrFactorType errorFactor) - : gen_(std::chrono::system_clock::now().time_since_epoch().count()), errFactor_{errorFactor} { +std::string& RandomGenerator::out() noexcept { + static std::string outStr; + return outStr; +} + +std::unique_ptr& RandomGenerator::in() noexcept { + static std::unique_ptr f; + return f; +} + +void RandomGenerator::SetOut(std::string o) { + ASSERT_TRUE(out().empty()); + ASSERT_FALSE(in()); + out() = std::move(o); + { + std::ifstream f{out()}; + ASSERT_FALSE(f.is_open()) << "File '" << out() << "' already exists"; + } +} + +void RandomGenerator::SetIn(const std::string& i) { + ASSERT_FALSE(in()); + ASSERT_TRUE(out().empty()); + in() = std::make_unique(i); + ASSERT_TRUE(in()->is_open()) << "Cannot open file '" << i << '\''; + in()->exceptions(std::ios_base::badbit | std::ios_base::failbit | std::ios_base::eofbit); +} + +RandomGenerator::RandomEngine RandomGenerator::createRandomEngine() { + if (in()) { + RandomEngine ret; + std::string buf; + std::getline(*in(), buf); + std::istringstream ss{buf}; + ss >> ret; + return ret; + } else { + RandomEngine ret(std::chrono::system_clock::now().time_since_epoch().count()); + if (!out().empty()) { + std::ofstream file{out(), std::ios_base::app}; + if (file.is_open()) { + file.exceptions(std::ios_base::badbit | std::ios_base::failbit | std::ios_base::eofbit); + file << ret << std::endl; + } else { + EXPECT_TRUE(false) << "Cannot open file '" << out() << '\''; + } + } + return ret; + } +} + +RandomGenerator::RandomGenerator(ErrFactorType errorFactor) : gen_{createRandomEngine()}, errFactor_{errorFactor} { assertrx(errFactor_.first < errFactor_.second); errParams_ = {static_cast(errFactor_.second - errFactor_.first), static_cast(errFactor_.first)}; - os << gen_ << std::endl; } -RandomGenerator::RandomGenerator(std::istream& is) { is >> gen_; } + +size_t RandomGenerator::FieldsCount(bool firstLevel) { + if (RndErr()) { + enum Err : uint8_t { Zero, TooMany, END = TooMany }; + switch (RndWhich()) { + case Zero: + return 0; + case TooMany: + return RndInt(0, 10'000); + default: + assertrx(0); + } + } + if (firstLevel) { + enum Size : uint8_t { Normal, Long, END = Long }; + switch (RndWhich()) { + case Normal: + return RndInt(1, 9); + case Long: + return RndInt(10, 100); + default: + assertrx(false); + std::abort(); + } + } + return RndInt(1, 5); +} std::string RandomGenerator::FieldName(std::unordered_set& generatedNames) { // TODO static constexpr char alfas[] = "_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; @@ -148,7 +228,7 @@ FieldPath RandomGenerator::RndScalarField(const NsScheme& nsScheme) { const int end = idx + size; while (idx < end) { res.back() = idx % size; - if (!nsScheme.IsArray(res) && !nsScheme.IsPoint(res)) break; + if (nsScheme.IsArray(res) == IsArray::No && !nsScheme.IsPoint(res)) break; ++idx; } if (idx == end) return {}; @@ -156,7 +236,7 @@ FieldPath RandomGenerator::RndScalarField(const NsScheme& nsScheme) { return res; } -std::string RandomGenerator::IndexFieldType(fuzzing::FieldType ft) { +std::string RandomGenerator::IndexFieldType(FieldType ft) { static const std::string types[] = {"bool", "int", "int64", "double", "string", "uuid", "point", "composite"}; if (RndErr()) { // TODO rnd string @@ -167,52 +247,158 @@ std::string RandomGenerator::IndexFieldType(fuzzing::FieldType ft) { return types[i]; } -std::string RandomGenerator::RndIndexType(fuzzing::FieldType ft, bool pk) { - static const std::string types[] = {"-", "hash", "tree", "ttl", "text", "fuzzytext", "rtree"}; - static const std::vector availableTypes[] = { - {0}, // Bool - {0, 1, 2}, // Int - {0, 1, 2, 3}, // Int64 - {0, 2}, // Double - {0, 1, 2 /*, 4, 5*/}, // String // TODO FT indexes - {1}, // Uuid - {6}, // Point - {1, 2 /*, 4, 5*/} // Struct // TODO FT indexes +IndexType RandomGenerator::RndIndexType(IndexType it) { + if (RndErr()) { + return RndWhich(); // TODO + } + return it; +} + +template * Availables> +IndexType RandomGenerator::rndIndexType(const std::vector& fieldTypes) { + if (RndErr()) { + // TODO rnd string + return RndWhich(); // TODO + } + assertrx(!fieldTypes.empty()); + std::vector availables; + { + const size_t f = static_cast(fieldTypes[0]); + assertrx(f < N); + availables = Availables[f]; + } + for (size_t i = 1, s = fieldTypes.size(); i < s; ++i) { + const size_t f = static_cast(fieldTypes[i]); + std::vector tmp; + tmp.reserve(availables.size()); + assertrx(f < N); + std::set_intersection(availables.begin(), availables.end(), Availables[f].begin(), Availables[f].end(), std::back_inserter(tmp)); + availables = tmp; + } + if (availables.empty()) { + return RndWhich(); // TODO + } else { + return RndWhich(availables); + } +} + +IndexType RandomGenerator::RndIndexType(const std::vector& fieldTypes) { + static const std::vector availableTypes[] = { + {IndexType::Store}, // Bool + {IndexType::Store, IndexType::Hash, IndexType::Tree}, // Int + {IndexType::Store, IndexType::Hash, IndexType::Tree, IndexType::Ttl}, // Int64 + {IndexType::Store, IndexType::Tree}, // Double + {IndexType::Store, IndexType::Hash, IndexType::Tree}, // String // TODO IndexType::FastFT IndexType::FuzzyFT + {IndexType::Hash}, // Uuid + {IndexType::RTree}, // Point + {IndexType::Hash, IndexType::Tree} // Struct // TODO IndexType::FastFT IndexType::FuzzyFT }; - static const std::vector availablePkTypes[] = { - {}, // Bool - {1, 2}, // Int - {1, 2, 3}, // Int64 - {2}, // Double - {1, 2}, // String - {1}, // Uuid - {}, // Point - {1, 2} // Struct + return rndIndexType, availableTypes>(fieldTypes); +} + +IndexType RandomGenerator::RndPkIndexType(const std::vector& fieldTypes) { + static const std::vector availablePkTypes[] = { + {}, // Bool + {IndexType::Hash, IndexType::Tree}, // Int + {IndexType::Hash, IndexType::Tree, IndexType::Ttl}, // Int64 + {IndexType::Tree}, // Double + {IndexType::Hash, IndexType::Tree}, // String + {IndexType::Hash}, // Uuid + {}, // Point + {IndexType::Hash, IndexType::Tree} // Struct }; + return rndIndexType, availablePkTypes>(fieldTypes); +} + +size_t RandomGenerator::ArraySize() { + if (RndErr()) return RndInt(0, 100'000); + enum Size : uint8_t { Short, Normal, Long, VeryLong, END = VeryLong }; + switch (RndWhich()) { + case Short: + return RndInt(0, 5); + case Normal: + return RndInt(6, 20); + case Long: + return RndInt(21, 200); + case VeryLong: + return RndInt(201, 10'000); + default: + assertrx(false); + std::abort(); + } +} + +size_t RandomGenerator::IndexesCount() { if (RndErr()) { - // TODO rnd string - return RndWhich(types); + enum Err : uint8_t { Zero, TooMany, END = TooMany }; + switch (RndWhich()) { + case Zero: + return 0; + case TooMany: + return RndInt(reindexer::kMaxIndexes, 5 + reindexer::kMaxIndexes); + default: + assertrx(0); + } } - const size_t i = static_cast(ft); - size_t n; - if (pk) { - assertrx(i < std::size(availablePkTypes)); - if (availablePkTypes[i].empty()) { - return RndWhich(types); + enum Count : uint8_t { Few, Normal, Many, TooMany, END = TooMany }; + switch (RndWhich()) { + case Few: + return RndInt(1, 3); + case Normal: + return RndInt(4, 20); + case Many: + return RndInt(21, 63); + case TooMany: + return RndInt(64, reindexer::kMaxIndexes); + default: + assertrx(false); + std::abort(); + } +} + +size_t RandomGenerator::compositeIndexSize(size_t scalarIndexesCount) { + if (RndErr()) { + enum Err : uint8_t { Zero, /*One,*/ TooMany, END = TooMany }; + switch (RndWhich()) { + case Zero: + return 0; + /*case One: + return 1;*/ + case TooMany: + return RndInt(0, 10'000); + default: + assertrx(0); + } + } + assertrx(scalarIndexesCount >= 1); + return RndInt(1, scalarIndexesCount); +} + +std::vector RandomGenerator::RndFieldsForCompositeIndex(const std::vector& scalarIndexes) { + std::vector result; + const size_t count = compositeIndexSize(scalarIndexes.size()); + result.reserve(count); + const bool uniqueFields = count <= scalarIndexes.size() && !RndErr(); + // TODO unexisted and not indexed fields + if (uniqueFields) { + auto scalars = scalarIndexes; + while (result.size() < count) { + const size_t idx = rndSize(0, scalars.size() - 1); + result.push_back(scalars[idx]); + scalars.erase(scalars.begin() + idx); } - n = RndWhich(availablePkTypes[i]); } else { - assertrx(i < std::size(availableTypes)); - n = RndWhich(availableTypes[i]); + while (result.size() < count) { + result.push_back(scalarIndexes[rndSize(0, scalarIndexes.size() - 1)]); + } } - assertrx(n < std::size(types)); - return types[n]; + return result; } template <> constexpr size_t RandomGenerator::itemsCount = CondType::CondDWithin + 1; -CondType RandomGenerator::rndCond(fuzzing::FieldType ft) { // TODO array +CondType RandomGenerator::rndCond(FieldType ft) { // TODO array if (RndErr()) { return RndWhich(); } @@ -240,8 +426,6 @@ std::string RandomGenerator::rndStrUuidValue(bool noErrors) { if (!noErrors && RndErr()) { err = RndWhich(); } - std::string res; - if (err == Empty) return res; size_t size = 32; switch (err) { case Short: @@ -253,15 +437,17 @@ std::string RandomGenerator::rndStrUuidValue(bool noErrors) { case TooLong: size = RndInt(51, 100'000); break; - case NoErrors: case Empty: + return {}; + case NoErrors: case WrongVariant: case WrongChar: break; default: - assert(0); + assertrx(0); abort(); } + std::string res; res.reserve(size + 4); if (RndBool(0.001)) { res = std::string(std::string::size_type{size}, '0'); @@ -293,94 +479,127 @@ std::string RandomGenerator::rndStrUuidValue(bool noErrors) { reindexer::Uuid RandomGenerator::rndUuidValue() { return reindexer::Uuid{rndStrUuidValue(true)}; } -void RandomGenerator::RndWhere(reindexer::Query& query, const std::string& field, - const std::vector& types) { // TODO array +int64_t RandomGenerator::RndTtlValue() { + const int64_t now = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + // TODO uncomment this after TTL subscribe done + /*enum Size : uint8_t { Negative, FarPast, Past, Now, Future, FarFuture, AnyShort, Any, END = Any }; + switch (RndWhich()) { case Negative: return rndInt64(std::numeric_limits::min(), 0); case FarPast: + return rndInt64(0, now - 10'000); + case Past: + return rndInt64(now - 10'000, now - 10); + case Now: + return rndInt64(now - 10, now + 10); + case Future: + return rndInt64(now + 10, now + 10'000); + case FarFuture: + return rndInt64(now + 10'000, std::numeric_limits::max()); + case AnyShort: + return rndInt64(-50, 50); + case Any: + return rndInt64(std::numeric_limits::min(), std::numeric_limits::max()); + default: + assertrx(false); + std::abort(); + }*/ + return rndInt64(now + 10'000, std::numeric_limits::max()); +} + +void RandomGenerator::RndWhere(reindexer::Query& query, const std::string& field, FieldType fieldType, + std::optional indexType) { // TODO array + if (RndErr()) { + return RndWhereComposite(query, field, RndFieldTypesArray({fieldType}), indexType); + } std::unordered_set generatedNames; - assertrx(!types.empty()); const std::string fldName = FieldName(field, generatedNames); - const auto type = types.size() > 1 ? fuzzing::FieldType::Struct : types[0]; - const auto cond = rndCond(type); - switch (RndFieldType(type)) { - case fuzzing::FieldType::Bool: + const auto cond = rndCond(fieldType); + switch (RndFieldType(fieldType)) { + case FieldType::Bool: query.Where(fldName, cond, RndBool(0.5)); break; - case fuzzing::FieldType::Int: + case FieldType::Int: query.Where(fldName, cond, RndIntValue()); break; - case fuzzing::FieldType::Int64: - query.Where(fldName, cond, RndInt64Value()); + case FieldType::Int64: + if (indexType == IndexType::Ttl) { + query.Where(fldName, cond, RndTtlValue()); + } else { + query.Where(fldName, cond, RndInt64Value()); + } break; - case fuzzing::FieldType::Double: + case FieldType::Double: query.Where(fldName, cond, RndDoubleValue()); break; - case fuzzing::FieldType::String: + case FieldType::String: query.Where(fldName, cond, RndStringValue()); break; - case fuzzing::FieldType::Uuid: + case FieldType::Uuid: if (RndBool(0.5)) { query.Where(fldName, cond, rndUuidValue()); } else { query.Where(fldName, cond, rndStrUuidValue(false)); } break; - case fuzzing::FieldType::Point: + case FieldType::Point: query.Where(fldName, cond, {reindexer::Variant{reindexer::Point{RndDoubleValue(), RndDoubleValue()}}, reindexer::Variant{RndErr() ? RndDoubleValue() : std::abs(RndDoubleValue())}}); break; - case fuzzing::FieldType::Struct: // TODO - if (type == fuzzing::FieldType::Struct) { - } else { - } + case FieldType::Struct: // TODO break; default: assertrx(0); } } -std::ostream& operator<<(std::ostream& os, FieldType ft) { - switch (ft) { - case FieldType::Bool: - return os << "bool"; - case FieldType::Int: - return os << "int"; - case FieldType::Int64: - return os << "int64"; - case FieldType::Double: - return os << "double"; - case FieldType::String: - return os << "string"; - case FieldType::Uuid: - return os << "uuid"; - case FieldType::Point: - return os << "point"; - case FieldType::Struct: - return os << "struct"; - default: - assertrx(0); +void RandomGenerator::RndWhereComposite(reindexer::Query& query, const std::string& field, std::vector&& fieldTypes, + std::optional indexType) { // TODO array + if (RndErr()) { + return RndWhere(query, field, RndFieldType(), indexType); } - return os; -} - -reindexer::KeyValueType ToKeyValueType(FieldType ft) { - switch (ft) { - case FieldType::Bool: - return reindexer::KeyValueType::Bool{}; - case FieldType::Int: - return reindexer::KeyValueType::Int{}; - case FieldType::Int64: - return reindexer::KeyValueType::Int64{}; - case FieldType::Double: - return reindexer::KeyValueType::Double{}; - case FieldType::String: - return reindexer::KeyValueType::String{}; - case FieldType::Uuid: - return reindexer::KeyValueType::Uuid{}; - case FieldType::Point: - case FieldType::Struct: - default: - assertrx(0); + std::unordered_set generatedNames; + const std::string fldName = FieldName(field, generatedNames); + fieldTypes = RndFieldTypesArray(std::move(fieldTypes)); + const auto cond = rndCond(FieldType::Struct); + reindexer::VariantArray keys; + keys.reserve(fieldTypes.size()); + for (const FieldType ft : fieldTypes) { + switch (ft) { + case FieldType::Bool: + keys.emplace_back(RndBool(0.5)); + break; + case FieldType::Int: + keys.emplace_back(RndIntValue()); + break; + case FieldType::Int64: + if (indexType == IndexType::Ttl) { + keys.emplace_back(RndTtlValue()); + } else { + keys.emplace_back(RndInt64Value()); + } + break; + case FieldType::Double: + keys.emplace_back(RndDoubleValue()); + break; + case FieldType::String: + keys.emplace_back(RndStringValue()); + break; + case FieldType::Uuid: + if (RndBool(0.5)) { + keys.emplace_back(rndUuidValue()); + } else { + keys.emplace_back(rndStrUuidValue(false)); + } + break; + case FieldType::Point: + keys.emplace_back(reindexer::Point{RndDoubleValue(), RndDoubleValue()}); + break; + case FieldType::Struct: // TODO + break; + default: + assertrx(0); + } } + query.WhereComposite(fldName, cond, {std::move(keys)}); } } // namespace fuzzing diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/random_generator.h b/cpp_src/gtests/tests/fixtures/fuzzing/random_generator.h index 7a628fe85..4960f28c3 100644 --- a/cpp_src/gtests/tests/fixtures/fuzzing/random_generator.h +++ b/cpp_src/gtests/tests/fixtures/fuzzing/random_generator.h @@ -1,10 +1,12 @@ #pragma once -#include +#include +#include #include #include #include "core/type_consts.h" #include "tools/assertrx.h" +#include "types.h" namespace reindexer { @@ -16,137 +18,69 @@ class KeyValueType; namespace fuzzing { -struct Index; class NsScheme; -enum class FieldType { Bool, Int, Int64, Double, String, Uuid, Point, Struct, END = Struct }; -reindexer::KeyValueType ToKeyValueType(FieldType); -std::ostream& operator<<(std::ostream&, FieldType); -using FieldPath = std::vector; - class RandomGenerator { using ErrFactorInt = uint32_t; public: using ErrFactorType = std::pair; - RandomGenerator(std::ostream&, ErrFactorType errorFactor); - RandomGenerator(std::istream&); + RandomGenerator(ErrFactorType errorFactor); - size_t FieldsCount(bool firstLevel) { - if (RndErr()) { - enum Err : uint8_t { Zero, TooMany, END = TooMany }; - switch (RndWhich()) { - case Zero: - return 0; - case TooMany: - return RndInt(0, 10'000); - default: - assertrx(0); - } - } - if (firstLevel) { - enum Size : uint8_t { Normal, Long, END = Long }; - switch (RndWhich()) { - case Normal: - return RndInt(1, 9); - case Long: - return RndInt(10, 100); - default: - assertrx(false); - std::abort(); - } - } - return RndInt(1, 5); - } - fuzzing::FieldType RndFieldType(unsigned level) { + size_t FieldsCount(bool firstLevel); + FieldType RndFieldType(unsigned level) { const bool withoutStruct = level > 2 && (level > 5 || !RndBool(1.0 / (2 << (2 * level)))); - return static_cast( - RndInt(static_cast(fuzzing::FieldType::Bool), static_cast(fuzzing::FieldType::Struct) - withoutStruct)); + return static_cast(RndInt(static_cast(FieldType::Bool), static_cast(FieldType::Struct) - withoutStruct)); } - fuzzing::FieldType RndFieldType() { - return static_cast( - RndInt(static_cast(fuzzing::FieldType::Bool), static_cast(fuzzing::FieldType::Point))); + FieldType RndFieldType() { + return static_cast(RndInt(static_cast(FieldType::Bool), static_cast(FieldType::Point))); } - fuzzing::FieldType RndPkIndexFieldType() { - return static_cast( - RndInt(static_cast(fuzzing::FieldType::Int), static_cast(fuzzing::FieldType::Uuid))); + FieldType RndPkIndexFieldType() { + return static_cast(RndInt(static_cast(FieldType::Int), static_cast(FieldType::Uuid))); } - std::string IndexFieldType(fuzzing::FieldType); - fuzzing::FieldType RndFieldType(fuzzing::FieldType type) { + std::string IndexFieldType(FieldType); + FieldType RndFieldType(FieldType type) { if (RndErr()) { - return RndWhich(); + return RndWhich(); } return type; } - std::string RndIndexType(fuzzing::FieldType, bool pk); - bool RndArrayField() { return RndBool(0.2); } - bool RndArrayField(bool array) { return RndErr() ? !array : array; } - size_t ArraySize() { - if (RndErr()) return RndInt(0, 100'000); - enum Size : uint8_t { Short, Normal, Long, VeryLong, END = VeryLong }; - switch (RndWhich()) { - case Short: - return RndInt(0, 5); - case Normal: - return RndInt(6, 20); - case Long: - return RndInt(21, 200); - case VeryLong: - return RndInt(201, 10'000); - default: - assertrx(false); - std::abort(); + std::vector RndFieldTypesArray(std::vector&& types) { + if (!RndErr()) { + return std::move(types); } - } - bool PkIndex(bool pk) { return RndErr() ? RndBool(0.5) : pk; } - bool SparseIndex(bool pk) { return pk ? RndErr() : RndBool(0.2); } - bool DenseIndex() { return RndBool(0.2); } - int64_t ExpiredIndex() { return RndInt(0, 100'000); } // TODO - size_t IndexesCount() { - if (RndErr()) { - enum Err : uint8_t { Zero, TooMany, END = TooMany }; - switch (RndWhich()) { - case Zero: - return 0; - case TooMany: - return RndInt(0, 1'000); - default: - assertrx(0); - } + if (RndBool(0.5)) { + types.resize(compositeIndexSize(types.size())); } - enum Count : uint8_t { Few, Normal, Many, TooMany, END = TooMany }; - switch (RndWhich()) { - case Few: - return RndInt(1, 3); - case Normal: - return RndInt(4, 6); - case Many: - return RndInt(7, 20); - case TooMany: - return RndInt(21, 63); - default: - assertrx(false); - std::abort(); + for (auto& t : types) { + t = RndFieldType(); } + return std::move(types); } - bool CompositeIndex() { return RndBool(0.2); } - bool UniqueName() { return RndBool(0.5); } - size_t CompositeIndexSize() { + IndexType RndIndexType(const std::vector&); + IndexType RndPkIndexType(const std::vector&); + IndexType RndIndexType(IndexType); + IsArray RndArrayField() { return RndBool(0.2) ? IsArray::Yes : IsArray::No; } + IsArray RndArrayField(IsArray array) { if (RndErr()) { - enum Err : uint8_t { Zero, One, TooMany, END = TooMany }; - switch (RndWhich()) { - case Zero: - return 0; - case One: - return 1; - case TooMany: - return RndInt(0, 10'000); - default: - assertrx(0); - } + return array == IsArray::Yes ? IsArray::No : IsArray::Yes; } - return RndInt(2, 5); + return array; + } + size_t ArraySize(); + bool PkIndex(bool pk) { return RndErr() ? RndBool(0.5) : pk; } + IsSparse RndSparseIndex(FieldType fldType) { + const bool couldBeSparse = fldType != FieldType::Struct && fldType != FieldType::Uuid; // TODO remove uuid #1470 + return (couldBeSparse ? RndBool(0.2) : RndErr()) ? IsSparse::Yes : IsSparse::No; } + bool RndSparseIndex(IsSparse isSparse) { return (isSparse == IsSparse::Yes) != RndErr(); } + bool DenseIndex() { return RndBool(0.2); } + int64_t ExpiredIndex() { return RndInt(0, 100'000); } // TODO + size_t IndexesCount(); + bool CompositeIndex(size_t scalarIndexesCount) { return scalarIndexesCount < 1 ? RndErr() : RndBool(0.2); } + bool UniqueName() { return RndBool(0.5); } + size_t compositeIndexSize(size_t scalarIndexesCount); + std::vector RndFieldsForCompositeIndex(const std::vector& scalarIndexes); std::string FieldName(std::unordered_set& generatedNames); std::string FieldName(const std::string& fieldName, std::unordered_set& generatedNames) { if (RndErr()) return FieldName(generatedNames); @@ -155,9 +89,12 @@ class RandomGenerator { FieldPath RndField(const NsScheme&); FieldPath RndScalarField(const NsScheme&); std::string IndexName(std::unordered_set& generatedNames) { return FieldName(generatedNames); } // TODO - std::string NsName(std::unordered_set& generatedNames) { return FieldName(generatedNames); } // TODO - std::string NsName(const std::string& nsName, std::unordered_set& generatedNames) { - if (RndErr()) return NsName(generatedNames); + std::string GenerateNsName() { // TODO + std::unordered_set generatedNames; + return FieldName(generatedNames); + } + std::string NsName(const std::string& nsName) { + if (RndErr()) return GenerateNsName(); return nsName; } int RndInt(int min, int max) { return rndInt_(gen_, IntRndParams(min, max)); } @@ -200,7 +137,7 @@ class RandomGenerator { return err; } char RndChar() { return rndChar_(gen_); } - bool NeedThisNode(bool sparse) { return sparse ? RndBool(0.5) : !RndErr(); } + bool NeedThisNode(IsSparse sparse) { return sparse == IsSparse::Yes ? RndBool(0.5) : !RndErr(); } int RndIntValue() { enum Size : uint8_t { Short, Long, END = Long }; switch (RndWhich()) { @@ -225,6 +162,7 @@ class RandomGenerator { std::abort(); } } + int64_t RndTtlValue(); bool RndBool(double p) { return rndBool_(gen_, BoolRndParams{p}); } double RndDoubleValue() { enum Size : uint8_t { Short, Long, END = Long }; @@ -273,14 +211,19 @@ class RandomGenerator { } template const auto& RndWhich(const Cont& cont) { - assert(!std::empty(cont)); + assertrx(!std::empty(cont)); auto it = std::begin(cont); std::advance(it, rndSize(0, std::size(cont) - 1)); return *it; } - void RndWhere(reindexer::Query&, const std::string& field, const std::vector&); + void RndWhere(reindexer::Query&, const std::string& field, FieldType, std::optional); + void RndWhereComposite(reindexer::Query&, const std::string& field, std::vector&&, std::optional); + + static void SetOut(std::string); + static void SetIn(const std::string&); private: + using RandomEngine = std::default_random_engine; using IntRndParams = std::uniform_int_distribution<>::param_type; using SizeRndParams = std::uniform_int_distribution::param_type; using Int64RndParams = std::uniform_int_distribution::param_type; @@ -291,11 +234,16 @@ class RandomGenerator { int rndInt(IntRndParams params) { return rndInt_(gen_, params); } int64_t rndInt64(int64_t min, int64_t max) { return rndInt64_(gen_, Int64RndParams(min, max)); } size_t rndSize(size_t min, size_t max) { return rndSize_(gen_, SizeRndParams(min, max)); } - CondType rndCond(fuzzing::FieldType); + CondType rndCond(FieldType); std::string rndStrUuidValue(bool noErrors); reindexer::Uuid rndUuidValue(); + template * Availables> + IndexType rndIndexType(const std::vector&); + static std::string& out() noexcept; + static std::unique_ptr& in() noexcept; + static RandomEngine createRandomEngine(); - std::default_random_engine gen_; + RandomEngine gen_; ErrFactorType errFactor_; ErrorParams errParams_; std::uniform_int_distribution<> rndInt_; diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/types.cc b/cpp_src/gtests/tests/fixtures/fuzzing/types.cc new file mode 100644 index 000000000..11b50b658 --- /dev/null +++ b/cpp_src/gtests/tests/fixtures/fuzzing/types.cc @@ -0,0 +1,90 @@ +#include "types.h" + +#include +#include +#include "core/key_value_type.h" + +namespace fuzzing { + +std::ostream& operator<<(std::ostream& os, FieldType ft) { + switch (ft) { + case FieldType::Bool: + return os << "bool"; + case FieldType::Int: + return os << "int"; + case FieldType::Int64: + return os << "int64"; + case FieldType::Double: + return os << "double"; + case FieldType::String: + return os << "string"; + case FieldType::Uuid: + return os << "uuid"; + case FieldType::Point: + return os << "point"; + case FieldType::Struct: + return os << "struct"; + default: + assertrx(0); + } + return os; +} + +reindexer::KeyValueType ToKeyValueType(FieldType ft) { + switch (ft) { + case FieldType::Bool: + return reindexer::KeyValueType::Bool{}; + case FieldType::Int: + return reindexer::KeyValueType::Int{}; + case FieldType::Int64: + return reindexer::KeyValueType::Int64{}; + case FieldType::Double: + return reindexer::KeyValueType::Double{}; + case FieldType::String: + return reindexer::KeyValueType::String{}; + case FieldType::Uuid: + return reindexer::KeyValueType::Uuid{}; + case FieldType::Point: + return reindexer::KeyValueType::Undefined{}; // TODO change to KeyValueType::Point #1352 + case FieldType::Struct: + default: + assertrx(0); + } +} + +std::ostream& operator<<(std::ostream& os, const FieldPath& fp) { + os << '['; + for (size_t i = 0, s = fp.size(); i < s; ++i) { + if (i != 0) { + os << ' '; + } + os << fp[i]; + } + return os << ']' << std::endl; +} + +std::string_view ToText(IndexType it) { + using namespace std::string_view_literals; + switch (it) { + case IndexType::Store: + return "-"sv; + case IndexType::Hash: + return "hash"sv; + case IndexType::Tree: + return "tree"sv; + case IndexType::Ttl: + return "ttl"sv; + case IndexType::FastFT: + return "text"sv; + case IndexType::FuzzyFT: + return "fuzzytext"sv; + case IndexType::RTree: + return "rtree"sv; + default: + assertrx(0); + } +} + +std::ostream& operator<<(std::ostream& os, IndexType it) { return os << ToText(it); } + +} // namespace fuzzing diff --git a/cpp_src/gtests/tests/fixtures/fuzzing/types.h b/cpp_src/gtests/tests/fixtures/fuzzing/types.h new file mode 100644 index 000000000..db97087c6 --- /dev/null +++ b/cpp_src/gtests/tests/fixtures/fuzzing/types.h @@ -0,0 +1,29 @@ +#pragma once + +#include +#include +#include + +namespace reindexer { + +class KeyValueType; + +} // namespace reindexer + +namespace fuzzing { + +enum class FieldType { Bool, Int, Int64, Double, String, Uuid, Point, Struct, END = Struct }; +reindexer::KeyValueType ToKeyValueType(FieldType); +std::ostream& operator<<(std::ostream&, FieldType); + +using FieldPath = std::vector; +std::ostream& operator<<(std::ostream&, const FieldPath&); + +enum class IndexType { Store, Hash, Tree, Ttl, FastFT, FuzzyFT, RTree, END = RTree }; +std::string_view ToText(IndexType); +std::ostream& operator<<(std::ostream&, IndexType); + +enum class IsArray : bool { Yes = true, No = false }; +enum class IsSparse : bool { Yes = true, No = false }; + +} // namespace fuzzing diff --git a/cpp_src/gtests/tests/fixtures/get_pk_api.h b/cpp_src/gtests/tests/fixtures/get_pk_api.h index ebe17f9ea..29e7ad876 100644 --- a/cpp_src/gtests/tests/fixtures/get_pk_api.h +++ b/cpp_src/gtests/tests/fixtures/get_pk_api.h @@ -90,7 +90,7 @@ class ExtractPK : public testing::Test { Error err = db_->Select(query, qres); if (!err.ok()) return ResultType(err, QueryResults{}); - if (print) printQueryResults(query._namespace, qres); + if (print) printQueryResults(query.NsName(), qres); return ResultType(err, std::move(qres)); } diff --git a/cpp_src/gtests/tests/fixtures/item_move_semantics_api.h b/cpp_src/gtests/tests/fixtures/item_move_semantics_api.h index 1c0434473..ba10e1784 100644 --- a/cpp_src/gtests/tests/fixtures/item_move_semantics_api.h +++ b/cpp_src/gtests/tests/fixtures/item_move_semantics_api.h @@ -6,9 +6,6 @@ #include "gason/gason.h" #include "reindexer_api.h" -using reindexer::Item; -using reindexer::ItemImpl; - class ItemMoveSemanticsApi : public ReindexerApi { protected: const std::string pkField = "bookid"; @@ -18,14 +15,22 @@ class ItemMoveSemanticsApi : public ReindexerApi { void SetUp() override { ReindexerApi::SetUp(); - rt.reindexer->OpenNamespace(default_namespace, StorageOpts().Enabled(false)); - rt.reindexer->AddIndex(default_namespace, {"bookid", "hash", "int", IndexOpts().PK()}); - rt.reindexer->AddIndex(default_namespace, {"title", "text", "string", IndexOpts()}); - rt.reindexer->AddIndex(default_namespace, {"pages", "hash", "int", IndexOpts().PK()}); - rt.reindexer->AddIndex(default_namespace, {"price", "hash", "int", IndexOpts().PK()}); - rt.reindexer->AddIndex(default_namespace, {"genreid_fk", "hash", "int", IndexOpts().PK()}); - rt.reindexer->AddIndex(default_namespace, {"authorid_fk", "hash", "int", IndexOpts().PK()}); - rt.reindexer->Commit(default_namespace); + auto err = rt.reindexer->OpenNamespace(default_namespace, StorageOpts().Enabled(false)); + ASSERT_TRUE(err.ok()) << err.what(); + err = rt.reindexer->AddIndex(default_namespace, {"bookid", "hash", "int", IndexOpts().PK()}); + ASSERT_TRUE(err.ok()) << err.what(); + err = rt.reindexer->AddIndex(default_namespace, {"title", "text", "string", IndexOpts()}); + ASSERT_TRUE(err.ok()) << err.what(); + err = rt.reindexer->AddIndex(default_namespace, {"pages", "hash", "int", IndexOpts()}); + ASSERT_TRUE(err.ok()) << err.what(); + err = rt.reindexer->AddIndex(default_namespace, {"price", "hash", "int", IndexOpts()}); + ASSERT_TRUE(err.ok()) << err.what(); + err = rt.reindexer->AddIndex(default_namespace, {"genreid_fk", "hash", "int", IndexOpts()}); + ASSERT_TRUE(err.ok()) << err.what(); + err = rt.reindexer->AddIndex(default_namespace, {"authorid_fk", "hash", "int", IndexOpts()}); + ASSERT_TRUE(err.ok()) << err.what(); + err = rt.reindexer->Commit(default_namespace); + ASSERT_TRUE(err.ok()) << err.what(); } void prepareItems() { @@ -48,7 +53,8 @@ class ItemMoveSemanticsApi : public ReindexerApi { ASSERT_TRUE(err.ok()) << err.what(); ASSERT_NO_THROW(gason::JsonParser().Parse(item.GetJSON())); } - rt.reindexer->Commit(default_namespace); + const auto err = rt.reindexer->Commit(default_namespace); + ASSERT_TRUE(err.ok()) << err.what(); } Item getItemById(int id) { diff --git a/cpp_src/gtests/tests/fixtures/join_on_conditions_api.h b/cpp_src/gtests/tests/fixtures/join_on_conditions_api.h index 6326c8faf..8834ba94e 100644 --- a/cpp_src/gtests/tests/fixtures/join_on_conditions_api.h +++ b/cpp_src/gtests/tests/fixtures/join_on_conditions_api.h @@ -23,7 +23,8 @@ class JoinOnConditionsApi : public JoinSelectsApi { builder.End(); err = item.FromJSON(ser.c_str()); ASSERT_TRUE(err.ok()) << err.what(); - rt.reindexer->Insert(leftNs, item); + err = rt.reindexer->Insert(leftNs, item); + ASSERT_TRUE(err.ok()) << err.what(); } for (unsigned int i = 0; i < rightNsData.size(); i++) { @@ -40,7 +41,8 @@ class JoinOnConditionsApi : public JoinSelectsApi { builder.End(); err = item.FromJSON(ser.c_str()); ASSERT_TRUE(err.ok()) << err.what(); - rt.reindexer->Insert(rightNs, item); + err = rt.reindexer->Insert(rightNs, item); + ASSERT_TRUE(err.ok()) << err.what(); } } diff --git a/cpp_src/gtests/tests/fixtures/queries_api.cc b/cpp_src/gtests/tests/fixtures/queries_api.cc index 9047a9627..a1be92c4c 100644 --- a/cpp_src/gtests/tests/fixtures/queries_api.cc +++ b/cpp_src/gtests/tests/fixtures/queries_api.cc @@ -281,6 +281,7 @@ void QueriesApi::initConditionsNs() { ASSERT_TRUE(err.ok()) << err.what(); err = rt.reindexer->AddIndex(conditionsNs, {kFieldNameId, "hash", "int", IndexOpts{}.PK()}); ASSERT_TRUE(err.ok()) << err.what(); + addIndexFields(conditionsNs, kFieldNameId, {{kFieldNameId, reindexer::KeyValueType::Int{}}}); for (const auto& fit : fieldIndexTypes) { for (const auto& it : fit.indexTypes) { for (const bool isArray : {true, false}) { @@ -291,8 +292,8 @@ void QueriesApi::initConditionsNs() { const std::string fieldType{fit.fieldType.Name()}; const std::string indexName{createIndexName(fieldType, it, isArray, isSparse)}; err = rt.reindexer->AddIndex(conditionsNs, {indexName, it, fieldType, IndexOpts{}.Array(isArray).Sparse(isSparse)}); - addIndexFields(conditionsNs, indexName, {{indexName, fit.fieldType}}); ASSERT_TRUE(err.ok()) << err.what(); + addIndexFields(conditionsNs, indexName, {{indexName, fit.fieldType}}); } } } @@ -300,6 +301,27 @@ void QueriesApi::initConditionsNs() { setPkFields(conditionsNs, {kFieldNameId}); } +void QueriesApi::initUUIDNs() { + const auto err = rt.reindexer->OpenNamespace(uuidNs); + ASSERT_TRUE(err.ok()) << err.what(); + DefineNamespaceDataset( + uuidNs, + { + IndexDeclaration{kFieldNameId, "hash", "int", IndexOpts{}.PK(), 0}, + IndexDeclaration{kFieldNameUuid, "hash", "uuid", IndexOpts{}, 0}, + /*IndexDeclaration{kFieldNameUuidSparse, "hash", "uuid", IndexOpts{}.Sparse(), 0}, // TODO uncomment this #1470 + IndexDeclaration{kFieldNameUuidNotIndex2, "hash", "uuid", IndexOpts{}, 0}, + IndexDeclaration{kFieldNameUuidNotIndex3, "hash", "uuid", IndexOpts{}.Sparse(), 0},*/ + IndexDeclaration{kFieldNameUuidArr, "hash", "uuid", IndexOpts{}.Array(), 0}, + // IndexDeclaration{kFieldNameUuidArrSparse, "hash", "uuid", IndexOpts{}.Array().Sparse(), 0} // TODO uncomment this #1470 + }); + for (const auto& idx : + {kFieldNameUuid, kFieldNameUuidArr /*, kFieldNameUuidSparse, kFieldNameUuidArrSparse*/}) { // TODO uncomment this #1470 + addIndexFields(uuidNs, idx, {{idx, reindexer::KeyValueType::Uuid{}}}); + } + setPkFields(uuidNs, {kFieldNameId}); +} + static reindexer::Variant createRandValue(int id, reindexer::KeyValueType fieldType) { using namespace reindexer; return fieldType.EvaluateOneOf(overloaded{ @@ -376,41 +398,19 @@ static reindexer::VariantArray createRandArrValues(size_t min, size_t max, int i } void QueriesApi::checkAllConditions(const std::string& fieldName, reindexer::KeyValueType fieldType, NullAllowed nullAllowed) { - for (const auto cond : {CondEq, CondSet, CondAllSet, CondLt, CondLe, CondGt, CondGe, CondRange, CondAny, CondEmpty, CondLike}) { - size_t min = 0, max = rand() % kMaxArraySize; - switch (cond) { - case CondEq: - case CondSet: - case CondAllSet: - break; - case CondLike: - if (!fieldType.Is()) { - continue; - } - [[fallthrough]]; - case CondLt: - case CondLe: - case CondGt: - case CondGe: - min = max = 1; - break; - case CondRange: - min = max = 2; - break; - case CondAny: - case CondEmpty: - if (nullAllowed == NullAllowed::No) { - continue; - } - min = max = 0; - break; - case CondDWithin: // TODO #1352 - assert(0); + for (const auto cond : {CondEq, CondSet, CondAllSet, CondLt, CondLe, CondGt, CondGe, CondRange, CondAny, CondEmpty, + CondLike}) { // TODO CondDWithin #1352 + if (cond == CondLike && !fieldType.Is()) { + continue; } + if (nullAllowed == NullAllowed::No && (cond == CondAny || cond == CondEmpty)) { + continue; + } + const auto argsCount = minMaxArgs(cond, 20); for (size_t i = 0; i < 3; ++i) { - ExecuteAndVerify( - reindexer::Query{conditionsNs}.Where(fieldName, cond, createRandArrValues(min, max, rand() % conditionsNsSize, fieldType))); - if (min <= 1 && max >= 1) { + ExecuteAndVerify(reindexer::Query{conditionsNs}.Where( + fieldName, cond, createRandArrValues(argsCount.min, argsCount.max, rand() % conditionsNsSize, fieldType))); + if (argsCount.min <= 1 && argsCount.max >= 1) { ExecuteAndVerify( reindexer::Query{conditionsNs}.Where(fieldName, cond, createRandValue(rand() % conditionsNsSize, fieldType))); } @@ -439,3 +439,99 @@ void QueriesApi::CheckConditions() { checkAllConditions(fieldType + "_array", fit.fieldType, NullAllowed::Yes); } } + +void QueriesApi::FillUUIDNs() { + static size_t lastId = 0; + reindexer::WrSerializer ser; + for (size_t i = lastId; i < uuidNsSize + lastId; ++i) { + Item item = rt.reindexer->NewItem(uuidNs); + ASSERT_TRUE(item.Status().ok()) << item.Status().what(); + if (rand() % 2) { + ser.Reset(); + { + reindexer::JsonBuilder json{ser}; + json.Put(kFieldNameId, i); + json.Put(kFieldNameUuid, randStrUuid()); + /*if (rand() % 2) { + json.Put(kFieldNameUuidSparse, randStrUuid()); // TODO uncomment this #1470 + }*/ + { + auto arr = json.Array(kFieldNameUuidArr); + for (size_t j = 0, s = rand() % 10; j < s; ++j) { + arr.Put({}, randStrUuid()); + } + } + /*if (rand() % 2) { + auto arr = json.Array(kFieldNameUuidArrSparse); // TODO uncomment this #1470 + for (size_t j = 0, s = rand() % 10; j < s; ++j) { + arr.Put({}, randStrUuid()); + } + }*/ + if (rand() % 2) { + json.Put(kFieldNameUuidNotIndex, randStrUuid()); + } + /*json.Put(kFieldNameUuidNotIndex2, randStrUuid()); // TODO uncomment this #1470 + if (rand() % 2) { + json.Put(kFieldNameUuidNotIndex3, randStrUuid()); + }*/ + if (rand() % 2) { + json.Put(kFieldNameRndString, RandString()); + } + } + const auto err = item.FromJSON(ser.Slice()); + ASSERT_TRUE(err.ok()) << err.what(); + } else { + item[kFieldNameId] = int(i); + if (rand() % 2) { + item[kFieldNameUuid] = randUuid(); + } else { + item[kFieldNameUuid] = randStrUuid(); + } + /*if (rand() % 2) { + item[kFieldNameUuidSparse] = randUuid(); // TODO uncomment this #1470 + }*/ + item[kFieldNameUuidArr] = randHeterogeneousUuidArray(0, 20); + /*if (rand() % 2) { + item[kFieldNameUuidArrSparse] = randHeterogeneousUuidArray(0, 20); // TODO uncomment this #1470 + } + if (rand() % 2) { + item[kFieldNameUuidNotIndex2] = randUuid(); + } else { + item[kFieldNameUuidNotIndex2] = randStrUuid(); + } + if (rand() % 2) { + if (rand() % 2) { + item[kFieldNameUuidNotIndex3] = randUuid(); + } else { + item[kFieldNameUuidNotIndex3] = randStrUuid(); + } + }*/ + } + ASSERT_TRUE(item.Status().ok()) << item.Status().what(); + Upsert(uuidNs, item); + saveItem(std::move(item), uuidNs); + } + const auto err = Commit(uuidNs); + ASSERT_TRUE(err.ok()) << err.what(); + lastId += uuidNsSize; +} + +void QueriesApi::CheckUUIDQueries() { + for (size_t i = 0; i < 10; ++i) { + for (const auto& field : { + kFieldNameUuid, kFieldNameUuidArr, kFieldNameUuidNotIndex, kFieldNameRndString /*, + kFieldNameUuidSparse, kFieldNameUuidArrSparse, kFieldNameUuidNotIndex2, kFieldNameUuidNotIndex3*/ + }) { // TODO uncomment this #1470 + for (auto cond : {CondEq, CondLe, CondLt, CondSet, CondGe, CondGt, CondAllSet, CondRange}) { + const auto argsCount = minMaxArgs(cond, 20); + if (argsCount.min <= 1 && argsCount.max >= 1) { + ExecuteAndVerify(Query(uuidNs).Where(field, cond, randUuid())); + ExecuteAndVerify(Query(uuidNs).Where(field, cond, randStrUuid())); + } + ExecuteAndVerify(Query(uuidNs).Where(field, cond, randUuidArray(argsCount.min, argsCount.max))); + ExecuteAndVerify(Query(uuidNs).Where(field, cond, randStrUuidArray(argsCount.min, argsCount.max))); + ExecuteAndVerify(Query(uuidNs).Where(field, cond, randHeterogeneousUuidArray(argsCount.min, argsCount.max))); + } + } + } +} diff --git a/cpp_src/gtests/tests/fixtures/queries_api.h b/cpp_src/gtests/tests/fixtures/queries_api.h index 20e4aa9af..b8306b844 100644 --- a/cpp_src/gtests/tests/fixtures/queries_api.h +++ b/cpp_src/gtests/tests/fixtures/queries_api.h @@ -9,14 +9,11 @@ #include #include "core/cjson/jsonbuilder.h" #include "core/keyvalue/geometry.h" -#include "core/nsselecter/sortexpression.h" #include "core/queryresults/joinresults.h" -#include "core/type_consts_helpers.h" #include "gtests/tools.h" #include "queries_verifier.h" #include "reindexer_api.h" #include "tools/random.h" -#include "tools/stringstools.h" class QueriesApi : public ReindexerApi, public QueriesVerifier { public: @@ -66,12 +63,33 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { IndexDeclaration{kCompositeFieldUuidName, "hash", "composite", IndexOpts{}, 0}, IndexDeclaration{kFieldNameYearSparse, "hash", "string", IndexOpts{}.Sparse(), 0}, }); + addIndexFields(default_namespace, kFieldNameId, {{kFieldNameId, reindexer::KeyValueType::Int{}}}); + addIndexFields(default_namespace, kFieldNameGenre, {{kFieldNameGenre, reindexer::KeyValueType::Int{}}}); + addIndexFields(default_namespace, kFieldNameYear, {{kFieldNameYear, reindexer::KeyValueType::Int{}}}); + addIndexFields(default_namespace, kFieldNamePackages, {{kFieldNamePackages, reindexer::KeyValueType::Int{}}}); + addIndexFields(default_namespace, kFieldNameName, {{kFieldNameName, reindexer::KeyValueType::String{}}}); + addIndexFields(default_namespace, kFieldNameCountries, {{kFieldNameCountries, reindexer::KeyValueType::String{}}}); + addIndexFields(default_namespace, kFieldNameAge, {{kFieldNameAge, reindexer::KeyValueType::Int{}}}); + addIndexFields(default_namespace, kFieldNameDescription, {{kFieldNameDescription, reindexer::KeyValueType::String{}}}); + addIndexFields(default_namespace, kFieldNameRate, {{kFieldNameRate, reindexer::KeyValueType::Double{}}}); + addIndexFields(default_namespace, kFieldNameIsDeleted, {{kFieldNameIsDeleted, reindexer::KeyValueType::Bool{}}}); + addIndexFields(default_namespace, kFieldNameActor, {{kFieldNameActor, reindexer::KeyValueType::String{}}}); + addIndexFields(default_namespace, kFieldNamePriceId, {{kFieldNamePriceId, reindexer::KeyValueType::Int{}}}); + addIndexFields(default_namespace, kFieldNameLocation, {{kFieldNameLocation, reindexer::KeyValueType::String{}}}); + addIndexFields(default_namespace, kFieldNameEndTime, {{kFieldNameEndTime, reindexer::KeyValueType::Int{}}}); + addIndexFields(default_namespace, kFieldNameStartTime, {{kFieldNameStartTime, reindexer::KeyValueType::Int{}}}); + addIndexFields(default_namespace, kFieldNameBtreeIdsets, {{kFieldNameBtreeIdsets, reindexer::KeyValueType::Int{}}}); + addIndexFields(default_namespace, kFieldNameTemp, {{kFieldNameTemp, reindexer::KeyValueType::String{}}}); + addIndexFields(default_namespace, kFieldNameNumeric, {{kFieldNameNumeric, reindexer::KeyValueType::String{}}}); + addIndexFields(default_namespace, kFieldNameUuid, {{kFieldNameUuid, reindexer::KeyValueType::Uuid{}}}); + addIndexFields(default_namespace, kFieldNameUuidArr, {{kFieldNameUuidArr, reindexer::KeyValueType::Uuid{}}}); addIndexFields(default_namespace, kCompositeFieldIdTemp, {{kFieldNameId, reindexer::KeyValueType::Int{}}, {kFieldNameTemp, reindexer::KeyValueType::String{}}}); addIndexFields(default_namespace, kCompositeFieldAgeGenre, {{kFieldNameAge, reindexer::KeyValueType::Int{}}, {kFieldNameGenre, reindexer::KeyValueType::Int{}}}); addIndexFields(default_namespace, kCompositeFieldUuidName, {{kFieldNameUuid, reindexer::KeyValueType::Uuid{}}, {kFieldNameName, reindexer::KeyValueType::String{}}}); + addIndexFields(default_namespace, kFieldNameYearSparse, {{kFieldNameYearSparse, reindexer::KeyValueType::String{}}}); err = rt.reindexer->OpenNamespace(joinNs); ASSERT_TRUE(err.ok()) << err.what(); @@ -81,6 +99,12 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { IndexDeclaration{kFieldNameName, "tree", "string", IndexOpts(), 0}, IndexDeclaration{kFieldNameDescription, "text", "string", IndexOpts{}, 0}, IndexDeclaration{kFieldNameYearSparse, "hash", "string", IndexOpts().Sparse(), 0}}); + addIndexFields(joinNs, kFieldNameId, {{kFieldNameId, reindexer::KeyValueType::Int{}}}); + addIndexFields(joinNs, kFieldNameYear, {{kFieldNameYear, reindexer::KeyValueType::Int{}}}); + addIndexFields(joinNs, kFieldNameAge, {{kFieldNameAge, reindexer::KeyValueType::Int{}}}); + addIndexFields(joinNs, kFieldNameName, {{kFieldNameName, reindexer::KeyValueType::String{}}}); + addIndexFields(joinNs, kFieldNameDescription, {{kFieldNameDescription, reindexer::KeyValueType::String{}}}); + addIndexFields(joinNs, kFieldNameYearSparse, {{kFieldNameYearSparse, reindexer::KeyValueType::String{}}}); err = rt.reindexer->OpenNamespace(testSimpleNs); ASSERT_TRUE(err.ok()) << err.what(); @@ -90,6 +114,10 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { IndexDeclaration{kFieldNameName, "hash", "string", IndexOpts(), 0}, IndexDeclaration{kFieldNamePhone, "hash", "string", IndexOpts(), 0}, }); + addIndexFields(testSimpleNs, kFieldNameId, {{kFieldNameId, reindexer::KeyValueType::Int{}}}); + addIndexFields(testSimpleNs, kFieldNameYear, {{kFieldNameYear, reindexer::KeyValueType::Int{}}}); + addIndexFields(testSimpleNs, kFieldNameName, {{kFieldNameName, reindexer::KeyValueType::String{}}}); + addIndexFields(testSimpleNs, kFieldNamePhone, {{kFieldNamePhone, reindexer::KeyValueType::String{}}}); err = rt.reindexer->OpenNamespace(compositeIndexesNs); ASSERT_TRUE(err.ok()) << err.what(); @@ -105,6 +133,12 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { IndexDeclaration{kCompositeFieldPriceTitle, "hash", "composite", IndexOpts(), 0}, IndexDeclaration{kCompositeFieldPagesTitle, "hash", "composite", IndexOpts(), 0}, IndexDeclaration{kCompositeFieldBookidBookid2, "hash", "composite", IndexOpts().PK(), 0}}); + addIndexFields(compositeIndexesNs, kFieldNameBookid, {{kFieldNameBookid, reindexer::KeyValueType::Int{}}}); + addIndexFields(compositeIndexesNs, kFieldNameBookid2, {{kFieldNameBookid2, reindexer::KeyValueType::Int{}}}); + addIndexFields(compositeIndexesNs, kFieldNameTitle, {{kFieldNameTitle, reindexer::KeyValueType::String{}}}); + addIndexFields(compositeIndexesNs, kFieldNamePages, {{kFieldNamePages, reindexer::KeyValueType::Int{}}}); + addIndexFields(compositeIndexesNs, kFieldNamePrice, {{kFieldNamePrice, reindexer::KeyValueType::Int{}}}); + addIndexFields(compositeIndexesNs, kFieldNameName, {{kFieldNameName, reindexer::KeyValueType::String{}}}); addIndexFields(compositeIndexesNs, kCompositeFieldPricePages, {{kFieldNamePrice, reindexer::KeyValueType::Int{}}, {kFieldNamePages, reindexer::KeyValueType::Int{}}}); addIndexFields(compositeIndexesNs, kCompositeFieldTitleName, @@ -127,12 +161,22 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { IndexDeclaration{kFieldNameColumnString, "-", "string", IndexOpts(), 0}, IndexDeclaration{kFieldNameColumnFullText, "text", "string", IndexOpts().SetConfig(R"xxx({"stemmers":[]})xxx"), 0}, IndexDeclaration{kFieldNameColumnStringNumeric, "-", "string", IndexOpts().SetCollateMode(CollateNumeric), 0}}); + addIndexFields(comparatorsNs, kFieldNameId, {{kFieldNameId, reindexer::KeyValueType::Int{}}}); + addIndexFields(comparatorsNs, kFieldNameColumnInt, {{kFieldNameColumnInt, reindexer::KeyValueType::Int{}}}); + addIndexFields(comparatorsNs, kFieldNameColumnInt64, {{kFieldNameColumnInt64, reindexer::KeyValueType::Int64{}}}); + addIndexFields(comparatorsNs, kFieldNameColumnDouble, {{kFieldNameColumnDouble, reindexer::KeyValueType::Double{}}}); + addIndexFields(comparatorsNs, kFieldNameColumnString, {{kFieldNameColumnString, reindexer::KeyValueType::String{}}}); + addIndexFields(comparatorsNs, kFieldNameColumnFullText, {{kFieldNameColumnFullText, reindexer::KeyValueType::String{}}}); + addIndexFields(comparatorsNs, kFieldNameColumnStringNumeric, {{kFieldNameColumnStringNumeric, reindexer::KeyValueType::String{}}}); err = rt.reindexer->OpenNamespace(forcedSortOffsetNs); ASSERT_TRUE(err.ok()) << err.what(); DefineNamespaceDataset(forcedSortOffsetNs, {IndexDeclaration{kFieldNameId, "hash", "int", IndexOpts().PK(), 0}, IndexDeclaration{kFieldNameColumnHash, "hash", "int", IndexOpts(), 0}, IndexDeclaration{kFieldNameColumnTree, "tree", "int", IndexOpts(), 0}}); + addIndexFields(forcedSortOffsetNs, kFieldNameId, {{kFieldNameId, reindexer::KeyValueType::Int{}}}); + addIndexFields(forcedSortOffsetNs, kFieldNameColumnHash, {{kFieldNameColumnHash, reindexer::KeyValueType::Int{}}}); + addIndexFields(forcedSortOffsetNs, kFieldNameColumnTree, {{kFieldNameColumnTree, reindexer::KeyValueType::Int{}}}); err = rt.reindexer->OpenNamespace(geomNs); ASSERT_TRUE(err.ok()) << err.what(); @@ -142,12 +186,20 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { IndexDeclaration{kFieldNamePointLinearRTree, "rtree", "point", IndexOpts{}.RTreeType(IndexOpts::Linear), 0}, IndexDeclaration{kFieldNamePointGreeneRTree, "rtree", "point", IndexOpts{}.RTreeType(IndexOpts::Greene), 0}, IndexDeclaration{kFieldNamePointRStarRTree, "rtree", "point", IndexOpts{}.RTreeType(IndexOpts::RStar), 0}}); + addIndexFields(geomNs, kFieldNameId, {{kFieldNameId, reindexer::KeyValueType::Int{}}}); + addIndexFields(geomNs, kFieldNamePointQuadraticRTree, {{kFieldNamePointQuadraticRTree, reindexer::KeyValueType::Double{}}}); + addIndexFields(geomNs, kFieldNamePointLinearRTree, {{kFieldNamePointLinearRTree, reindexer::KeyValueType::Double{}}}); + addIndexFields(geomNs, kFieldNamePointGreeneRTree, {{kFieldNamePointGreeneRTree, reindexer::KeyValueType::Double{}}}); + addIndexFields(geomNs, kFieldNamePointRStarRTree, {{kFieldNamePointRStarRTree, reindexer::KeyValueType::Double{}}}); err = rt.reindexer->OpenNamespace(btreeIdxOptNs); ASSERT_TRUE(err.ok()) << err.what(); DefineNamespaceDataset(btreeIdxOptNs, {IndexDeclaration{kFieldNameId, "tree", "int", IndexOpts().PK(), 0}, IndexDeclaration{kFieldNameStartTime, "tree", "int", IndexOpts(), 0}}); + addIndexFields(btreeIdxOptNs, kFieldNameId, {{kFieldNameId, reindexer::KeyValueType::Int{}}}); + addIndexFields(btreeIdxOptNs, kFieldNameStartTime, {{kFieldNameStartTime, reindexer::KeyValueType::Int{}}}); initConditionsNs(); + initUUIDNs(); } void initConditionsNs(); @@ -155,6 +207,9 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { void CheckConditions(); enum class NullAllowed : bool { Yes = true, No = false }; void checkAllConditions(const std::string& fieldName, reindexer::KeyValueType fieldType, NullAllowed); + void initUUIDNs(); + void FillUUIDNs(); + void CheckUUIDQueries(); template void ExecuteAndVerify(const Query& query, T... args) { @@ -515,8 +570,9 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { tr.Insert(std::move(item)); } QueryResults res; - rt.reindexer->CommitTransaction(tr, res); - const auto err = Commit(default_namespace); + auto err = rt.reindexer->CommitTransaction(tr, res); + ASSERT_TRUE(err.ok()) << err.what(); + err = Commit(default_namespace); ASSERT_TRUE(err.ok()) << err.what(); } @@ -699,8 +755,6 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { static const std::vector distincts = {"", kFieldNameYear, kFieldNameRate}; static const std::vector sortOrders = {true, false}; - static const std::string compositeIndexName(kFieldNameAge + compositePlus + kFieldNameGenre); - for (const bool sortOrder : sortOrders) { for (const auto& sortIdx : sortIdxs) { for (const std::string& distinct : distincts) { @@ -870,7 +924,7 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { // end of check substituteCompositIndexes ExecuteAndVerify(Query(default_namespace) - .Where(kFieldNamePackages, CondEmpty, 0) + .Where(kFieldNamePackages, CondEmpty, {}) .Distinct(distinct.c_str()) .Sort(sortIdx, sortOrder)); @@ -900,7 +954,7 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { ExecuteAndVerify(Query(default_namespace).Sort(kFieldNameGenre, true, {10, 20, 30})); ExecuteAndVerify(Query(default_namespace) - .Where(kFieldNamePackages, CondAny, 0) + .Where(kFieldNamePackages, CondAny, {}) .Distinct(distinct.c_str()) .Sort(sortIdx, sortOrder)); @@ -966,7 +1020,7 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { .Where(kFieldNameGenre, CondEq, 3) .Where(kFieldNamePackages, CondSet, RandIntVector(5, 10000, 50)) .Or() - .Where(kFieldNamePackages, CondEmpty, 0) + .Where(kFieldNamePackages, CondEmpty, {}) .Debug(LogTrace)); ExecuteAndVerify(Query(default_namespace) @@ -976,7 +1030,7 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { .Where(kFieldNameGenre, CondSet, {5, 1, 7}) .Where(kFieldNameYear, CondLt, 2010) .Or() - .Where(kFieldNamePackages, CondAny, 0) + .Where(kFieldNamePackages, CondAny, {}) .Where(kFieldNamePackages, CondSet, RandIntVector(5, 10000, 50)) .Debug(LogTrace)); @@ -1112,7 +1166,7 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { .ReqTotal() .Distinct(distinct) .Sort(sortIdx, sortOrder) - .WhereComposite(compositeIndexName.c_str(), CondLe, {{Variant(27), Variant(10000)}})); + .WhereComposite(kCompositeFieldAgeGenre, CondLe, {{Variant(27), Variant(10000)}})); ExecuteAndVerify(Query(default_namespace) .ReqTotal() @@ -1125,7 +1179,7 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { .ReqTotal() .Distinct(distinct) .Sort(sortIdx, sortOrder) - .WhereComposite(compositeIndexName.c_str(), CondEq, {{Variant(rand() % 10), Variant(rand() % 50)}})); + .WhereComposite(kCompositeFieldAgeGenre, CondEq, {{Variant(rand() % 10), Variant(rand() % 50)}})); ExecuteAndVerify(Query(default_namespace) .InnerJoin(kFieldNameYear, kFieldNameYear, CondEq, Query(joinNs)) @@ -1461,83 +1515,64 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { .CloseBracket() .Distinct(distinct)); - for (CondType cond : {CondEq, CondSet, CondLt, CondLe, CondGt, CondGe}) { - const bool multyArgCond = cond == CondEq || cond == CondSet; - ExecuteAndVerify(Query(default_namespace) - .Where(kFieldNameUuid, cond, randUuid()) - .Distinct(distinct.c_str()) - .Sort(sortIdx, sortOrder)); - - ExecuteAndVerify(Query(default_namespace) - .Where(kFieldNameUuid, cond, randStrUuid()) - .Distinct(distinct.c_str()) - .Sort(sortIdx, sortOrder)); - - ExecuteAndVerify(Query(default_namespace) - .Where(kFieldNameUuid, cond, - multyArgCond ? VariantArray::Create(randUuid(), randStrUuid(), randUuid(), - randStrUuid(), randUuid()) - : VariantArray::Create(randUuid())) - .Distinct(distinct.c_str()) - .Sort(sortIdx, sortOrder)); - - ExecuteAndVerify(Query(default_namespace) - .Where(kFieldNameUuidArr, cond, randUuid()) - .Distinct(distinct.c_str()) - .Sort(sortIdx, sortOrder)); + for (CondType cond : {CondEq, CondSet, CondLt, CondLe, CondGt, CondGe, CondRange}) { + const auto argsCount = minMaxArgs(cond, 20); + if (argsCount.min <= 1 && argsCount.max >= 1) { + ExecuteAndVerify(Query(default_namespace) + .Where(kFieldNameUuid, cond, randUuid()) + .Distinct(distinct.c_str()) + .Sort(sortIdx, sortOrder)); + + ExecuteAndVerify(Query(default_namespace) + .Where(kFieldNameUuid, cond, randStrUuid()) + .Distinct(distinct.c_str()) + .Sort(sortIdx, sortOrder)); + + ExecuteAndVerify(Query(default_namespace) + .Where(kFieldNameUuidArr, cond, randUuid()) + .Distinct(distinct.c_str()) + .Sort(sortIdx, sortOrder)); + + ExecuteAndVerify(Query(default_namespace) + .Where(kFieldNameUuidArr, cond, randStrUuid()) + .Distinct(distinct.c_str()) + .Sort(sortIdx, sortOrder)); + } ExecuteAndVerify(Query(default_namespace) - .Where(kFieldNameUuidArr, cond, randStrUuid()) + .Where(kFieldNameUuid, cond, randHeterogeneousUuidArray(argsCount.min, argsCount.max)) .Distinct(distinct.c_str()) .Sort(sortIdx, sortOrder)); ExecuteAndVerify(Query(default_namespace) - .Where(kFieldNameUuidArr, cond, - multyArgCond ? VariantArray::Create(randUuid(), randStrUuid(), randUuid(), - randStrUuid(), randUuid(), randStrUuid()) - : VariantArray::Create(randUuid())) + .Where(kFieldNameUuidArr, cond, randHeterogeneousUuidArray(argsCount.min, argsCount.max)) .Distinct(distinct.c_str()) .Sort(sortIdx, sortOrder)); + std::vector compositeKeyValues; + VariantArray hetUuidArray = randHeterogeneousUuidArray(argsCount.min, argsCount.max); + compositeKeyValues.reserve(hetUuidArray.size()); + std::transform(std::make_move_iterator(hetUuidArray.begin()), std::make_move_iterator(hetUuidArray.end()), + std::back_inserter(compositeKeyValues), + [this](Variant&& uuid) { return VariantArray::Create(std::move(uuid), RandString()); }); ExecuteAndVerify(Query(default_namespace) - .WhereComposite(kFieldNameUuid + compositePlus + kFieldNameName, cond, - multyArgCond - ? std::vector{VariantArray::Create(randUuid(), RandString()), - VariantArray::Create(randStrUuid(), RandString()), - VariantArray::Create(randUuid(), RandString()), - VariantArray::Create(randStrUuid(), RandString()), - VariantArray::Create(randUuid(), RandString()), - VariantArray::Create(randStrUuid(), RandString()), - VariantArray::Create(randUuid(), RandString()), - VariantArray::Create(randStrUuid(), RandString()), - VariantArray::Create(randUuid(), RandString()), - VariantArray::Create(randStrUuid(), RandString()), - VariantArray::Create(randUuid(), RandString()), - VariantArray::Create(randStrUuid(), RandString()), - VariantArray::Create(randUuid(), RandString()), - VariantArray::Create(randStrUuid(), RandString()), - VariantArray::Create(randUuid(), RandString()), - VariantArray::Create(randStrUuid(), RandString()), - VariantArray::Create(randUuid(), RandString()), - VariantArray::Create(randStrUuid(), RandString()), - VariantArray::Create(randUuid(), RandString())} - : std::vector{VariantArray::Create(randUuid(), RandString())}) + .WhereComposite(kCompositeFieldUuidName, cond, compositeKeyValues) .Distinct(distinct.c_str()) .Sort(sortIdx, sortOrder)); } ExecuteAndVerify(Query(default_namespace) - .Where(kFieldNameUuid, CondRange, {nilUuid(), randUuid()}) + .Where(kFieldNameUuid, CondRange, randHeterogeneousUuidArray(2, 2)) .Distinct(distinct.c_str()) .Sort(sortIdx, sortOrder)); ExecuteAndVerify(Query(default_namespace) - .Where(kFieldNameUuidArr, CondRange, {nilUuid(), randUuid()}) + .Where(kFieldNameUuidArr, CondRange, randHeterogeneousUuidArray(2, 2)) .Distinct(distinct.c_str()) .Sort(sortIdx, sortOrder)); ExecuteAndVerify(Query(default_namespace) - .WhereComposite(kFieldNameUuid + compositePlus + kFieldNameName, CondRange, + .WhereComposite(kCompositeFieldUuidName, CondRange, {VariantArray::Create(nilUuid(), RandString()), VariantArray::Create(randUuid(), RandString())}) .Distinct(distinct.c_str()) @@ -2178,7 +2213,13 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { const char* kFieldNamePages = "pages"; const char* kFieldNamePrice = "price"; const char* kFieldNameUuid = "uuid"; + const char* kFieldNameUuidSparse = "uuid_sparse"; const char* kFieldNameUuidArr = "uuid_arr"; + const char* kFieldNameUuidArrSparse = "uuid_arr_sparse"; + const char* kFieldNameUuidNotIndex = "uuid_not_index"; + const char* kFieldNameUuidNotIndex2 = "uuid_not_index_2"; + const char* kFieldNameUuidNotIndex3 = "uuid_not_index_3"; + const char* kFieldNameRndString = "rndString"; const char* kFieldNameBtreeIdsets = "btree_idsets"; const char* kFieldNamePointQuadraticRTree = "point_quadratic_rtree"; const char* kFieldNamePointLinearRTree = "point_linear_rtree"; @@ -2204,6 +2245,7 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { const std::string forcedSortOffsetNs = "forced_sort_offset_namespace"; const std::string nsWithObject = "namespace_with_object"; const std::string geomNs = "geom_namespace"; + const std::string uuidNs = "uuid_namespace"; const std::string btreeIdxOptNs = "btree_idx_opt_namespace"; const std::string conditionsNs = "conditions_namespace"; @@ -2221,6 +2263,7 @@ class QueriesApi : public ReindexerApi, public QueriesVerifier { static constexpr size_t forcedSortOffsetNsSize = 1000; static constexpr int forcedSortOffsetMaxValue = 1000; static constexpr size_t geomNsSize = 10000; + static constexpr size_t uuidNsSize = 10000; static constexpr int btreeIdxOptNsSize = 10000; size_t conditionsNsSize = 0; std::vector> forcedSortOffsetValues; diff --git a/cpp_src/gtests/tests/fixtures/queries_verifier.h b/cpp_src/gtests/tests/fixtures/queries_verifier.h index 861c05861..ed717fdca 100644 --- a/cpp_src/gtests/tests/fixtures/queries_verifier.h +++ b/cpp_src/gtests/tests/fixtures/queries_verifier.h @@ -26,6 +26,8 @@ class QueriesVerifier : public virtual ::testing::Test { std::string name; reindexer::KeyValueType type; }; + using IndexesData = std::unordered_map>; + void Verify(const reindexer::QueryResults& qr, const reindexer::Query& query, reindexer::Reindexer& rx) { std::unordered_set, PkHash> pks; std::unordered_map> distincts; @@ -40,14 +42,14 @@ class QueriesVerifier : public virtual ::testing::Test { ASSERT_TRUE(err.ok()) << err.what(); Verify(js.QueryResults(), js.JoinQuery(), rx); } - const auto& indexesFields = indexesFields_[query._namespace]; + const auto& indexesFields = indexesFields_[query.NsName()]; for (size_t i = 0; i < qr.Count(); ++i) { reindexer::Item itemr(qr[i].GetItem(false)); - auto pk = getPk(itemr, query._namespace); + auto pk = getPk(itemr, query.NsName()); EXPECT_TRUE(pks.insert(pk).second) << "Duplicated primary key: " + getPkString(pk); - InsertedItemsByPk& insertedItemsByPk = insertedItems_[query._namespace]; + InsertedItemsByPk& insertedItemsByPk = insertedItems_[query.NsName()]; auto itInsertedItem = insertedItemsByPk.find(pk); EXPECT_NE(itInsertedItem, insertedItemsByPk.end()) << "Item with such PK has not been inserted yet: " + getPkString(pk); if (itInsertedItem != insertedItemsByPk.end()) { @@ -142,14 +144,14 @@ class QueriesVerifier : public virtual ::testing::Test { // Check non found items, to not match conditions // If query has limit and offset, skip verification - if (query.start != 0 || query.count != reindexer::QueryEntry::kDefaultLimit) return; + if (query.HasOffset() || query.HasLimit()) return; // If query has distinct, skip verification for (const auto& agg : query.aggregations_) { if (agg.Type() == AggDistinct) return; } - for (auto& insertedItem : insertedItems_[query._namespace]) { + for (auto& insertedItem : insertedItems_[query.NsName()]) { if (pks.find(insertedItem.first) != pks.end()) continue; bool conditionsSatisfied = checkConditions(insertedItem.second, query.entries.cbegin(), query.entries.cend(), joinedSelectors, indexesFields); @@ -161,7 +163,7 @@ class QueriesVerifier : public virtual ::testing::Test { } auto aggResults = qr.GetAggregationResults(); - if (query.calcTotal != ModeNoTotal) { + if (query.CalcTotal() != ModeNoTotal) { // calcTotal from version 3.0.2 also return total count in aggregations, so we have remove it from here for // clean compare aggresults with aggregations aggResults.pop_back(); @@ -232,7 +234,7 @@ class QueriesVerifier : public virtual ::testing::Test { private: bool checkConditions(const reindexer::Item& item, reindexer::QueryEntries::const_iterator it, reindexer::QueryEntries::const_iterator to, const std::vector& joinedSelectors, - const std::unordered_map>& indexesFields) { + const IndexesData& indexesFields) { bool result = true; for (; it != to; ++it) { OpType op = it->operation; @@ -247,7 +249,7 @@ class QueriesVerifier : public virtual ::testing::Test { return checkConditions(item, it.cbegin(), it.cend(), joinedSelectors, indexesFields); }, [&](const reindexer::QueryEntry& qe) { - if ((op == OpOr && result) || qe.distinct) { + if ((op == OpOr && result) || qe.Distinct()) { skip = true; return false; } @@ -287,8 +289,7 @@ class QueriesVerifier : public virtual ::testing::Test { return result; } - static std::string getFieldName(const std::string& indexName, - const std::unordered_map>& indexesFields) { + static std::string getFieldName(const std::string& indexName, const IndexesData& indexesFields) { if (const auto it = indexesFields.find(indexName); it == indexesFields.end()) { return indexName; } else { @@ -300,15 +301,15 @@ class QueriesVerifier : public virtual ::testing::Test { static bool checkDistincts(reindexer::Item& item, const reindexer::Query& qr, std::unordered_map>& distincts, - const std::unordered_map>& indexesFields) { + const IndexesData& indexesFields) { bool result = true; // check only on root level for (auto it = qr.entries.cbegin(); it != qr.entries.cend(); ++it) { if (!it->HoldsOrReferTo()) continue; const reindexer::QueryEntry& qentry = it->Value(); - if (!qentry.distinct) continue; + if (!qentry.Distinct()) continue; - const std::string fieldName = getFieldName(qentry.index, indexesFields); + const std::string fieldName = getFieldName(qentry.FieldName(), indexesFields); reindexer::VariantArray fieldValue = item[fieldName]; EXPECT_EQ(fieldValue.size(), 1) << "Distinct field's size cannot be > 1"; if (fieldValue.empty()) return false; @@ -316,30 +317,30 @@ class QueriesVerifier : public virtual ::testing::Test { std::unordered_set& values = distincts[fieldName]; reindexer::Variant keyValue(fieldValue[0]); bool inserted = values.insert(keyValue.As()).second; - EXPECT_TRUE(inserted) << "Duplicate distinct item for index: " << keyValue.As() << ", " << qentry.idxNo; + EXPECT_TRUE(inserted) << "Duplicate distinct item for index: " << keyValue.As() << ", " << qentry.FieldName() + << " (" << qentry.IndexNo() << ')'; result &= inserted; } return result; } - bool checkCondition(const reindexer::Item& item, const JoinedSelectorMock& joinedSelector, - const std::unordered_map>& leftIndexesFields, - const std::unordered_map>& rightIndexesFields) { + bool checkCondition(const reindexer::Item& item, const JoinedSelectorMock& joinedSelector, const IndexesData& leftIndexesFields, + const IndexesData& rightIndexesFields) { for (auto it : joinedSelector.QueryResults()) { const reindexer::Item& rightItem = it.GetItem(false); bool result = true; const auto& joinEntries{joinedSelector.JoinQuery().joinEntries_}; assertrx(!joinEntries.empty()); - assertrx(joinEntries[0].op_ != OpOr); + assertrx(joinEntries[0].Operation() != OpOr); for (const auto& je : joinEntries) { - if (je.op_ == OpOr) { + if (je.Operation() == OpOr) { if (result) continue; } else if (!result) { break; } - const bool curResult = - checkOnCondition(item, rightItem, je.index_, je.joinIndex_, je.condition_, leftIndexesFields, rightIndexesFields); - switch (je.op_) { + const bool curResult = checkOnCondition(item, rightItem, je.LeftFieldName(), je.RightFieldName(), je.Condition(), + leftIndexesFields, rightIndexesFields); + switch (je.Operation()) { case OpAnd: result = curResult; break; @@ -359,9 +360,8 @@ class QueriesVerifier : public virtual ::testing::Test { } bool checkOnCondition(const reindexer::Item& leftItem, const reindexer::Item& rightItem, const std::string& leftIndexName, - const std::string& rightIndexName, CondType cond, - const std::unordered_map>& leftIndexesFields, - const std::unordered_map>& rightIndexesFields) { + const std::string& rightIndexName, CondType cond, const IndexesData& leftIndexesFields, + const IndexesData& rightIndexesFields) { const CollateOpts& collate = indexesCollates[leftIndexName]; const std::string leftFieldName = getFieldName(leftIndexName, leftIndexesFields); const std::string rightFieldName = getFieldName(rightIndexName, rightIndexesFields); @@ -375,21 +375,20 @@ class QueriesVerifier : public virtual ::testing::Test { return false; } - bool checkCondition(const reindexer::Item& item, const reindexer::QueryEntry& qentry, - const std::unordered_map>& indexesFields) { + bool checkCondition(const reindexer::Item& item, const reindexer::QueryEntry& qentry, const IndexesData& indexesFields) { EXPECT_GT(item.NumFields(), 0); - if (isGeomConditions(qentry.condition)) { + if (isGeomConditions(qentry.Condition())) { return checkGeomConditions(item, qentry, indexesFields); } - const CollateOpts& collate = indexesCollates[qentry.index]; + const CollateOpts& collate = indexesCollates[qentry.FieldName()]; - if (isIndexComposite(item, qentry)) { - return checkCompositeValues(item, qentry, collate, indexesFields); + if (isIndexComposite(qentry.FieldName(), indexesFields)) { + return checkCompositeCondition(item, qentry, collate, indexesFields); } else { std::string fieldName; reindexer::KeyValueType fieldType = reindexer::KeyValueType::Undefined{}; - if (const auto it = indexesFields.find(qentry.index); it == indexesFields.end()) { - fieldName = qentry.index; + if (const auto it = indexesFields.find(qentry.FieldName()); it == indexesFields.end()) { + fieldName = qentry.FieldName(); } else { EXPECT_EQ(it->second.size(), 1); assertrx(!it->second.empty()); @@ -397,13 +396,13 @@ class QueriesVerifier : public virtual ::testing::Test { fieldType = it->second[0].type; } reindexer::VariantArray fieldValues = item[fieldName]; - switch (qentry.condition) { + switch (qentry.Condition()) { case CondEmpty: return fieldValues.size() == 0; case CondAny: return fieldValues.size() > 0; case CondAllSet: - return checkAllSet(fieldValues, qentry.values, collate, fieldType); + return checkAllSet(fieldValues, qentry.Values(), collate, fieldType); case CondEq: case CondLt: case CondLe: @@ -414,7 +413,8 @@ class QueriesVerifier : public virtual ::testing::Test { case CondLike: case CondDWithin: for (const reindexer::Variant& fieldValue : fieldValues) { - if (compareValue(fieldValue, qentry.condition, qentry.values, collate, fieldType)) return true; + if (compareValue(fieldValue, qentry.Condition(), qentry.Values(), collate, fieldType)) + return true; } } } @@ -424,16 +424,15 @@ class QueriesVerifier : public virtual ::testing::Test { static bool isGeomConditions(CondType cond) noexcept { return cond == CondType::CondDWithin; } - static bool checkGeomConditions(const reindexer::Item& item, const reindexer::QueryEntry& qentry, - const std::unordered_map>& indexesFields) { - assertrx(qentry.values.size() == 2); - const reindexer::VariantArray coordinates = item[getFieldName(qentry.index, indexesFields)]; + static bool checkGeomConditions(const reindexer::Item& item, const reindexer::QueryEntry& qentry, const IndexesData& indexesFields) { + assertrx(qentry.Values().size() == 2); + const reindexer::VariantArray coordinates = item[getFieldName(qentry.FieldName(), indexesFields)]; if (coordinates.empty()) return false; assertrx(coordinates.size() == 2); const double x = coordinates[0].As(); const double y = coordinates[1].As(); - if (qentry.condition == CondDWithin) { - return DWithin(reindexer::Point{x, y}, qentry.values[0].As(), qentry.values[1].As()); + if (qentry.Condition() == CondDWithin) { + return DWithin(reindexer::Point{x, y}, qentry.Values()[0].As(), qentry.Values()[1].As()); } else { assertrx(0); abort(); @@ -448,25 +447,24 @@ class QueriesVerifier : public virtual ::testing::Test { return kvalues; } - static std::vector getCompositeFields(const std::string& indexName, - const std::unordered_map>& indexesFields) { + static const std::vector& getCompositeFields(const std::string& indexName, const IndexesData& indexesFields) { const auto it = indexesFields.find(indexName); assert(it != indexesFields.end()); return it->second; } - static bool checkCompositeValues(const reindexer::Item& item, const reindexer::QueryEntry& qentry, const CollateOpts& opts, - const std::unordered_map>& indexesFields) { - const auto fields = getCompositeFields(qentry.index, indexesFields); + static bool checkCompositeCondition(const reindexer::Item& item, const reindexer::QueryEntry& qentry, const CollateOpts& opts, + const IndexesData& indexesFields) { + const auto fields = getCompositeFields(qentry.FieldName(), indexesFields); const reindexer::VariantArray& indexesValues = getValues(item, fields); - const reindexer::VariantArray& keyValues = qentry.values; + const reindexer::VariantArray& keyValues = qentry.Values(); - switch (qentry.condition) { + switch (qentry.Condition()) { case CondEmpty: - return indexesValues.size() == 0; + return indexesValues.empty(); case CondAny: - return indexesValues.size() > 0; + return !indexesValues.empty(); case CondGe: assert(!keyValues.empty()); return compareCompositeValues(indexesValues, keyValues[0], opts) >= 0; @@ -490,10 +488,10 @@ class QueriesVerifier : public virtual ::testing::Test { } return false; case CondAllSet: - for (const reindexer::Variant& kv : indexesValues) { + for (const reindexer::Variant& kv : keyValues) { if (compareCompositeValues(indexesValues, kv, opts) != 0) return false; } - return !indexesValues.empty(); + return !keyValues.empty(); case CondLike: case CondDWithin: default: @@ -640,51 +638,42 @@ class QueriesVerifier : public virtual ::testing::Test { } bool checkCompositeCondition(const reindexer::Item& item, const reindexer::BetweenFieldsQueryEntry& qentry, - const std::unordered_map>& indexesFields) { - const auto firstFields = getCompositeFields(qentry.firstIndex, indexesFields); - const auto secondFields = getCompositeFields(qentry.secondIndex, indexesFields); + const IndexesData& indexesFields) { + const auto& firstFields = getCompositeFields(qentry.LeftFieldName(), indexesFields); + const auto& secondFields = getCompositeFields(qentry.RightFieldName(), indexesFields); assertrx(firstFields.size() == secondFields.size()); - reindexer::BetweenFieldsQueryEntry qe{qentry}; for (size_t i = 0; i < firstFields.size(); ++i) { - qe.firstIndex = firstFields[i].name; - qe.secondIndex = secondFields[i].name; - if (!checkCondition(item, qe, indexesFields)) return false; + if (!checkCondition(item, + reindexer::BetweenFieldsQueryEntry{std::string{firstFields[i].name}, qentry.Condition(), + std::string{secondFields[i].name}}, + indexesFields)) + return false; } return !firstFields.empty(); } - static bool isIndexComposite(const reindexer::BetweenFieldsQueryEntry& qe, - const std::unordered_map>& indexesFields) { - if (qe.firstIndex.find('+') != std::string::npos || qe.secondIndex.find('+') != std::string::npos) return true; - if (const auto it = indexesFields.find(qe.firstIndex); it != indexesFields.end() && it->second.size() > 1) return true; - if (const auto it = indexesFields.find(qe.secondIndex); it != indexesFields.end() && it->second.size() > 1) return true; - return false; + static bool isIndexComposite(const std::string& indexName, const IndexesData& indexesFields) { + const auto it = indexesFields.find(indexName); + return it != indexesFields.end() && it->second.size() > 1; } - static bool isIndexComposite(const reindexer::Item& item, const reindexer::QueryEntry& qentry) { - if (qentry.values.empty()) return false; - if (qentry.idxNo < 0) { - return qentry.values.size() && (qentry.values[0].Type().Is() || - qentry.values[0].Type().Is()); - } - const auto indexType = item.GetIndexType(qentry.idxNo); - return indexType.Is() || indexType.Is(); + static bool isIndexComposite(const reindexer::BetweenFieldsQueryEntry& qe, const IndexesData& indexesFields) { + return isIndexComposite(qe.LeftFieldName(), indexesFields) || isIndexComposite(qe.RightFieldName(), indexesFields); } - bool checkCondition(const reindexer::Item& item, const reindexer::BetweenFieldsQueryEntry& qentry, - const std::unordered_map>& indexesFields) { + bool checkCondition(const reindexer::Item& item, const reindexer::BetweenFieldsQueryEntry& qentry, const IndexesData& indexesFields) { EXPECT_GT(item.NumFields(), 0); assertrx(!isGeomConditions(qentry.Condition())); - const CollateOpts& collate = indexesCollates[qentry.firstIndex]; + const CollateOpts& collate = indexesCollates[qentry.LeftFieldName()]; if (isIndexComposite(qentry, indexesFields)) { return checkCompositeCondition(item, qentry, indexesFields); } - const std::string firstField = getFieldName(qentry.firstIndex, indexesFields); - const std::string secondField = getFieldName(qentry.secondIndex, indexesFields); + const std::string firstField = getFieldName(qentry.LeftFieldName(), indexesFields); + const std::string secondField = getFieldName(qentry.RightFieldName(), indexesFields); reindexer::VariantArray lValues = item[firstField]; reindexer::VariantArray rValues = item[secondField]; switch (qentry.Condition()) { @@ -811,8 +800,8 @@ class QueriesVerifier : public virtual ::testing::Test { std::vector result; result.reserve(query.joinQueries_.size()); for (auto jq : query.joinQueries_) { - jq.count = reindexer::QueryEntry::kDefaultLimit; - jq.start = reindexer::QueryEntry::kDefaultOffset; + jq.Limit(reindexer::QueryEntry::kDefaultLimit); + jq.Offset(reindexer::QueryEntry::kDefaultOffset); jq.sortingEntries_.clear(); jq.forcedSortOrder_.clear(); result.emplace_back(InnerJoin, std::move(jq)); @@ -915,5 +904,5 @@ class QueriesVerifier : public virtual ::testing::Test { } std::unordered_map> ns2pk_; - std::unordered_map>> indexesFields_; + std::unordered_map indexesFields_; }; diff --git a/cpp_src/gtests/tests/fixtures/reindexer_api.h b/cpp_src/gtests/tests/fixtures/reindexer_api.h index 0cc0c4c84..194c32c72 100644 --- a/cpp_src/gtests/tests/fixtures/reindexer_api.h +++ b/cpp_src/gtests/tests/fixtures/reindexer_api.h @@ -6,16 +6,12 @@ #include #include -#include "core/keyvalue/key_string.h" #include "core/keyvalue/variant.h" #include "core/query/query.h" #include "core/reindexer.h" #include "reindexertestapi.h" #include "servercontrol.h" #include "tools/errors.h" -#include "tools/serializer.h" -#include "tools/stringstools.h" -#include "vendor/utf8cpp/utf8.h" using reindexer::Error; using reindexer::Item; diff --git a/cpp_src/gtests/tests/fixtures/servercontrol.cc b/cpp_src/gtests/tests/fixtures/servercontrol.cc index 59cfe9e97..bcbfc80f2 100644 --- a/cpp_src/gtests/tests/fixtures/servercontrol.cc +++ b/cpp_src/gtests/tests/fixtures/servercontrol.cc @@ -124,9 +124,31 @@ void ServerControl::Interface::WriteServerConfig(const std::string& configYaml) void ServerControl::Interface::SetWALSize(int64_t size, std::string_view nsName) { setNamespaceConfigItem(nsName, "wal_size", size); } +void ServerControl::Interface::SetTxAlwaysCopySize(int64_t size, std::string_view nsName) { + setNamespaceConfigItem(nsName, "tx_size_to_always_copy", size); +} + void ServerControl::Interface::SetOptmizationSortWorkers(size_t cnt, std::string_view nsName) { setNamespaceConfigItem(nsName, "optimization_sort_workers", cnt); } + +void ServerControl::Interface::EnableAllProfilings() { + constexpr std::string_view kJsonCfgProfiling = R"json({ + "type":"profiling", + "profiling":{ + "queriesperfstats":true, + "queries_threshold_us":0, + "perfstats":true, + "memstats":true + } + })json"; + auto item = api.reindexer->NewItem(kConfigNs); + ASSERT_TRUE(item.Status().ok()) << item.Status().what(); + auto err = item.FromJSON(kJsonCfgProfiling); + ASSERT_TRUE(err.ok()) << err.what(); + err = api.reindexer->Upsert(kConfigNs, item); + ASSERT_TRUE(err.ok()) << err.what(); +} void ServerControl::Interface::Init() { stopped_ = false; YAML::Node y; diff --git a/cpp_src/gtests/tests/fixtures/servercontrol.h b/cpp_src/gtests/tests/fixtures/servercontrol.h index b268f0b26..86a4154ab 100644 --- a/cpp_src/gtests/tests/fixtures/servercontrol.h +++ b/cpp_src/gtests/tests/fixtures/servercontrol.h @@ -112,8 +112,9 @@ class ServerControl { void WriteServerConfig(const std::string& configYaml); // set server's WAL size void SetWALSize(int64_t size, std::string_view nsName); - + void SetTxAlwaysCopySize(int64_t size, std::string_view nsName); void SetOptmizationSortWorkers(size_t cnt, std::string_view nsName); + void EnableAllProfilings(); reindexer_server::Server srv; #ifndef _WIN32 diff --git a/cpp_src/gtests/tests/fixtures/storage_lazy_load.h b/cpp_src/gtests/tests/fixtures/storage_lazy_load.h index 4feca8291..c32ec7caf 100644 --- a/cpp_src/gtests/tests/fixtures/storage_lazy_load.h +++ b/cpp_src/gtests/tests/fixtures/storage_lazy_load.h @@ -86,7 +86,10 @@ class DISABLED_StorageLazyLoadApi : public ReindexerApi { ASSERT_TRUE(err.ok()) << err.what(); } - void dropNs() { rt.reindexer->DropNamespace(default_namespace); } + void dropNs() { + const auto err = rt.reindexer->DropNamespace(default_namespace); + ASSERT_TRUE(err.ok()) << err.what(); + } int64_t getItemsCount(bool& storageLoaded) { QueryResults qr; diff --git a/cpp_src/gtests/tests/fixtures/ttl_index_api.h b/cpp_src/gtests/tests/fixtures/ttl_index_api.h index 9e6768fef..5026ed6e0 100644 --- a/cpp_src/gtests/tests/fixtures/ttl_index_api.h +++ b/cpp_src/gtests/tests/fixtures/ttl_index_api.h @@ -11,7 +11,8 @@ class TtlIndexApi : public ReindexerApi { DefineNamespaceDataset(default_namespace, {IndexDeclaration{kFieldId, "hash", "int", IndexOpts().PK(), 0}, IndexDeclaration{kFieldData, "tree", "int", IndexOpts().Array(), 0}, IndexDeclaration{kFieldData, "tree", "int", IndexOpts().Array(), 0}}); - rt.reindexer->AddIndex(default_namespace, reindexer::IndexDef(kFieldDate, {kFieldDate}, "ttl", "int64", IndexOpts(), 1)); + err = rt.reindexer->AddIndex(default_namespace, reindexer::IndexDef(kFieldDate, {kFieldDate}, "ttl", "int64", IndexOpts(), 1)); + ASSERT_TRUE(err.ok()) << err.what(); AddDataToNs(3000); } diff --git a/cpp_src/gtests/tests/fuzzing/fuzzing.cc b/cpp_src/gtests/tests/fuzzing/fuzzing.cc index 5491b6c69..fb71502d8 100644 --- a/cpp_src/gtests/tests/fuzzing/fuzzing.cc +++ b/cpp_src/gtests/tests/fuzzing/fuzzing.cc @@ -1,4 +1,6 @@ #include "fuzzing/fuzzing.h" +#include "args/args.hpp" +#include "fuzzing/index.h" #include "fuzzing/ns.h" #include "fuzzing/query_generator.h" @@ -6,23 +8,25 @@ TEST_F(Fuzzing, BaseTest) { try { const fuzzing::RandomGenerator::ErrFactorType errorFactor{0, 1}; reindexer::WrSerializer ser; - std::unordered_set generatedNames; - fuzzing::RandomGenerator rnd(std::cout, errorFactor); - std::vector namespaces_; + fuzzing::RandomGenerator rnd(errorFactor); + std::vector namespaces; const size_t N = 1; for (size_t i = 0; i < N; ++i) { - namespaces_.emplace_back(rnd.NsName(generatedNames), std::cout, errorFactor); - fuzzing::Ns& ns = namespaces_.back(); + namespaces.emplace_back(rnd.GenerateNsName(), errorFactor); + fuzzing::Ns& ns = namespaces.back(); auto err = rx_.OpenNamespace(ns.GetName()); EXPECT_TRUE(err.ok()) << err.what(); - if (!err.ok()) continue; + if (!err.ok()) { + continue; + } auto& indexes = ns.GetIndexes(); - for (size_t i = 0; i < indexes.size();) { - const auto idxDef = indexes[i].IndexDef(ns.GetRandomGenerator(), ns.GetScheme()); + for (size_t j = 0; j < indexes.size();) { + const fuzzing::Index& idx = indexes[j]; + const auto idxDef = idx.IndexDef(ns.GetRandomGenerator(), ns.GetScheme(), indexes); err = rx_.AddIndex(ns.GetName(), idxDef); EXPECT_TRUE(err.ok()) << err.what(); if (err.ok()) { - ns.AddIndex(indexes[i], !idxDef.opts_.IsDense() && idxDef.opts_.IsSparse() && !idxDef.opts_.IsPK()); + ns.AddIndexToScheme(idx, j); // TODO move to fuzzing::Ns std::vector fields; std::visit(reindexer::overloaded{ [&](const fuzzing::Index::Child& c) { @@ -35,23 +39,22 @@ TEST_F(Fuzzing, BaseTest) { ToKeyValueType(ns.GetScheme().GetFieldType(child.fieldPath))}); } }}, - indexes[i].content); - if (indexes[i].isPk) { + idx.Content()); + if (idx.IsPk()) { setPkFields(ns.GetName(), fields); } - - addIndexFields(ns.GetName(), indexes[i].name, std::move(fields)); - ++i; + addIndexFields(ns.GetName(), idx.Name(), std::move(fields)); + ++j; } else { - indexes.erase(indexes.begin() + i); + indexes.erase(indexes.begin() + j); } } - for (size_t i = 0, s = ns.GetRandomGenerator().RndItemsCount(); i < s; ++i) { + for (size_t j = 0, s = ns.GetRandomGenerator().RndItemsCount(); j < s; ++j) { auto item = rx_.NewItem(ns.GetName()); err = item.Status(); EXPECT_TRUE(err.ok()) << err.what(); if (!err.ok()) continue; - ns.NewItem(ser); + ns.NewItem(ser); // TODO not json err = item.FromJSON(ser.Slice()); EXPECT_TRUE(err.ok()) << err.what() << std::endl << "size: " << ser.Slice().size() << std::endl << ser.Slice(); if (!err.ok()) continue; @@ -61,26 +64,26 @@ TEST_F(Fuzzing, BaseTest) { enum Op : uint8_t { Insert, Upsert, Update, Delete, END = Delete }; switch (rnd.RndWhich()) { case Insert: - err = rx_.Insert(rnd.NsName(ns.GetName(), generatedNames), item); + err = rx_.Insert(rnd.NsName(ns.GetName()), item); if (err.ok() && item.GetID() != -1) { saveItem(std::move(item), ns.GetName()); } break; case Upsert: - err = rx_.Upsert(rnd.NsName(ns.GetName(), generatedNames), item); + err = rx_.Upsert(rnd.NsName(ns.GetName()), item); if (err.ok()) { saveItem(std::move(item), ns.GetName()); } break; case Update: - err = rx_.Update(rnd.NsName(ns.GetName(), generatedNames), item); + err = rx_.Update(rnd.NsName(ns.GetName()), item); if (err.ok() && item.GetID() != -1) { saveItem(std::move(item), ns.GetName()); } break; case Delete: { const auto id = item.GetID(); - err = rx_.Delete(rnd.NsName(ns.GetName(), generatedNames), item); + err = rx_.Delete(rnd.NsName(ns.GetName()), item); if (err.ok() && item.GetID() != id) { deleteItem(item, ns.GetName()); } @@ -94,7 +97,7 @@ TEST_F(Fuzzing, BaseTest) { err = rx_.Select(reindexer::Query(ns.GetName()).ReqTotal(), qr); EXPECT_TRUE(err.ok()) << err.what(); } - fuzzing::QueryGenerator queryGenerator{namespaces_, std::cout, errorFactor}; + fuzzing::QueryGenerator queryGenerator{namespaces, errorFactor}; for (size_t i = 0; i < 100; ++i) { auto query = queryGenerator(); reindexer::QueryResults qr; @@ -113,5 +116,40 @@ TEST_F(Fuzzing, BaseTest) { int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); + + args::ArgumentParser parser("Reindexer fuzzing tests"); + args::HelpFlag help(parser, "help", "show this message", {'h', "help"}); + args::Group progOptions("options"); + args::ValueFlag dbDsn(progOptions, "DSN", + "DSN to 'reindexer'. Can be 'cproto://:/' or 'builtin://'", {'d', "dsn"}, + args::Options::Single | args::Options::Global); + args::ValueFlag output(progOptions, "FILENAME", "A file for saving initial states of random engines", {'s', "save"}, + args::Options::Single | args::Options::Global); + args::ValueFlag input(progOptions, "FILENAME", "A file for initial states of random engines recovery", {'r', "repeat"}, + args::Options::Single | args::Options::Global); + args::GlobalOptions globals(parser, progOptions); + try { + parser.ParseCLI(argc, argv); + } catch (const args::Help&) { + std::cout << parser.Help() << std::endl; + return 1; + } catch (const args::Error& e) { + std::cerr << "ERROR: " << e.what() << std::endl; + std::cout << parser.Help() << std::endl; + return 1; + } + std::string out = args::get(output); + if (!out.empty()) { + fuzzing::RandomGenerator::SetOut(std::move(out)); + } + const std::string in = args::get(input); + if (!in.empty()) { + fuzzing::RandomGenerator::SetIn(in); + } + std::string dsn = args::get(dbDsn); + if (!dsn.empty()) { + Fuzzing::SetDsn(std::move(dsn)); + } + return RUN_ALL_TESTS(); } diff --git a/cpp_src/gtests/tests/unit/composite_indexes_api.h b/cpp_src/gtests/tests/unit/composite_indexes_api.h index a555bb162..e78f25c55 100644 --- a/cpp_src/gtests/tests/unit/composite_indexes_api.h +++ b/cpp_src/gtests/tests/unit/composite_indexes_api.h @@ -91,16 +91,24 @@ class CompositeIndexesApi : public ReindexerApi { QueryResults qr; auto err = rt.reindexer->Select(query, qr); EXPECT_TRUE(err.ok()) << err.what(); + assert(err.ok()); QueryResults qrSql; auto sqlQuery = query.GetSQL(); err = rt.reindexer->Select(query.GetSQL(), qrSql); EXPECT_TRUE(err.ok()) << err.what(); + assert(err.ok()); EXPECT_EQ(qr.Count(), qrSql.Count()) << "SQL: " << sqlQuery; for (auto it = qr.begin(), itSql = qrSql.begin(); it != qr.end() && itSql != qrSql.end(); ++it, ++itSql) { + EXPECT_TRUE(it.Status().ok()) << it.Status().what(); + assert(it.Status().ok()); reindexer::WrSerializer ser, serSql; - it.GetCJSON(ser); - itSql.GetCJSON(serSql); + err = it.GetCJSON(ser); + EXPECT_TRUE(err.ok()) << err.what(); + assert(err.ok()); + err = itSql.GetCJSON(serSql); + EXPECT_TRUE(err.ok()) << err.what(); + assert(err.ok()); EXPECT_EQ(ser.Slice(), serSql.Slice()) << "SQL: " << sqlQuery; } return qr; diff --git a/cpp_src/gtests/tests/unit/composite_indexes_test.cc b/cpp_src/gtests/tests/unit/composite_indexes_test.cc index b6ab2b70c..c5579d1b5 100644 --- a/cpp_src/gtests/tests/unit/composite_indexes_test.cc +++ b/cpp_src/gtests/tests/unit/composite_indexes_test.cc @@ -46,11 +46,13 @@ TEST_F(CompositeIndexesApi, AddIndexWithExistingCompositeIndex) { static void selectAll(reindexer::Reindexer* reindexer, const std::string& ns) { QueryResults qr; Error err = reindexer->Select(Query(ns, 0, 1000, ModeAccurateTotal), qr); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); for (auto it : qr) { + ASSERT_TRUE(it.Status().ok()) << it.Status().what(); reindexer::WrSerializer wrser; - it.GetJSON(wrser, false); + err = it.GetJSON(wrser, false); + ASSERT_TRUE(err.ok()) << err.what(); } } @@ -64,7 +66,7 @@ TEST_F(CompositeIndexesApi, DropTest2) { for (int i = 0; i < 1000; ++i) { Item item = NewItem(test_ns); - EXPECT_TRUE(!!item); + EXPECT_FALSE(!item); EXPECT_TRUE(item.Status().ok()) << item.Status().what(); item["id"] = i + 1; @@ -118,7 +120,7 @@ TEST_F(CompositeIndexesApi, CompositeIndexesSelectTest) { auto qr = execAndCompareQuery( Query(default_namespace).WhereComposite(compositeIndexName.c_str(), CondEq, {{Variant(priceValue), Variant(pagesValue)}})); - EXPECT_TRUE(qr.Count() == 1); + ASSERT_EQ(qr.Count(), 1); Item pricePageRow = qr.begin().GetItem(false); Variant selectedPrice = pricePageRow[kFieldNamePrice]; @@ -129,8 +131,8 @@ TEST_F(CompositeIndexesApi, CompositeIndexesSelectTest) { Item titleNameRow = qr.begin().GetItem(false); Variant selectedTitle = titleNameRow[kFieldNameTitle]; Variant selectedName = titleNameRow[kFieldNameName]; - EXPECT_TRUE(static_cast(selectedTitle)->compare(std::string(titleValue)) == 0); - EXPECT_TRUE(static_cast(selectedName)->compare(std::string(nameValue)) == 0); + EXPECT_EQ(static_cast(selectedTitle)->compare(std::string(titleValue)), 0); + EXPECT_EQ(static_cast(selectedName)->compare(std::string(nameValue)), 0); execAndCompareQuery(Query(default_namespace).WhereComposite(compositeIndexName, CondLt, {{Variant(priceValue), Variant(pagesValue)}})); execAndCompareQuery(Query(default_namespace).WhereComposite(compositeIndexName, CondLe, {{Variant(priceValue), Variant(pagesValue)}})); diff --git a/cpp_src/gtests/tests/unit/dsl_parser_test.cc b/cpp_src/gtests/tests/unit/dsl_parser_test.cc index 0daee4f86..2fb4bb4c7 100644 --- a/cpp_src/gtests/tests/unit/dsl_parser_test.cc +++ b/cpp_src/gtests/tests/unit/dsl_parser_test.cc @@ -1,17 +1,20 @@ #include "join_selects_api.h" +static void checkQueryDslParse(const reindexer::Query& q) { + const std::string dsl = q.GetJSON(); + Query parsedQuery; + Error err = parsedQuery.FromJSON(dsl); + ASSERT_TRUE(err.ok()) << err.what() << "\nDSL:\n" << dsl; + ASSERT_EQ(q, parsedQuery) << "DSL:\n" << dsl << "\nOriginal query:\n" << q.GetSQL() << "\nParsed query:\n" << parsedQuery.GetSQL(); +} + TEST_F(JoinSelectsApi, JoinsDSLTest) { Query queryGenres(genres_namespace); Query queryAuthors(authors_namespace); Query queryBooks{Query(books_namespace, 0, 10).Where(price, CondGe, 500)}; queryBooks.OrInnerJoin(genreId_fk, genreid, CondEq, std::move(queryGenres)); queryBooks.LeftJoin(authorid_fk, authorid, CondEq, std::move(queryAuthors)); - - std::string dsl = queryBooks.GetJSON(); - Query testLoadDslQuery; - Error err = testLoadDslQuery.FromJSON(dsl); - ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(queryBooks == testLoadDslQuery); + checkQueryDslParse(queryBooks); } TEST_F(JoinSelectsApi, EqualPositionDSLTest) { @@ -22,12 +25,7 @@ TEST_F(JoinSelectsApi, EqualPositionDSLTest) { query.OpenBracket().Where("f4", CondEq, 4).Where("f5", CondLt, 10); query.AddEqualPosition({"f4", "f5"}); query.CloseBracket(); - - std::string dsl = query.GetJSON(); - Query testLoadDslQuery; - Error err = testLoadDslQuery.FromJSON(dsl); - ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(query == testLoadDslQuery); + checkQueryDslParse(query); } TEST_F(JoinSelectsApi, MergedQueriesDSLTest) { @@ -37,28 +35,15 @@ TEST_F(JoinSelectsApi, MergedQueriesDSLTest) { mainBooksQuery.mergeQueries_.emplace_back(Merge, std::move(firstMergedQuery)); mainBooksQuery.mergeQueries_.emplace_back(Merge, std::move(secondMergedQuery)); - - std::string dsl = mainBooksQuery.GetJSON(); - Query testLoadDslQuery; - Error err = testLoadDslQuery.FromJSON(dsl); - ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(mainBooksQuery == testLoadDslQuery); + checkQueryDslParse(mainBooksQuery); } TEST_F(JoinSelectsApi, AggregateFunctonsDSLTest) { Query query{Query(books_namespace, 10, 100).Where(pages, CondGe, 150)}; - query.aggregations_.push_back({AggAvg, {price}}); - query.aggregations_.push_back({AggSum, {pages}}); - query.aggregations_.push_back({AggFacet, {title, pages}, {{{title, true}}}, 100, 10}); - - std::string dsl = query.GetJSON(); - Query testLoadDslQuery; - Error err = testLoadDslQuery.FromJSON(dsl); - ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(query == testLoadDslQuery); + checkQueryDslParse(query); } TEST_F(JoinSelectsApi, SelectFilterDSLTest) { @@ -66,12 +51,7 @@ TEST_F(JoinSelectsApi, SelectFilterDSLTest) { query.selectFilter_.push_back(price); query.selectFilter_.push_back(pages); query.selectFilter_.push_back(title); - - std::string dsl = query.GetJSON(); - Query testLoadDslQuery; - Error err = testLoadDslQuery.FromJSON(dsl); - ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(query == testLoadDslQuery); + checkQueryDslParse(query); } TEST_F(JoinSelectsApi, SelectFilterInJoinDSLTest) { @@ -85,35 +65,18 @@ TEST_F(JoinSelectsApi, SelectFilterInJoinDSLTest) { queryBooks.LeftJoin(authorid_fk, authorid, CondEq, std::move(queryAuthors)); } - std::string dsl = queryBooks.GetJSON(); - Query testLoadDslQuery; - Error err = testLoadDslQuery.FromJSON(dsl); - ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_EQ(queryBooks, testLoadDslQuery); + checkQueryDslParse(queryBooks); } TEST_F(JoinSelectsApi, ReqTotalDSLTest) { Query query{Query(books_namespace, 10, 100, ModeNoTotal).Where(pages, CondGe, 150)}; - - std::string dsl1 = query.GetJSON(); - Query testLoadDslQuery1; - Error err = testLoadDslQuery1.FromJSON(dsl1); - ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(query == testLoadDslQuery1); + checkQueryDslParse(query); query.CachedTotal(); - std::string dsl2 = query.GetJSON(); - Query testLoadDslQuery2; - err = testLoadDslQuery2.FromJSON(dsl2); - ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(query == testLoadDslQuery2); + checkQueryDslParse(query); query.ReqTotal(); - std::string dsl3 = query.GetJSON(); - Query testLoadDslQuery3; - err = testLoadDslQuery3.FromJSON(dsl3); - ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_EQ(query, testLoadDslQuery3); + checkQueryDslParse(query); } TEST_F(JoinSelectsApi, SelectFunctionsDSLTest) { @@ -121,22 +84,13 @@ TEST_F(JoinSelectsApi, SelectFunctionsDSLTest) { query.AddFunction("f1()"); query.AddFunction("f2()"); query.AddFunction("f3()"); - - std::string dsl = query.GetJSON(); - Query testLoadDslQuery; - Error err = testLoadDslQuery.FromJSON(dsl); - ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(query == testLoadDslQuery); + checkQueryDslParse(query); } TEST_F(JoinSelectsApi, CompositeValuesDSLTest) { std::string pagesBookidIndex = pages + std::string("+") + bookid; Query query{Query(books_namespace).WhereComposite(pagesBookidIndex.c_str(), CondGe, {{Variant(500), Variant(10)}})}; - std::string dsl = query.GetJSON(); - Query testLoadDslQuery; - Error err = testLoadDslQuery.FromJSON(dsl); - ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(query == testLoadDslQuery); + checkQueryDslParse(query); } TEST_F(JoinSelectsApi, GeneralDSLTest) { @@ -153,12 +107,7 @@ TEST_F(JoinSelectsApi, GeneralDSLTest) { testDslQuery.selectFilter_.push_back(authorid_fk); testDslQuery.AddFunction("f1()"); testDslQuery.AddFunction("f2()"); - testDslQuery.aggregations_.push_back({AggDistinct, {bookid}}); - Query testLoadDslQuery; - const std::string dsl1 = testDslQuery.GetJSON(); - Error err = testLoadDslQuery.FromJSON(dsl1); - EXPECT_TRUE(err.ok()) << err.what(); - EXPECT_TRUE(testDslQuery == testLoadDslQuery); + checkQueryDslParse(testDslQuery); } diff --git a/cpp_src/gtests/tests/unit/ft/ft_generic.cc b/cpp_src/gtests/tests/unit/ft/ft_generic.cc index ac58f97de..e2def8c9e 100644 --- a/cpp_src/gtests/tests/unit/ft/ft_generic.cc +++ b/cpp_src/gtests/tests/unit/ft/ft_generic.cc @@ -534,7 +534,8 @@ TEST_P(FTGenericApi, DeleteTest) { // Delete(data[1].first); // Delete(data[1].first); - Delete(data.find("In law, a legal entity is an entity that is capable of bearing legal rights")->second); + const auto err = Delete(data.find("In law, a legal entity is an entity that is capable of bearing legal rights")->second); + ASSERT_TRUE(err.ok()) << err.what(); res = SimpleSelect("entity"); // for (auto it : res) { @@ -572,7 +573,8 @@ TEST_P(FTGenericApi, RebuildAfterDeletion) { auto res = selectF("entity"); ASSERT_EQ(res.Count(), 3); - Delete(data.find("In law, a legal entity is an entity that is capable of bearing legal rights")->second); + err = Delete(data.find("In law, a legal entity is an entity that is capable of bearing legal rights")->second); + ASSERT_TRUE(err.ok()) << err.what(); res = selectF("entity"); ASSERT_EQ(res.Count(), 2); } diff --git a/cpp_src/gtests/tests/unit/ft/ft_stress.cc b/cpp_src/gtests/tests/unit/ft/ft_stress.cc index f25e5429c..e0a0d35e7 100644 --- a/cpp_src/gtests/tests/unit/ft/ft_stress.cc +++ b/cpp_src/gtests/tests/unit/ft/ft_stress.cc @@ -10,7 +10,9 @@ class FTStressApi : public FTApi { }; TEST_P(FTStressApi, BasicStress) { - Init(GetDefaultConfig()); + const std::string kStorage = reindexer::fs::JoinPath(reindexer::fs::GetTempDir(), "reindex_FTApi/BasicStress"); + reindexer::fs::RmDirAll(kStorage); + Init(GetDefaultConfig(), NS1, kStorage); std::vector data; std::vector phrase; @@ -29,7 +31,8 @@ TEST_P(FTStressApi, BasicStress) { std::thread statsThread([&] { while (!terminate) { reindexer::QueryResults qr; - rt.reindexer->Select(reindexer::Query("#memstats"), qr); + const auto err = rt.reindexer->Select(reindexer::Query("#memstats"), qr); + ASSERT_TRUE(err.ok()) << err.what(); std::this_thread::sleep_for(std::chrono::milliseconds(10)); } }); @@ -89,7 +92,8 @@ TEST_P(FTStressApi, ConcurrencyCheck) { lck.unlock(); while (!terminate) { reindexer::QueryResults qr; - rt.reindexer->Select(reindexer::Query("#memstats"), qr); + const auto err = rt.reindexer->Select(reindexer::Query("#memstats"), qr); + ASSERT_TRUE(err.ok()) << err.what(); } }); } else { diff --git a/cpp_src/gtests/tests/unit/join_test.cc b/cpp_src/gtests/tests/unit/join_test.cc index 35f4f46f1..17aee3df6 100644 --- a/cpp_src/gtests/tests/unit/join_test.cc +++ b/cpp_src/gtests/tests/unit/join_test.cc @@ -4,6 +4,7 @@ #include #include "core/itemimpl.h" #include "core/nsselecter/joinedselector.h" +#include "core/type_consts_helpers.h" #include "join_on_conditions_api.h" #include "join_selects_api.h" #include "test_helpers.h" @@ -33,8 +34,8 @@ TEST_F(JoinSelectsApi, JoinsAsWhereConditionsTest) { QueryWatcher watcher{queryBooks}; reindexer::QueryResults qr; Error err = rt.reindexer->Select(queryBooks, qr); - EXPECT_TRUE(err.ok()) << err.what(); - EXPECT_TRUE(qr.Count() <= 50); + ASSERT_TRUE(err.ok()) << err.what(); + EXPECT_LE(qr.Count(), 50); CheckJoinsInComplexWhereCondition(qr); } @@ -47,7 +48,7 @@ TEST_F(JoinSelectsApi, JoinsLockWithCache_364) { for (int i = 0; i < 10; ++i) { reindexer::QueryResults qr; Error err = rt.reindexer->Select(queryBooks, qr); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); } } @@ -67,8 +68,8 @@ TEST_F(JoinSelectsApi, JoinsAsWhereConditionsTest2) { QueryWatcher watcher{query}; reindexer::QueryResults qr; Error err = rt.reindexer->Select(query, qr); - EXPECT_TRUE(err.ok()) << err.what(); - EXPECT_TRUE(qr.Count() <= 50); + ASSERT_TRUE(err.ok()) << err.what(); + EXPECT_LE(qr.Count(), 50); CheckJoinsInComplexWhereCondition(qr); } @@ -91,14 +92,16 @@ TEST_F(JoinSelectsApi, SqlPasringTest) { Query dstQuery; dstQuery.FromSQL(wrser.Slice()); - ASSERT_TRUE(srcQuery == dstQuery); + ASSERT_EQ(srcQuery, dstQuery); wrser.Reset(); srcQuery.Serialize(wrser); Query deserializedQuery; reindexer::Serializer ser(wrser.Buf(), wrser.Len()); deserializedQuery.Deserialize(ser); - ASSERT_TRUE(srcQuery == deserializedQuery); + ASSERT_EQ(srcQuery, deserializedQuery) << "Original query:\n" + << srcQuery.GetSQL() << "\nDeserialized query:\n" + << deserializedQuery.GetSQL(); } TEST_F(JoinSelectsApi, InnerJoinTest) { @@ -109,14 +112,14 @@ TEST_F(JoinSelectsApi, InnerJoinTest) { reindexer::QueryResults joinQueryRes; Error err = rt.reindexer->Select(joinQuery, joinQueryRes); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); err = VerifyResJSON(joinQueryRes); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); reindexer::QueryResults pureSelectRes; err = rt.reindexer->Select(queryBooks, pureSelectRes); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); QueryResultRows joinSelectRows; QueryResultRows pureSelectRows; @@ -129,7 +132,7 @@ TEST_F(JoinSelectsApi, InnerJoinTest) { reindexer::QueryResults authorsSelectRes; Query authorsQuery{Query(authors_namespace).Where(authorid, CondEq, authorIdKeyRef)}; err = rt.reindexer->Select(authorsQuery, authorsSelectRes); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); if (err.ok()) { int bookId = booksItem[bookid].Get(); @@ -152,7 +155,7 @@ TEST_F(JoinSelectsApi, LeftJoinTest) { Query booksQuery{Query(books_namespace).Where(price, CondGe, 500)}; reindexer::QueryResults booksQueryRes; Error err = rt.reindexer->Select(booksQuery, booksQueryRes); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); QueryResultRows pureSelectRows; if (err.ok()) { @@ -169,10 +172,10 @@ TEST_F(JoinSelectsApi, LeftJoinTest) { QueryWatcher watcher{joinQuery}; reindexer::QueryResults joinQueryRes; err = rt.reindexer->Select(joinQuery, joinQueryRes); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); err = VerifyResJSON(joinQueryRes); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); if (err.ok()) { std::unordered_set presentedAuthorIds; @@ -188,7 +191,7 @@ TEST_F(JoinSelectsApi, LeftJoinTest) { for (auto joinedFieldIt = itemIt.begin(); joinedFieldIt != itemIt.end(); ++joinedFieldIt) { reindexer::ItemImpl item2(joinedFieldIt.GetItem(0, joinQueryRes.getPayloadType(1), joinQueryRes.getTagsMatcher(1))); Variant authorIdKeyRef2 = item2.GetField(joinQueryRes.getPayloadType(1).FieldByName(authorid_fk)); - EXPECT_TRUE(authorIdKeyRef1 == authorIdKeyRef2); + EXPECT_EQ(authorIdKeyRef1, authorIdKeyRef2); } presentedAuthorIds.insert(static_cast(authorIdKeyRef1)); @@ -208,15 +211,15 @@ TEST_F(JoinSelectsApi, LeftJoinTest) { int authorId = static_cast(authorIdKeyRef1); auto itAutorid(presentedAuthorIds.find(authorId)); - EXPECT_TRUE(itAutorid != presentedAuthorIds.end()); + EXPECT_NE(itAutorid, presentedAuthorIds.end()); auto itRowidIndex(rowidsIndexes.find(rowid)); - EXPECT_TRUE(itRowidIndex != rowidsIndexes.end()); + EXPECT_NE(itRowidIndex, rowidsIndexes.end()); if (itRowidIndex != rowidsIndexes.end()) { Item item2((joinQueryRes.begin() + rowid).GetItem(false)); Variant authorIdKeyRef2 = item2[authorid]; - EXPECT_TRUE(authorIdKeyRef1 == authorIdKeyRef2); + EXPECT_EQ(authorIdKeyRef1, authorIdKeyRef2); } } } @@ -236,10 +239,10 @@ TEST_F(JoinSelectsApi, OrInnerJoinTest) { reindexer::QueryResults queryRes; Error err = rt.reindexer->Select(orInnerJoinQuery, queryRes); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); err = VerifyResJSON(queryRes); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); if (err.ok()) { for (auto rowIt : queryRes) { @@ -251,7 +254,7 @@ TEST_F(JoinSelectsApi, OrInnerJoinTest) { for (int i = 0; i < authorIdIt.ItemsCount(); ++i) { reindexer::ItemImpl authorsItem(authorIdIt.GetItem(i, queryRes.getPayloadType(1), queryRes.getTagsMatcher(1))); Variant authorIdKeyRef2 = authorsItem.GetField(queryRes.getPayloadType(1).FieldByName(authorid)); - EXPECT_TRUE(authorIdKeyRef1 == authorIdKeyRef2); + EXPECT_EQ(authorIdKeyRef1, authorIdKeyRef2); } reindexer::joins::JoinedFieldIterator genreIdIt = itemIt.at(genresNsJoinIndex); @@ -259,7 +262,7 @@ TEST_F(JoinSelectsApi, OrInnerJoinTest) { for (int i = 0; i < genreIdIt.ItemsCount(); ++i) { reindexer::ItemImpl genresItem = genreIdIt.GetItem(i, queryRes.getPayloadType(2), queryRes.getTagsMatcher(2)); Variant genresIdKeyRef2 = genresItem.GetField(queryRes.getPayloadType(2).FieldByName(genreid)); - EXPECT_TRUE(genresIdKeyRef1 == genresIdKeyRef2); + EXPECT_EQ(genresIdKeyRef1, genresIdKeyRef2); } } } @@ -292,7 +295,7 @@ TEST_F(JoinSelectsApi, JoinTestSorting) { for (auto rowIt : joinQueryRes) { Item item = rowIt.GetItem(false); if (!prevField.Type().Is()) { - ASSERT_TRUE(prevField.Compare(item[age]) <= 0); + ASSERT_LE(prevField.Compare(item[age]), 0); } Variant key = item[authorid]; @@ -304,14 +307,14 @@ TEST_F(JoinSelectsApi, JoinTestSorting) { for (int i = 0; i < joinedFieldIt.ItemsCount(); ++i) { reindexer::ItemImpl joinItem(joinedFieldIt.GetItem(i, joinQueryRes.getPayloadType(1), joinQueryRes.getTagsMatcher(1))); Variant fkey = joinItem.GetField(joinQueryRes.getPayloadType(1).FieldByName(authorid_fk)); - ASSERT_TRUE(key.Compare(fkey) == 0) << key.As() << " " << fkey.As(); + ASSERT_EQ(key.Compare(fkey), 0) << key.As() << " " << fkey.As(); Variant recentJoinedValue = joinItem.GetField(joinQueryRes.getPayloadType(1).FieldByName(price)); - ASSERT_TRUE(recentJoinedValue.As() >= 200); + ASSERT_GE(recentJoinedValue.As(), 200); if (!prevJoinedValue.Type().Is()) { - ASSERT_TRUE(prevJoinedValue.Compare(recentJoinedValue) >= 0); + ASSERT_GE(prevJoinedValue.Compare(recentJoinedValue), 0); } Variant pagesValue = joinItem.GetField(joinQueryRes.getPayloadType(1).FieldByName(pages)); - ASSERT_TRUE(pagesValue.As() >= 100); + ASSERT_GE(pagesValue.As(), 100); prevJoinedValue = recentJoinedValue; } prevField = item[age]; @@ -350,7 +353,7 @@ TEST_F(JoinSelectsApi, TestSortingByJoinedNs) { const Variant recentValue = joinItem.GetField(joinQueryRes2.getPayloadType(1).FieldByName(age)); if (!prevValue.Type().Is()) { reindexer::WrSerializer ser; - ASSERT_TRUE(prevValue.Compare(recentValue) <= 0) << (prevValue.Dump(ser), ser << ' ', recentValue.Dump(ser), ser.Slice()); + ASSERT_LE(prevValue.Compare(recentValue), 0) << (prevValue.Dump(ser), ser << ' ', recentValue.Dump(ser), ser.Slice()); } prevValue = recentValue; } @@ -365,12 +368,12 @@ TEST_F(JoinSelectsApi, JoinTestSelectNonIndexedField) { qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() == 1) << err.what(); + ASSERT_EQ(qr.Count(), 1); Item theOnlyItem = qr[0].GetItem(false); VariantArray krefs = theOnlyItem[title]; - ASSERT_TRUE(krefs.size() == 1); - ASSERT_TRUE(krefs[0].As() == "Crime and Punishment"); + ASSERT_EQ(krefs.size(), 1); + ASSERT_EQ(krefs[0].As(), "Crime and Punishment"); } TEST_F(JoinSelectsApi, JoinByNonIndexedField) { @@ -400,7 +403,7 @@ TEST_F(JoinSelectsApi, JoinByNonIndexedField) { qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() == 1) << err.what(); + ASSERT_EQ(qr.Count(), 1); // And backwards even! reindexer::QueryResults qr2; @@ -411,7 +414,7 @@ TEST_F(JoinSelectsApi, JoinByNonIndexedField) { qr2); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr2.Count() == 1) << err.what(); + ASSERT_EQ(qr2.Count(), 1); } TEST_F(JoinSelectsApi, JoinsEasyStressTest) { @@ -426,15 +429,15 @@ TEST_F(JoinSelectsApi, JoinsEasyStressTest) { for (size_t i = 0; i < 10; ++i) { reindexer::QueryResults queryRes; Error err = rt.reindexer->Select(orInnerJoinQuery, queryRes); - EXPECT_TRUE(err.ok()) << err.what(); - EXPECT_TRUE(queryRes.Count() > 0); + ASSERT_TRUE(err.ok()) << err.what(); + EXPECT_GT(queryRes.Count(), 0); } }; auto removeTh = [this]() { QueryResults qres; Error err = rt.reindexer->Delete(Query(books_namespace, 0, 10).Where(price, CondGe, 5000), qres); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); }; int32_t since = 0, count = 1000; @@ -493,22 +496,20 @@ TEST_F(JoinSelectsApi, JoinPreResultStoreValuesOptimizationStressTest) { QueryResults qres; while (!start) std::this_thread::sleep_for(std::chrono::milliseconds(1)); Error err = rt.reindexer->Select(q, qres); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); }); } start = true; for (auto& th : threads) th.join(); } -static bool checkForAllowedJsonTags(const std::vector& tags, gason::JsonValue jsonValue) { +static void checkForAllowedJsonTags(const std::vector& tags, gason::JsonValue jsonValue) { size_t count = 0; for (const auto& elem : jsonValue) { - if (std::find(tags.begin(), tags.end(), std::string_view(elem.key)) == tags.end()) { - return false; - } + ASSERT_NE(std::find(tags.begin(), tags.end(), std::string_view(elem.key)), tags.end()); ++count; } - return (count == tags.size()); + ASSERT_EQ(count, tags.size()); } TEST_F(JoinSelectsApi, JoinWithSelectFilter) { @@ -525,22 +526,26 @@ TEST_F(JoinSelectsApi, JoinWithSelectFilter) { ASSERT_TRUE(err.ok()) << err.what(); for (auto it : qr) { + ASSERT_TRUE(it.Status().ok()) << it.Status().what(); reindexer::WrSerializer wrser; - it.GetJSON(wrser, false); + err = it.GetJSON(wrser, false); + ASSERT_TRUE(err.ok()) << err.what(); reindexer::joins::ItemIterator joinIt = it.GetJoined(); gason::JsonParser jsonParser; gason::JsonNode root = jsonParser.Parse(reindexer::giftStr(wrser.Slice())); - EXPECT_TRUE(checkForAllowedJsonTags({title, price, "joined_authors_namespace"}, root.value)); + checkForAllowedJsonTags({title, price, "joined_authors_namespace"}, root.value); for (auto fieldIt = joinIt.begin(); fieldIt != joinIt.end(); ++fieldIt) { QueryResults jqr = fieldIt.ToQueryResults(); jqr.addNSContext(qr.getPayloadType(1), qr.getTagsMatcher(1), qr.getFieldsFilter(1), qr.getSchema(1)); for (auto jit : jqr) { + ASSERT_TRUE(jit.Status().ok()) << jit.Status().what(); wrser.Reset(); - jit.GetJSON(wrser, false); + err = jit.GetJSON(wrser, false); + ASSERT_TRUE(err.ok()) << err.what(); root = jsonParser.Parse(reindexer::giftStr(wrser.Slice())); - EXPECT_TRUE(checkForAllowedJsonTags({name, age}, root.value)); + checkForAllowedJsonTags({name, age}, root.value); } } } @@ -573,7 +578,7 @@ TEST_F(JoinSelectsApi, TestMergeWithJoins) { for (auto it : qr) { Item item = it.GetItem(false); auto joined = it.GetJoined(); - ASSERT_TRUE(joined.getJoinedFieldsCount() == 1); + ASSERT_EQ(joined.getJoinedFieldsCount(), 1); bool booksItem = (rowId <= 10000); QueryResults jqr = joined.begin().ToQueryResults(); @@ -585,14 +590,14 @@ TEST_F(JoinSelectsApi, TestMergeWithJoins) { for (auto jit : jqr) { Item jItem = jit.GetItem(false); Variant value = jItem[authorid]; - ASSERT_TRUE(value == fkValue); + ASSERT_EQ(value, fkValue); } } else { Variant fkValue = item[locationid_fk]; for (auto jit : jqr) { Item jItem = jit.GetItem(false); Variant value = jItem[locationid]; - ASSERT_TRUE(value == fkValue); + ASSERT_EQ(value, fkValue); } } @@ -634,30 +639,29 @@ TEST_F(JoinSelectsApi, TestNestedMergesInMergesError) { TEST_F(JoinOnConditionsApi, TestGeneralConditions) { const std::string sqlTemplate = R"(select * from books_namespace inner join books_namespace on (books_namespace.authorid_fk = books_namespace.authorid_fk and books_namespace.pages %s books_namespace.pages);)"; - std::vector conditionsSet = {CondLt, CondLe, CondGt, CondGe, CondEq}; - for (size_t i = 0; i < conditionsSet.size(); ++i) { - CondType condition = conditionsSet[i]; + for (CondType condition : {CondLt, CondLe, CondGt, CondGe, CondEq}) { Query queryBooks; queryBooks.FromSQL(GetSql(sqlTemplate, condition)); QueryResults qr; Error err = rt.reindexer->Select(queryBooks, qr); ASSERT_TRUE(err.ok()) << err.what(); for (auto it : qr) { - auto item = it.GetItem(); + const auto item = it.GetItem(); ASSERT_TRUE(item.Status().ok()) << item.Status().what(); - auto joined = it.GetJoined(); - ASSERT_TRUE(joined.getJoinedFieldsCount() == 1); + const Variant authorid1 = item[authorid_fk]; + const Variant pages1 = item[pages]; + const auto joined = it.GetJoined(); + ASSERT_EQ(joined.getJoinedFieldsCount(), 1); QueryResults jqr = joined.begin().ToQueryResults(); jqr.addNSContext(qr.getPayloadType(0), qr.getTagsMatcher(0), qr.getFieldsFilter(0), qr.getSchema(0)); for (auto jit : jqr) { auto joinedItem = jit.GetItem(); ASSERT_TRUE(joinedItem.Status().ok()) << joinedItem.Status().what(); - Variant authorid1 = item[authorid_fk]; Variant authorid2 = joinedItem[authorid_fk]; - ASSERT_TRUE(authorid1 == authorid2); - Variant pages1 = item[pages]; + ASSERT_EQ(authorid1, authorid2); Variant pages2 = joinedItem[pages]; - ASSERT_TRUE(CompareVariants(pages1, pages2, condition)); + ASSERT_TRUE(CompareVariants(pages1, pages2, condition)) + << pages1.As() << ' ' << reindexer::CondTypeToStr(condition) << ' ' << pages2.As(); } } } @@ -691,14 +695,14 @@ TEST_F(JoinOnConditionsApi, TestComparisonConditions) { auto item1 = it1.GetItem(); ASSERT_TRUE(item1.Status().ok()) << item1.Status().what(); auto joined1 = it1.GetJoined(); - ASSERT_TRUE(joined1.getJoinedFieldsCount() == 1); + ASSERT_EQ(joined1.getJoinedFieldsCount(), 1); QueryResults jqr1 = joined1.begin().ToQueryResults(); jqr1.addNSContext(qr1.getPayloadType(1), qr1.getTagsMatcher(1), qr1.getFieldsFilter(1), qr1.getSchema(0)); auto item2 = it2.GetItem(); ASSERT_TRUE(item2.Status().ok()) << item2.Status().what(); auto joined2 = it2.GetJoined(); - ASSERT_TRUE(joined2.getJoinedFieldsCount() == 1); + ASSERT_EQ(joined2.getJoinedFieldsCount(), 1); QueryResults jqr2 = joined2.begin().ToQueryResults(); jqr2.addNSContext(qr2.getPayloadType(1), qr2.getTagsMatcher(1), qr2.getFieldsFilter(1), qr2.getSchema(0)); @@ -748,8 +752,10 @@ TEST_F(JoinOnConditionsApi, TestLeftJoinOnCondSet) { ASSERT_EQ(qr.Count(), results.size()); int k = 0; for (auto it = qr.begin(); it != qr.end(); ++it, ++k) { + ASSERT_TRUE(it.Status().ok()) << it.Status().what(); reindexer::WrSerializer ser; - it.GetJSON(ser, false); + err = it.GetJSON(ser, false); + ASSERT_TRUE(err.ok()) << err.what(); ASSERT_EQ(ser.c_str(), results[k]); } }; @@ -787,7 +793,7 @@ TEST_F(JoinOnConditionsApi, TestInvalidConditions) { } QueryResults qr; Error err = rt.reindexer->Select(Query(books_namespace).InnerJoin(authorid_fk, authorid, CondAllSet, Query(authors_namespace)), qr); - EXPECT_TRUE(!err.ok()); + EXPECT_FALSE(err.ok()); err = rt.reindexer->Select(Query(books_namespace).InnerJoin(authorid_fk, authorid, CondLike, Query(authors_namespace)), qr); - EXPECT_TRUE(!err.ok()); + EXPECT_FALSE(err.ok()); } diff --git a/cpp_src/gtests/tests/unit/namespace_test.cc b/cpp_src/gtests/tests/unit/namespace_test.cc index 4de15ddae..e8bffaa96 100644 --- a/cpp_src/gtests/tests/unit/namespace_test.cc +++ b/cpp_src/gtests/tests/unit/namespace_test.cc @@ -339,12 +339,12 @@ TEST_F(NsApi, QueryperfstatsNsDummyTest) { QueryResults qr; err = rt.reindexer->Select(Query("#queriesperfstats"), qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() > 0) << "#queriesperfstats table is empty!"; + ASSERT_GT(qr.Count(), 0) << "#queriesperfstats table is empty!"; for (size_t i = 0; i < qr.Count(); ++i) { std::cout << qr[i].GetItem(false).GetJSON() << std::endl; } } - ASSERT_TRUE(qres.Count() == 1) << "Expected 1 row for this query, got " << qres.Count(); + ASSERT_EQ(qres.Count(), 1); Item item = qres[0].GetItem(false); Variant val; val = item["latency_stddev"]; @@ -415,12 +415,12 @@ TEST_F(NsApi, TestUpdateNonindexedField) { Query updateQuery{Query(default_namespace).Where("id", CondGe, Variant("1500")).Set("nested.bonus", static_cast(100500))}; Error err = rt.reindexer->Update(updateQuery, qrUpdate); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qrUpdate.Count() == 500) << qrUpdate.Count(); + ASSERT_EQ(qrUpdate.Count(), 500); QueryResults qrAll; err = rt.reindexer->Select(Query(default_namespace).Where("id", CondGe, Variant("1500")), qrAll); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qrAll.Count() == 500) << qrAll.Count(); + ASSERT_EQ(qrAll.Count(), 500); for (auto it : qrAll) { Item item = it.GetItem(false); @@ -439,12 +439,12 @@ TEST_F(NsApi, TestUpdateSparseField) { Query updateQuery{Query(default_namespace).Where("id", CondGe, Variant("1500")).Set("sparse_field", static_cast(100500))}; Error err = rt.reindexer->Update(updateQuery, qrUpdate); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qrUpdate.Count() == 500) << qrUpdate.Count(); + ASSERT_EQ(qrUpdate.Count(), 500); QueryResults qrAll; err = rt.reindexer->Select(Query(default_namespace).Where("id", CondGe, Variant("1500")), qrAll); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qrAll.Count() == 500) << qrAll.Count(); + ASSERT_EQ(qrAll.Count(), 500); for (auto it : qrAll) { Item item = it.GetItem(false); @@ -473,7 +473,7 @@ TEST_F(NsApi, TestUpdateTwoFields) { // Make sure query worked well ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qrUpdate.Count() == 1) << qrUpdate.Count(); + ASSERT_EQ(qrUpdate.Count(), 1); // Make sure: // 1. JSON of the item is correct @@ -500,12 +500,12 @@ static void updateArrayField(const std::shared_ptr &reinde Query updateQuery{Query(ns).Where("id", CondGe, Variant("500")).Set(updateFieldPath, values)}; Error err = reindexer->Update(updateQuery, qrUpdate); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qrUpdate.Count() > 0) << qrUpdate.Count(); + ASSERT_GT(qrUpdate.Count(), 0); QueryResults qrAll; err = reindexer->Select(Query(ns).Where("id", CondGe, Variant("500")), qrAll); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qrAll.Count() == qrUpdate.Count()) << qrAll.Count(); + ASSERT_EQ(qrAll.Count(), qrUpdate.Count()); for (auto it : qrAll) { Item item = it.GetItem(false); @@ -537,7 +537,7 @@ TEST_F(NsApi, TestUpdateNonindexedArrayField2) { QueryResults qr; Error err = rt.reindexer->Select(R"(update test_namespace set nested.bonus=[{"first":1,"second":2,"third":3}] where id = 1000;)", qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() == 1) << qr.Count(); + ASSERT_EQ(qr.Count(), 1); Item item = qr[0].GetItem(false); std::string_view json = item.GetJSON(); @@ -553,7 +553,7 @@ TEST_F(NsApi, TestUpdateNonindexedArrayField3) { Error err = rt.reindexer->Select(R"(update test_namespace set nested.bonus=[{"id":1},{"id":2},{"id":3},{"id":4}] where id = 1000;)", qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() == 1) << qr.Count(); + ASSERT_EQ(qr.Count(), 1); Item item = qr[0].GetItem(false); VariantArray val = item["nested.bonus"]; @@ -576,12 +576,12 @@ TEST_F(NsApi, TestUpdateNonindexedArrayField4) { QueryResults qr; Error err = rt.reindexer->Select(R"(update test_namespace set nested.bonus=[0] where id = 1000;)", qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() == 1) << qr.Count(); + ASSERT_EQ(qr.Count(), 1); Item item = qr[0].GetItem(false); std::string_view json = item.GetJSON(); size_t pos = json.find(R"("nested":{"bonus":[0])"); - ASSERT_TRUE(pos != std::string::npos) << "'nested.bonus' was not updated properly" << json; + ASSERT_NE(pos, std::string::npos) << "'nested.bonus' was not updated properly" << json; } TEST_F(NsApi, TestUpdateNonindexedArrayField5) { @@ -611,12 +611,12 @@ TEST_F(NsApi, TestUpdateIndexedArrayField2) { Query q{Query(default_namespace).Where(idIdxName, CondEq, static_cast(1000)).Set(indexedArrayField, std::move(value.MarkArray()))}; Error err = rt.reindexer->Update(q, qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() == 1) << qr.Count(); + ASSERT_EQ(qr.Count(), 1); Item item = qr[0].GetItem(false); std::string_view json = item.GetJSON(); size_t pos = json.find(R"("indexed_array_field":[77])"); - ASSERT_TRUE(pos != std::string::npos) << "'indexed_array_field' was not updated properly" << json; + ASSERT_NE(pos, std::string::npos) << "'indexed_array_field' was not updated properly" << json; } static void addAndSetNonindexedField(const std::shared_ptr &reindexer, const std::string &ns, @@ -844,6 +844,7 @@ TEST_F(NsApi, DropArrayField3) { DropArrayItem(rt.reindexer, default_namespace, "nested.nested_array[*].prices[*]", "nested.nested_array.prices"); } +#if (0) // #1500 TEST_F(NsApi, DropArrayField4) { // 1. Define NS // 2. Fill NS @@ -853,6 +854,7 @@ TEST_F(NsApi, DropArrayField4) { DropArrayItem(rt.reindexer, default_namespace, "nested.nested_array[0].prices[((2+4)*2)/6]", "nested.nested_array.prices", 0, ((2 + 4) * 2) / 6); } +#endif TEST_F(NsApi, SetArrayFieldWithSql) { // 1. Define NS @@ -1692,12 +1694,12 @@ static void checkFieldConversion(const std::shared_ptr &re ASSERT_TRUE(!err.ok()); } else { ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qrUpdate.Count() > 0) << qrUpdate.Count(); + ASSERT_GT(qrUpdate.Count(), 0); QueryResults qrAll; err = reindexer->Select(selectQuery, qrAll); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qrAll.Count() == qrUpdate.Count()) << qrAll.Count(); + ASSERT_EQ(qrAll.Count(), qrUpdate.Count()); for (auto it : qrAll) { Item item = it.GetItem(false); @@ -1845,7 +1847,7 @@ TEST_F(NsApi, TestUpdatePkFieldNoConditions) { QueryResults qr; Error err = rt.reindexer->Select("update test_namespace set id = id + 1;", qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() > 0); + ASSERT_GT(qr.Count(), 0); int i = 1; for (auto &it : qr) { @@ -1862,7 +1864,7 @@ TEST_F(NsApi, TestUpdateIndexArrayWithNull) { QueryResults qr; Error err = rt.reindexer->Select("update test_namespace set indexed_array_field = null where id = 1;", qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() == 1); + ASSERT_EQ(qr.Count(), 1); for (auto &it : qr) { Item item = it.GetItem(false); @@ -1983,7 +1985,7 @@ TEST_F(NsApi, TestUpdateNonIndexFieldWithNull) { QueryResults qr; Error err = rt.reindexer->Select("update test_namespace set extra = null where id = 1001;", qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() == 1); + ASSERT_EQ(qr.Count(), 1); for (auto &it : qr) { Item item = it.GetItem(false); @@ -2008,7 +2010,7 @@ TEST_F(NsApi, TestUpdateEmptyArrayField) { QueryResults qr; Error err = rt.reindexer->Select("update test_namespace set indexed_array_field = [] where id = 1;", qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() == 1); + ASSERT_EQ(qr.Count(), 1); Item item = qr[0].GetItem(false); Variant idFieldVal = item[idIdxName]; @@ -2070,12 +2072,12 @@ TEST_F(NsApi, TestUpdateEmptyIndexedField) { .Set(indexedArrayField, {Variant(static_cast(4)), Variant(static_cast(5)), Variant(static_cast(6))}); Error err = rt.reindexer->Update(q, qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() == 1); + ASSERT_EQ(qr.Count(), 1); QueryResults qr2; err = rt.reindexer->Select("select * from test_namespace where id = 1001;", qr2); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr2.Count() == 1); + ASSERT_EQ(qr2.Count(), 1); for (auto it : qr2) { Item item = it.GetItem(false); @@ -2100,7 +2102,7 @@ TEST_F(NsApi, TestDropField) { QueryResults qr; Error err = rt.reindexer->Select("update test_namespace drop extra where id >= 1000 and id < 1010;", qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() == 10) << qr.Count(); + ASSERT_EQ(qr.Count(), 10); for (auto it : qr) { Item item = it.GetItem(false); @@ -2112,7 +2114,7 @@ TEST_F(NsApi, TestDropField) { QueryResults qr2; err = rt.reindexer->Select("update test_namespace drop nested.bonus where id >= 1005 and id < 1010;", qr2); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr2.Count() == 5); + ASSERT_EQ(qr2.Count(), 5); for (auto it : qr2) { Item item = it.GetItem(false); @@ -2144,7 +2146,7 @@ TEST_F(NsApi, TestUpdateFieldWithFunction) { Error err = rt.reindexer->Select( "update test_namespace set int_field = SERIAL(), extra = SERIAL(), nested.timeField = NOW(msec) where id >= 0;", qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() > 0); + ASSERT_GT(qr.Count(), 0); int i = 1; for (auto &it : qr) { @@ -2168,7 +2170,7 @@ TEST_F(NsApi, TestUpdateFieldWithExpressions) { "0;", qr); ASSERT_TRUE(err.ok()) << err.what(); - ASSERT_TRUE(qr.Count() > 0) << qr.Count(); + ASSERT_GT(qr.Count(), 0); int i = 1; for (auto &it : qr) { @@ -2207,22 +2209,22 @@ static void checkQueryDsl(const Query &src) { } } if (objectValues) { - EXPECT_TRUE(src.entries == dst.entries); - EXPECT_TRUE(src.aggregations_ == dst.aggregations_); - EXPECT_TRUE(src._namespace == dst._namespace); - EXPECT_TRUE(src.sortingEntries_ == dst.sortingEntries_); - EXPECT_TRUE(src.calcTotal == dst.calcTotal); - EXPECT_TRUE(src.start == dst.start); - EXPECT_TRUE(src.count == dst.count); - EXPECT_TRUE(src.debugLevel == dst.debugLevel); - EXPECT_TRUE(src.strictMode == dst.strictMode); - EXPECT_TRUE(src.forcedSortOrder_ == dst.forcedSortOrder_); - EXPECT_TRUE(src.selectFilter_ == dst.selectFilter_); - EXPECT_TRUE(src.selectFunctions_ == dst.selectFunctions_); - EXPECT_TRUE(src.joinQueries_ == dst.joinQueries_); - EXPECT_TRUE(src.mergeQueries_ == dst.mergeQueries_); + EXPECT_EQ(src.entries, dst.entries); + EXPECT_EQ(src.aggregations_, dst.aggregations_); + EXPECT_EQ(src.NsName(), dst.NsName()); + EXPECT_EQ(src.sortingEntries_, dst.sortingEntries_); + EXPECT_EQ(src.CalcTotal(), dst.CalcTotal()); + EXPECT_EQ(src.Offset(), dst.Offset()); + EXPECT_EQ(src.Limit(), dst.Limit()); + EXPECT_EQ(src.debugLevel, dst.debugLevel); + EXPECT_EQ(src.strictMode, dst.strictMode); + EXPECT_EQ(src.forcedSortOrder_, dst.forcedSortOrder_); + EXPECT_EQ(src.selectFilter_, dst.selectFilter_); + EXPECT_EQ(src.selectFunctions_, dst.selectFunctions_); + EXPECT_EQ(src.joinQueries_, dst.joinQueries_); + EXPECT_EQ(src.mergeQueries_, dst.mergeQueries_); } else { - EXPECT_TRUE(dst == src); + EXPECT_EQ(dst, src); } } @@ -2379,12 +2381,13 @@ TEST_F(NsApi, MsgPackEncodingTest) { ASSERT_TRUE(err.ok()) << err.what(); std::string json(item.GetJSON()); - ASSERT_TRUE(json == items[i++]); + ASSERT_EQ(json, items[i++]); } reindexer::WrSerializer wrSer3; for (size_t i = 0; i < qr.Count(); ++i) { - qr[i].GetMsgPack(wrSer3, false); + const auto err = qr[i].GetMsgPack(wrSer3, false); + ASSERT_TRUE(err.ok()) << err.what(); } i = 0; @@ -2397,7 +2400,7 @@ TEST_F(NsApi, MsgPackEncodingTest) { ASSERT_TRUE(err.ok()) << err.what(); std::string json(item.GetJSON()); - ASSERT_TRUE(json == items[i++]); + ASSERT_EQ(json, items[i++]); } } diff --git a/cpp_src/gtests/tests/unit/queries_test.cc b/cpp_src/gtests/tests/unit/queries_test.cc index 5a9ac6eb7..725fa47d6 100644 --- a/cpp_src/gtests/tests/unit/queries_test.cc +++ b/cpp_src/gtests/tests/unit/queries_test.cc @@ -32,7 +32,7 @@ TEST_F(QueriesApi, QueriesStandardTestSet) { auto& items = insertedItems_[default_namespace]; for (auto it = items.begin(); it != items.end();) { Error err = rt.reindexer->Delete(default_namespace, it->second); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); it = items.erase(it); if (++itemsCount == 4000) break; } @@ -43,7 +43,7 @@ TEST_F(QueriesApi, QueriesStandardTestSet) { itemsCount = 0; for (auto it = items.begin(); it != items.end();) { Error err = rt.reindexer->Delete(default_namespace, it->second); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); it = items.erase(it); if (++itemsCount == 5000) break; } @@ -52,7 +52,7 @@ TEST_F(QueriesApi, QueriesStandardTestSet) { auto itToRemove = items.begin(); if (itToRemove != items.end()) { Error err = rt.reindexer->Delete(default_namespace, itToRemove->second); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); items.erase(itToRemove); } FillDefaultNamespace(rand() % 100, 1, 0); @@ -62,7 +62,7 @@ TEST_F(QueriesApi, QueriesStandardTestSet) { std::advance(itToRemove, rand() % std::min(100, int(items.size()))); if (itToRemove != items.end()) { Error err = rt.reindexer->Delete(default_namespace, itToRemove->second); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); items.erase(itToRemove); } } @@ -70,7 +70,7 @@ TEST_F(QueriesApi, QueriesStandardTestSet) { for (auto it = items.begin(); it != items.end();) { Error err = rt.reindexer->Delete(default_namespace, it->second); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); it = items.erase(it); } @@ -105,6 +105,18 @@ TEST_F(QueriesApi, QueriesConditions) { CheckConditions(); } +#if !defined(REINDEX_WITH_TSAN) +TEST_F(QueriesApi, UuidQueries) { + FillUUIDNs(); + // hack to obtain not index not string uuid fields + /*auto err = rt.reindexer->DropIndex(uuidNs, {kFieldNameUuidNotIndex2}); // TODO uncomment this #1470 + ASSERT_TRUE(err.ok()) << err.what(); + err = rt.reindexer->DropIndex(uuidNs, {kFieldNameUuidNotIndex3}); + ASSERT_TRUE(err.ok()) << err.what();*/ + CheckUUIDQueries(); +} +#endif + TEST_F(QueriesApi, IndexCacheInvalidationTest) { std::vector> data{{0, 10}, {1, 9}, {2, 8}, {3, 7}, {4, 6}, {5, 5}, {6, 4}, {7, 3}, {8, 2}, {9, 1}, {10, 0}, {11, -1}}; @@ -340,15 +352,15 @@ TEST_F(QueriesApi, StrictModeTest) { const std::string kNotExistingField = "some_random_name123"; QueryResults qr; { - Query query = Query(testSimpleNs).Where(kNotExistingField, CondEmpty, 0); + Query query = Query(testSimpleNs).Where(kNotExistingField, CondEmpty, {}); Error err = rt.reindexer->Select(query.Strict(StrictModeNames), qr); - EXPECT_EQ(err.code(), errParams); + EXPECT_EQ(err.code(), errQueryExec); qr.Clear(); err = rt.reindexer->Select(query.Strict(StrictModeIndexes), qr); - EXPECT_EQ(err.code(), errParams); + EXPECT_EQ(err.code(), errQueryExec); qr.Clear(); err = rt.reindexer->Select(query.Strict(StrictModeNone), qr); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); Verify(qr, Query(testSimpleNs), *rt.reindexer); qr.Clear(); } @@ -356,13 +368,13 @@ TEST_F(QueriesApi, StrictModeTest) { { Query query = Query(testSimpleNs).Where(kNotExistingField, CondEq, 0); Error err = rt.reindexer->Select(query.Strict(StrictModeNames), qr); - EXPECT_EQ(err.code(), errParams); + EXPECT_EQ(err.code(), errQueryExec); qr.Clear(); err = rt.reindexer->Select(query.Strict(StrictModeIndexes), qr); - EXPECT_EQ(err.code(), errParams); + EXPECT_EQ(err.code(), errQueryExec); qr.Clear(); err = rt.reindexer->Select(query.Strict(StrictModeNone), qr); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); EXPECT_EQ(qr.Count(), 0); } } @@ -453,6 +465,7 @@ TEST_F(QueriesApi, JoinByNotIndexField) { ASSERT_EQ(qr.Count(), sizeof(expectedIds) / sizeof(int)); for (size_t i = 0; i < qr.Count(); ++i) { Item item = qr[i].GetItem(false); + ASSERT_TRUE(item.Status().ok()) << item.Status().what(); VariantArray values = item["id"]; ASSERT_EQ(values.size(), 1); EXPECT_EQ(values[0].As(), expectedIds[i]); @@ -807,14 +820,18 @@ TEST_F(QueriesApi, ConvertationStringToDoubleDuringSorting) { std::string print(const reindexer::Query& q, reindexer::QueryResults::Iterator& currIt, reindexer::QueryResults::Iterator& prevIt, const reindexer::QueryResults& qr) { + assertrx(currIt.Status().ok()); std::string res = '\n' + q.GetSQL() + "\ncurr: "; reindexer::WrSerializer ser; - currIt.GetJSON(ser, false); + const auto err = currIt.GetJSON(ser, false); + assertrx(err.ok()); res += ser.Slice(); if (prevIt != qr.end()) { + assertrx(prevIt.Status().ok()); res += "\nprev: "; ser.Reset(); - prevIt.GetJSON(ser, false); + const auto err = prevIt.GetJSON(ser, false); + assertrx(err.ok()); res += ser.Slice(); } return res; diff --git a/cpp_src/gtests/tests/unit/replication_master_master_test.cc b/cpp_src/gtests/tests/unit/replication_master_master_test.cc index 96aa46cb4..b620b6109 100644 --- a/cpp_src/gtests/tests/unit/replication_master_master_test.cc +++ b/cpp_src/gtests/tests/unit/replication_master_master_test.cc @@ -431,6 +431,105 @@ TEST_F(ReplicationSlaveSlaveApi, TransactionTest) { for (auto& node : nodes) node.Stop(); } +TEST_F(ReplicationSlaveSlaveApi, TransactionCopyPolicyForceSync) { + // Check transactions copy policy after force sync + /* + m + | + 1 + | + 2 + */ + constexpr std::string_view kJsonCfgNss = R"=({ + "namespaces": [ + { + "namespace": "*", + "start_copy_policy_tx_size": 10000, + "copy_policy_multiplier": 5, + "tx_size_to_always_copy": 100000 + }, + { + "namespace": "ns1", + "start_copy_policy_tx_size": 10000, + "copy_policy_multiplier": 5, + "tx_size_to_always_copy": 1 + } + ], + "type": "namespaces" + })="; + constexpr int port = 9999; + const std::string kBaseDbPath(fs::JoinPath(kBaseTestsetDbPath, "TransactionCopyPolicyForceSync")); + const std::string kDbPathMaster(kBaseDbPath + "/test_"); + constexpr int serverId = 5; + constexpr size_t kRows = 100; + const std::string nsName("ns1"); + + std::vector slaveConfiguration = {-1, port, port + 1}; + std::vector nodes; + for (size_t i = 0; i < slaveConfiguration.size(); i++) { + nodes.emplace_back(); + nodes.back().InitServer(i, port + i, port + 1000 + i, kDbPathMaster + std::to_string(i), "db", true); + nodes.back().Get()->EnableAllProfilings(); + } + + // Set tx copy policy for the node '2' to 'always copy' + { + auto item = nodes[2].Get()->api.reindexer->NewItem("#config"); + ASSERT_TRUE(item.Status().ok()) << item.Status().what(); + auto err = item.FromJSON(kJsonCfgNss); + ASSERT_TRUE(err.ok()) << err.what(); + err = nodes[2].Get()->api.reindexer->Upsert("#config", item); + ASSERT_TRUE(err.ok()) << err.what(); + } + + for (size_t i = 0; i < slaveConfiguration.size(); i++) { + if (i == 0) { + ReplicationConfigTest config("master"); + nodes[i].Get()->MakeMaster(config); + } else { + std::string masterDsn = "cproto://127.0.0.1:" + std::to_string(slaveConfiguration[i]) + "/db"; + ReplicationConfigTest config("slave", false, true, serverId + i, masterDsn); + nodes[i].Get()->MakeSlave(slaveConfiguration[i], config); + } + } + nodes[2].Drop(); + + ServerControl& master = nodes[0]; + TestNamespace1 ns1(master, nsName); + WaitSync(nodes[0], nodes[1], nsName); + + // Restart node '2' + nodes[2].InitServer(2, port + 2, port + 1000 + 2, kDbPathMaster + std::to_string(2), "db", true); + std::string masterDsn = "cproto://127.0.0.1:" + std::to_string(slaveConfiguration[2]) + "/db"; + ReplicationConfigTest config("slave", false, true, serverId + 2, masterDsn); + nodes[2].Get()->MakeSlave(1, config); + WaitSync(nodes[0], nodes[2], nsName); + + // Apply tx + reindexer::client::SyncCoroTransaction tr = master.Get()->api.reindexer->NewTransaction(nsName); + for (unsigned int i = 0; i < kRows; i++) { + reindexer::client::Item item = tr.NewItem(); + auto err = item.FromJSON("{\"id\":" + std::to_string(i + kRows * 10) + "}"); + tr.Upsert(std::move(item)); + } + master.Get()->api.reindexer->CommitTransaction(tr); + WaitSync(nodes[0], nodes[2], nsName); + + // Check copy tx event in the perfstats + client::SyncCoroQueryResults qr(nodes[2].Get()->api.reindexer.get()); + auto err = nodes[2].Get()->api.reindexer->Select("select * from #perfstats", qr); + ASSERT_TRUE(err.ok()) << err.what(); + ASSERT_EQ(qr.Count(), 1); + WrSerializer ser; + err = qr.begin().GetJSON(ser, false); + ASSERT_TRUE(err.ok()) << err.what(); + gason::JsonParser parser; + auto resJS = parser.Parse(ser.Slice()); + ASSERT_EQ(resJS["transactions"]["total_copy_count"].As(-1), 1) << ser.Slice(); + + for (auto& node : nodes) node.Stop(); +} + TEST_F(ReplicationSlaveSlaveApi, ForceSync3Node) { // Check force-sync for cascade setup /* diff --git a/cpp_src/gtests/tests/unit/rpcclient_test.cc b/cpp_src/gtests/tests/unit/rpcclient_test.cc index a6c1027ee..9ea0b80b9 100644 --- a/cpp_src/gtests/tests/unit/rpcclient_test.cc +++ b/cpp_src/gtests/tests/unit/rpcclient_test.cc @@ -20,11 +20,11 @@ TEST_F(RPCClientTestApi, ConnectTimeout) { config.RequestTimeout = seconds(5); reindexer::client::Reindexer rx(config); auto res = rx.Connect(std::string("cproto://") + kDefaultRPCServerAddr + "/test_db"); - EXPECT_TRUE(res.ok()) << res.what(); + ASSERT_TRUE(res.ok()) << res.what(); res = rx.AddNamespace(reindexer::NamespaceDef("MyNamespace")); EXPECT_EQ(res.code(), errTimeout); res = StopServer(); - EXPECT_TRUE(res.ok()) << res.what(); + ASSERT_TRUE(res.ok()) << res.what(); } TEST_F(RPCClientTestApi, RequestTimeout) { @@ -35,14 +35,14 @@ TEST_F(RPCClientTestApi, RequestTimeout) { config.RequestTimeout = seconds(3); reindexer::client::Reindexer rx(config); auto res = rx.Connect(std::string("cproto://") + kDefaultRPCServerAddr + "/test_db"); - EXPECT_TRUE(res.ok()) << res.what(); + ASSERT_TRUE(res.ok()) << res.what(); const std::string kNamespaceName = "MyNamespace"; res = rx.AddNamespace(reindexer::NamespaceDef(kNamespaceName)); EXPECT_EQ(res.code(), errTimeout); res = rx.DropNamespace(kNamespaceName); - EXPECT_TRUE(res.ok()) << res.what(); + ASSERT_TRUE(res.ok()) << res.what(); res = StopServer(); - EXPECT_TRUE(res.ok()) << res.what(); + ASSERT_TRUE(res.ok()) << res.what(); } TEST_F(RPCClientTestApi, RequestCancels) { @@ -50,7 +50,7 @@ TEST_F(RPCClientTestApi, RequestCancels) { StartServer(); reindexer::client::Reindexer rx; auto res = rx.Connect(std::string("cproto://") + kDefaultRPCServerAddr + "/test_db"); - EXPECT_TRUE(res.ok()) << res.what(); + ASSERT_TRUE(res.ok()) << res.what(); { CancelRdxContext ctx; @@ -72,7 +72,7 @@ TEST_F(RPCClientTestApi, RequestCancels) { } res = StopServer(); - EXPECT_TRUE(res.ok()) << res.what(); + ASSERT_TRUE(res.ok()) << res.what(); } TEST_F(RPCClientTestApi, SuccessfullRequestWithTimeout) { @@ -83,22 +83,23 @@ TEST_F(RPCClientTestApi, SuccessfullRequestWithTimeout) { config.RequestTimeout = seconds(6); reindexer::client::Reindexer rx(config); auto res = rx.Connect(std::string("cproto://") + kDefaultRPCServerAddr + "/test_db"); - EXPECT_TRUE(res.ok()) << res.what(); + ASSERT_TRUE(res.ok()) << res.what(); res = rx.AddNamespace(reindexer::NamespaceDef("MyNamespace")); - EXPECT_TRUE(res.ok()) << res.what(); + ASSERT_TRUE(res.ok()) << res.what(); res = StopServer(); - EXPECT_TRUE(res.ok()) << res.what(); + ASSERT_TRUE(res.ok()) << res.what(); } TEST_F(RPCClientTestApi, ErrorLoginResponse) { AddFakeServer(); StartServer(kDefaultRPCServerAddr, errForbidden); reindexer::client::Reindexer rx; - rx.Connect(std::string("cproto://") + kDefaultRPCServerAddr + "/test_db"); - auto res = rx.AddNamespace(reindexer::NamespaceDef("MyNamespace")); - EXPECT_EQ(res.code(), errForbidden) << res.what(); - res = StopServer(); - EXPECT_TRUE(res.ok()) << res.what(); + auto err = rx.Connect(std::string("cproto://") + kDefaultRPCServerAddr + "/test_db"); + ASSERT_TRUE(err.ok()) << err.what(); + err = rx.AddNamespace(reindexer::NamespaceDef("MyNamespace")); + EXPECT_EQ(err.code(), errForbidden) << err.what(); + err = StopServer(); + ASSERT_TRUE(err.ok()) << err.what(); } TEST_F(RPCClientTestApi, SeveralDsnReconnect) { @@ -126,7 +127,7 @@ TEST_F(RPCClientTestApi, SeveralDsnReconnect) { connectData.emplace_back(std::string().append(cprotoIdentifier).append(uri).append(dbName), reindexer::client::ConnectOpts()); } auto res = rx.Connect(connectData); - EXPECT_TRUE(res.ok()) << res.what(); + ASSERT_TRUE(res.ok()) << res.what(); for (size_t i = 0; i < 100; ++i) { if (CheckIfFakeServerConnected(uris[0])) break; @@ -147,7 +148,7 @@ TEST_F(RPCClientTestApi, SeveralDsnReconnect) { if (res.ok()) break; std::this_thread::sleep_for(std::chrono::milliseconds(100)); } - EXPECT_TRUE(res.ok()) << res.what(); + ASSERT_TRUE(res.ok()) << res.what(); } StopAllServers(); } @@ -209,7 +210,7 @@ TEST_F(RPCClientTestApi, SelectFromClosedNamespace) { loop.run(); ASSERT_TRUE(finished); Error err = StopServer(); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); } TEST_F(RPCClientTestApi, RenameNamespace) { @@ -288,7 +289,7 @@ TEST_F(RPCClientTestApi, RenameNamespace) { loop.run(); ASSERT_TRUE(finished); Error err = StopServer(); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); } TEST_F(RPCClientTestApi, CoroRequestTimeout) { @@ -305,19 +306,19 @@ TEST_F(RPCClientTestApi, CoroRequestTimeout) { config.RequestTimeout = seconds(1); reindexer::client::CoroReindexer rx(config); auto err = rx.Connect(std::string("cproto://") + kDefaultRPCServerAddr + "/test_db", loop); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); const std::string kNamespaceName = "MyNamespace"; err = rx.AddNamespace(reindexer::NamespaceDef(kNamespaceName)); EXPECT_EQ(err.code(), errTimeout); loop.sleep(std::chrono::seconds(4)); err = rx.DropNamespace(kNamespaceName); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); finished = true; }); loop.run(); ASSERT_TRUE(finished); Error err = StopServer(); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); } static std::chrono::seconds GetMaxTimeForCoroSelectTimeout(unsigned requests, std::chrono::seconds delay) { @@ -391,16 +392,16 @@ TEST_F(RPCClientTestApi, CoroSelectTimeout) { [&] { return server.CloseQRRequestsCount() >= kCorCount * kQueriesCount; }); EXPECT_EQ(server.CloseQRRequestsCount(), kCorCount * kQueriesCount); err = rx.AddNamespace(reindexer::NamespaceDef(kNamespaceName + std::to_string(index))); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); finished[index] = true; }); } loop.run(); for (size_t i = 0; i < kCorCount; ++i) { - EXPECT_TRUE(finished[i]); + ASSERT_TRUE(finished[i]); } Error const err = StopServer(); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); } TEST_F(RPCClientTestApi, CoroRequestCancels) { @@ -412,7 +413,7 @@ TEST_F(RPCClientTestApi, CoroRequestCancels) { loop.spawn([&loop, &finished] { reindexer::client::CoroReindexer rx; auto err = rx.Connect(std::string("cproto://") + kDefaultRPCServerAddr + "/test_db", loop); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); { CancelRdxContext ctx; @@ -440,7 +441,7 @@ TEST_F(RPCClientTestApi, CoroRequestCancels) { loop.run(); ASSERT_TRUE(finished); Error err = StopServer(); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); } TEST_F(RPCClientTestApi, CoroSuccessfullRequestWithTimeout) { @@ -455,15 +456,15 @@ TEST_F(RPCClientTestApi, CoroSuccessfullRequestWithTimeout) { config.RequestTimeout = seconds(6); reindexer::client::CoroReindexer rx(config); auto err = rx.Connect(std::string("cproto://") + kDefaultRPCServerAddr + "/test_db", loop); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); err = rx.AddNamespace(reindexer::NamespaceDef("MyNamespace")); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); finished = true; }); loop.run(); ASSERT_TRUE(finished); Error err = StopServer(); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); } TEST_F(RPCClientTestApi, CoroErrorLoginResponse) { @@ -483,7 +484,7 @@ TEST_F(RPCClientTestApi, CoroErrorLoginResponse) { loop.run(); ASSERT_TRUE(finished); Error err = StopServer(); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); } TEST_F(RPCClientTestApi, CoroStatus) { @@ -504,7 +505,7 @@ TEST_F(RPCClientTestApi, CoroStatus) { err = rx.Status(); ASSERT_TRUE(err.ok()) << err.what(); err = StopServer(); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); loop.sleep(std::chrono::milliseconds(20)); // Allow reading coroutine to handle disconnect err = rx.Status(); ASSERT_EQ(err.code(), errNetwork) << err.what(); @@ -621,8 +622,7 @@ TEST_F(RPCClientTestApi, CoroUpserts) { for (auto& it : qr) { ASSERT_TRUE(it.Status().ok()) << it.Status().what(); } - err = rx.Stop(); - ASSERT_TRUE(err.ok()) << err.what(); + rx.Stop(); finished = true; }); @@ -713,8 +713,7 @@ TEST_F(RPCClientTestApi, ServerRestart) { ready = true; wg.wait(); - err = rx.Stop(); - ASSERT_TRUE(err.ok()) << err.what(); + rx.Stop(); finished = true; }); @@ -728,7 +727,7 @@ TEST_F(RPCClientTestApi, ServerRestart) { // Shutdown server step = Step::ShutdownInProgress; Error err = StopServer(); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); step = Step::ShutdownDone; std::this_thread::sleep_for(std::chrono::milliseconds(300)); @@ -861,8 +860,7 @@ TEST_F(RPCClientTestApi, CoroUpdatesFilteringByNs) { ASSERT_TRUE(reciever2.AwaitNamespaces(0)); ASSERT_TRUE(reciever3.AwaitNamespaces(0)); } - err = rx.Stop(); - ASSERT_TRUE(err.ok()) << err.what(); + rx.Stop(); err = rx.Status(); ASSERT_FALSE(err.ok()) << err.what(); @@ -964,8 +962,7 @@ TEST_F(RPCClientTestApi, FetchingWithJoin) { EXPECT_EQ(ser.Slice(), expected); i++; } - err = rx.Stop(); - ASSERT_TRUE(err.ok()) << err.what(); + rx.Stop(); }); loop.run(); @@ -1044,8 +1041,7 @@ TEST_F(RPCClientTestApi, AggregationsFetching) { } } - err = rx.Stop(); - ASSERT_TRUE(err.ok()) << err.what(); + rx.Stop(); }); loop.run(); diff --git a/cpp_src/gtests/tests/unit/selector_plan_test.cc b/cpp_src/gtests/tests/unit/selector_plan_test.cc index 5c4c09368..89a08333a 100644 --- a/cpp_src/gtests/tests/unit/selector_plan_test.cc +++ b/cpp_src/gtests/tests/unit/selector_plan_test.cc @@ -421,7 +421,7 @@ TEST_F(SelectorPlanTest, ConditionsMergeIntoEmptyCondition) { .Where("id", CondSet, {32, 33, 34}) .Where("id", CondEq, 310) .Where("id", CondSet, {35, 36, 37}) - .Where("value", CondAny, 0) + .Where("value", CondAny, {}) .Explain(); QueryResults qr; err = rt.reindexer->Select(q, qr); @@ -439,7 +439,7 @@ TEST_F(SelectorPlanTest, ConditionsMergeIntoEmptyCondition) { .Where("id", CondEq, 39) .Where("id", CondSet, {32, 39, 34}) .Where("id", CondSet, {}) - .Where("value", CondAny, 0) + .Where("value", CondAny, {}) .Explain(); QueryResults qr; err = rt.reindexer->Select(q, qr); @@ -454,7 +454,7 @@ TEST_F(SelectorPlanTest, ConditionsMergeIntoEmptyCondition) { { // Query with multiple empty sets const auto q = - Query(nsName).Where("id", CondEq, 45).Where("id", CondSet, {}).Where("id", CondSet, {}).Where("value", CondAny, 0).Explain(); + Query(nsName).Where("id", CondEq, 45).Where("id", CondSet, {}).Where("id", CondSet, {}).Where("value", CondAny, {}).Explain(); QueryResults qr; err = rt.reindexer->Select(q, qr); ASSERT_TRUE(err.ok()) << err.what(); diff --git a/cpp_src/gtests/tests/unit/value_by_json_path.cc b/cpp_src/gtests/tests/unit/value_by_json_path.cc index 5d2797dbc..5bc6f8662 100644 --- a/cpp_src/gtests/tests/unit/value_by_json_path.cc +++ b/cpp_src/gtests/tests/unit/value_by_json_path.cc @@ -1,16 +1,15 @@ #include "core/cjson/jsonbuilder.h" #include "reindexer_api.h" -#include "tools/logger.h" TEST_F(ReindexerApi, GetValueByJsonPath) { Error err = rt.reindexer->OpenNamespace(default_namespace, StorageOpts().Enabled(false)); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); err = rt.reindexer->AddIndex(default_namespace, {"id", "hash", "string", IndexOpts().PK()}); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); err = rt.reindexer->Commit(default_namespace); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); struct Data { std::string id; @@ -26,7 +25,7 @@ TEST_F(ReindexerApi, GetValueByJsonPath) { for (int i = 0; i < 100; ++i) { Item item = rt.reindexer->NewItem(default_namespace); - EXPECT_TRUE(item.Status().ok()) << item.Status().what(); + ASSERT_TRUE(item.Status().ok()) << item.Status().what(); Data data = {"pk" + std::to_string(i), i + 1, "str" + std::to_string(i + 2), {{i + 3, i + 4, i + 5}}, i + 6, i + 7, i + 8}; char json[1024]; @@ -34,58 +33,58 @@ TEST_F(ReindexerApi, GetValueByJsonPath) { data.intArray[1], data.intArray[2], data.firstInner, data.secondInner, data.thirdInner); err = item.FromJSON(json); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); err = rt.reindexer->Upsert(default_namespace, item); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); err = rt.reindexer->Commit(default_namespace); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); VariantArray intField = item["inner.intField"]; - EXPECT_TRUE(intField.size() == 1); - EXPECT_TRUE(static_cast(intField[0]) == data.intField); + ASSERT_EQ(intField.size(), 1); + EXPECT_EQ(static_cast(intField[0]), data.intField); VariantArray stringField = item["inner.stringField"]; - EXPECT_TRUE(stringField.size() == 1); - EXPECT_TRUE(stringField[0].As().compare(data.stringField) == 0); + ASSERT_EQ(stringField.size(), 1); + EXPECT_EQ(stringField[0].As().compare(data.stringField), 0); VariantArray intArray = item["inner.inner2.intArray"]; - EXPECT_TRUE(intArray.size() == 3); + ASSERT_EQ(intArray.size(), data.intArray.size()); for (size_t j = 0; j < intArray.size(); ++j) { - EXPECT_TRUE(static_cast(intArray[j]) == data.intArray[j]); + EXPECT_EQ(static_cast(intArray[j]), data.intArray[j]); } VariantArray firstInner = item["inner.inner2.inner3.first"]; - EXPECT_TRUE(firstInner.size() == 1); - EXPECT_TRUE(static_cast(firstInner[0]) == data.firstInner); + ASSERT_EQ(firstInner.size(), 1); + EXPECT_EQ(static_cast(firstInner[0]), data.firstInner); VariantArray secondInner = item["inner.inner2.inner3.second"]; - EXPECT_TRUE(secondInner.size() == 1); - EXPECT_TRUE(static_cast(secondInner[0]) == data.secondInner); + ASSERT_EQ(secondInner.size(), 1); + EXPECT_EQ(static_cast(secondInner[0]), data.secondInner); VariantArray thirdInner = item["inner.inner2.inner3.third"]; - EXPECT_TRUE(thirdInner.size() == 1); - EXPECT_TRUE(static_cast(thirdInner[0]) == data.thirdInner); + ASSERT_EQ(thirdInner.size(), 1); + EXPECT_EQ(static_cast(thirdInner[0]), data.thirdInner); } } TEST_F(ReindexerApi, SelectByJsonPath) { Error err = rt.reindexer->OpenNamespace(default_namespace, StorageOpts().Enabled(false)); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); err = rt.reindexer->AddIndex(default_namespace, {"id", "hash", "string", IndexOpts().PK()}); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); err = rt.reindexer->Commit(default_namespace); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); const char jsonPattern[] = R"xxx({"id": "%s", "nested": {"string": "%s", "int": %d, "intarray" : [1,2,3]}})xxx"; std::vector properIntValues; for (int i = 0; i < 15; ++i) { Item item = rt.reindexer->NewItem(default_namespace); - EXPECT_TRUE(item.Status().ok()) << item.Status().what(); + ASSERT_TRUE(item.Status().ok()) << item.Status().what(); char json[512]; auto pk = "pk" + std::to_string(i); @@ -95,97 +94,97 @@ TEST_F(ReindexerApi, SelectByJsonPath) { if (i >= 5) properIntValues.push_back(i); err = item.FromJSON(json); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); err = rt.reindexer->Upsert(default_namespace, item); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); err = rt.reindexer->Commit(default_namespace); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); } QueryResults qr1; Variant strValueToFind("str_pk1"); Query query1{Query(default_namespace).Where("nested.string", CondEq, strValueToFind)}; err = rt.reindexer->Select(query1, qr1); - EXPECT_TRUE(err.ok()) << err.what(); - EXPECT_TRUE(qr1.Count() == 1); + ASSERT_TRUE(err.ok()) << err.what(); + ASSERT_EQ(qr1.Count(), 1); Item theOnlyItem = qr1[0].GetItem(false); VariantArray krefs = theOnlyItem["nested.string"]; - EXPECT_TRUE(krefs.size() == 1); - EXPECT_TRUE(krefs[0].As() == strValueToFind.As()); + ASSERT_EQ(krefs.size(), 1); + EXPECT_EQ(krefs[0].As(), strValueToFind.As()); QueryResults qr2; Variant intValueToFind(static_cast(5)); Query query2{Query(default_namespace).Where("nested.int", CondGe, intValueToFind)}; err = rt.reindexer->Select(query2, qr2); - EXPECT_TRUE(err.ok()) << err.what(); - EXPECT_TRUE(qr2.Count() == 10); + ASSERT_TRUE(err.ok()) << err.what(); + ASSERT_EQ(qr2.Count(), properIntValues.size()); - EXPECT_TRUE(properIntValues.size() == qr2.Count()); + ASSERT_EQ(properIntValues.size(), qr2.Count()); for (size_t i = 0; i < properIntValues.size(); ++i) { Item item = qr2[i].GetItem(false); VariantArray krefs = item["nested.int"]; - EXPECT_TRUE(krefs.size() == 1); - EXPECT_TRUE(static_cast(krefs[0]) == properIntValues[i]); + ASSERT_EQ(krefs.size(), 1); + EXPECT_EQ(static_cast(krefs[0]), properIntValues[i]); } QueryResults qr3; Variant arrayItemToFind(static_cast(2)); Query query3{Query(default_namespace).Where("nested.intarray", CondGe, arrayItemToFind)}; err = rt.reindexer->Select(query3, qr3); - EXPECT_TRUE(err.ok()) << err.what(); - EXPECT_TRUE(qr3.Count() == 15); + ASSERT_TRUE(err.ok()) << err.what(); + EXPECT_EQ(qr3.Count(), 15); } TEST_F(ReindexerApi, CompositeFTSelectByJsonPath) { Error err = rt.reindexer->OpenNamespace(default_namespace, StorageOpts().Enabled(false)); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); err = rt.reindexer->AddIndex(default_namespace, {"id", "hash", "string", IndexOpts().PK()}); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); err = rt.reindexer->AddIndex(default_namespace, {"locale", "hash", "string", IndexOpts()}); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); err = rt.reindexer->Commit(default_namespace); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); const char jsonPattern[] = R"xxx({"id": "key%d", "locale" : "%s", "nested": {"name": "name%d", "count": %ld}})xxx"; for (int i = 0; i < 20'000; ++i) { Item item = rt.reindexer->NewItem(default_namespace); - EXPECT_TRUE(item.Status().ok()) << item.Status().what(); + ASSERT_TRUE(item.Status().ok()) << item.Status().what(); char json[1024]; long count = i; snprintf(json, sizeof(json) - 1, jsonPattern, i, i % 2 ? "en" : "ru", i, count); err = item.Unsafe(true).FromJSON(json); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); err = rt.reindexer->Upsert(default_namespace, item); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); err = rt.reindexer->Commit(default_namespace); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); } err = rt.reindexer->AddIndex(default_namespace, {"composite_ft", {"nested.name", "id", "locale"}, "text", "composite", IndexOpts()}); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); err = rt.reindexer->Commit(default_namespace); - EXPECT_TRUE(err.ok()) << err.what(); + ASSERT_TRUE(err.ok()) << err.what(); QueryResults qr; Query query{Query(default_namespace).Where("composite_ft", CondEq, "name2")}; err = rt.reindexer->Select(query, qr); - EXPECT_TRUE(err.ok()) << err.what(); - EXPECT_TRUE(qr.Count() == 1); + ASSERT_TRUE(err.ok()) << err.what(); + EXPECT_EQ(qr.Count(), 1); for (auto it : qr) { Item ritem(it.GetItem(false)); auto json = ritem.GetJSON(); - EXPECT_TRUE(json == R"xxx({"id":"key2","locale":"ru","nested":{"name":"name2","count":2}})xxx"); + EXPECT_EQ(json, R"xxx({"id":"key2","locale":"ru","nested":{"name":"name2","count":2}})xxx"); } } diff --git a/cpp_src/gtests/tools.h b/cpp_src/gtests/tools.h index 65f3a90b9..7095193b9 100644 --- a/cpp_src/gtests/tools.h +++ b/cpp_src/gtests/tools.h @@ -27,3 +27,67 @@ inline std::string randStrUuid() { inline reindexer::Uuid randUuid() { return reindexer::Uuid{randStrUuid()}; } inline reindexer::Uuid nilUuid() { return reindexer::Uuid{nilUUID}; } + +template +inline reindexer::VariantArray randUuidArrayImpl(Fn fillFn, size_t min, size_t max) { + assert(min <= max); + reindexer::VariantArray ret; + const size_t count = min == max ? min : min + rand() % (max - min); + ret.reserve(count); + for (size_t i = 0; i < count; ++i) { + fillFn(ret); + } + return ret; +} + +inline reindexer::VariantArray randUuidArray(size_t min, size_t max) { + return randUuidArrayImpl([](auto& v) { v.emplace_back(randUuid()); }, min, max); +} + +inline reindexer::VariantArray randStrUuidArray(size_t min, size_t max) { + return randUuidArrayImpl([](auto& v) { v.emplace_back(randStrUuid()); }, min, max); +} + +inline reindexer::VariantArray randHeterogeneousUuidArray(size_t min, size_t max) { + return randUuidArrayImpl( + [](auto& v) { + if (rand() % 2) { + v.emplace_back(randStrUuid()); + } else { + v.emplace_back(randUuid()); + } + }, + min, max); +} + +inline auto minMaxArgs(CondType cond, size_t max) { + struct { + size_t min; + size_t max; + } res; + switch (cond) { + case CondEq: + case CondSet: + case CondAllSet: + res.min = 0; + res.max = max; + break; + case CondLike: + case CondLt: + case CondLe: + case CondGt: + case CondGe: + res.min = res.max = 1; + break; + case CondRange: + res.min = res.max = 2; + break; + case CondAny: + case CondEmpty: + res.min = res.max = 0; + break; + case CondDWithin: + assert(0); + } + return res; +} diff --git a/cpp_src/net/http/router.cc b/cpp_src/net/http/router.cc index ef56ddff2..2797d979c 100644 --- a/cpp_src/net/http/router.cc +++ b/cpp_src/net/http/router.cc @@ -50,9 +50,12 @@ HttpStatusCode HttpStatus::errCodeToHttpStatus(int errCode) { case errParams: case errParseSQL: case errParseDSL: + case errQueryExec: return StatusBadRequest; case errForbidden: return StatusForbidden; + case errTimeout: + return StatusRequestTimeout; default: return StatusInternalServerError; } diff --git a/cpp_src/net/http/router.h b/cpp_src/net/http/router.h index 6fc5fca6b..8f1fe2b97 100644 --- a/cpp_src/net/http/router.h +++ b/cpp_src/net/http/router.h @@ -63,7 +63,7 @@ enum HttpMethod : int { typedef std::string_view UrlParam; struct HttpStatus { - HttpStatus() { code = StatusOK; } + HttpStatus() noexcept : code(StatusOK) {} HttpStatus(HttpStatusCode httpcode, std::string httpwhat) : code(httpcode), what(std::move(httpwhat)) {} explicit HttpStatus(const Error &err) : what(err.what()) { code = errCodeToHttpStatus(err.code()); } diff --git a/cpp_src/replicator/replicator.cc b/cpp_src/replicator/replicator.cc index 4717903e3..5bc0b37fb 100644 --- a/cpp_src/replicator/replicator.cc +++ b/cpp_src/replicator/replicator.cc @@ -15,8 +15,6 @@ namespace reindexer { using namespace net; using namespace std::string_view_literals; -static constexpr size_t kTmpNsPostfixLen = 20; - Replicator::Replicator(ReindexerImpl *slave) : slave_(slave), resyncUpdatesLostFlag_(false), @@ -515,9 +513,9 @@ Error Replicator::syncNamespaceByWAL(const NamespaceDef &nsDef) { case errQrUIDMissmatch: case errSystem: case errAssert: - default: - return err; + break; } + return err; } // Forced namespace sync @@ -535,7 +533,7 @@ Error Replicator::syncNamespaceForced(const NamespaceDef &ns, std::string_view r tmpNsDef.storage = StorageOpts().SlaveMode(); } - tmpNsDef.name = "@" + ns.name + "_tmp_" + randStringAlph(kTmpNsPostfixLen); + tmpNsDef.name = createTmpNamespaceName(ns.name); auto dropTmpNs = [this, &tmpNsDef] { auto tmpNs = slave_->getNamespaceNoThrow(tmpNsDef.name, dummyCtx_); if (tmpNs) { diff --git a/cpp_src/replicator/walrecord.cc b/cpp_src/replicator/walrecord.cc index a9daa40f4..436a7044f 100644 --- a/cpp_src/replicator/walrecord.cc +++ b/cpp_src/replicator/walrecord.cc @@ -1,7 +1,6 @@ #include "walrecord.h" #include "core/cjson/baseencoder.h" -#include "core/transactionimpl.h" #include "tools/logger.h" #include "tools/serializer.h" @@ -148,7 +147,7 @@ static std::string_view wrecType2Str(WALRecType t) { } } -WrSerializer &WALRecord::Dump(WrSerializer &ser, const std::function& cjsonViewer) const { +WrSerializer &WALRecord::Dump(WrSerializer &ser, const std::function &cjsonViewer) const { ser << wrecType2Str(type); if (inTransaction) ser << " InTransaction"; switch (type) { @@ -182,7 +181,7 @@ WrSerializer &WALRecord::Dump(WrSerializer &ser, const std::function& cjsonViewer) const { +void WALRecord::GetJSON(JsonBuilder &jb, const std::function &cjsonViewer) const { jb.Put("type", wrecType2Str(type)); jb.Put("in_transaction", inTransaction); diff --git a/cpp_src/replicator/walselecter.cc b/cpp_src/replicator/walselecter.cc index d853a90d3..23da38c5c 100644 --- a/cpp_src/replicator/walselecter.cc +++ b/cpp_src/replicator/walselecter.cc @@ -1,4 +1,3 @@ - #include "walselecter.h" #include "core/cjson/jsonbuilder.h" #include "core/namespace/namespaceimpl.h" @@ -15,8 +14,8 @@ WALSelecter::WALSelecter(const NamespaceImpl *ns) : ns_(ns) {} void WALSelecter::operator()(QueryResults &result, SelectCtx ¶ms) { using namespace std::string_view_literals; const Query &q = params.query; - int count = q.count; - int start = q.start; + int count = q.Limit(); + int start = q.Offset(); result.totalCount = 0; if (!q.IsWALQuery()) { @@ -31,20 +30,20 @@ void WALSelecter::operator()(QueryResults &result, SelectCtx ¶ms) { q.entries.InvokeAppropriate( i, [&lsnIdx, &versionIdx, i](const QueryEntry &qe) { - if ("#lsn"sv == qe.index) { + if ("#lsn"sv == qe.FieldName()) { lsnIdx = i; - } else if ("#slave_version"sv == qe.index) { + } else if ("#slave_version"sv == qe.FieldName()) { versionIdx = i; } else { - throw Error(errLogic, "Unexpected index in WAL select query: %s", qe.index); + throw Error(errLogic, "Unexpected index in WAL select query: %s", qe.FieldName()); } }, [&q](const auto &) { throw Error(errLogic, "Unexpected WAL select query: %s", q.GetSQL()); }); } - auto slaveVersion = versionIdx < 0 ? SemVersion() : SemVersion(q.entries.Get(versionIdx).values[0].As()); + auto slaveVersion = versionIdx < 0 ? SemVersion() : SemVersion(q.entries.Get(versionIdx).Values()[0].As()); auto &lsnEntry = q.entries.Get(lsnIdx); - if (lsnEntry.values.size() == 1 && lsnEntry.condition == CondGt) { - lsn_t fromLSN = lsn_t(std::min(lsnEntry.values[0].As(), std::numeric_limits::max() - 1)); + if (lsnEntry.Values().size() == 1 && lsnEntry.Condition() == CondGt) { + lsn_t fromLSN = lsn_t(std::min(lsnEntry.Values()[0].As(), std::numeric_limits::max() - 1)); if (fromLSN.Server() != ns_->serverId_) throw Error(errOutdatedWAL, "Query to WAL with incorrect LSN %ld, LSN counter %ld", int64_t(fromLSN), ns_->wal_.LSNCounter()); if (ns_->wal_.LSNCounter() != (fromLSN.Counter() + 1) && ns_->wal_.is_outdated(fromLSN.Counter() + 1) && count) @@ -73,7 +72,7 @@ void WALSelecter::operator()(QueryResults &result, SelectCtx ¶ms) { if (versionIdx < 0) { break; } - if (q.entries.Get(versionIdx).condition != CondEq || slaveVersion < kMinUnknownReplSupportRxVersion) { + if (q.entries.Get(versionIdx).Condition() != CondEq || slaveVersion < kMinUnknownReplSupportRxVersion) { break; } // fall-through @@ -110,7 +109,7 @@ void WALSelecter::operator()(QueryResults &result, SelectCtx ¶ms) { std::abort(); } } - } else if (lsnEntry.condition == CondAny) { + } else if (lsnEntry.Condition() == CondAny) { if (start == 0 && !(slaveVersion < kMinUnknownReplSupportRxVersion)) { auto addSpRecord = [&result](const WALRecord &wrec) { PackedWALRecord wr; diff --git a/cpp_src/server/config.cc b/cpp_src/server/config.cc index 08523c4c5..dfee6dd83 100644 --- a/cpp_src/server/config.cc +++ b/cpp_src/server/config.cc @@ -43,6 +43,8 @@ void ServerConfig::Reset() { EnableConnectionsStats = true; TxIdleTimeout = std::chrono::seconds(600); RPCQrIdleTimeout = std::chrono::seconds(600); + HttpReadTimeout = std::chrono::seconds(0); + HttpWriteTimeout = std::chrono::seconds(0); MaxUpdatesSize = 1024 * 1024 * 1024; EnableGRPC = false; MaxHttpReqSize = 2 * 1024 * 1024; @@ -52,7 +54,6 @@ void ServerConfig::Reset() { const std::string ServerConfig::kDedicatedThreading = "dedicated"; const std::string ServerConfig::kSharedThreading = "shared"; -const std::string ServerConfig::kPoolThreading = "pool"; reindexer::Error ServerConfig::ParseYaml(const std::string &yaml) { Error err; @@ -121,6 +122,11 @@ Error ServerConfig::ParseCmd(int argc, char *argv[]) { #endif args::ValueFlag webRootF(netGroup, "PATH", "web root. This path if set overrides linked-in resources", {'w', "webroot"}, WebRoot, args::Options::Single); + args::ValueFlag httpReadTimeoutF(netGroup, "", "timeout (s) for HTTP read operations (i.e. selects, get meta and others)", + {"http-read-timeout"}, HttpReadTimeout.count(), args::Options::Single); + args::ValueFlag httpWriteTimeoutF(netGroup, "", + "timeout (s) for HTTP write operations (i.e. update, delete, put meta, add index and others)", + {"http-write-timeout"}, HttpWriteTimeout.count(), args::Options::Single); args::ValueFlag maxUpdatesSizeF(netGroup, "", "Maximum cached updates size", {"updatessize"}, MaxUpdatesSize, args::Options::Single); args::Flag pprofF(netGroup, "", "Enable pprof http handler", {'f', "pprof"}); @@ -231,6 +237,8 @@ Error ServerConfig::ParseCmd(int argc, char *argv[]) { if (prometheusF) EnablePrometheus = args::get(prometheusF); if (prometheusPeriodF) PrometheusCollectPeriod = std::chrono::milliseconds(args::get(prometheusPeriodF)); if (clientsConnectionsStatF) EnableConnectionsStats = args::get(clientsConnectionsStatF); + if (httpReadTimeoutF) HttpReadTimeout = std::chrono::seconds(args::get(httpReadTimeoutF)); + if (httpWriteTimeoutF) HttpWriteTimeout = std::chrono::seconds(args::get(httpWriteTimeoutF)); if (logAllocsF) DebugAllocs = args::get(logAllocsF); if (txIdleTimeoutF) TxIdleTimeout = std::chrono::seconds(args::get(txIdleTimeoutF)); if (rpcQrIdleTimeoutF) RPCQrIdleTimeout = std::chrono::seconds(args::get(rpcQrIdleTimeoutF)); @@ -261,6 +269,8 @@ reindexer::Error ServerConfig::fromYaml(YAML::Node &root) { EnableGRPC = root["net"]["grpc"].as(EnableGRPC); GRPCAddr = root["net"]["grpcaddr"].as(GRPCAddr); TxIdleTimeout = std::chrono::seconds(root["net"]["tx_idle_timeout"].as(TxIdleTimeout.count())); + HttpReadTimeout = std::chrono::seconds(root["net"]["http_read_timeout"].as(HttpReadTimeout.count())); + HttpWriteTimeout = std::chrono::seconds(root["net"]["http_write_timeout"].as(HttpWriteTimeout.count())); RPCQrIdleTimeout = std::chrono::seconds(root["net"]["rpc_qr_idle_timeout"].as(RPCQrIdleTimeout.count())); MaxHttpReqSize = root["net"]["max_http_body_size"].as(MaxHttpReqSize); EnablePrometheus = root["metrics"]["prometheus"].as(EnablePrometheus); diff --git a/cpp_src/server/config.h b/cpp_src/server/config.h index 147b10ccc..3cd2ad9dd 100644 --- a/cpp_src/server/config.h +++ b/cpp_src/server/config.h @@ -63,6 +63,8 @@ struct ServerConfig { std::chrono::milliseconds PrometheusCollectPeriod; bool DebugAllocs; std::chrono::seconds TxIdleTimeout; + std::chrono::seconds HttpReadTimeout; + std::chrono::seconds HttpWriteTimeout; size_t MaxUpdatesSize; bool EnableGRPC; std::string GRPCAddr; @@ -73,7 +75,6 @@ struct ServerConfig { static const std::string kDedicatedThreading; static const std::string kSharedThreading; - static const std::string kPoolThreading; protected: Error fromYaml(YAML::Node& root); diff --git a/cpp_src/server/contrib/server.md b/cpp_src/server/contrib/server.md index 7b092c81c..42753b29c 100644 --- a/cpp_src/server/contrib/server.md +++ b/cpp_src/server/contrib/server.md @@ -1584,7 +1584,7 @@ This operation pareses SQL query, and suggests autocompletion variants |Type|Name|Description|Schema| |---|---|---|---| |**Path**|**database**
*required*|Database name|string| -|**Query**|**line**
*optional*|Cursor line for suggest|integer| +|**Query**|**line**
*required*|Cursor line for suggest|integer| |**Query**|**pos**
*required*|Cursor position for suggest|integer| |**Query**|**q**
*required*|SQL query|string| diff --git a/cpp_src/server/contrib/server.yml b/cpp_src/server/contrib/server.yml index d2906ac08..7e043c5ba 100644 --- a/cpp_src/server/contrib/server.yml +++ b/cpp_src/server/contrib/server.yml @@ -54,6 +54,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -83,6 +85,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -111,6 +115,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -144,6 +150,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" get: @@ -178,6 +186,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -211,6 +221,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" delete: @@ -242,6 +254,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -273,6 +287,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -310,6 +326,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -369,6 +387,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -407,6 +427,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -446,6 +468,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -498,6 +522,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" post: @@ -557,6 +583,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" put: @@ -616,6 +644,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -676,6 +706,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" get: @@ -744,6 +776,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -777,6 +811,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" post: @@ -813,6 +849,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" put: @@ -849,6 +887,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -886,6 +926,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -923,6 +965,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" get: @@ -953,6 +997,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -989,6 +1035,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -1052,6 +1100,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" post: @@ -1104,6 +1154,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" put: @@ -1134,6 +1186,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" delete: @@ -1164,6 +1218,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -1204,6 +1260,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -1233,6 +1291,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -1262,6 +1322,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -1313,6 +1375,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" post: @@ -1371,6 +1435,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" put: @@ -1429,6 +1495,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" patch: @@ -1487,6 +1555,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -1538,6 +1608,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" delete: @@ -1579,6 +1651,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -1610,7 +1684,7 @@ paths: in: query type: integer description: "Cursor line for suggest" - required: false + required: true responses: 200: description: "successful operation" @@ -1622,6 +1696,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -1676,6 +1752,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -1697,6 +1775,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -1724,6 +1804,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -1752,6 +1834,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -1779,6 +1863,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -1806,6 +1892,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -1833,6 +1921,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -1860,6 +1950,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -1895,6 +1987,8 @@ paths: $ref: "#/responses/Forbidden" 404: $ref: "#/responses/NotFound" + 408: + $ref: "#/responses/RequestTimeout" 500: $ref: "#/responses/UnexpectedError" @@ -1907,6 +2001,10 @@ responses: description: "Invalid arguments supplied" schema: $ref: "#/definitions/StatusResponse" + RequestTimeout: + description: "Context timeout" + schema: + $ref: "#/definitions/StatusResponse" Forbidden: description: "Forbidden" schema: @@ -3109,6 +3207,12 @@ definitions: field: type: string description: "Field or index name" + field_type: + type: string + enum: + - "non-indexed" + - "indexed" + description: "Shows which kind of the field was used for the filtration. Non-indexed fields are usually really slow for 'scan' and should be avoided" items: type: integer description: "Count of scanned documents by this selector" @@ -3130,7 +3234,7 @@ definitions: explain_select: description: "One of selects in joined namespace execution explainings" $ref: "#/definitions/ExplainDef" - join_on_conditions: + on_conditions_injections: type: array description: "Describes Join ON conditions injections" items: @@ -3262,32 +3366,9 @@ definitions: properties: success: type: boolean - description: "Status of operation" response_code: type: integer - enum: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18] - description: > - Error code: - * 0 - errOK - * 1 - errParseSQL - * 2 - errQueryExec - * 3 - errParams - * 4 - errLogic - * 5 - errParseJson - * 6 - errParseDSL - * 7 - errConflict - * 8 - errParseBin - * 9 - errForbidden - * 10 - errWasRelock - * 11 - errNotValid - * 12 - errNetwork - * 13 - errNotFound - * 14 - errStateInvalidated - * 15 - errBadTransaction - * 16 - errOutdatedWAL - * 17 - errNoWAL - * 18 - errDataHashMismatch - + description: "Duplicates HTTP response code" description: type: string description: "Text description of error details" diff --git a/cpp_src/server/grpc/reindexerservice.cc b/cpp_src/server/grpc/reindexerservice.cc index a3a66b58c..22085eedd 100644 --- a/cpp_src/server/grpc/reindexerservice.cc +++ b/cpp_src/server/grpc/reindexerservice.cc @@ -980,7 +980,7 @@ Error ReindexerService::execSqlQueryByType(QueryResults& res, const SelectSqlReq return rx->Update(q, res); } case QueryTruncate: { - return rx->TruncateNamespace(q._namespace); + return rx->TruncateNamespace(q.NsName()); } default: return Error(errParams, "unknown query type %d", q.Type()); diff --git a/cpp_src/server/httpserver.cc b/cpp_src/server/httpserver.cc index a6720bfbf..87d43fca0 100644 --- a/cpp_src/server/httpserver.cc +++ b/cpp_src/server/httpserver.cc @@ -54,25 +54,16 @@ Error HTTPServer::execSqlQueryByType(std::string_view sqlQuery, reindexer::Query reindexer::Query q; q.FromSQL(sqlQuery); switch (q.Type()) { - case QuerySelect: { - auto db = getDB(ctx, kRoleDataRead); - return db.Select(q, res); - } - case QueryDelete: { - auto db = getDB(ctx, kRoleDataWrite); - return db.Delete(q, res); - } - case QueryUpdate: { - auto db = getDB(ctx, kRoleDataWrite); - return db.Update(q, res); - } - case QueryTruncate: { - auto db = getDB(ctx, kRoleDBAdmin); - return db.TruncateNamespace(q._namespace); - } - default: - throw Error(errParams, "unknown query type %d", q.Type()); + case QuerySelect: + return getDB(ctx).Select(q, res); + case QueryDelete: + return getDB(ctx).Delete(q, res); + case QueryUpdate: + return getDB(ctx).Update(q, res); + case QueryTruncate: + return getDB(ctx).TruncateNamespace(q.NsName()); } + throw Error(errParams, "unknown query type %d", q.Type()); } int HTTPServer::GetSQLQuery(http::Context &ctx) { @@ -85,7 +76,7 @@ int HTTPServer::GetSQLQuery(http::Context &ctx) { unsigned offset = prepareOffset(offsetParam); if (sqlQuery.empty()) { - return status(ctx, http::HttpStatus(http::StatusBadRequest, "Missed `q` parameter")); + return status(ctx, http::HttpStatus(http::StatusBadRequest, "Missing `q` parameter")); } reindexer::QueryResults res; auto ret = execSqlQueryByType(sqlQuery, res, ctx); @@ -99,11 +90,17 @@ int HTTPServer::GetSQLQuery(http::Context &ctx) { int HTTPServer::GetSQLSuggest(http::Context &ctx) { std::string sqlQuery = urldecode2(ctx.request->params.Get("q")); if (sqlQuery.empty()) { - return jsonStatus(ctx, http::HttpStatus(http::StatusBadRequest, "Missed `q` parameter")); + return jsonStatus(ctx, http::HttpStatus(http::StatusBadRequest, "Missing `q` parameter")); } std::string_view posParam = ctx.request->params.Get("pos"); + if (posParam.empty()) { + return jsonStatus(ctx, http::HttpStatus(http::StatusBadRequest, "Missing `pos` parameter")); + } std::string_view lineParam = ctx.request->params.Get("line"); + if (lineParam.empty()) { + return jsonStatus(ctx, http::HttpStatus(http::StatusBadRequest, "Missing `line` parameter")); + } int pos = stoi(posParam); if (pos < 0) { return jsonStatus(ctx, http::HttpStatus(http::StatusBadRequest, "`pos` parameter should be >= 0")); @@ -122,8 +119,10 @@ int HTTPServer::GetSQLSuggest(http::Context &ctx) { logPrintf(LogTrace, "GetSQLSuggest() incoming data: %s, %d", sqlQuery, bytePos); std::vector suggestions; - auto db = getDB(ctx, kRoleDataRead); - db.GetSqlSuggestions(sqlQuery, bytePos, suggestions); + err = getDB(ctx).GetSqlSuggestions(sqlQuery, bytePos, suggestions); + if (!err.ok()) { + return jsonStatus(ctx, http::HttpStatus(http::StatusBadRequest, err.what())); + } WrSerializer ser(ctx.writer->GetChunk()); reindexer::JsonBuilder builder(ser); @@ -143,13 +142,13 @@ int HTTPServer::PostSQLQuery(http::Context &ctx) { } auto ret = execSqlQueryByType(sqlQuery, res, ctx); if (!ret.ok()) { - return status(ctx, http::HttpStatus(http::StatusInternalServerError, ret.what())); + return status(ctx, http::HttpStatus(ret)); } return queryResults(ctx, res, true); } int HTTPServer::PostQuery(http::Context &ctx) { - auto db = getDB(ctx, kRoleDataRead); + auto db = getDB(ctx); reindexer::QueryResults res; std::string dsl = ctx.body->Read(); @@ -167,7 +166,7 @@ int HTTPServer::PostQuery(http::Context &ctx) { } int HTTPServer::DeleteQuery(http::Context &ctx) { - auto db = getDB(ctx, kRoleDataWrite); + auto db = getDB(ctx); std::string dsl = ctx.body->Read(); reindexer::Query q; @@ -190,7 +189,7 @@ int HTTPServer::DeleteQuery(http::Context &ctx) { } int HTTPServer::UpdateQuery(http::Context &ctx) { - auto db = getDB(ctx, kRoleDataWrite); + auto db = getDB(ctx); std::string dsl = ctx.body->Read(); reindexer::Query q; @@ -303,12 +302,13 @@ int HTTPServer::DeleteDatabase(http::Context &ctx) { } int HTTPServer::GetNamespaces(http::Context &ctx) { - auto db = getDB(ctx, kRoleDataRead); - std::string_view sortOrder = ctx.request->params.Get("sort_order"); std::vector nsDefs; - db.EnumNamespaces(nsDefs, EnumNamespacesOpts().OnlyNames()); + const auto err = getDB(ctx).EnumNamespaces(nsDefs, EnumNamespacesOpts().OnlyNames()); + if (!err.ok()) { + return jsonStatus(ctx, http::HttpStatus(err)); + } int sortDirection = 0; if (sortOrder == "asc") { @@ -342,7 +342,7 @@ int HTTPServer::GetNamespaces(http::Context &ctx) { } int HTTPServer::GetNamespace(http::Context &ctx) { - auto db = getDB(ctx, kRoleDataRead); + auto db = getDB(ctx); std::string nsName = urldecode2(ctx.request->urlParams[1]); @@ -351,7 +351,10 @@ int HTTPServer::GetNamespace(http::Context &ctx) { } std::vector nsDefs; - db.EnumNamespaces(nsDefs, EnumNamespacesOpts().WithFilter(nsName)); + const auto err = db.EnumNamespaces(nsDefs, EnumNamespacesOpts().WithFilter(nsName)); + if (!err.ok()) { + return jsonStatus(ctx, http::HttpStatus(err)); + } if (nsDefs.empty()) { return jsonStatus(ctx, http::HttpStatus(http::StatusNotFound, "Namespace is not found")); @@ -363,7 +366,7 @@ int HTTPServer::GetNamespace(http::Context &ctx) { } int HTTPServer::PostNamespace(http::Context &ctx) { - auto db = getDB(ctx, kRoleDBAdmin); + auto db = getDB(ctx); reindexer::NamespaceDef nsdef(""); std::string body = ctx.body->Read(); @@ -381,7 +384,7 @@ int HTTPServer::PostNamespace(http::Context &ctx) { } int HTTPServer::DeleteNamespace(http::Context &ctx) { - auto db = getDB(ctx, kRoleDBAdmin); + auto db = getDB(ctx); std::string nsName = urldecode2(ctx.request->urlParams[1]); if (nsName.empty()) { @@ -399,7 +402,7 @@ int HTTPServer::DeleteNamespace(http::Context &ctx) { } int HTTPServer::TruncateNamespace(http::Context &ctx) { - auto db = getDB(ctx, kRoleDBAdmin); + auto db = getDB(ctx); std::string nsName = urldecode2(ctx.request->urlParams[1]); if (nsName.empty()) { @@ -417,7 +420,7 @@ int HTTPServer::TruncateNamespace(http::Context &ctx) { } int HTTPServer::RenameNamespace(http::Context &ctx) { - auto db = getDB(ctx, kRoleDBAdmin); + auto db = getDB(ctx); std::string srcNsName = urldecode2(ctx.request->urlParams[1]); std::string dstNsName = urldecode2(ctx.request->urlParams[2]); @@ -440,7 +443,7 @@ int HTTPServer::RenameNamespace(http::Context &ctx) { } int HTTPServer::GetItems(http::Context &ctx) { - auto db = getDB(ctx, kRoleDataRead); + auto db = getDB(ctx); std::string nsName = urldecode2(ctx.request->urlParams[1]); @@ -490,7 +493,7 @@ int HTTPServer::GetItems(http::Context &ctx) { reindexer::QueryResults res; auto ret = db.Select(q, res); if (!ret.ok()) { - return status(ctx, http::HttpStatus(http::StatusInternalServerError, ret.what())); + return status(ctx, http::HttpStatus(ret)); } return queryResults(ctx, res); @@ -503,7 +506,7 @@ int HTTPServer::PostItems(http::Context &ctx) { return modifyItems(ctx, ModeInse int HTTPServer::PatchItems(http::Context &ctx) { return modifyItems(ctx, ModeUpsert); } int HTTPServer::GetMetaList(http::Context &ctx) { - auto db = getDB(ctx, kRoleDataRead); + auto db = getDB(ctx); const std::string nsName = urldecode2(ctx.request->urlParams[1]); if (!nsName.length()) { return jsonStatus(ctx, http::HttpStatus(http::StatusBadRequest, "Namespace is not specified")); @@ -578,7 +581,7 @@ int HTTPServer::GetMetaList(http::Context &ctx) { } int HTTPServer::GetMetaByKey(http::Context &ctx) { - auto db = getDB(ctx, kRoleDataRead); + auto db = getDB(ctx); const std::string nsName = urldecode2(ctx.request->urlParams[1]); const std::string key = urldecode2(ctx.request->urlParams[2]); if (!nsName.length()) { @@ -598,7 +601,7 @@ int HTTPServer::GetMetaByKey(http::Context &ctx) { } int HTTPServer::PutMetaByKey(http::Context &ctx) { - auto db = getDB(ctx, kRoleDataWrite); + auto db = getDB(ctx); const std::string nsName = urldecode2(ctx.request->urlParams[1]); if (!nsName.length()) { return jsonStatus(ctx, http::HttpStatus(http::StatusBadRequest, "Namespace is not specified")); @@ -620,7 +623,7 @@ int HTTPServer::PutMetaByKey(http::Context &ctx) { } int HTTPServer::GetIndexes(http::Context &ctx) { - auto db = getDB(ctx, kRoleDataRead); + auto db = getDB(ctx); std::string nsName = urldecode2(ctx.request->urlParams[1]); @@ -629,8 +632,10 @@ int HTTPServer::GetIndexes(http::Context &ctx) { } std::vector nsDefs; - db.EnumNamespaces(nsDefs, EnumNamespacesOpts().WithFilter(nsName)); - + const auto err = db.EnumNamespaces(nsDefs, EnumNamespacesOpts().WithFilter(nsName)); + if (!err.ok()) { + return jsonStatus(ctx, http::HttpStatus(err)); + } if (nsDefs.empty()) { return jsonStatus(ctx, http::HttpStatus(http::StatusNotFound, "Namespace is not found")); } @@ -649,7 +654,7 @@ int HTTPServer::GetIndexes(http::Context &ctx) { } int HTTPServer::PostIndex(http::Context &ctx) { - auto db = getDB(ctx, kRoleDBAdmin); + auto db = getDB(ctx); std::string nsName = urldecode2(ctx.request->urlParams[1]); if (!nsName.length()) { @@ -660,10 +665,13 @@ int HTTPServer::PostIndex(http::Context &ctx) { std::string newIdxName = getNameFromJson(json); std::vector nsDefs; - db.EnumNamespaces(nsDefs, EnumNamespacesOpts().WithFilter(nsName)); + auto err = db.EnumNamespaces(nsDefs, EnumNamespacesOpts().WithFilter(nsName)); + if (!err.ok()) { + return jsonStatus(ctx, http::HttpStatus(err)); + } reindexer::IndexDef idxDef; - auto err = idxDef.FromJSON(giftStr(json)); + err = idxDef.FromJSON(giftStr(json)); if (!err.ok()) { return jsonStatus(ctx, http::HttpStatus{err}); } @@ -686,7 +694,7 @@ int HTTPServer::PostIndex(http::Context &ctx) { } int HTTPServer::PutIndex(http::Context &ctx) { - auto db = getDB(ctx, kRoleDBAdmin); + auto db = getDB(ctx); std::string nsName = urldecode2(ctx.request->urlParams[1]); if (!nsName.length()) { @@ -707,7 +715,7 @@ int HTTPServer::PutIndex(http::Context &ctx) { } int HTTPServer::PutSchema(http::Context &ctx) { - auto db = getDB(ctx, kRoleDBAdmin); + auto db = getDB(ctx); std::string nsName = urldecode2(ctx.request->urlParams[1]); if (nsName.empty()) { @@ -723,7 +731,7 @@ int HTTPServer::PutSchema(http::Context &ctx) { } int HTTPServer::GetSchema(http::Context &ctx) { - auto db = getDB(ctx, kRoleDataRead); + auto db = getDB(ctx); std::string nsName = urldecode2(ctx.request->urlParams[1]); if (nsName.empty()) { @@ -740,7 +748,7 @@ int HTTPServer::GetSchema(http::Context &ctx) { } int HTTPServer::GetProtobufSchema(http::Context &ctx) { - Reindexer db = getDB(ctx, kRoleDataRead); + Reindexer db = getDB(ctx); std::vector nses; for (auto &p : ctx.request->params) { @@ -757,7 +765,7 @@ int HTTPServer::GetProtobufSchema(http::Context &ctx) { } int HTTPServer::DeleteIndex(http::Context &ctx) { - auto db = getDB(ctx, kRoleDBAdmin); + auto db = getDB(ctx); std::string nsName = urldecode2(ctx.request->urlParams[1]); IndexDef idef(urldecode2(ctx.request->urlParams[2])); @@ -1029,7 +1037,7 @@ Error HTTPServer::modifyItem(Reindexer &db, std::string &nsName, Item &item, Que } int HTTPServer::modifyItemsJSON(http::Context &ctx, std::string &nsName, const std::vector &precepts, ItemModifyMode mode) { - auto db = getDB(ctx, kRoleDataWrite); + auto db = getDB(ctx); std::string itemJson = ctx.body->Read(); int cnt = 0; std::vector updatedItems; @@ -1066,7 +1074,6 @@ int HTTPServer::modifyItemsJSON(http::Context &ctx, std::string &nsName, const s if (!precepts.empty()) updatedItems.emplace_back(item.GetJSON()); } } - db.Commit(nsName); } WrSerializer ser(ctx.writer->GetChunk()); @@ -1087,7 +1094,7 @@ int HTTPServer::modifyItemsMsgPack(http::Context &ctx, std::string &nsName, cons QueryResults qr; int totalItems = 0; - auto db = getDB(ctx, kRoleDataWrite); + auto db = getDB(ctx); std::string sbuffer = ctx.body->Read(); size_t length = sbuffer.size(); @@ -1120,7 +1127,10 @@ int HTTPServer::modifyItemsMsgPack(http::Context &ctx, std::string &nsName, cons if (!precepts.empty()) { auto itemsArray = msgpackBuilder.Array(kParamItems, qr.Count()); for (size_t i = 0; i < qr.Count(); ++i) { - qr[i].GetMsgPack(wrSer, false); + const auto err = qr[i].GetMsgPack(wrSer, false); + if (!err.ok()) { + return msgpackStatus(ctx, http::HttpStatus(err)); + } } itemsArray.End(); } @@ -1144,7 +1154,7 @@ int HTTPServer::modifyItemsProtobuf(http::Context &ctx, std::string &nsName, con return ctx.Protobuf(reindexer::net::http::HttpStatus::errCodeToHttpStatus(err.code()), wrSer.DetachChunk()); }; - auto db = getDB(ctx, kRoleDataWrite); + auto db = getDB(ctx); Item item = db.NewItem(nsName); if (!item.Status().ok()) return sendResponse(0, item.Status()); @@ -1246,7 +1256,7 @@ int HTTPServer::modifyItems(http::Context &ctx, ItemModifyMode mode) { int HTTPServer::modifyItemsTx(http::Context &ctx, ItemModifyMode mode) { std::string dbName; - auto db = getDB(ctx, kRoleDataWrite, &dbName); + auto db = getDB(ctx, &dbName); std::string txId = urldecode2(ctx.request->urlParams[1]); if (txId.empty()) { return status(ctx, http::HttpStatus(http::StatusBadRequest, "Tx ID is not specified")); @@ -1271,10 +1281,14 @@ int HTTPServer::queryResultsJSON(http::Context &ctx, const reindexer::QueryResul auto iarray = builder.Array(kParamItems); const bool isWALQuery = res.IsWALQuery(); + std::optional db; for (size_t i = offset; i < res.Count() && i < offset + limit; ++i) { if (!isWALQuery) { iarray.Raw(nullptr, ""); - res[i].GetJSON(wrSer, false); + const auto err = res[i].GetJSON(wrSer, false); + if (!err.ok()) { + return jsonStatus(ctx, http::HttpStatus(err)); + } } else { auto obj = iarray.Object(nullptr); { @@ -1284,11 +1298,17 @@ int HTTPServer::queryResultsJSON(http::Context &ctx, const reindexer::QueryResul } if (!res[i].IsRaw()) { iarray.Raw(kWALParamItem, ""); - res[i].GetJSON(wrSer, false); + const auto err = res[i].GetJSON(wrSer, false); + if (!err.ok()) { + return jsonStatus(ctx, http::HttpStatus(err)); + } } else { reindexer::WALRecord rec(res[i].GetRaw()); - rec.GetJSON(obj, [this, &res, &ctx](std::string_view cjson) { - auto item = getDB(ctx, kRoleDataRead).NewItem(res.GetNamespaces()[0]); + rec.GetJSON(obj, [this, &ctx, &res, &db](std::string_view cjson) { + if (!db.has_value()) { + db.emplace(getDB(ctx)); + } + auto item = db->NewItem(res.GetNamespaces()[0]); auto err = item.FromCJSON(cjson); if (!err.ok()) { throw Error(err.code(), "Unable to parse CJSON for WAL item: %s", err.what()); @@ -1395,7 +1415,10 @@ int HTTPServer::queryResultsMsgPack(http::Context &ctx, const reindexer::QueryRe auto itemsArray = msgpackBuilder.Array(kParamItems, std::min(size_t(limit), size_t(res.Count() - offset))); for (size_t i = offset; i < res.Count() && i < offset + limit; i++) { - res[i].GetMsgPack(wrSer, false); + const auto err = res[i].GetMsgPack(wrSer, false); + if (!err.ok()) { + return msgpackStatus(ctx, http::HttpStatus(err)); + } } itemsArray.End(); @@ -1422,7 +1445,10 @@ int HTTPServer::queryResultsProtobuf(http::Context &ctx, const reindexer::QueryR auto item = protobufBuilder.Object(itemsField); auto it = res[i]; auto i1 = item.Object(res.getNsNumber(it.GetItemRef().Nsid()) + 1); - it.GetProtobuf(wrSer, false); + const auto err = it.GetProtobuf(wrSer, false); + if (!err.ok()) { + return ctx.Protobuf(err.code(), wrSer.DetachChunk()); + } i1.End(); item.End(); } @@ -1522,7 +1548,7 @@ void HTTPServer::queryResultParams(Builder &builder, const reindexer::QueryResul int HTTPServer::queryResults(http::Context &ctx, reindexer::QueryResults &res, bool isQueryResults, unsigned limit, unsigned offset) { std::string_view widthParam = ctx.request->params.Get("width"sv); - int width = stoi(widthParam); + int width = widthParam.empty() ? 0 : stoi(widthParam); std::string_view format = ctx.request->params.Get("format"); std::string_view withColumnsParam = ctx.request->params.Get("with_columns"); @@ -1619,7 +1645,8 @@ int HTTPServer::modifyQueryTxImpl(http::Context &ctx, const std::string &dbName, return status(ctx); } -Reindexer HTTPServer::getDB(http::Context &ctx, UserRole role, std::string *dbNameOut) { +template +Reindexer HTTPServer::getDB(http::Context &ctx, std::string *dbNameOut) { (void)ctx; Reindexer *db = nullptr; @@ -1647,8 +1674,27 @@ Reindexer HTTPServer::getDB(http::Context &ctx, UserRole role, std::string *dbNa throw http::HttpStatus(status); } assertrx(db); - return db->NeedTraceActivity() ? db->WithActivityTracer(ctx.request->clientAddr, std::string(ctx.request->headers.Get("User-Agent"))) - : *db; + std::string_view timeoutHeader = ctx.request->headers.Get("Request-Timeout"); + std::optional timeoutSec; + if (!timeoutHeader.empty()) { + timeoutSec = try_stoi(timeoutHeader); + if rx_unlikely (!timeoutSec.has_value()) { + logger_.warn("Unable to get integer value from 'Request-Timeout'-header('%s'). Using default value", timeoutHeader); + } + } + std::chrono::seconds timeout; + + if constexpr (role == kUnauthorized || role == kRoleNone) { + throw Error(errLogic, "Unexpected user's role"); + } else if constexpr (role == kRoleDataRead) { + timeout = timeoutSec.has_value() ? std::chrono::seconds(timeoutSec.value()) : serverConfig_.HttpReadTimeout; + } else if constexpr (role >= kRoleDataWrite) { + timeout = timeoutSec.has_value() ? std::chrono::seconds(timeoutSec.value()) : serverConfig_.HttpWriteTimeout; + } + return db->NeedTraceActivity() + ? db->WithContextParams(timeout, ctx.request->clientAddr, std::string(ctx.request->headers.Get("User-Agent")), + InternalRdxContext::kNoConnectionId) + : db->WithTimeout(timeout); } std::string HTTPServer::getNameFromJson(std::string_view json) { @@ -1662,7 +1708,7 @@ std::string HTTPServer::getNameFromJson(std::string_view json) { } std::shared_ptr HTTPServer::getTx(const std::string &dbName, std::string_view txId) { - std::lock_guard lck(txMtx_); + std::lock_guard lck(txMtx_); auto found = txMap_.find(txId); if (found == txMap_.end()) { throw http::HttpStatus(Error(errNotFound, "Invalid tx id"sv)); @@ -1681,7 +1727,8 @@ std::string HTTPServer::addTx(std::string dbName, Transaction &&tx) { txInfo.tx = std::make_shared(std::move(tx)); txInfo.dbName = std::move(dbName); txInfo.txDeadline = TxDeadlineClock::now() + serverConfig_.TxIdleTimeout; - std::lock_guard lck(txMtx_); + + std::lock_guard lck(txMtx_); auto result = txMap_.try_emplace(txId, std::move(txInfo)); if (!result.second) { throw Error(errLogic, "Tx id conflict"); @@ -1690,7 +1737,7 @@ std::string HTTPServer::addTx(std::string dbName, Transaction &&tx) { } void HTTPServer::removeTx(const std::string &dbName, std::string_view txId) { - std::lock_guard lck(txMtx_); + std::lock_guard lck(txMtx_); auto found = txMap_.find(txId); if (found == txMap_.end() || !iequals(found.value().dbName, dbName)) { throw Error(errNotFound, "Invalid tx id"); @@ -1699,8 +1746,9 @@ void HTTPServer::removeTx(const std::string &dbName, std::string_view txId) { } void HTTPServer::removeExpiredTx() { - auto now = TxDeadlineClock::now(); - std::lock_guard lck(txMtx_); + const auto now = TxDeadlineClock::now(); + + std::lock_guard lck(txMtx_); for (auto it = txMap_.begin(); it != txMap_.end();) { if (it->second.txDeadline <= now) { auto ctx = MakeSystemAuthContext(); @@ -1708,9 +1756,9 @@ void HTTPServer::removeExpiredTx() { if (status.ok()) { reindexer::Reindexer *db = nullptr; status = ctx.GetDB(kRoleSystem, &db); - if (db) { + if (db && status.ok()) { logger_.info("Rollback tx {} on idle deadline", it->first); - db->RollBackTransaction(*it->second.tx); + status = db->RollBackTransaction(*it->second.tx); } } it = txMap_.erase(it); @@ -1761,8 +1809,7 @@ int HTTPServer::BeginTx(http::Context &ctx) { } std::string dbName; - auto db = getDB(ctx, kRoleDataWrite, &dbName); - auto tx = db.NewTransaction(nsName); + auto tx = getDB(ctx, &dbName).NewTransaction(nsName); if (!tx.Status().ok()) { return status(ctx, http::HttpStatus(tx.Status())); } @@ -1789,12 +1836,12 @@ int HTTPServer::CommitTx(http::Context &ctx) { } std::string dbName; - auto db = getDB(ctx, kRoleDataWrite, &dbName); + auto db = getDB(ctx, &dbName); auto tx = getTx(dbName, txId); QueryResults qr; auto ret = db.CommitTransaction(*tx, qr); if (!ret.ok()) { - return status(ctx, http::HttpStatus(http::StatusInternalServerError, ret.what())); + return status(ctx, http::HttpStatus(ret)); } removeTx(dbName, txId); return queryResults(ctx, qr); @@ -1807,7 +1854,7 @@ int HTTPServer::RollbackTx(http::Context &ctx) { } std::string dbName; - auto db = getDB(ctx, kRoleDataWrite, &dbName); + auto db = getDB(ctx, &dbName); auto tx = getTx(dbName, txId); QueryResults qr; auto ret = db.RollBackTransaction(*tx); @@ -1828,7 +1875,7 @@ int HTTPServer::DeleteItemsTx(http::Context &ctx) { return modifyItemsTx(ctx, Mo int HTTPServer::GetSQLQueryTx(http::Context &ctx) { std::string dbName; - auto db = getDB(ctx, kRoleDataRead, &dbName); + auto db = getDB(ctx, &dbName); std::string txId = urldecode2(ctx.request->urlParams[1]); if (txId.empty()) { return status(ctx, http::HttpStatus(http::StatusBadRequest, "Tx ID is not specified")); @@ -1836,7 +1883,7 @@ int HTTPServer::GetSQLQueryTx(http::Context &ctx) { reindexer::QueryResults res; std::string sqlQuery = urldecode2(ctx.request->params.Get("q")); if (sqlQuery.empty()) { - return status(ctx, http::HttpStatus(http::StatusBadRequest, "Missed `q` parameter")); + return status(ctx, http::HttpStatus(http::StatusBadRequest, "Missing `q` parameter")); } try { @@ -1849,9 +1896,8 @@ int HTTPServer::GetSQLQueryTx(http::Context &ctx) { case QuerySelect: case QueryTruncate: return status(ctx, http::HttpStatus(http::StatusInternalServerError, "Transactions support update/delete queries only")); - default: - abort(); } + return status(ctx, http::HttpStatus(Error(errLogic, "Unexpected query type: %d", q.type_))); } catch (const Error &e) { return status(ctx, http::HttpStatus(e)); } @@ -1859,7 +1905,7 @@ int HTTPServer::GetSQLQueryTx(http::Context &ctx) { int HTTPServer::DeleteQueryTx(http::Context &ctx) { std::string dbName; - auto db = getDB(ctx, kRoleDataWrite, &dbName); + auto db = getDB(ctx, &dbName); std::string dsl = ctx.body->Read(); reindexer::Query q; diff --git a/cpp_src/server/httpserver.h b/cpp_src/server/httpserver.h index 647c8c7c9..15420a135 100644 --- a/cpp_src/server/httpserver.h +++ b/cpp_src/server/httpserver.h @@ -18,7 +18,7 @@ struct IStatsWatcher; using namespace reindexer::net; -struct HTTPClientData : public http::ClientData { +struct HTTPClientData final : public http::ClientData { AuthContext auth; }; @@ -108,7 +108,8 @@ class HTTPServer { unsigned prepareOffset(std::string_view offsetParam, int offsetDefault = kDefaultOffset); int modifyQueryTxImpl(http::Context &ctx, const std::string &dbName, std::string_view txId, Query &q); - Reindexer getDB(http::Context &ctx, UserRole role, std::string *dbNameOut = nullptr); + template + Reindexer getDB(http::Context &ctx, std::string *dbNameOut = nullptr); std::string getNameFromJson(std::string_view json); constexpr static std::string_view statsSourceName() { return std::string_view{"http"}; } diff --git a/cpp_src/server/rpcserver.cc b/cpp_src/server/rpcserver.cc index c0731aa5d..43c7ab32c 100644 --- a/cpp_src/server/rpcserver.cc +++ b/cpp_src/server/rpcserver.cc @@ -207,7 +207,7 @@ Error RPCServer::execSqlQueryByType(std::string_view sqlQuery, QueryResults &res case QueryUpdate: return getDB(ctx, kRoleDataWrite).Update(q, res); case QueryTruncate: - return getDB(ctx, kRoleDBAdmin).TruncateNamespace(q._namespace); + return getDB(ctx, kRoleDBAdmin).TruncateNamespace(q.NsName()); default: return Error(errParams, "unknown query type %d", q.Type()); } diff --git a/cpp_src/tools/errors.h b/cpp_src/tools/errors.h index 7ff62211b..639c2cb0a 100644 --- a/cpp_src/tools/errors.h +++ b/cpp_src/tools/errors.h @@ -110,11 +110,11 @@ void assertf_fmt(const char *fmt, const Args &...args) { fmt::fprintf(std::cerr, fmt, args...); } -#define assertf(e, fmt, ...) \ - if (!(e)) { \ - assertf_fmt("%s:%d: failed assertion '%s':\n" fmt, __FILE__, __LINE__, #e, __VA_ARGS__); \ - debug::print_crash_query(std::cerr); \ - abort(); \ +#define assertf(e, fmt, ...) \ + if (!(e)) { \ + reindexer::assertf_fmt("%s:%d: failed assertion '%s':\n" fmt, __FILE__, __LINE__, #e, __VA_ARGS__); \ + reindexer::debug::print_crash_query(std::cerr); \ + abort(); \ } #endif // NDEBUG #endif // REINDEX_CORE_BUILD diff --git a/cpp_src/tools/logger.cc b/cpp_src/tools/logger.cc index 4c0700d78..f4936ab86 100644 --- a/cpp_src/tools/logger.cc +++ b/cpp_src/tools/logger.cc @@ -45,11 +45,6 @@ void logInstallWriter(LogWriter writer, LoggerPolicy policy) { #if defined(REINDEX_WITH_ASAN) || defined(REINDEX_WITH_TSAN) || defined(RX_WITH_STDLIB_DEBUG) std::abort(); #else - lck.unlock(); - if (writer) { - errorText.append(". THIS logger is not active"); - writer(LogError, &errorText[0]); - } return; #endif } diff --git a/cpp_src/tools/stringstools.cc b/cpp_src/tools/stringstools.cc index ab9cfa83d..b5c4c9547 100644 --- a/cpp_src/tools/stringstools.cc +++ b/cpp_src/tools/stringstools.cc @@ -147,7 +147,7 @@ bool is_number(std::string_view str) { } void split(std::string_view str, std::string &buf, std::vector &words, const std::string &extraWordSymbols) { - //assuming that the 'ToLower' function and the 'check for replacement' function should not change the character size in bytes + // assuming that the 'ToLower' function and the 'check for replacement' function should not change the character size in bytes buf.resize(str.length()); words.resize(0); auto bufIt = buf.begin(); @@ -609,14 +609,27 @@ int getUTF8StringCharactersCount(std::string_view str) noexcept { int stoi(std::string_view sl) { bool valid; - return jsteemann::atoi(sl.data(), sl.data() + sl.size(), valid); + const int res = jsteemann::atoi(sl.data(), sl.data() + sl.size(), valid); + if (!valid) { + throw Error(errParams, "Can't convert '%s' to number", sl); + } + return res; +} + +std::optional try_stoi(std::string_view sl) { + bool valid; + const int res = jsteemann::atoi(sl.data(), sl.data() + sl.size(), valid); + if (!valid) { + return std::nullopt; + } + return res; } int64_t stoll(std::string_view sl) { bool valid; auto ret = jsteemann::atoi(sl.data(), sl.data() + sl.size(), valid); if (!valid) { - throw Error(errParams, "Can't convert %s to number", sl); + throw Error(errParams, "Can't convert '%s' to number", sl); } return ret; } diff --git a/cpp_src/tools/stringstools.h b/cpp_src/tools/stringstools.h index 80731c581..27f4f87bb 100644 --- a/cpp_src/tools/stringstools.h +++ b/cpp_src/tools/stringstools.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -27,6 +28,12 @@ inline std::string_view skipSpace(std::string_view str) { return str.substr(i); } +template +bool strEmpty(const Str& str) noexcept { + return str.empty(); +} +inline bool strEmpty(const char* str) noexcept { return str[0] == '\0'; } + template Container& split(const typename Container::value_type& str, std::string_view delimiters, bool trimEmpty, Container& tokens) { tokens.resize(0); @@ -136,11 +143,12 @@ int fast_strftime(char* buf, const tm* tm); std::string urldecode2(std::string_view str); int stoi(std::string_view sl); +std::optional try_stoi(std::string_view sl); int64_t stoll(std::string_view sl); bool validateObjectName(std::string_view name, bool allowSpecialChars) noexcept; bool validateUserNsName(std::string_view name) noexcept; -RX_ALWAYS_INLINE bool isSystemNamespaceNameFast(std::string_view name) noexcept { return name.size() && name[0] == '#'; } +RX_ALWAYS_INLINE bool isSystemNamespaceNameFast(std::string_view name) noexcept { return !name.empty() && name[0] == '#'; } LogLevel logLevelFromString(const std::string& strLogLevel); StrictMode strictModeFromString(const std::string& strStrictMode); std::string_view strictModeToString(StrictMode mode); @@ -232,9 +240,29 @@ struct hash_str { size_t operator()(const std::string& hs) const noexcept { return collateHash(hs); } }; -inline void deepCopy(std::string& dst, const std::string& src) { +RX_ALWAYS_INLINE void deepCopy(std::string& dst, const std::string& src) { dst.resize(src.size()); std::memcpy(&dst[0], &src[0], src.size()); } +constexpr size_t kTmpNsPostfixLen = 20; +constexpr std::string_view kTmpNsSuffix = "_tmp_"; +constexpr char kTmpNsPrefix = '@'; +RX_ALWAYS_INLINE bool isTmpNamespaceNameFast(std::string_view name) noexcept { return !name.empty() && name[0] == kTmpNsPrefix; } +[[nodiscard]] inline std::string createTmpNamespaceName(std::string_view baseName) { + return std::string({kTmpNsPrefix}).append(baseName).append(kTmpNsSuffix).append(randStringAlph(kTmpNsPostfixLen)); +} +[[nodiscard]] inline std::string_view demangleTmpNamespaceName(std::string_view tmpNsName) noexcept { + if (tmpNsName.size() < kTmpNsPostfixLen + kTmpNsSuffix.size() + 1) { + return tmpNsName; + } + if (tmpNsName[0] != kTmpNsPrefix) { + return tmpNsName; + } + if (tmpNsName.substr(tmpNsName.size() - kTmpNsPostfixLen - kTmpNsSuffix.size(), kTmpNsSuffix.size()) != kTmpNsSuffix) { + return tmpNsName; + } + return tmpNsName.substr(1, tmpNsName.size() - kTmpNsPostfixLen - 1 - kTmpNsSuffix.size()); +} + } // namespace reindexer diff --git a/cpp_src/tools/verifying_updater.h b/cpp_src/tools/verifying_updater.h new file mode 100644 index 000000000..8bd0967f3 --- /dev/null +++ b/cpp_src/tools/verifying_updater.h @@ -0,0 +1,30 @@ +#pragma once + +#include + +namespace reindexer { + +template +class VerifyingUpdater { + using BaseType = BT; + using FieldType = FT; + +public: + VerifyingUpdater(BaseType& base) noexcept : base_{base} {} + operator FieldType&() & noexcept { return Get(); } + FieldType& Get() & noexcept { + touched_ = true; + return base_.*field; + } + ~VerifyingUpdater() noexcept(false) { + if (touched_ && std::uncaught_exceptions() == 0) { + (base_.*Verify)(); + } + } + +private: + BaseType& base_; + bool touched_{false}; +}; + +} // namespace reindexer diff --git a/dsl/dsl.go b/dsl/dsl.go index 8d6cd5e49..0b9509219 100644 --- a/dsl/dsl.go +++ b/dsl/dsl.go @@ -305,7 +305,10 @@ func (f *Filter) parseValue(data string) error { f.Value, err = f.parseValuesArray(rawValues) return err case "any", "empty": - f.Value = 0 + if len(data) != 0 && data != `""` && data != "null" { + return fmt.Errorf("filter expects no arguments or null for '%s' condition", f.Cond) + } + f.Value = nil default: return fmt.Errorf("cond type '%s' not found", f.Cond) } diff --git a/reindexer_impl.go b/reindexer_impl.go index c1afe3fb7..b545e2c92 100644 --- a/reindexer_impl.go +++ b/reindexer_impl.go @@ -816,7 +816,7 @@ func (db *reindexerImpl) addFilterDSL(filter *dsl.Filter, q *Query) error { if err != nil { return err } - if filter.Value != nil { + if filter.Value != nil || cond == bindings.ANY || cond == bindings.EMPTY { // Skip filter if value is nil (for backwards compatibility reasons) q.Where(filter.Field, cond, filter.Value) } else { diff --git a/test/multiple_json_paths_test.go b/test/multiple_json_paths_test.go new file mode 100644 index 000000000..886bd147a --- /dev/null +++ b/test/multiple_json_paths_test.go @@ -0,0 +1,532 @@ +package reindexer + +import ( + "math" + "math/rand" + "sort" + "strconv" + "testing" + + "github.com/restream/reindexer/v3" + "github.com/stretchr/testify/require" +) + +type TestItemAppendable struct { + ID int `json:"id" reindex:"id,,pk"` + Field1 int `json:"field1,omitempty" reindex:"idx,,appendable"` + Field2 int `json:"field2,omitempty" reindex:"idx,,appendable"` +} + +type TestArrItemAppendable struct { + ID int `json:"id" reindex:"id,,pk"` + ArrField1 []int `json:"arrfield1,omitempty" reindex:"arridx,,appendable"` + ArrField2 []int `json:"arrfield2,omitempty" reindex:"arridx,,appendable"` +} + +type TestJoinItemAppendable struct { + ID int `json:"id" reindex:"id,,pk"` + Field1 int `json:"field1,omitempty" reindex:"idx,,appendable"` + Field2 int `json:"field2,omitempty" reindex:"idx,,appendable"` + TestItemJoined []*TestItemAppendable `reindex:"test_joined,,joined"` +} + +type TestJoinArrItemAppendable struct { + ID int `json:"id" reindex:"id,,pk"` + ArrField1 []int `json:"arrfield1,omitempty" reindex:"arridx,,appendable"` + ArrField2 []int `json:"arrfield2,omitempty" reindex:"arridx,,appendable"` + TestArrItemJoined []*TestArrItemAppendable `reindex:"test_arr_joined,,joined"` +} + +type TestItemNestedAppendable struct { + ID int `json:"id" reindex:"id,,pk"` + NField int `json:"nfield,omitempty" reindex:"idx,,appendable"` + TestNested1 *TestItemNestedAppendableN1 `json:"test_nested_1,omitempty"` + TestNested2 *TestItemNestedAppendableN2 `json:"test_nested_2,omitempty"` +} + +type TestItemNestedAppendableN1 struct { + ID int `json:"id"` + NField int `json:"nfield,omitempty" reindex:"idx,,appendable"` +} + +type TestItemNestedAppendableN2 struct { + ID int `json:"id"` + NField int `json:"nfield,omitempty" reindex:"idx,,appendable"` +} + +type TestItemNestedArrAppendable struct { + ID int `json:"id" reindex:"id,,pk"` + NFieldArr []int `json:"nfield_arr,omitempty" reindex:"arridx,,appendable"` + TestNested1 *TestItemNestedArrAppendableN1 `json:"test_nested_1,omitempty"` + TestNested2 *TestItemNestedArrAppendableN2 `json:"test_nested_2,omitempty"` +} + +type TestItemNestedArrAppendableN1 struct { + ID int `json:"id"` + NFieldArr []int `json:"nfield_arr,omitempty" reindex:"arridx,,appendable"` +} + +type TestItemNestedArrAppendableN2 struct { + ID int `json:"id"` + NFieldArr []int `json:"nfield_arr,omitempty" reindex:"arridx,,appendable"` +} + +const TestSelectWithMultipleJsonPathsNs = "test_select_with_multiple_json_paths" +const TestSelectArrWithMultipleJsonPathsNs = "test_select_arr_with_multiple_json_paths" + +const TestJoinWithMultipleJsonPathsNs = "test_join_with_multiple_json_paths" +const TestJoinedWithMultipleJsonPathsNs = "test_joined_with_multiple_json_paths" +const TestJoinArrWithMultipleJsonPathsNs = "test_join_arr_with_multiple_json_paths" +const TestJoinedArrWithMultipleJsonPathsNs = "test_joined_arr_with_multiple_json_paths" + +const TestAggregWithMultipleJsonPathsNs = "test_aggreg_with_multiple_json_paths" +const TestAggregArrWithMultipleJsonPathsNs = "test_aggreg_arr_with_multiple_json_paths" + +const TestNestedWithMultipleJsonPathsNs = "test_nested_with_multiple_json_paths" +const TestNestedArrWithMultipleJsonPathsNs = "test_nested_arr_with_multiple_json_paths" + +func init() { + tnamespaces[TestSelectWithMultipleJsonPathsNs] = TestItemAppendable{} + tnamespaces[TestSelectArrWithMultipleJsonPathsNs] = TestArrItemAppendable{} + + tnamespaces[TestJoinWithMultipleJsonPathsNs] = TestJoinItemAppendable{} + tnamespaces[TestJoinedWithMultipleJsonPathsNs] = TestItemAppendable{} + tnamespaces[TestJoinArrWithMultipleJsonPathsNs] = TestJoinArrItemAppendable{} + tnamespaces[TestJoinedArrWithMultipleJsonPathsNs] = TestArrItemAppendable{} + + tnamespaces[TestAggregWithMultipleJsonPathsNs] = TestItemAppendable{} + tnamespaces[TestAggregArrWithMultipleJsonPathsNs] = TestArrItemAppendable{} + + tnamespaces[TestNestedWithMultipleJsonPathsNs] = TestItemNestedAppendable{} + tnamespaces[TestNestedArrWithMultipleJsonPathsNs] = TestItemNestedArrAppendable{} +} + +func checkResultItem(t *testing.T, it *reindexer.Iterator, item interface{}) { + defer it.Close() + require.Equal(t, 1, it.Count()) + for it.Next() { + require.EqualValues(t, item, it.Object()) + } +} + +func TestSelectWithMultipleJsonPaths(t *testing.T) { + t.Parallel() + + const ns = TestSelectWithMultipleJsonPathsNs + const ns2 = TestSelectArrWithMultipleJsonPathsNs + + testItem1 := TestItemAppendable{ID: 1, Field1: 10} + testItem2 := TestItemAppendable{ID: 2, Field2: 20} + testItem3 := TestItemAppendable{ID: 3, Field1: 30} + testItem4 := TestItemAppendable{ID: 4, Field2: 30} + + for _, item := range []TestItemAppendable{testItem1, testItem2, testItem3, testItem4} { + err := DB.Upsert(ns, item) + require.NoError(t, err) + } + + t.Run("test select with index multiple json paths", func(t *testing.T) { + it1 := DBD.Query(ns).Where("idx", reindexer.EQ, testItem1.Field1).MustExec() + checkResultItem(t, it1, &testItem1) + + it2 := DBD.Query(ns).Where("idx", reindexer.EQ, testItem2.Field2).MustExec() + checkResultItem(t, it2, &testItem2) + + items, err := DBD.Query(ns). + Where("idx", reindexer.EQ, testItem3.Field1).MustExec().FetchAll() + require.NoError(t, err) + expectedItems := []TestItemAppendable{testItem3, testItem4} + for i, v := range items { + require.EqualValues(t, &expectedItems[i], v.(*TestItemAppendable)) + } + }) + + testArrItem1 := TestArrItemAppendable{ID: 5, ArrField1: []int{50, 51}} + testArrItem2 := TestArrItemAppendable{ID: 6, ArrField2: []int{60, 61}} + testArrItem3 := TestArrItemAppendable{ID: 7, ArrField1: []int{70, 71}, ArrField2: []int{72, 73}} + + for _, item := range []TestArrItemAppendable{testArrItem1, testArrItem2, testArrItem3} { + err := DB.Upsert(ns2, item) + require.NoError(t, err) + } + + t.Run("test select with array index multiple json paths", func(t *testing.T) { + it1 := DBD.Query(ns2).Where("arridx", reindexer.EQ, testArrItem1.ArrField1[0]).MustExec() + checkResultItem(t, it1, &testArrItem1) + + it2 := DBD.Query(ns2).Where("arridx", reindexer.EQ, testArrItem2.ArrField2[0]).MustExec() + checkResultItem(t, it2, &testArrItem2) + + it3 := DBD.Query(ns2).Where("arridx", reindexer.EQ, testArrItem3.ArrField1[0]).MustExec() + checkResultItem(t, it3, &testArrItem3) + + it4 := DBD.Query(ns2).Where("arridx", reindexer.EQ, testArrItem3.ArrField2[0]).MustExec() + checkResultItem(t, it4, &testArrItem3) + }) + + t.Run("can't sort with appendable tag", func(t *testing.T) { + _, err := DBD.Query(ns).Sort("idx", false).Exec().FetchAll() + require.ErrorContains(t, err, "Sorting cannot be applied to array field.") + + _, err = DBD.Query(ns2).Sort("arridx", false).Exec().FetchAll() + require.ErrorContains(t, err, "Sorting cannot be applied to array field.") + }) +} + +func TestJoinWithMultipleJsonPaths(t *testing.T) { + t.Parallel() + + const ns = TestJoinWithMultipleJsonPathsNs + const nsj = TestJoinedWithMultipleJsonPathsNs + + testItem11 := TestJoinItemAppendable{ID: 1, Field1: 10} + testItem12 := TestJoinItemAppendable{ID: 2, Field2: 20} + testItem13 := TestJoinItemAppendable{ID: 3, Field1: 30} + for _, item := range []TestJoinItemAppendable{testItem11, testItem12, testItem13} { + err := DB.Upsert(ns, item) + require.NoError(t, err) + } + + testItem21 := TestItemAppendable{ID: 1, Field1: 10} + testItem22 := TestItemAppendable{ID: 2, Field2: 20} + testItem23 := TestItemAppendable{ID: 3, Field2: 30} + for _, item := range []TestItemAppendable{testItem21, testItem22, testItem23} { + err := DB.Upsert(nsj, item) + require.NoError(t, err) + } + + testItem11.TestItemJoined = []*TestItemAppendable{&testItem21} + testItem12.TestItemJoined = []*TestItemAppendable{&testItem22} + testItem13.TestItemJoined = []*TestItemAppendable{&testItem23} + + expectedItems1 := []TestJoinItemAppendable{testItem11, testItem12, testItem13} + + t.Run("test inner join with index multiple json paths", func(t *testing.T) { + qjoin := DBD.Query(nsj) + items, err := DBD.Query(ns).InnerJoin(qjoin, "test_joined"). + On("idx", reindexer.EQ, "idx").MustExec().FetchAll() + require.NoError(t, err) + + for i, v := range items { + require.EqualValues(t, &expectedItems1[i], v.(*TestJoinItemAppendable)) + } + }) + + t.Run("test left join with index multiple json paths", func(t *testing.T) { + qjoin := DBD.Query(nsj) + items, err := DBD.Query(ns).LeftJoin(qjoin, "test_joined"). + On("idx", reindexer.EQ, "idx").MustExec().FetchAll() + require.NoError(t, err) + + for i, v := range items { + require.EqualValues(t, &expectedItems1[i], v.(*TestJoinItemAppendable)) + } + }) + + const ns2 = TestJoinArrWithMultipleJsonPathsNs + const ns2j = TestJoinedArrWithMultipleJsonPathsNs + + testArrItem11 := TestJoinArrItemAppendable{ID: 1, ArrField1: []int{10, 11}} + testArrItem12 := TestJoinArrItemAppendable{ID: 2, ArrField2: []int{20, 21, 22}} + testArrItem13 := TestJoinArrItemAppendable{ID: 3, ArrField1: []int{30, 31, 32}, ArrField2: []int{32, 33}} + for _, item := range []TestJoinArrItemAppendable{testArrItem11, testArrItem12, testArrItem13} { + err := DB.Upsert(ns2, item) + require.NoError(t, err) + } + + testArrItem21 := TestArrItemAppendable{ID: 1, ArrField1: []int{10, 11, 12}} + testArrItem22 := TestArrItemAppendable{ID: 2, ArrField2: []int{20, 21}} + testArrItem23 := TestArrItemAppendable{ID: 3, ArrField2: []int{30, 31}, ArrField1: []int{32, 33, 34}} + for _, item := range []TestArrItemAppendable{testArrItem21, testArrItem22, testArrItem23} { + err := DB.Upsert(ns2j, item) + require.NoError(t, err) + } + + testArrItem11.TestArrItemJoined = []*TestArrItemAppendable{&testArrItem21} + testArrItem12.TestArrItemJoined = []*TestArrItemAppendable{&testArrItem22} + testArrItem13.TestArrItemJoined = []*TestArrItemAppendable{&testArrItem23} + + expectedItems2 := []TestJoinArrItemAppendable{testArrItem11, testArrItem12, testArrItem13} + + t.Run("test inner join with array index multiple json paths", func(t *testing.T) { + qjoin := DBD.Query(ns2j) + items, err := DBD.Query(ns2).InnerJoin(qjoin, "test_arr_joined"). + On("arridx", reindexer.EQ, "arridx").MustExec().FetchAll() + require.NoError(t, err) + + for i, v := range items { + require.EqualValues(t, &expectedItems2[i], v.(*TestJoinArrItemAppendable)) + } + }) + + t.Run("test left join with array index multiple json paths", func(t *testing.T) { + qjoin := DBD.Query(ns2j) + items, err := DBD.Query(ns2).LeftJoin(qjoin, "test_arr_joined"). + On("arridx", reindexer.EQ, "arridx").MustExec().FetchAll() + require.NoError(t, err) + + for i, v := range items { + require.EqualValues(t, &expectedItems2[i], v.(*TestJoinArrItemAppendable)) + } + }) +} + +type aggValuesStruct struct { + Min float64 `json:"min"` + Max float64 `json:"max"` + Sum float64 `json:"sum"` + Avg float64 `json:"avg"` +} + +func addValuesFromArrToMap(m map[string]int, arrField []int) { + for _, arr := range arrField { + m[strconv.Itoa(arr)] += 1 + } +} + +func TestAggregationsWithMultipleJsonPaths(t *testing.T) { + t.Parallel() + + const ns = TestAggregWithMultipleJsonPathsNs + const ns2 = TestAggregArrWithMultipleJsonPathsNs + + start, end := 0, 10 + minValue, maxRandValue := 1, 20 + + aggValues := aggValuesStruct{ + Min: float64(math.MaxFloat64), Max: float64(0), Sum: 0, Avg: 0, + } + + fieldsSet := make(map[string]int) + for i := start; i < end; i++ { + item1 := TestItemAppendable{ID: i, Field1: minValue + rand.Intn(maxRandValue)} + err := DB.Upsert(ns, item1) + require.NoError(t, err) + value1 := strconv.Itoa(item1.Field1) + fieldsSet[value1] += 1 + item2 := TestItemAppendable{ID: i + 10, Field2: minValue + rand.Intn(maxRandValue)} + err = DB.Upsert(ns, item2) + require.NoError(t, err) + value2 := strconv.Itoa(item2.Field2) + fieldsSet[value2] += 1 + + val1, val2 := float64(item1.Field1), float64(item2.Field2) + if aggValues.Min > val1 || aggValues.Min > val2 { + aggValues.Min = float64(min(item1.Field1, item2.Field2)) + } + if aggValues.Max < val1 || aggValues.Max < val2 { + aggValues.Max = float64(max(item1.Field1, item2.Field2)) + } + aggValues.Sum += float64(item1.Field1 + item2.Field2) + } + aggValues.Avg = aggValues.Sum / float64(end*2) + + fieldValues := []string{} + for k := range fieldsSet { + fieldValues = append(fieldValues, k) + } + sort.Strings(fieldValues) + + t.Run("test aggreations with index multiple json paths", func(t *testing.T) { + q := DBD.Query(ns) + q.AggregateMin("idx") + q.AggregateMax("idx") + q.AggregateSum("idx") + q.AggregateAvg("idx") + + it := q.MustExec() + require.NoError(t, it.Error()) + defer it.Close() + + aggResults := it.AggResults() + require.Equal(t, 4, len(aggResults)) + require.Equal(t, aggValues.Min, *aggResults[0].Value) + require.Equal(t, aggValues.Max, *aggResults[1].Value) + require.Equal(t, aggValues.Sum, *aggResults[2].Value) + require.Equal(t, aggValues.Avg, *aggResults[3].Value) + }) + + t.Run("test distinct with index multiple json paths", func(t *testing.T) { + it := DBD.Query(ns).Distinct("idx").MustExec() + require.NoError(t, it.Error()) + defer it.Close() + require.Equal(t, len(fieldValues), it.Count()) + + aggResults := it.AggResults() + require.Equal(t, 1, len(aggResults)) + sort.Strings(aggResults[0].Distincts) + require.Equal(t, fieldValues, aggResults[0].Distincts) + }) + + t.Run("test facet with index multiple json paths", func(t *testing.T) { + q := DBD.Query(ns) + q.AggregateFacet("idx") + it := q.MustExec() + require.NoError(t, it.Error()) + defer it.Close() + require.Equal(t, 0, it.Count()) + + aggResults := it.AggResults() + require.Equal(t, 1, len(aggResults)) + for _, facet := range aggResults[0].Facets { + require.Equal(t, fieldsSet[facet.Values[0]], facet.Count) + } + }) + + aggArrValues := aggValuesStruct{ + Min: float64(math.MaxFloat64), Max: float64(0), Sum: 0, Avg: 0, + } + + intValues := []int{} + fieldsArrSet := make(map[string]int) + for i := start; i < end; i++ { + item1 := TestArrItemAppendable{ID: i, ArrField1: randIntArr(2, minValue, maxRandValue)} + err := DB.Upsert(ns2, item1) + require.NoError(t, err) + addValuesFromArrToMap(fieldsArrSet, item1.ArrField1) + item2 := TestArrItemAppendable{ID: i + 10, ArrField2: randIntArr(2, minValue, maxRandValue)} + err = DB.Upsert(ns2, item2) + require.NoError(t, err) + addValuesFromArrToMap(fieldsArrSet, item2.ArrField2) + item3 := TestArrItemAppendable{ID: i + 20, ArrField1: randIntArr(2, minValue, maxRandValue), + ArrField2: randIntArr(2, 0, minValue)} + err = DB.Upsert(ns2, item3) + require.NoError(t, err) + addValuesFromArrToMap(fieldsArrSet, item3.ArrField1) + addValuesFromArrToMap(fieldsArrSet, item3.ArrField2) + + arrFields := [][]int{item1.ArrField1, item2.ArrField2, item3.ArrField1, item3.ArrField2} + for _, intArrValue := range arrFields { + for _, intValue := range intArrValue { + intValues = append(intValues, intValue) + aggArrValues.Sum += float64(intValue) + } + } + } + sort.Ints(intValues) + + aggArrValues.Min = float64(intValues[0]) + aggArrValues.Max = float64(intValues[len(intValues)-1]) + aggArrValues.Avg = aggArrValues.Sum / float64(len(intValues)) + + fieldValues2 := []string{} + for k := range fieldsArrSet { + fieldValues2 = append(fieldValues2, k) + } + sort.Strings(fieldValues2) + + t.Run("test aggreations with array index multiple json paths", func(t *testing.T) { + q := DBD.Query(ns2) + q.AggregateMin("arridx") + q.AggregateMax("arridx") + q.AggregateSum("arridx") + q.AggregateAvg("arridx") + + it := q.MustExec() + require.NoError(t, it.Error()) + defer it.Close() + aggResults := it.AggResults() + require.Equal(t, 4, len(aggResults)) + require.Equal(t, aggArrValues.Min, *aggResults[0].Value) + require.Equal(t, aggArrValues.Max, *aggResults[1].Value) + require.Equal(t, aggArrValues.Sum, *aggResults[2].Value) + require.Equal(t, aggArrValues.Avg, *aggResults[3].Value) + }) + + t.Run("test distinct with array index multiple json paths", func(t *testing.T) { + it := DBD.Query(ns2).Distinct("arridx").MustExec() + require.NoError(t, it.Error()) + defer it.Close() + // require.Equal(t, end*3, it.Count()) TODO: 1526 + + aggResults := it.AggResults() + require.Equal(t, 1, len(aggResults)) + sort.Strings(aggResults[0].Distincts) + require.Equal(t, fieldValues2, aggResults[0].Distincts) + }) + + t.Run("test facet with array index multiple json paths", func(t *testing.T) { + q := DBD.Query(ns2) + q.AggregateFacet("arridx") + it := q.MustExec() + require.NoError(t, it.Error()) + defer it.Close() + require.Equal(t, 0, it.Count()) + + aggResults := it.AggResults() + require.Equal(t, 1, len(aggResults)) + for _, facet := range aggResults[0].Facets { + require.Equal(t, fieldsArrSet[facet.Values[0]], facet.Count) + } + }) +} + +func TestNestedWithMultipleJsonPaths(t *testing.T) { + t.Parallel() + + const ns = TestNestedWithMultipleJsonPathsNs + const ns2 = TestNestedArrWithMultipleJsonPathsNs + + testItem1 := TestItemNestedAppendable{ID: 1, NField: 10, + TestNested1: &TestItemNestedAppendableN1{ID: 1, NField: 30}, + TestNested2: &TestItemNestedAppendableN2{ID: 1, NField: 10}} + testItem2 := TestItemNestedAppendable{ID: 2, NField: 20, + TestNested1: &TestItemNestedAppendableN1{ID: 2, NField: 21}, + TestNested2: &TestItemNestedAppendableN2{ID: 2, NField: 30}} + testItem3 := TestItemNestedAppendable{ID: 3, NField: 30, + TestNested1: &TestItemNestedAppendableN1{ID: 3, NField: 31}, + TestNested2: &TestItemNestedAppendableN2{ID: 3, NField: 30}} + + for _, item := range []TestItemNestedAppendable{testItem1, testItem2, testItem3} { + err := DB.Upsert(ns, item) + require.NoError(t, err) + } + + t.Run("test select with nested index multiple json paths", func(t *testing.T) { + it1 := DBD.Query(ns).Where("idx", reindexer.EQ, testItem1.NField).MustExec() + checkResultItem(t, it1, &testItem1) + + it2 := DBD.Query(ns).Where("idx", reindexer.EQ, testItem2.TestNested1.NField).MustExec() + checkResultItem(t, it2, &testItem2) + + items, err := DBD.Query(ns). + Where("idx", reindexer.EQ, testItem3.NField).MustExec().FetchAll() + require.NoError(t, err) + expectedItems := []TestItemNestedAppendable{testItem1, testItem2, testItem3} + for i, v := range items { + require.EqualValues(t, &expectedItems[i], v.(*TestItemNestedAppendable)) + } + }) + + testArrItem1 := TestItemNestedArrAppendable{ID: 1, NFieldArr: []int{10, 11}, + TestNested1: &TestItemNestedArrAppendableN1{ID: 1, NFieldArr: []int{30, 11}}, + TestNested2: &TestItemNestedArrAppendableN2{ID: 1, NFieldArr: []int{10, 11}}} + testArrItem2 := TestItemNestedArrAppendable{ID: 2, NFieldArr: []int{20, 11}, + TestNested1: &TestItemNestedArrAppendableN1{ID: 1, NFieldArr: []int{21, 11}}, + TestNested2: &TestItemNestedArrAppendableN2{ID: 1, NFieldArr: []int{31, 30}}} + testArrItem3 := TestItemNestedArrAppendable{ID: 3, NFieldArr: []int{30, 11}, + TestNested1: &TestItemNestedArrAppendableN1{ID: 1, NFieldArr: []int{31, 11}}, + TestNested2: &TestItemNestedArrAppendableN2{ID: 1, NFieldArr: []int{30, 11}}} + + for _, item := range []TestItemNestedArrAppendable{testArrItem1, testArrItem2, testArrItem3} { + err := DB.Upsert(ns2, item) + require.NoError(t, err) + } + + t.Run("test select with nested array index multiple json paths", func(t *testing.T) { + it1 := DBD.Query(ns2).Where("arridx", reindexer.EQ, testArrItem1.NFieldArr[0]).MustExec() + checkResultItem(t, it1, &testArrItem1) + + it2 := DBD.Query(ns2).Where("arridx", reindexer.EQ, testArrItem2.TestNested1.NFieldArr[0]).MustExec() + checkResultItem(t, it2, &testArrItem2) + + items, err := DBD.Query(ns2). + Where("arridx", reindexer.EQ, testArrItem3.NFieldArr[0]).MustExec().FetchAll() + require.NoError(t, err) + expectedItems := []TestItemNestedArrAppendable{testArrItem1, testArrItem2, testArrItem3} + for i, v := range items { + require.EqualValues(t, &expectedItems[i], v.(*TestItemNestedArrAppendable)) + } + }) + +} diff --git a/test/queries_test.go b/test/queries_test.go index c89451d7f..440217696 100644 --- a/test/queries_test.go +++ b/test/queries_test.go @@ -113,23 +113,23 @@ type TestItemWithSparse struct { Prices []*TestJoinItem `reindex:"prices,,joined"` Pricesx []*TestJoinItem `reindex:"pricesx,,joined"` ID int `reindex:"id,-"` - Genre int64 `reindex:"genre,tree,sparse"` + Genre int64 `reindex:"genre,tree"` Year int `reindex:"year,tree,sparse"` Packages []int `reindex:"packages,hash,sparse"` Name string `reindex:"name,tree,sparse"` Countries []string `reindex:"countries,tree,sparse"` - Age int `reindex:"age,hash,sparse"` + Age int `reindex:"age,hash"` AgeLimit int64 `json:"age_limit" reindex:"age_limit,hash,sparse"` CompanyName string `json:"company_name" reindex:"company_name,hash,sparse"` Address string `json:"address"` PostalCode int `json:"postal_code"` Description string `reindex:"description,fuzzytext"` - Rate float64 `reindex:"rate,tree,sparse"` + Rate float64 `reindex:"rate,tree"` ExchangeRate float64 `json:"exchange_rate"` PollutionRate float32 `json:"pollution_rate"` IsDeleted bool `reindex:"isdeleted,-"` Actor Actor `reindex:"actor"` - PricesIDs []int `reindex:"price_id"` + PricesIDs []int `reindex:"price_id,,sparse"` LocationID string `reindex:"location"` EndTime int `reindex:"end_time,-"` StartTime int `reindex:"start_time,tree"` @@ -636,7 +636,7 @@ func TestWALQueries(t *testing.T) { }) t.Run("JSON WAL query with ANY", func(t *testing.T) { - jsonIt := DBD.Query(ns).Where("#lsn", reindexer.ANY, 0).ExecToJson() + jsonIt := DBD.Query(ns).Where("#lsn", reindexer.ANY, nil).ExecToJson() validateJson(t, jsonIt) }) @@ -646,7 +646,7 @@ func TestWALQueries(t *testing.T) { }) t.Run("CJSON WAL query with ANY (expecting error)", func(t *testing.T) { - it := DBD.Query(ns).Where("#lsn", reindexer.ANY, 0).Exec() + it := DBD.Query(ns).Where("#lsn", reindexer.ANY, nil).Exec() assert.Error(t, it.Error()) }) } @@ -873,8 +873,8 @@ func callQueriesSequence(t *testing.T, namespace string, distinct []string, sort newTestQuery(DB, namespace).Where("name", reindexer.LIKE, makeLikePattern(randString())).ExecAndVerify(t) newTestQuery(DB, namespace).Where("packages", reindexer.SET, randIntArr(10, 10000, 50)).Distinct(distinct).Sort(sort, desc).ExecAndVerify(t) - newTestQuery(DB, namespace).Where("packages", reindexer.EMPTY, 0).Distinct(distinct).Sort(sort, desc).ExecAndVerify(t) - newTestQuery(DB, namespace).Where("packages", reindexer.ANY, 0).Distinct(distinct).Sort(sort, desc).ExecAndVerify(t) + newTestQuery(DB, namespace).Where("packages", reindexer.EMPTY, nil).Distinct(distinct).Sort(sort, desc).ExecAndVerify(t) + newTestQuery(DB, namespace).Where("packages", reindexer.ANY, nil).Distinct(distinct).Sort(sort, desc).ExecAndVerify(t) newTestQuery(DB, namespace).Where("isdeleted", reindexer.EQ, true).Distinct(distinct).Sort(sort, desc).ExecAndVerify(t) @@ -929,12 +929,12 @@ func callQueriesSequence(t *testing.T, namespace string, distinct []string, sort newTestQuery(DB, namespace).Distinct(distinct).Sort(sort, desc).ReqTotal(). Where("genre", reindexer.SET, []int{5, 1, 7}). Where("year", reindexer.LT, 2010).Or().Where("genre", reindexer.EQ, 3). - Where("packages", reindexer.SET, randIntArr(5, 10000, 50)).Or().Where("packages", reindexer.EMPTY, 0).Debug(reindexer.TRACE). + Where("packages", reindexer.SET, randIntArr(5, 10000, 50)).Or().Where("packages", reindexer.EMPTY, nil).Debug(reindexer.TRACE). ExecAndVerify(t) newTestQuery(DB, namespace).Distinct(distinct).Sort(sort, desc).ReqTotal(). Where("genre", reindexer.SET, []int{5, 1, 7}). - Where("year", reindexer.LT, 2010).Or().Where("packages", reindexer.ANY, 0). + Where("year", reindexer.LT, 2010).Or().Where("packages", reindexer.ANY, nil). Where("packages", reindexer.SET, randIntArr(5, 10000, 50)).Debug(reindexer.TRACE). ExecAndVerify(t) @@ -1349,12 +1349,12 @@ func CheckTestItemsDSLQueries(t *testing.T) { { Field: "PACKAGES", Cond: "ANY", - Value: 0, + Value: nil, }, { Field: "countries", Cond: "EMPTY", - Value: 0, + Value: nil, }, { Field: "isdeleted", @@ -1381,8 +1381,8 @@ func CheckTestItemsDSLQueries(t *testing.T) { newTestQuery(DB, "test_items"). Where("year", reindexer.GT, 2016). Where("genre", reindexer.SET, []int{1, 2, 3}). - Where("packages", reindexer.ANY, 0). - Where("countries", reindexer.EMPTY, 0). + Where("packages", reindexer.ANY, nil). + Where("countries", reindexer.EMPTY, nil). Where("isdeleted", reindexer.EQ, true). Where("company_name", reindexer.LIKE, likePattern). Sort("year", true). @@ -1700,13 +1700,13 @@ func TestStrictMode(t *testing.T) { t.Run("Strict filtering/sort by folded fields (empty namespace)", func(t *testing.T) { { - itNames := DBD.Query(namespace).Strict(reindexer.QueryStrictModeNames).Where("nested.Name", reindexer.ANY, 0).Sort("nested.Name", false).MustExec() + itNames := DBD.Query(namespace).Strict(reindexer.QueryStrictModeNames).Where("nested.Name", reindexer.ANY, nil).Sort("nested.Name", false).MustExec() assert.Equal(t, itNames.Count(), 0) itNames.Close() - itNone := DBD.Query(namespace).Strict(reindexer.QueryStrictModeNone).Where("nested.Name", reindexer.ANY, 0).Sort("nested.Name", false).MustExec() + itNone := DBD.Query(namespace).Strict(reindexer.QueryStrictModeNone).Where("nested.Name", reindexer.ANY, nil).Sort("nested.Name", false).MustExec() assert.Equal(t, itNone.Count(), 0) itNone.Close() - itIndexes := DBD.Query(namespace).Strict(reindexer.QueryStrictModeIndexes).Where("nested.Name", reindexer.ANY, 0).Sort("nested.Name", false).Exec() + itIndexes := DBD.Query(namespace).Strict(reindexer.QueryStrictModeIndexes).Where("nested.Name", reindexer.ANY, nil).Sort("nested.Name", false).Exec() assert.Error(t, itIndexes.Error()) itIndexes.Close() } @@ -1833,25 +1833,25 @@ func TestStrictMode(t *testing.T) { assert.Error(t, itIndexes1.Error()) itIndexes1.Close() - itNone3 := DBD.Query(namespace).Where("unknown_field", reindexer.EMPTY, 0).Sort("year", true). + itNone3 := DBD.Query(namespace).Where("unknown_field", reindexer.EMPTY, nil).Sort("year", true). Sort("name", false).Strict(reindexer.QueryStrictModeNone).MustExec() assert.Equal(t, itNone3.Count(), itemsCount) itNone3.Close() } { - itNames := DBD.Query(namespace).Where("unknown_field", reindexer.EMPTY, 0).Sort("year", true). + itNames := DBD.Query(namespace).Where("unknown_field", reindexer.EMPTY, nil).Sort("year", true). Sort("name", false).Strict(reindexer.QueryStrictModeNames).Exec() assert.Error(t, itNames.Error()) itNames.Close() - itNone := DBD.Query(namespace).Where("unknown_field", reindexer.EMPTY, 0).Sort("year", true). + itNone := DBD.Query(namespace).Where("unknown_field", reindexer.EMPTY, nil).Sort("year", true). Sort("name", false).Strict(reindexer.QueryStrictModeNone).MustExec() itNone.Close() itAll := DBD.Query(namespace).Sort("year", true). Sort("name", false).Strict(reindexer.QueryStrictModeNone).MustExec() assert.Equal(t, itNone.Count(), itAll.Count()) itAll.Close() - itIndexes := DBD.Query(namespace).Where("unknown_field", reindexer.EMPTY, 0).Sort("year", true). + itIndexes := DBD.Query(namespace).Where("unknown_field", reindexer.EMPTY, nil).Sort("year", true). Sort("name", false).Strict(reindexer.QueryStrictModeIndexes).Exec() assert.Error(t, itIndexes.Error()) itIndexes.Close() diff --git a/test/query_test.go b/test/query_test.go index 9b7daea79..924523459 100644 --- a/test/query_test.go +++ b/test/query_test.go @@ -395,12 +395,14 @@ func (qt *queryTest) DeepReplEqual() *queryTest { func (qt *queryTest) where(index string, condition int, keys interface{}) *queryTest { qte := queryTestEntry{index: index, condition: condition, ikeys: keys} - if reflect.TypeOf(keys).Kind() == reflect.Slice || reflect.TypeOf(keys).Kind() == reflect.Array { - for i := 0; i < reflect.ValueOf(keys).Len(); i++ { - qte.keys = append(qte.keys, reflect.ValueOf(keys).Index(i)) + if keys != nil { + if reflect.TypeOf(keys).Kind() == reflect.Slice || reflect.TypeOf(keys).Kind() == reflect.Array { + for i := 0; i < reflect.ValueOf(keys).Len(); i++ { + qte.keys = append(qte.keys, reflect.ValueOf(keys).Index(i)) + } + } else { + qte.keys = append(qte.keys, reflect.ValueOf(keys)) } - } else { - qte.keys = append(qte.keys, reflect.ValueOf(keys)) } qte.fieldIdx, _ = qt.ns.getField(index) qt.entries.addEntry(qte, qt.nextOp) @@ -1343,6 +1345,13 @@ func min(a, b int) int { return b } +func max(a, b int) int { + if a > b { + return a + } + return b +} + func checkResult(cmpRes int, cond int) bool { result := false switch cond {