diff --git a/bindings/builtin/builtin.go b/bindings/builtin/builtin.go index bfd8b17ed..59eacc4a9 100644 --- a/bindings/builtin/builtin.go +++ b/bindings/builtin/builtin.go @@ -31,8 +31,13 @@ type Logger interface { Printf(level int, fmt string, msg ...interface{}) } -var logger Logger +// Separate mutexes for logger object itself and for reindexer_enable_logger call: +// logMtx provides safe access to the logger +// logEnableMtx provides atomic logic for (enable + set) and (disable + reset) procedures var logMtx sync.RWMutex +var logEnableMtx sync.Mutex +var logger Logger + var enableDebug bool var bufPool sync.Pool @@ -600,18 +605,24 @@ func CGoLogger(level int, msg string) { } } -func (binding *Builtin) EnableLogger(log bindings.Logger) { +func (binding *Builtin) setLogger(log bindings.Logger) { logMtx.Lock() defer logMtx.Unlock() logger = log +} + +func (binding *Builtin) EnableLogger(log bindings.Logger) { + logEnableMtx.Lock() + defer logEnableMtx.Unlock() + binding.setLogger(log) C.reindexer_enable_go_logger() } func (binding *Builtin) DisableLogger() { - logMtx.Lock() - defer logMtx.Unlock() + logEnableMtx.Lock() + defer logEnableMtx.Unlock() C.reindexer_disable_go_logger() - logger = nil + binding.setLogger(nil) } func (binding *Builtin) ReopenLogFiles() error { diff --git a/bindings/builtinserver/config/config.go b/bindings/builtinserver/config/config.go index 30ffe0516..8b3cf0783 100644 --- a/bindings/builtinserver/config/config.go +++ b/bindings/builtinserver/config/config.go @@ -14,10 +14,12 @@ type StorageConf struct { } type NetConf struct { - HTTPAddr string `yaml:"httpaddr"` - RPCAddr string `yaml:"rpcaddr"` - WebRoot string `yaml:"webroot"` - Security bool `yaml:"security"` + HTTPAddr string `yaml:"httpaddr"` + RPCAddr string `yaml:"rpcaddr"` + WebRoot string `yaml:"webroot"` + Security bool `yaml:"security"` + HttpReadTimeoutSec int `yaml:"http_read_timeout,omitempty"` + HttpWriteTimeoutSec int `yaml:"http_write_timeout,omitempty"` } type LoggerConf struct { diff --git a/changelog.md b/changelog.md index 9c3b0da69..479e723fb 100644 --- a/changelog.md +++ b/changelog.md @@ -1905,5 +1905,3 @@ - [ref] EnableStorage method was deprecated - [fix] Query builder did not reset opOR after InnerJoin -## Misc - diff --git a/cjson/decoder.go b/cjson/decoder.go index 3cc9bd183..dc9667ced 100644 --- a/cjson/decoder.go +++ b/cjson/decoder.go @@ -711,7 +711,7 @@ func (dec *Decoder) Decode(cjson []byte, dest interface{}) (err error) { } }() - fieldsoutcnt := make([]int, 64, 64) + fieldsoutcnt := make([]int, MaxIndexes) ctagsPath := make([]int, 0, 8) dec.decodeValue(nil, ser, reflect.ValueOf(dest), fieldsoutcnt, ctagsPath) diff --git a/cpp_src/CMakeLists.txt b/cpp_src/CMakeLists.txt index df73c53cd..82ede719d 100644 --- a/cpp_src/CMakeLists.txt +++ b/cpp_src/CMakeLists.txt @@ -89,6 +89,10 @@ endif () set (EXTRA_FLAGS "") +if (WITH_ASAN AND WITH_TSAN) + message(FATAL_ERROR "You cannot use the ASAN and TSAN options at the same time, CMake will exit.") +endif() + if (WITH_ASAN) set (EXTRA_FLAGS "-fsanitize=address") add_definitions(-DREINDEX_WITH_ASAN) @@ -272,6 +276,9 @@ else() list(APPEND SRCS ${KOISHI_PATH}/fcontext/fcontext.c ${KOISHI_PATH}/fcontext/fcontext.hpp) endif() +# Static LevelDB v1.23 is built with -fno-rtti by default. To inherit our logger from leveldb's logger, this file must be built with -fno-rtti to +set_source_files_properties(${REINDEXER_SOURCE_PATH}/core/storage/leveldblogger.cc PROPERTIES COMPILE_FLAGS "-fno-rtti") + list(APPEND REINDEXER_LIBRARIES reindexer) add_library(${TARGET} STATIC ${HDRS} ${SRCS} ${VENDORS}) add_definitions(-DREINDEX_CORE_BUILD=1) @@ -695,6 +702,7 @@ if (NOT WIN32) SET(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME "server") SET(DIST_INCLUDE_FILES "tools/errors.h" "tools/serializer.h" "tools/varint.h" "tools/stringstools.h" "tools/customhash.h" "tools/assertrx.h" "tools/jsonstring.h" + "tools/verifying_updater.h" "core/reindexer.h" "core/type_consts.h" "core/item.h" "core/payload/payloadvalue.h" "core/payload/payloadiface.h" "core/indexopts.h" "core/namespacedef.h" "core/keyvalue/variant.h" "core/keyvalue/geometry.h" "core/sortingprioritiestable.h" "core/rdxcontext.h" "core/activity_context.h" "core/type_consts_helpers.h" "core/payload/fieldsset.h" "core/payload/payloadtype.h" @@ -702,8 +710,8 @@ if (NOT WIN32) "core/query/query.h" "core/query/queryentry.h" "core/queryresults/queryresults.h" "core/indexdef.h" "core/queryresults/aggregationresult.h" "core/queryresults/itemref.h" "core/namespace/stringsholder.h" "core/keyvalue/key_string.h" "core/key_value_type.h" "core/keyvalue/uuid.h" "core/expressiontree.h" "core/lsn.h" "core/cjson/tagspath.h" "core/cjson/ctag.h" - "estl/cow.h" "estl/overloaded.h" "estl/one_of.h" "estl/h_vector.h" "estl/mutex.h" "estl/intrusive_ptr.h" "estl/trivial_reverse_iterator.h" - "estl/span.h" "estl/chunk.h" "estl/fast_hash_traits.h" "estl/debug_macros.h" "estl/defines.h" + "estl/cow.h" "estl/overloaded.h" "estl/one_of.h" "estl/h_vector.h" "estl/mutex.h" "estl/intrusive_ptr.h" "estl/trivial_reverse_iterator.h" + "estl/span.h" "estl/chunk.h" "estl/fast_hash_traits.h" "estl/debug_macros.h" "estl/defines.h" "estl/template.h" "estl/comparation_result.h" "client/reindexer.h" "client/item.h" "client/reindexerconfig.h" "client/queryresults.h" "client/resultserializer.h" "client/internalrdxcontext.h" "client/transaction.h" "client/cororeindexer.h" "client/coroqueryresults.h" "client/corotransaction.h" diff --git a/cpp_src/client/coroqueryresults.cc b/cpp_src/client/coroqueryresults.cc index fdc295c75..5c0d2c62c 100644 --- a/cpp_src/client/coroqueryresults.cc +++ b/cpp_src/client/coroqueryresults.cc @@ -3,8 +3,6 @@ #include "core/cjson/baseencoder.h" #include "core/keyvalue/p_string.h" #include "net/cproto/coroclientconnection.h" -#include "server/rpcqrwatcher.h" -#include "tools/logger.h" namespace reindexer { namespace client { diff --git a/cpp_src/client/cororeindexer.cc b/cpp_src/client/cororeindexer.cc index 4cb747cf5..1d919d93c 100644 --- a/cpp_src/client/cororeindexer.cc +++ b/cpp_src/client/cororeindexer.cc @@ -1,7 +1,6 @@ #include "client/cororeindexer.h" #include "client/cororpcclient.h" #include "tools/cpucheck.h" -#include "tools/logger.h" namespace reindexer { namespace client { @@ -33,7 +32,7 @@ CoroReindexer& CoroReindexer::operator=(CoroReindexer&& rdx) noexcept { Error CoroReindexer::Connect(const std::string& dsn, dynamic_loop& loop, const client::ConnectOpts& opts) { return impl_->Connect(dsn, loop, opts); } -Error CoroReindexer::Stop() { return impl_->Stop(); } +void CoroReindexer::Stop() { impl_->Stop(); } Error CoroReindexer::AddNamespace(const NamespaceDef& nsDef) { return impl_->AddNamespace(nsDef, ctx_); } Error CoroReindexer::OpenNamespace(std::string_view nsName, const StorageOpts& storage) { return impl_->OpenNamespace(nsName, ctx_, storage); diff --git a/cpp_src/client/cororeindexer.h b/cpp_src/client/cororeindexer.h index da211a6aa..9d9410902 100644 --- a/cpp_src/client/cororeindexer.h +++ b/cpp_src/client/cororeindexer.h @@ -46,7 +46,7 @@ class CoroReindexer { /// @param opts - Connect options. May contaion any of
Error Connect(const std::string &dsn, dynamic_loop &loop, const client::ConnectOpts &opts = client::ConnectOpts()); /// Stop - shutdown connector - Error Stop(); + void Stop(); /// Open or create namespace /// @param nsName - Name of namespace /// @param opts - Storage options. Can be one of
diff --git a/cpp_src/client/cororpcclient.cc b/cpp_src/client/cororpcclient.cc index 91ab0114b..9b3236b68 100644 --- a/cpp_src/client/cororpcclient.cc +++ b/cpp_src/client/cororpcclient.cc @@ -46,13 +46,12 @@ Error CoroRPCClient::Connect(const std::string& dsn, ev::dynamic_loop& loop, con return errOK; } -Error CoroRPCClient::Stop() { +void CoroRPCClient::Stop() { terminate_ = true; conn_.Stop(); resubWg_.wait(); loop_ = nullptr; terminate_ = false; - return errOK; } Error CoroRPCClient::AddNamespace(const NamespaceDef& nsDef, const InternalRdxContext& ctx) { @@ -236,7 +235,7 @@ Error CoroRPCClient::Delete(const Query& query, CoroQueryResults& result, const query.Serialize(ser); NsArray nsArray; - query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q._namespace)); }); + query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q.NsName())); }); result = CoroQueryResults(&conn_, std::move(nsArray), 0, config_.FetchAmount, config_.RequestTimeout); @@ -257,7 +256,7 @@ Error CoroRPCClient::Update(const Query& query, CoroQueryResults& result, const query.Serialize(ser); NsArray nsArray; - query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q._namespace)); }); + query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q.NsName())); }); result = CoroQueryResults(&conn_, std::move(nsArray), 0, config_.FetchAmount, config_.RequestTimeout); @@ -322,7 +321,7 @@ Error CoroRPCClient::selectImpl(const Query& query, CoroQueryResults& result, se } NsArray nsArray; query.Serialize(qser); - query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q._namespace)); }); + query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q.NsName())); }); h_vector vers; for (auto& ns : nsArray) { vers.push_back(ns->tagsMatcher_.version() ^ ns->tagsMatcher_.stateToken()); diff --git a/cpp_src/client/cororpcclient.h b/cpp_src/client/cororpcclient.h index 8c217aa3d..93a776c92 100644 --- a/cpp_src/client/cororpcclient.h +++ b/cpp_src/client/cororpcclient.h @@ -8,7 +8,6 @@ #include "client/item.h" #include "client/namespace.h" #include "client/reindexerconfig.h" -#include "core/keyvalue/p_string.h" #include "core/namespacedef.h" #include "core/query/query.h" #include "coroutine/waitgroup.h" @@ -16,7 +15,6 @@ #include "net/cproto/coroclientconnection.h" #include "replicator/updatesobserver.h" #include "tools/errors.h" -#include "urlparser/urlparser.h" namespace reindexer { @@ -36,7 +34,7 @@ class CoroRPCClient { ~CoroRPCClient(); Error Connect(const std::string &dsn, ev::dynamic_loop &loop, const client::ConnectOpts &opts); - Error Stop(); + void Stop(); Error OpenNamespace(std::string_view nsName, const InternalRdxContext &ctx, const StorageOpts &opts = StorageOpts().Enabled().CreateIfMissing()); diff --git a/cpp_src/client/item.h b/cpp_src/client/item.h index 3f1bbd684..618398a5c 100644 --- a/cpp_src/client/item.h +++ b/cpp_src/client/item.h @@ -60,13 +60,13 @@ class Item { /// Get status of item /// @return data slice with JSON. Returned slice is allocated in temporary Item's buffer, and can be invalidated by any next operation /// with Item - Error Status() { return status_; } + Error Status() const noexcept { return status_; } /// Get internal ID of item /// @return ID of item int GetID() const noexcept { return id_; } - /// Get internal version of item - /// @return version of item - int NumFields(); + /// Get count of indexed fields + /// @return count of indexed fields + int NumFields() const noexcept; /// Set additional percepts for modify operation /// @param precepts - strings in format "fieldName=Func()" void SetPrecepts(const std::vector &precepts); diff --git a/cpp_src/client/itemimpl.h b/cpp_src/client/itemimpl.h index 313d381e4..fba2bfa1e 100644 --- a/cpp_src/client/itemimpl.h +++ b/cpp_src/client/itemimpl.h @@ -3,10 +3,8 @@ #include #include #include "core/cjson/tagsmatcher.h" -#include "core/keyvalue/key_string.h" #include "core/keyvalue/variant.h" #include "core/payload/payloadiface.h" -#include "gason/gason.h" #include "tools/serializer.h" namespace reindexer { diff --git a/cpp_src/client/queryresults.cc b/cpp_src/client/queryresults.cc index 7f0b070b1..9d57ce9f9 100644 --- a/cpp_src/client/queryresults.cc +++ b/cpp_src/client/queryresults.cc @@ -3,7 +3,6 @@ #include "core/cjson/baseencoder.h" #include "core/keyvalue/p_string.h" #include "net/cproto/clientconnection.h" -#include "tools/logger.h" namespace reindexer { namespace client { @@ -114,8 +113,6 @@ void QueryResults::fetchNextResults() { rawResult_.assign(rawResult.begin() + ser.Pos(), rawResult.end()); } -QueryResults::~QueryResults() {} - h_vector QueryResults::GetNamespaces() const { h_vector ret; ret.reserve(nsArray_.size()); @@ -310,8 +307,5 @@ QueryResults::Iterator &QueryResults::Iterator::operator++() { return *this; } -bool QueryResults::Iterator::operator!=(const Iterator &other) const { return idx_ != other.idx_; } -bool QueryResults::Iterator::operator==(const Iterator &other) const { return idx_ == other.idx_; } - } // namespace client } // namespace reindexer diff --git a/cpp_src/client/queryresults.h b/cpp_src/client/queryresults.h index 97c2a1835..e8b059568 100644 --- a/cpp_src/client/queryresults.h +++ b/cpp_src/client/queryresults.h @@ -26,7 +26,7 @@ class QueryResults { QueryResults(int fetchFlags = 0); QueryResults(const QueryResults&) = delete; QueryResults(QueryResults&&) noexcept; - ~QueryResults(); + ~QueryResults() = default; QueryResults& operator=(const QueryResults&) = delete; QueryResults& operator=(QueryResults&& obj) noexcept; @@ -40,10 +40,10 @@ class QueryResults { bool IsRaw(); std::string_view GetRaw(); Iterator& operator++(); - Error Status() { return qr_->status_; } - bool operator!=(const Iterator&) const; - bool operator==(const Iterator&) const; - Iterator& operator*() { return *this; } + Error Status() const noexcept { return qr_->status_; } + bool operator==(const Iterator& other) const noexcept { return idx_ == other.idx_; } + bool operator!=(const Iterator& other) const noexcept { return !operator==(other); } + Iterator& operator*() noexcept { return *this; } void readNext(); void getJSONFromCJSON(std::string_view cjson, WrSerializer& wrser, bool withHdrLen = true); @@ -55,13 +55,15 @@ class QueryResults { Iterator begin() const { return Iterator{this, 0, 0, 0, {}}; } Iterator end() const { return Iterator{this, queryParams_.qcount, 0, 0, {}}; } - size_t Count() const { return queryParams_.qcount; } - int TotalCount() const { return queryParams_.totalcount; } - bool HaveRank() const { return queryParams_.flags & kResultsWithRank; } - bool NeedOutputRank() const { return queryParams_.flags & kResultsNeedOutputRank; } - const std::string& GetExplainResults() const { return queryParams_.explainResults; } - const std::vector& GetAggregationResults() const { return queryParams_.aggResults; } - Error Status() { return status_; } + size_t Count() const noexcept { return queryParams_.qcount; } + int TotalCount() const noexcept { return queryParams_.totalcount; } + bool HaveRank() const noexcept { return queryParams_.flags & kResultsWithRank; } + bool NeedOutputRank() const noexcept { return queryParams_.flags & kResultsNeedOutputRank; } + const std::string& GetExplainResults() const& noexcept { return queryParams_.explainResults; } + const std::string& GetExplainResults() const&& = delete; + const std::vector& GetAggregationResults() const& noexcept { return queryParams_.aggResults; } + const std::vector& GetAggregationResults() const&& = delete; + Error Status() const noexcept { return status_; } h_vector GetNamespaces() const; bool IsCacheEnabled() const { return queryParams_.flags & kResultsWithItemID; } diff --git a/cpp_src/client/reindexer.cc b/cpp_src/client/reindexer.cc index e28041713..f885918de 100644 --- a/cpp_src/client/reindexer.cc +++ b/cpp_src/client/reindexer.cc @@ -1,6 +1,5 @@ #include "client/reindexer.h" #include "client/rpcclient.h" -#include "tools/logger.h" namespace reindexer { namespace client { @@ -29,7 +28,7 @@ Error Reindexer::Connect(const std::string& dsn, const client::ConnectOpts& opts Error Reindexer::Connect(const std::vector>& connectData) { return impl_->Connect(connectData); } -Error Reindexer::Stop() { return impl_->Stop(); } +void Reindexer::Stop() { impl_->Stop(); } Error Reindexer::AddNamespace(const NamespaceDef& nsDef) { return impl_->AddNamespace(nsDef, ctx_); } Error Reindexer::OpenNamespace(std::string_view nsName, const StorageOpts& storage) { return impl_->OpenNamespace(nsName, ctx_, storage); } Error Reindexer::DropNamespace(std::string_view nsName) { return impl_->DropNamespace(nsName, ctx_); } diff --git a/cpp_src/client/reindexer.h b/cpp_src/client/reindexer.h index 705de78e3..8a8780f2e 100644 --- a/cpp_src/client/reindexer.h +++ b/cpp_src/client/reindexer.h @@ -48,7 +48,7 @@ class Reindexer { /// @param connectData - list of server dsn + it's ConnectOpts Error Connect(const std::vector> &connectData); /// Stop - shutdown connector - Error Stop(); + void Stop(); /// Open or create namespace /// @param nsName - Name of namespace /// @param opts - Storage options. Can be one of
diff --git a/cpp_src/client/resultserializer.cc b/cpp_src/client/resultserializer.cc index cc23e0be8..d048a93d5 100644 --- a/cpp_src/client/resultserializer.cc +++ b/cpp_src/client/resultserializer.cc @@ -1,6 +1,5 @@ #include "resultserializer.h" #include "core/payload/payloadtypeimpl.h" -#include "vendor/msgpack/msgpack.h" namespace reindexer { namespace client { diff --git a/cpp_src/client/rpcclient.cc b/cpp_src/client/rpcclient.cc index 29b6f5ccc..3c7d7089c 100644 --- a/cpp_src/client/rpcclient.cc +++ b/cpp_src/client/rpcclient.cc @@ -76,8 +76,8 @@ Error RPCClient::Connect(const std::vector vers; for (auto& ns : nsArray) { shared_lock lck(ns->lck_); diff --git a/cpp_src/client/rpcclient.h b/cpp_src/client/rpcclient.h index 238b16a12..8bc892ec5 100644 --- a/cpp_src/client/rpcclient.h +++ b/cpp_src/client/rpcclient.h @@ -12,7 +12,6 @@ #include "client/queryresults.h" #include "client/reindexerconfig.h" #include "client/transaction.h" -#include "core/keyvalue/p_string.h" #include "core/namespacedef.h" #include "core/query/query.h" #include "estl/fast_hash_map.h" @@ -20,7 +19,6 @@ #include "net/cproto/clientconnection.h" #include "replicator/updatesobserver.h" #include "tools/errors.h" -#include "urlparser/urlparser.h" namespace reindexer { @@ -41,7 +39,7 @@ class RPCClient { Error Connect(const std::string &dsn, const client::ConnectOpts &opts); Error Connect(const std::vector> &connectData); - Error Stop(); + void Stop(); Error OpenNamespace(std::string_view nsName, const InternalRdxContext &ctx, const StorageOpts &opts = StorageOpts().Enabled().CreateIfMissing()); diff --git a/cpp_src/client/rpcclientmock.cc b/cpp_src/client/rpcclientmock.cc index f2cb5635e..e01b6a0dd 100644 --- a/cpp_src/client/rpcclientmock.cc +++ b/cpp_src/client/rpcclientmock.cc @@ -2,10 +2,7 @@ #include #include "client/itemimpl.h" #include "core/namespacedef.h" -#include "gason/gason.h" #include "tools/errors.h" -#include "tools/logger.h" -#include "vendor/gason/gason.h" namespace reindexer { @@ -37,7 +34,7 @@ Error RPCClientMock::Delete(const Query& query, QueryResults& result, const Inte auto conn = getConn(); NsArray nsArray; - query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q._namespace)); }); + query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q.NsName())); }); result = QueryResults(conn, std::move(nsArray), nullptr, 0, config_.FetchAmount, config_.RequestTimeout); @@ -69,7 +66,7 @@ Error RPCClientMock::Update(const Query& query, QueryResults& result, const Inte auto conn = getConn(); NsArray nsArray; - query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q._namespace)); }); + query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q.NsName())); }); result = QueryResults(conn, std::move(nsArray), nullptr, 0, config_.FetchAmount, config_.RequestTimeout); @@ -308,7 +305,7 @@ Error RPCClientMock::selectImpl(const Query& query, QueryResults& result, cproto NsArray nsArray; query.Serialize(qser); - query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q._namespace)); }); + query.WalkNested(true, true, [this, &nsArray](const Query& q) { nsArray.push_back(getNamespace(q.NsName())); }); h_vector vers; for (auto& ns : nsArray) { shared_lock lck(ns->lck_); diff --git a/cpp_src/client/synccororeindexer.cc b/cpp_src/client/synccororeindexer.cc index 9315e98b3..11167a519 100644 --- a/cpp_src/client/synccororeindexer.cc +++ b/cpp_src/client/synccororeindexer.cc @@ -1,6 +1,4 @@ #include "synccororeindexer.h" -#include "client/cororeindexer.h" -#include "client/cororpcclient.h" #include "synccororeindexerimpl.h" namespace reindexer { diff --git a/cpp_src/client/synccororeindexer.h b/cpp_src/client/synccororeindexer.h index 79257bb86..c22c1ee27 100644 --- a/cpp_src/client/synccororeindexer.h +++ b/cpp_src/client/synccororeindexer.h @@ -5,7 +5,6 @@ #include "core/indexdef.h" #include "core/namespacedef.h" #include "internalrdxcontext.h" -#include "net/ev/ev.h" #include "reindexerconfig.h" namespace reindexer { diff --git a/cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh b/cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh new file mode 100644 index 000000000..2c014c6a5 --- /dev/null +++ b/cpp_src/cmd/reindexer_server/test/test_storage_compatibility.sh @@ -0,0 +1,194 @@ +#!/bin/bash +# Task: https://github.com/restream/reindexer/-/issues/1188 +set -e + +function KillAndRemoveServer { + local pid=$1 + kill $pid + wait $pid + yum remove -y 'reindexer*' > /dev/null +} + +function WaitForDB { + # wait until DB is loaded + set +e # disable "exit on error" so the script won't stop when DB's not loaded yet + is_connected=$(reindexer_tool --dsn $ADDRESS --command '\databases list'); + while [[ $is_connected != "test" ]] + do + sleep 2 + is_connected=$(reindexer_tool --dsn $ADDRESS --command '\databases list'); + done + set -e +} + +function CompareNamespacesLists { + local ns_list_actual=$1 + local ns_list_expected=$2 + local pid=$3 + + diff=$(echo ${ns_list_actual[@]} ${ns_list_expected[@]} | tr ' ' '\n' | sort | uniq -u) # compare in any order + if [ "$diff" == "" ]; then + echo "## PASS: namespaces list not changed" + else + echo "##### FAIL: namespaces list was changed" + echo "expected: $ns_list_expected" + echo "actual: $ns_list_actual" + KillAndRemoveServer $pid; + exit 1 + fi +} + +function CompareMemstats { + local actual=$1 + local expected=$2 + local pid=$3 + diff=$(echo ${actual[@]} ${expected[@]} | tr ' ' '\n' | sed 's/\(.*\),$/\1/' | sort | uniq -u) # compare in any order + if [ "$diff" == "" ]; then + echo "## PASS: memstats not changed" + else + echo "##### FAIL: memstats was changed" + echo "expected: $expected" + echo "actual: $actual" + KillAndRemoveServer $pid; + exit 1 + fi +} + + +RX_SERVER_CURRENT_VERSION_RPM="$(basename build/reindexer-*server*.rpm)" +VERSION_FROM_RPM=$(echo "$RX_SERVER_CURRENT_VERSION_RPM" | grep -o '.*server-..') +VERSION=$(echo ${VERSION_FROM_RPM: -2:1}) # one-digit version + +if [ $VERSION == 3 ]; then + LATEST_RELEASE=$(curl -s http://repo.restream.ru/itv-api-ng/7/x86_64/ | grep -o '>reindexer-server-.*.rpm' | tail -n1 | cut -c 2-) + namespaces_list_expected=$'purchase_options_ext_dict\nchild_account_recommendations\n#config\n#activitystats\nradio_channels\ncollections\n#namespaces\nwp_imports_tasks\nepg_genres\nrecom_media_items_personal\nrecom_epg_archive_default\n#perfstats\nrecom_epg_live_default\nmedia_view_templates\nasset_video_servers\nwp_tasks_schedule\nadmin_roles\n#clientsstats\nrecom_epg_archive_personal\nrecom_media_items_similars\nmenu_items\naccount_recommendations\nkaraoke_items\nmedia_items\nbanners\n#queriesperfstats\nrecom_media_items_default\nrecom_epg_live_personal\nservices\n#memstats\nchannels\nmedia_item_recommendations\nwp_tasks_tasks\nepg' +elif [ $VERSION == 4 ]; then + LATEST_RELEASE=$(curl -s http://repo.restream.ru/itv-api-ng/7/x86_64/ | grep -o '>reindexer-4-server-.*.rpm' | tail -n1 | cut -c 2-) + # replicationstats ns added for v4 + namespaces_list_expected=$'purchase_options_ext_dict\nchild_account_recommendations\n#config\n#activitystats\n#replicationstats\nradio_channels\ncollections\n#namespaces\nwp_imports_tasks\nepg_genres\nrecom_media_items_personal\nrecom_epg_archive_default\n#perfstats\nrecom_epg_live_default\nmedia_view_templates\nasset_video_servers\nwp_tasks_schedule\nadmin_roles\n#clientsstats\nrecom_epg_archive_personal\nrecom_media_items_similars\nmenu_items\naccount_recommendations\nkaraoke_items\nmedia_items\nbanners\n#queriesperfstats\nrecom_media_items_default\nrecom_epg_live_personal\nservices\n#memstats\nchannels\nmedia_item_recommendations\nwp_tasks_tasks\nepg' +else + echo "Unknown version" + exit 1 +fi + +echo "## downloading latest release rpm file: $LATEST_RELEASE" +curl "http://repo.itv.restr.im/itv-api-ng/7/x86_64/$LATEST_RELEASE" --output $LATEST_RELEASE; +echo "## downloading example DB" +curl "https://git.restream.ru/MaksimKravchuk/reindexer_testdata/-/raw/master/big.zip" --output big.zip; +unzip -o big.zip # unzips into mydb_big.rxdump; + +ADDRESS="cproto://127.0.0.1:6534/" +DB_NAME="test" + +memstats_expected=$'[ +{"replication":{"data_hash":24651210926,"data_count":3}}, +{"replication":{"data_hash":6252344969,"data_count":1}}, +{"replication":{"data_hash":37734732881,"data_count":28}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":1024095024522,"data_count":1145}}, +{"replication":{"data_hash":8373644068,"data_count":1315}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":7404222244,"data_count":97}}, +{"replication":{"data_hash":94132837196,"data_count":4}}, +{"replication":{"data_hash":1896088071,"data_count":2}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":-672103903,"data_count":33538}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":6833710705,"data_count":1}}, +{"replication":{"data_hash":5858155773472,"data_count":4500}}, +{"replication":{"data_hash":-473221280268823592,"data_count":65448}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":8288213744,"data_count":3}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":0,"data_count":0}}, +{"replication":{"data_hash":354171024786967,"data_count":3941}}, +{"replication":{"data_hash":-6520334670,"data_count":35886}}, +{"replication":{"data_hash":112772074632,"data_count":281}}, +{"replication":{"data_hash":-12679568198538,"data_count":1623116}} +] +Returned 27 rows' + +echo "##### Forward compatibility test #####" + +DB_PATH=$(pwd)"/rx_db" + +echo "Database: "$DB_PATH + +echo "## installing latest release: $LATEST_RELEASE" +yum install -y $LATEST_RELEASE > /dev/null; +# run RX server with disabled logging +reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +reindexer_tool --dsn $ADDRESS$DB_NAME -f mydb_big.rxdump --createdb; +sleep 1; + +namespaces_1=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_1; +CompareNamespacesLists "${namespaces_1[@]}" "${namespaces_list_expected[@]}" $server_pid; + +memstats_1=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats'); +CompareMemstats "${memstats_1[@]}" "${memstats_expected[@]}" $server_pid; + +KillAndRemoveServer $server_pid; + +echo "## installing current version: $RX_SERVER_CURRENT_VERSION_RPM" +yum install -y build/*.rpm > /dev/null; +reindexer_server -l0 --corelog=none --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +WaitForDB + +namespaces_2=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_2; +CompareNamespacesLists "${namespaces_2[@]}" "${namespaces_1[@]}" $server_pid; + +memstats_2=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats'); +CompareMemstats "${memstats_2[@]}" "${memstats_1[@]}" $server_pid; + +KillAndRemoveServer $server_pid; +rm -rf $DB_PATH; +sleep 1; + +echo "##### Backward compatibility test #####" + +echo "## installing current version: $RX_SERVER_CURRENT_VERSION_RPM" +yum install -y build/*.rpm > /dev/null; +reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +reindexer_tool --dsn $ADDRESS$DB_NAME -f mydb_big.rxdump --createdb; +sleep 1; + +namespaces_3=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_3; +CompareNamespacesLists "${namespaces_3[@]}" "${namespaces_list_expected[@]}" $server_pid; + +memstats_3=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats'); +CompareMemstats "${memstats_3[@]}" "${memstats_expected[@]}" $server_pid; + +KillAndRemoveServer $server_pid; + +echo "## installing latest release: $LATEST_RELEASE" +yum install -y $LATEST_RELEASE > /dev/null; +reindexer_server -l warning --httplog=none --rpclog=none --db $DB_PATH & +server_pid=$! +sleep 2; + +WaitForDB + +namespaces_4=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command '\namespaces list'); +echo $namespaces_4; +CompareNamespacesLists "${namespaces_4[@]}" "${namespaces_3[@]}" $server_pid; + +memstats_4=$(reindexer_tool --dsn $ADDRESS$DB_NAME --command 'select replication.data_hash, replication.data_count from #memstats'); +CompareMemstats "${memstats_4[@]}" "${memstats_3[@]}" $server_pid; + +KillAndRemoveServer $server_pid; +rm -rf $DB_PATH; diff --git a/cpp_src/cmd/reindexer_tool/commandsexecutor.cc b/cpp_src/cmd/reindexer_tool/commandsexecutor.cc index 8509155f5..eee2a4f62 100644 --- a/cpp_src/cmd/reindexer_tool/commandsexecutor.cc +++ b/cpp_src/cmd/reindexer_tool/commandsexecutor.cc @@ -47,19 +47,18 @@ Error CommandsExecutor::Run(const std::string& } template -void CommandsExecutor::GetSuggestions(const std::string& input, std::vector& suggestions) { +Error CommandsExecutor::GetSuggestions(const std::string& input, std::vector& suggestions) { OutParamCommand> cmd( - [this, &input](std::vector& suggestions) { - getSuggestions(input, suggestions); - return errOK; - }, - suggestions); - execCommand(cmd); + [this, &input](std::vector& suggestions) { return getSuggestions(input, suggestions); }, suggestions); + return execCommand(cmd); } template Error CommandsExecutor::Stop() { - GenericCommand cmd([this] { return stop(true); }); + GenericCommand cmd([this] { + stop(true); + return Error{}; + }); auto err = execCommand(cmd); if (err.ok() && executorThr_.joinable()) { executorThr_.join(); @@ -475,15 +474,14 @@ Error CommandsExecutor::processImpl(const std::string& command) noe } template <> -Error CommandsExecutor::stop(bool terminate) { +void CommandsExecutor::stop(bool terminate) { if (terminate) { stopCh_.close(); } - return Error(); } template <> -Error CommandsExecutor::stop(bool terminate) { +void CommandsExecutor::stop(bool terminate) { if (terminate) { stopCh_.close(); } @@ -491,11 +489,17 @@ Error CommandsExecutor::stop(bool terminate) { } template -void CommandsExecutor::getSuggestions(const std::string& input, std::vector& suggestions) { - if (!input.empty() && input[0] != '\\') db().GetSqlSuggestions(input, input.length() - 1, suggestions); +Error CommandsExecutor::getSuggestions(const std::string& input, std::vector& suggestions) { + if (!input.empty() && input[0] != '\\') { + auto err = db().GetSqlSuggestions(input, input.length() - 1, suggestions); + if (!err.ok()) { + return err; + } + } if (suggestions.empty()) { addCommandsSuggestions(input, suggestions); } + return {}; } template @@ -761,7 +765,8 @@ Error CommandsExecutor::commandDump(const std::string& command) { return Error(errCanceled, "Canceled"); } wrser << "\\UPSERT " << reindexer::escapeString(nsDef.name) << ' '; - it.GetJSON(wrser, false); + err = it.GetJSON(wrser, false); + if (!err.ok()) return err; wrser << '\n'; if (wrser.Len() > 0x100000) { output_() << wrser.Slice(); @@ -847,9 +852,11 @@ Error CommandsExecutor::commandMeta(const std::string& command) { auto nsName = reindexer::unescapeString(parser.NextToken()); std::vector allMeta; auto err = db().EnumMeta(nsName, allMeta); + if (!err.ok()) return err; for (auto& metaKey : allMeta) { std::string metaData; - db().GetMeta(nsName, metaKey, metaData); + err = db().GetMeta(nsName, metaKey, metaData); + if (!err.ok()) return err; output_() << metaKey << " = " << metaData << std::endl; } return err; @@ -917,15 +924,18 @@ Error CommandsExecutor::commandBench(const std::string& command) { LineParser parser(command); parser.NextToken(); - int benchTime = reindexer::stoi(parser.NextToken()); - if (benchTime == 0) benchTime = kBenchDefaultTime; + const std::string_view benchTimeToken = parser.NextToken(); + const int benchTime = benchTimeToken.empty() ? kBenchDefaultTime : reindexer::stoi(benchTimeToken); - db().DropNamespace(kBenchNamespace); + auto err = db().DropNamespace(kBenchNamespace); + if (!err.ok() && err.code() != errNotFound) { + return err; + } NamespaceDef nsDef(kBenchNamespace); nsDef.AddIndex("id", "hash", "int", IndexOpts().PK()); - auto err = db().AddNamespace(nsDef); + err = db().AddNamespace(nsDef); if (!err.ok()) return err; output_() << "Seeding " << kBenchItemsCount << " documents to bench namespace..." << std::endl; @@ -1014,19 +1024,17 @@ Error CommandsExecutor::commandProcessDatabase return Error(); } else if (subCommand == "use") { std::string currentDsn = getCurrentDsn() + std::string(parser.NextToken()); - Error err = stop(false); - if (!err.ok()) return err; - err = db().Connect(currentDsn, loop_); + stop(false); + auto err = db().Connect(currentDsn, loop_); if (err.ok()) err = db().Status(); if (err.ok()) output_() << "Succesfully connected to " << currentDsn << std::endl; return err; } else if (subCommand == "create") { auto dbName = parser.NextToken(); std::string currentDsn = getCurrentDsn() + std::string(dbName); - Error err = stop(false); - if (!err.ok()) return err; + stop(false); output_() << "Creating database '" << dbName << "'" << std::endl; - err = db().Connect(currentDsn, loop_, reindexer::client::ConnectOpts().CreateDBIfMissing()); + auto err = db().Connect(currentDsn, loop_, reindexer::client::ConnectOpts().CreateDBIfMissing()); if (!err.ok()) { std::cerr << "Error on database '" << dbName << "' creation" << std::endl; return err; @@ -1145,11 +1153,14 @@ std::function CommandsExecutor Error Run(const std::string& dsn, const Args&... args); - void GetSuggestions(const std::string& input, std::vector& suggestions); + Error GetSuggestions(const std::string& input, std::vector& suggestions); Error Stop(); Error Process(const std::string& command); Error FromFile(std::istream& in); @@ -78,8 +78,8 @@ class CommandsExecutor : public reindexer::IUpdatesObserver { std::vector& suggestions); Error processImpl(const std::string& command) noexcept; - Error stop(bool terminate); - void getSuggestions(const std::string& input, std::vector& suggestions); + void stop(bool terminate); + Error getSuggestions(const std::string& input, std::vector& suggestions); Error commandSelect(const std::string& command); Error commandUpsert(const std::string& command); Error commandUpdateSQL(const std::string& command); diff --git a/cpp_src/cmd/reindexer_tool/commandsprocessor.cc b/cpp_src/cmd/reindexer_tool/commandsprocessor.cc index 17ce39931..f084ea387 100644 --- a/cpp_src/cmd/reindexer_tool/commandsprocessor.cc +++ b/cpp_src/cmd/reindexer_tool/commandsprocessor.cc @@ -3,8 +3,8 @@ #include #include "client/cororeindexer.h" #include "core/reindexer.h" -#include "tableviewscroller.h" #include "tools/fsops.h" +#include "tools/terminalutils.h" namespace reindexer_tool { @@ -30,9 +30,13 @@ template void CommandsProcessor::setCompletionCallback(T& rx, void (T::*set_completion_callback)(new_v_callback_t const&)) { (rx.*set_completion_callback)([this](std::string const& input, int) -> replxx::Replxx::completions_t { std::vector completions; - executor_.GetSuggestions(input, completions); + const auto err = executor_.GetSuggestions(input, completions); replxx::Replxx::completions_t result; - for (const std::string& suggestion : completions) result.emplace_back(suggestion); + if (err.ok()) { + for (const std::string& suggestion : completions) { + result.emplace_back(suggestion); + } + } return result; }); } @@ -43,7 +47,8 @@ void CommandsProcessor::setCompletionCallback(T& rx, void (T::*set_ (rx.*set_completion_callback)( [this](std::string const& input, int, void*) -> replxx::Replxx::completions_t { std::vector completions; - executor_.GetSuggestions(input, completions); + const auto err = executor_.GetSuggestions(input, completions); + if (!err.ok()) return {}; return completions; }, nullptr); diff --git a/cpp_src/cmd/reindexer_tool/iotools.h b/cpp_src/cmd/reindexer_tool/iotools.h index a9ac704c4..2d0f04159 100644 --- a/cpp_src/cmd/reindexer_tool/iotools.h +++ b/cpp_src/cmd/reindexer_tool/iotools.h @@ -4,7 +4,6 @@ #include #include "tools/errors.h" -#include "tools/serializer.h" namespace reindexer_tool { diff --git a/cpp_src/cmd/reindexer_tool/reindexer_tool.cc b/cpp_src/cmd/reindexer_tool/reindexer_tool.cc index cb5be7eb0..7db936d27 100644 --- a/cpp_src/cmd/reindexer_tool/reindexer_tool.cc +++ b/cpp_src/cmd/reindexer_tool/reindexer_tool.cc @@ -9,7 +9,6 @@ #include "repair_tool.h" #include "tools/cpucheck.h" #include "tools/logger.h" -#include "tools/stringstools.h" namespace reindexer_tool { diff --git a/cpp_src/cmd/reindexer_tool/tableviewscroller.cc b/cpp_src/cmd/reindexer_tool/tableviewscroller.cc index 6ed4812b6..75ebf609e 100644 --- a/cpp_src/cmd/reindexer_tool/tableviewscroller.cc +++ b/cpp_src/cmd/reindexer_tool/tableviewscroller.cc @@ -1,6 +1,6 @@ #include "tableviewscroller.h" -#include "client/cororeindexer.h" -#include "core/reindexer.h" +#include "client/coroqueryresults.h" +#include "core/queryresults/queryresults.h" #include "iotools.h" #include "tools/oscompat.h" diff --git a/cpp_src/core/cancelcontextpool.h b/cpp_src/core/cancelcontextpool.h index c2850f625..6568ff6cf 100644 --- a/cpp_src/core/cancelcontextpool.h +++ b/cpp_src/core/cancelcontextpool.h @@ -1,7 +1,6 @@ #pragma once #include "core/rdxcontext.h" -#include "core/reindexer.h" #include "estl/atomic_unique_ptr.h" #include diff --git a/cpp_src/core/cbinding/cgocancelcontextpool.h b/cpp_src/core/cbinding/cgocancelcontextpool.h index 68282e3d0..899a47c78 100644 --- a/cpp_src/core/cbinding/cgocancelcontextpool.h +++ b/cpp_src/core/cbinding/cgocancelcontextpool.h @@ -2,6 +2,7 @@ #include "core/cancelcontextpool.h" #include "core/cbinding/reindexer_ctypes.h" +#include "core/reindexer.h" namespace reindexer { diff --git a/cpp_src/core/cbinding/reindexer_c.cc b/cpp_src/core/cbinding/reindexer_c.cc index 134937df5..cfac85159 100644 --- a/cpp_src/core/cbinding/reindexer_c.cc +++ b/cpp_src/core/cbinding/reindexer_c.cc @@ -7,15 +7,11 @@ #include "cgocancelcontextpool.h" #include "core/cjson/baseencoder.h" -#include "core/selectfunc/selectfuncparser.h" -#include "core/transactionimpl.h" -#include "debug/allocdebug.h" #include "estl/syncpool.h" #include "reindexer_version.h" #include "resultserializer.h" #include "tools/logger.h" #include "tools/semversion.h" -#include "tools/stringstools.h" using namespace reindexer; const int kQueryResultsPoolSize = 1024; diff --git a/cpp_src/core/cbinding/reindexer_c.h b/cpp_src/core/cbinding/reindexer_c.h index a88452564..cf650dda2 100644 --- a/cpp_src/core/cbinding/reindexer_c.h +++ b/cpp_src/core/cbinding/reindexer_c.h @@ -6,6 +6,7 @@ extern "C" { #include #include "reindexer_ctypes.h" +#include "core/type_consts.h" uintptr_t init_reindexer(); uintptr_t init_reindexer_with_config(reindexer_config config); diff --git a/cpp_src/core/cbinding/reindexer_ctypes.h b/cpp_src/core/cbinding/reindexer_ctypes.h index 1d700a262..32e4188c7 100644 --- a/cpp_src/core/cbinding/reindexer_ctypes.h +++ b/cpp_src/core/cbinding/reindexer_ctypes.h @@ -5,7 +5,6 @@ extern "C" { #endif #include -#include "core/type_consts.h" typedef struct reindexer_config { int64_t allocator_cache_limit; diff --git a/cpp_src/core/cbinding/resultserializer.h b/cpp_src/core/cbinding/resultserializer.h index 3138582a3..d4ae982c2 100644 --- a/cpp_src/core/cbinding/resultserializer.h +++ b/cpp_src/core/cbinding/resultserializer.h @@ -1,5 +1,4 @@ #pragma once -#include "estl/h_vector.h" #include "estl/span.h" #include "tools/serializer.h" namespace reindexer { diff --git a/cpp_src/core/cjson/baseencoder.h b/cpp_src/core/cjson/baseencoder.h index 0e2536316..e76c665b8 100644 --- a/cpp_src/core/cjson/baseencoder.h +++ b/cpp_src/core/cjson/baseencoder.h @@ -1,7 +1,6 @@ #pragma once #include "cjsonbuilder.h" -#include "core/keyvalue/key_string.h" #include "core/payload/payloadiface.h" #include "fieldextractor.h" #include "jsonbuilder.h" diff --git a/cpp_src/core/cjson/cjsondecoder.cc b/cpp_src/core/cjson/cjsondecoder.cc index b74b2bcba..51f243b3b 100644 --- a/cpp_src/core/cjson/cjsondecoder.cc +++ b/cpp_src/core/cjson/cjsondecoder.cc @@ -1,6 +1,5 @@ #include "cjsondecoder.h" #include "cjsontools.h" -#include "core/keyvalue/p_string.h" #include "tagsmatcher.h" #include "tools/flagguard.h" #include "tools/serializer.h" diff --git a/cpp_src/core/cjson/cjsondecoder.h b/cpp_src/core/cjson/cjsondecoder.h index a128fe487..e69e6a239 100644 --- a/cpp_src/core/cjson/cjsondecoder.h +++ b/cpp_src/core/cjson/cjsondecoder.h @@ -1,6 +1,9 @@ #pragma once +#include "core/cjson/tagspath.h" +#include "core/payload/fieldsset.h" #include "core/payload/payloadiface.h" +#include "core/type_consts.h" namespace reindexer { diff --git a/cpp_src/core/cjson/cjsonmodifier.cc b/cpp_src/core/cjson/cjsonmodifier.cc index 8409dbeeb..66a7c2545 100644 --- a/cpp_src/core/cjson/cjsonmodifier.cc +++ b/cpp_src/core/cjson/cjsonmodifier.cc @@ -1,5 +1,5 @@ #include "cjsonmodifier.h" -#include "core/keyvalue/p_string.h" +#include "cjsontools.h" #include "core/type_consts_helpers.h" #include "jsondecoder.h" #include "tagsmatcher.h" diff --git a/cpp_src/core/cjson/cjsonmodifier.h b/cpp_src/core/cjson/cjsonmodifier.h index 642e2908b..5e3c12c7b 100644 --- a/cpp_src/core/cjson/cjsonmodifier.h +++ b/cpp_src/core/cjson/cjsonmodifier.h @@ -1,9 +1,13 @@ #pragma once -#include "cjsontools.h" +#include "core/cjson/tagspath.h" +#include "core/payload/payloadiface.h" +#include "core/payload/payloadtype.h" namespace reindexer { +class TagsMatcher; + class CJsonModifier { public: CJsonModifier(TagsMatcher &tagsMatcher, PayloadType pt) noexcept : pt_(std::move(pt)), tagsMatcher_(tagsMatcher) {} @@ -28,4 +32,5 @@ class CJsonModifier { IndexedTagsPath fieldPath_, tagsPath_; TagsMatcher &tagsMatcher_; }; + } // namespace reindexer diff --git a/cpp_src/core/cjson/cjsontools.cc b/cpp_src/core/cjson/cjsontools.cc index 0294772ff..b9d09a24b 100644 --- a/cpp_src/core/cjson/cjsontools.cc +++ b/cpp_src/core/cjson/cjsontools.cc @@ -1,4 +1,5 @@ #include "cjsontools.h" +#include "cjsonbuilder.h" #include "core/type_consts_helpers.h" namespace reindexer { diff --git a/cpp_src/core/cjson/cjsontools.h b/cpp_src/core/cjson/cjsontools.h index 69070c1e8..e19e5c118 100644 --- a/cpp_src/core/cjson/cjsontools.h +++ b/cpp_src/core/cjson/cjsontools.h @@ -1,6 +1,5 @@ #pragma once -#include "cjsonbuilder.h" #include "core/payload/payloadiface.h" namespace reindexer { diff --git a/cpp_src/core/cjson/csvbuilder.cc b/cpp_src/core/cjson/csvbuilder.cc index 1485f9ad6..25b2eca49 100644 --- a/cpp_src/core/cjson/csvbuilder.cc +++ b/cpp_src/core/cjson/csvbuilder.cc @@ -1,5 +1,4 @@ #include "csvbuilder.h" -#include "tools/json2kv.h" namespace reindexer { diff --git a/cpp_src/core/cjson/csvbuilder.h b/cpp_src/core/cjson/csvbuilder.h index 0df66cddf..7ab394f2b 100644 --- a/cpp_src/core/cjson/csvbuilder.h +++ b/cpp_src/core/cjson/csvbuilder.h @@ -5,7 +5,6 @@ #include "objtype.h" #include "tagslengths.h" #include "tagsmatcher.h" -#include "vendor/gason/gason.h" namespace reindexer { diff --git a/cpp_src/core/cjson/fieldextractor.h b/cpp_src/core/cjson/fieldextractor.h index 6b11f179a..43ac72d64 100644 --- a/cpp_src/core/cjson/fieldextractor.h +++ b/cpp_src/core/cjson/fieldextractor.h @@ -16,7 +16,7 @@ class FieldsExtractor { }; FieldsExtractor() = default; - FieldsExtractor(VariantArray *va, KeyValueType expectedType, int expectedPathDepth, FieldsSet *filter = nullptr, + FieldsExtractor(VariantArray *va, KeyValueType expectedType, int expectedPathDepth, const FieldsSet *filter, FieldParams *params = nullptr) noexcept : values_(va), expectedType_(expectedType), expectedPathDepth_(expectedPathDepth), filter_(filter), params_(params) {} FieldsExtractor(FieldsExtractor &&other) = default; @@ -176,7 +176,7 @@ class FieldsExtractor { VariantArray *values_ = nullptr; KeyValueType expectedType_{KeyValueType::Undefined{}}; int expectedPathDepth_ = 0; - FieldsSet *filter_; + const FieldsSet *filter_; FieldParams *params_; }; diff --git a/cpp_src/core/cjson/jschemachecker.cc b/cpp_src/core/cjson/jschemachecker.cc index c0300a590..b36864211 100644 --- a/cpp_src/core/cjson/jschemachecker.cc +++ b/cpp_src/core/cjson/jschemachecker.cc @@ -6,7 +6,6 @@ #include "estl/h_vector.h" #include "gason/gason.h" #include "tools/errors.h" -#include "tools/stringstools.h" namespace reindexer { diff --git a/cpp_src/core/cjson/jsonbuilder.cc b/cpp_src/core/cjson/jsonbuilder.cc index bb8905d7a..cc958fd01 100644 --- a/cpp_src/core/cjson/jsonbuilder.cc +++ b/cpp_src/core/cjson/jsonbuilder.cc @@ -1,5 +1,4 @@ #include "jsonbuilder.h" -#include "tools/json2kv.h" namespace reindexer { diff --git a/cpp_src/core/cjson/jsonbuilder.h b/cpp_src/core/cjson/jsonbuilder.h index ccdc04b07..7bb6e73c5 100644 --- a/cpp_src/core/cjson/jsonbuilder.h +++ b/cpp_src/core/cjson/jsonbuilder.h @@ -4,7 +4,6 @@ #include "objtype.h" #include "tagslengths.h" #include "tagsmatcher.h" -#include "vendor/gason/gason.h" namespace reindexer { class JsonBuilder { diff --git a/cpp_src/core/cjson/msgpackbuilder.cc b/cpp_src/core/cjson/msgpackbuilder.cc index 719aa7800..32338967e 100644 --- a/cpp_src/core/cjson/msgpackbuilder.cc +++ b/cpp_src/core/cjson/msgpackbuilder.cc @@ -2,7 +2,6 @@ #include "core/type_consts_helpers.h" #include "tools/serializer.h" #include "vendor/gason/gason.h" -#include "vendor/msgpack/msgpack.h" namespace reindexer { diff --git a/cpp_src/core/cjson/msgpackdecoder.cc b/cpp_src/core/cjson/msgpackdecoder.cc index e29178282..965fd2694 100644 --- a/cpp_src/core/cjson/msgpackdecoder.cc +++ b/cpp_src/core/cjson/msgpackdecoder.cc @@ -4,7 +4,6 @@ #include "core/cjson/objtype.h" #include "core/cjson/tagsmatcher.h" #include "tools/flagguard.h" -#include "vendor/msgpack/msgpack.h" namespace reindexer { diff --git a/cpp_src/core/cjson/protobufdecoder.cc b/cpp_src/core/cjson/protobufdecoder.cc index 2a1478cd0..a8313ab01 100644 --- a/cpp_src/core/cjson/protobufdecoder.cc +++ b/cpp_src/core/cjson/protobufdecoder.cc @@ -1,6 +1,5 @@ #include "protobufdecoder.h" #include "core/schema.h" -#include "core/type_consts_helpers.h" #include "estl/protobufparser.h" #include "protobufbuilder.h" diff --git a/cpp_src/core/cjson/protobufschemabuilder.h b/cpp_src/core/cjson/protobufschemabuilder.h index d2b0495eb..b05eae923 100644 --- a/cpp_src/core/cjson/protobufschemabuilder.h +++ b/cpp_src/core/cjson/protobufschemabuilder.h @@ -1,6 +1,7 @@ #pragma once -#include "core/cjson/tagsmatcherimpl.h" +#include +#include "core/key_value_type.h" #include "objtype.h" namespace reindexer { diff --git a/cpp_src/core/cjson/tagsmatcher.h b/cpp_src/core/cjson/tagsmatcher.h index 8bb8786b0..a70cff548 100644 --- a/cpp_src/core/cjson/tagsmatcher.h +++ b/cpp_src/core/cjson/tagsmatcher.h @@ -4,7 +4,6 @@ #include "core/cjson/tagsmatcherimpl.h" #include "core/payload/payloadtype.h" #include "estl/cow.h" -#include "tagspathcache.h" #include "tools/serializer.h" namespace reindexer { diff --git a/cpp_src/core/cjson/tagsmatcherimpl.h b/cpp_src/core/cjson/tagsmatcherimpl.h index e96109868..07037b2ee 100644 --- a/cpp_src/core/cjson/tagsmatcherimpl.h +++ b/cpp_src/core/cjson/tagsmatcherimpl.h @@ -4,7 +4,6 @@ #include #include -#include "core/keyvalue/key_string.h" #include "core/payload/payloadtype.h" #include "core/payload/payloadtypeimpl.h" #include "ctag.h" @@ -75,25 +74,29 @@ class TagsMatcherImpl { if (content == "*"sv) { node.MarkAllItems(true); } else { - int index = stoi(content); - if (index == 0 && content != "0"sv && ev) { - VariantArray values = ev(content); - if (values.size() != 1) { - throw Error(errParams, "Index expression_ has wrong syntax: '%s'", content); + auto index = try_stoi(content); + if (!index) { + if (ev) { + VariantArray values = ev(content); + if (values.size() != 1) { + throw Error(errParams, "Index expression_ has wrong syntax: '%s'", content); + } + values.front().Type().EvaluateOneOf( + [](OneOf) noexcept {}, + [&](OneOf) { + throw Error(errParams, "Wrong type of index: '%s'", content); + }); + node.SetExpression(content); + index = values.front().As(); + } else { + throw Error(errParams, "Can't convert '%s' to number", content); } - values.front().Type().EvaluateOneOf( - [](OneOf) noexcept {}, - [&](OneOf) { - throw Error(errParams, "Wrong type of index: '%s'", content); - }); - node.SetExpression(content); - index = values.front().As(); } if (index < 0) { throw Error(errLogic, "Array index value cannot be negative"); } - node.SetIndex(index); + node.SetIndex(*index); } field = field.substr(0, openBracketPos); } diff --git a/cpp_src/core/cjson/tagspathcache.h b/cpp_src/core/cjson/tagspathcache.h index fecd8d3de..daf58b986 100644 --- a/cpp_src/core/cjson/tagspathcache.h +++ b/cpp_src/core/cjson/tagspathcache.h @@ -5,7 +5,6 @@ #include #include #include "estl/h_vector.h" -#include "tools/errors.h" namespace reindexer { @@ -51,7 +50,7 @@ class TagsPathCache { } } - void walk(int16_t *path, int depth, const std::function& visitor) const { + void walk(int16_t *path, int depth, const std::function &visitor) const { int16_t &i = path[depth]; for (i = 0; i < int(entries_.size()); i++) { if (entries_[i].field_ > 0) visitor(depth + 1, entries_[i].field_); diff --git a/cpp_src/core/comparator.cc b/cpp_src/core/comparator.cc deleted file mode 100644 index 08d001339..000000000 --- a/cpp_src/core/comparator.cc +++ /dev/null @@ -1,213 +0,0 @@ -#include "core/comparator.h" -#include "core/payload/payloadiface.h" - -namespace reindexer { - -Comparator::Comparator(CondType cond, KeyValueType type, const VariantArray &values, bool isArray, bool distinct, PayloadType payloadType, - const FieldsSet &fields, void *rawData, const CollateOpts &collateOpts) - : ComparatorVars(cond, type, isArray, std::move(payloadType), fields, rawData, collateOpts), - cmpBool(distinct), - cmpInt(distinct), - cmpInt64(distinct), - cmpDouble(distinct), - cmpString(distinct), - cmpGeom(distinct), - cmpUuid(distinct) { - if (type.Is()) assertrx(fields_.size() > 0); - switch (cond) { - case CondEq: - if (values.size() != 1) { - cond_ = CondSet; - } - setValues(values); - break; - case CondSet: - case CondAllSet: - if (values.size() == 1) { - cond_ = CondEq; - } - [[fallthrough]]; - case CondLt: - case CondLe: - case CondGt: - case CondGe: - case CondLike: - case CondRange: - setValues(values); - break; - case CondDWithin: - cmpGeom.SetValues(values); - break; - case CondEmpty: - case CondAny: - break; - } -} - -void Comparator::setValues(const VariantArray &values) { - if (values.size() > 0) { - valuesType_ = values.front().Type(); - } - if (fields_.getTagsPathsLength() > 0) { - cmpInt.SetValues(cond_, values); - cmpBool.SetValues(cond_, values); - cmpInt64.SetValues(cond_, values); - cmpDouble.SetValues(cond_, values); - cmpString.SetValues(cond_, values, collateOpts_); - cmpUuid.SetValues(cond_, values); - } else { - type_.EvaluateOneOf([&](KeyValueType::Bool) { cmpBool.SetValues(cond_, values); }, - [&](KeyValueType::Int) { cmpInt.SetValues(cond_, values); }, - [&](KeyValueType::Int64) { cmpInt64.SetValues(cond_, values); }, - [&](KeyValueType::Double) { cmpDouble.SetValues(cond_, values); }, - [&](KeyValueType::String) { cmpString.SetValues(cond_, values, collateOpts_); }, - [&](KeyValueType::Composite) { cmpComposite.SetValues(cond_, values, *this); }, - [&](KeyValueType::Uuid) { cmpUuid.SetValues(cond_, values); }, - [](OneOf) noexcept { - assertrx(0); - abort(); - }); - } - bool isRegularIndex = fields_.size() > 0 && fields_.getTagsPathsLength() == 0 && fields_[0] < payloadType_.NumFields(); - if (isArray_ && isRegularIndex && payloadType_->Field(fields_[0]).IsArray()) { - offset_ = payloadType_->Field(fields_[0]).Offset(); - sizeof_ = payloadType_->Field(fields_[0]).ElemSizeof(); - } -} - -void Comparator::Bind(const PayloadType &type, int field) { - if (!type_.Is()) { - offset_ = type->Field(field).Offset(); - sizeof_ = type->Field(field).ElemSizeof(); - } -} - -void Comparator::BindEqualPosition(int field, const VariantArray &val, CondType cond) { cmpEqualPosition.BindField(field, val, cond); } - -void Comparator::BindEqualPosition(const TagsPath &tagsPath, const VariantArray &val, CondType cond) { - cmpEqualPosition.BindField(tagsPath, val, cond); -} - -bool Comparator::isNumericComparison(const VariantArray &values) const { - if (valuesType_.Is() || values.empty()) return false; - const KeyValueType keyType{values.front().Type()}; - return !valuesType_.IsSame(keyType) && (valuesType_.Is() || keyType.Is()); -} - -bool Comparator::Compare(const PayloadValue &data, int rowId) { - if (cmpEqualPosition.IsBinded()) { - return cmpEqualPosition.Compare(data, *this); - } - if (fields_.getTagsPathsLength() == 0) { - if (cond_ == CondAllSet) clearIndividualAllSetValues(); - // Comparing field from payload by offset (fast path) - - // Special case: compare by composite condition. Pass pointer to PayloadValue - if (type_.Is()) return compare(&data); - - // Check if we have column (rawData_), then go to fastest path with column - if (rawData_) return compare(rawData_ + rowId * sizeof_); - - // Not array: Just compare field by offset in PayloadValue - if (!isArray_) return compare(data.Ptr() + offset_); - - const PayloadFieldValue::Array *arr = reinterpret_cast(data.Ptr() + offset_); - - switch (cond_) { - case CondEmpty: - return arr->len == 0; - case CondAny: - if (arr->len == 0) return false; - break; - case CondEq: - case CondLt: - case CondLe: - case CondGt: - case CondGe: - case CondRange: - case CondSet: - case CondAllSet: - case CondLike: - case CondDWithin: - break; - } - - const uint8_t *ptr = data.Ptr() + arr->offset; - if (cond_ == CondDWithin) { - if (arr->len != 2 || !type_.Is()) throw Error(errQueryExec, "DWithin with not point data"); - return cmpGeom.Compare(Point{*reinterpret_cast(ptr), *reinterpret_cast(ptr + sizeof_)}); - } - - for (int i = 0; i < arr->len; ++i, ptr += sizeof_) { - if (compare(ptr)) return true; - } - } else { - VariantArray rhs; - ConstPayload(payloadType_, data).GetByJsonPath(fields_.getTagsPath(0), rhs, type_); - if (isNumericComparison(rhs)) { - // Numeric comparison is not allowed - return false; - } - switch (cond_) { - case CondEmpty: - return rhs.empty() || rhs[0].Type().Is(); - case CondDWithin: - return cmpGeom.Compare(static_cast(rhs)); - case CondAllSet: - clearAllSetValues(); - break; - case CondAny: - if (rhs.empty() || rhs[0].Type().Is()) return false; - break; - case CondEq: - case CondLt: - case CondLe: - case CondGt: - case CondGe: - case CondRange: - case CondSet: - case CondLike: - break; - } - for (const Variant &kr : rhs) { - if (compare(kr)) return true; - } - } - - return false; -} - -void Comparator::ExcludeDistinct(const PayloadValue &data, int rowId) { - assertrx(!cmpEqualPosition.IsBinded()); - if (fields_.getTagsPathsLength() > 0) { - // Exclude field by CJSON path (slow path) - VariantArray rhs; - ConstPayload(payloadType_, data).GetByJsonPath(fields_.getTagsPath(0), rhs, type_); - if (cond_ == CondDWithin) { - cmpGeom.ExcludeDistinct(static_cast(rhs)); - } else { - for (const auto &v : rhs) excludeDistinct(v); - } - } else { - // Exclude field from payload by offset (fast path) - - assertrx(!type_.Is()); - - // Check if we have column (rawData_), then go to fastest path with column - if (rawData_) return excludeDistinct(rawData_ + rowId * sizeof_); - - // Not array: Just exclude field by offset in PayloadValue - if (!isArray_) return excludeDistinct(data.Ptr() + offset_); - - PayloadFieldValue::Array *arr = reinterpret_cast(data.Ptr() + offset_); - uint8_t *ptr = data.Ptr() + arr->offset; - if (cond_ == CondDWithin) { - if (arr->len != 2 || !type_.Is()) throw Error(errQueryExec, "DWithin with not point data"); - return cmpGeom.ExcludeDistinct(Point{*reinterpret_cast(ptr), *reinterpret_cast(ptr + sizeof_)}); - } - - for (int i = 0; i < arr->len; i++, ptr += sizeof_) excludeDistinct(ptr); - } -} - -} // namespace reindexer diff --git a/cpp_src/core/comparator.h b/cpp_src/core/comparator.h deleted file mode 100644 index 3760077c8..000000000 --- a/cpp_src/core/comparator.h +++ /dev/null @@ -1,123 +0,0 @@ -#pragma once - -#include "comparatorimpl.h" -#include "compositearraycomparator.h" - -namespace reindexer { - -class Comparator : public ComparatorVars { -public: - Comparator() = delete; - Comparator(CondType cond, KeyValueType type, const VariantArray &values, bool isArray, bool distinct, PayloadType payloadType, - const FieldsSet &fields, void *rawData = nullptr, const CollateOpts &collateOpts = CollateOpts()); - ~Comparator() = default; - - bool Compare(const PayloadValue &lhs, int rowId); - void ExcludeDistinct(const PayloadValue &, int rowId); - void Bind(const PayloadType &type, int field); - void BindEqualPosition(int field, const VariantArray &val, CondType cond); - void BindEqualPosition(const TagsPath &tagsPath, const VariantArray &val, CondType cond); - void ClearDistinct() { - cmpInt.ClearDistinct(); - cmpBool.ClearDistinct(); - cmpInt64.ClearDistinct(); - cmpDouble.ClearDistinct(); - cmpString.ClearDistinct(); - cmpGeom.ClearDistinct(); - cmpUuid.ClearDistinct(); - } - bool HasJsonPaths() const noexcept { return fields_.getTagsPathsLength(); } - -private: - bool compare(const Variant &kr) { - return kr.Type().EvaluateOneOf( - [&](KeyValueType::Null) noexcept { return cond_ == CondEmpty; }, - [&](KeyValueType::Int) { return cmpInt.Compare(cond_, static_cast(kr)); }, - [&](KeyValueType::Bool) { return cmpBool.Compare(cond_, static_cast(kr)); }, - [&](KeyValueType::Int64) { return cmpInt64.Compare(cond_, static_cast(kr)); }, - [&](KeyValueType::Double) { return cmpDouble.Compare(cond_, static_cast(kr)); }, - [&](KeyValueType::String) { return cmpString.Compare(cond_, static_cast(kr), collateOpts_); }, - [&](KeyValueType::Composite) { return cmpComposite.Compare(cond_, static_cast(kr), *this); }, - [&](KeyValueType::Uuid) { return cmpUuid.Compare(cond_, Uuid{kr}); }, - [](OneOf) noexcept -> bool { abort(); }); - } - - bool compare(const void *ptr) { - return type_.EvaluateOneOf( - [&](KeyValueType::Null) noexcept { return cond_ == CondEmpty; }, - [&](KeyValueType::Bool) { return cmpBool.Compare(cond_, *static_cast(ptr)); }, - [&](KeyValueType::Int) { return cmpInt.Compare(cond_, *static_cast(ptr)); }, - [&](KeyValueType::Int64) { return cmpInt64.Compare(cond_, *static_cast(ptr)); }, - [&](KeyValueType::Double) { return cmpDouble.Compare(cond_, *static_cast(ptr)); }, - [&](KeyValueType::String) { return cmpString.Compare(cond_, *static_cast(ptr), collateOpts_); }, - [&](KeyValueType::Uuid) { return cmpUuid.Compare(cond_, *static_cast(ptr)); }, - [&](KeyValueType::Composite) { return cmpComposite.Compare(cond_, *static_cast(ptr), *this); }, - [](OneOf) noexcept -> bool { - assertrx(0); - abort(); - }); - } - - void excludeDistinct(const Variant &kr) { - kr.Type().EvaluateOneOf([&](KeyValueType::Int) { cmpInt.ExcludeDistinct(static_cast(kr)); }, - [&](KeyValueType::Bool) { cmpBool.ExcludeDistinct(static_cast(kr)); }, - [&](KeyValueType::Int64) { cmpInt64.ExcludeDistinct(static_cast(kr)); }, - [&](KeyValueType::Double) { cmpDouble.ExcludeDistinct(static_cast(kr)); }, - [&](KeyValueType::String) { cmpString.ExcludeDistinct(static_cast(kr)); }, - [&](KeyValueType::Uuid) { cmpUuid.ExcludeDistinct(Uuid{kr}); }, - [](KeyValueType::Composite) { throw Error(errQueryExec, "Distinct by composite index"); }, - [](OneOf) noexcept {}); - } - - void excludeDistinct(const void *ptr) { - type_.EvaluateOneOf([&](KeyValueType::Bool) { cmpBool.ExcludeDistinct(*static_cast(ptr)); }, - [&](KeyValueType::Int) { cmpInt.ExcludeDistinct(*static_cast(ptr)); }, - [&](KeyValueType::Int64) { cmpInt64.ExcludeDistinct(*static_cast(ptr)); }, - [&](KeyValueType::Double) { cmpDouble.ExcludeDistinct(*static_cast(ptr)); }, - [&](KeyValueType::String) { cmpString.ExcludeDistinct(*static_cast(ptr)); }, - [&](KeyValueType::Uuid) { cmpUuid.ExcludeDistinct(*static_cast(ptr)); }, - [](KeyValueType::Composite) { throw Error(errQueryExec, "Distinct by composite index"); }, - [](KeyValueType::Null) noexcept {}, - [](OneOf) noexcept { - assertrx(0); - abort(); - }); - } - - void setValues(const VariantArray &values); - bool isNumericComparison(const VariantArray &values) const; - - void clearAllSetValues() { - cmpInt.ClearAllSetValues(); - cmpBool.ClearAllSetValues(); - cmpInt64.ClearAllSetValues(); - cmpDouble.ClearAllSetValues(); - cmpString.ClearAllSetValues(); - } - - void clearIndividualAllSetValues() { - type_.EvaluateOneOf([&](KeyValueType::Bool) noexcept { cmpBool.ClearAllSetValues(); }, - [&](KeyValueType::Int) noexcept { cmpInt.ClearAllSetValues(); }, - [&](KeyValueType::Int64) noexcept { cmpInt64.ClearAllSetValues(); }, - [&](KeyValueType::Double) noexcept { cmpDouble.ClearAllSetValues(); }, - [&](KeyValueType::String) noexcept { cmpString.ClearAllSetValues(); }, - [&](KeyValueType::Composite) noexcept { cmpComposite.ClearAllSetValues(); }, [](KeyValueType::Null) noexcept {}, - [&](KeyValueType::Uuid) noexcept { cmpUuid.ClearAllSetValues(); }, - [](OneOf) noexcept { - assertrx(0); - abort(); - }); - } - ComparatorImpl cmpBool; - ComparatorImpl cmpInt; - ComparatorImpl cmpInt64; - ComparatorImpl cmpDouble; - ComparatorImpl cmpString; - ComparatorImpl cmpComposite; - ComparatorImpl cmpGeom; - ComparatorImpl cmpUuid; - CompositeArrayComparator cmpEqualPosition; - KeyValueType valuesType_{KeyValueType::Undefined{}}; -}; - -} // namespace reindexer diff --git a/cpp_src/core/compositearraycomparator.h b/cpp_src/core/compositearraycomparator.h deleted file mode 100644 index d646ed6b4..000000000 --- a/cpp_src/core/compositearraycomparator.h +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include "comparatorimpl.h" -#include "core/payload/fieldsset.h" - -namespace reindexer { - -class CompositeArrayComparator { -public: - CompositeArrayComparator(); - - void BindField(int field, const VariantArray &values, CondType condType); - void BindField(const TagsPath &tagsPath, const VariantArray &values, CondType condType); - bool Compare(const PayloadValue &pv, const ComparatorVars &vars); - bool IsBinded() { return !ctx_.empty(); } - -private: - bool compareField(size_t field, const Variant &v, const ComparatorVars &vars); - - struct Context { - CondType cond; - ComparatorImpl cmpBool; - ComparatorImpl cmpInt; - ComparatorImpl cmpInt64; - ComparatorImpl cmpDouble; - ComparatorImpl cmpString; - ComparatorImpl cmpUuid; - }; - - std::vector ctx_; - FieldsSet fields_; -}; -} // namespace reindexer diff --git a/cpp_src/core/dbconfig.cc b/cpp_src/core/dbconfig.cc index d2b4aa462..0e9d2255f 100644 --- a/cpp_src/core/dbconfig.cc +++ b/cpp_src/core/dbconfig.cc @@ -4,10 +4,9 @@ #include "cjson/jsonbuilder.h" #include "estl/smart_lock.h" #include "gason/gason.h" -#include "tools/jsontools.h" #include "tools/serializer.h" #include "tools/stringstools.h" -#include "yaml-cpp/yaml.h" +#include "vendor/yaml-cpp/yaml.h" namespace reindexer { @@ -45,8 +44,7 @@ Error DBConfigProvider::FromJSON(const gason::JsonNode &root) { LongTxLoggingParams{profilingNode["long_queries_logging"]["transaction"]["threshold_us"].As(), profilingNode["long_queries_logging"]["transaction"]["avg_step_threshold_us"].As()}); } - auto it = handlers_.find(ProfilingConf); - if (it != handlers_.end()) (it->second)(); + if (handlers_[ProfilingConf]) (handlers_[ProfilingConf])(); } auto &namespacesNode = root["namespaces"]; @@ -75,8 +73,7 @@ Error DBConfigProvider::FromJSON(const gason::JsonNode &root) { data.syncStorageFlushLimit = nsNode["sync_storage_flush_limit"].As(data.syncStorageFlushLimit, 0); namespacesData_.emplace(nsNode["namespace"].As(), std::move(data)); // NOLINT(performance-move-const-arg) } - auto it = handlers_.find(NamespaceDataConf); - if (it != handlers_.end()) (it->second)(); + if (handlers_[NamespaceDataConf]) (handlers_[NamespaceDataConf])(); } auto &replicationNode = root["replication"]; @@ -84,8 +81,7 @@ Error DBConfigProvider::FromJSON(const gason::JsonNode &root) { auto err = replicationData_.FromJSON(replicationNode); if (!err.ok()) return err; - auto it = handlers_.find(ReplicationConf); - if (it != handlers_.end()) (it->second)(); + if (handlers_[ReplicationConf]) (handlers_[ReplicationConf])(); } return errOK; } catch (const Error &err) { @@ -105,11 +101,11 @@ ReplicationConfigData DBConfigProvider::GetReplicationConfig() { return replicationData_; } -bool DBConfigProvider::GetNamespaceConfig(const std::string &nsName, NamespaceConfigData &data) { +bool DBConfigProvider::GetNamespaceConfig(std::string_view nsName, NamespaceConfigData &data) { shared_lock lk(mtx_); auto it = namespacesData_.find(nsName); if (it == namespacesData_.end()) { - it = namespacesData_.find("*"); + it = namespacesData_.find(std::string_view("*")); } if (it == namespacesData_.end()) { data = {}; diff --git a/cpp_src/core/dbconfig.h b/cpp_src/core/dbconfig.h index cbb4b1bed..6b9bd9ac5 100644 --- a/cpp_src/core/dbconfig.h +++ b/cpp_src/core/dbconfig.h @@ -2,9 +2,8 @@ #include #include -#include +#include "estl/fast_hash_map.h" #include "estl/fast_hash_set.h" -#include "estl/mutex.h" #include "estl/shared_mutex.h" #include "tools/errors.h" #include "tools/stringstools.h" @@ -18,7 +17,13 @@ class JsonBuilder; class RdxContext; class WrSerializer; -enum ConfigType { ProfilingConf, NamespaceDataConf, ReplicationConf }; +enum ConfigType { + ProfilingConf = 0, + NamespaceDataConf, + ReplicationConf, + // + kConfigTypesTotalCount +}; class LongQueriesLoggingParams { public: @@ -133,7 +138,7 @@ class DBConfigProvider { void setHandler(ConfigType cfgType, std::function handler); ReplicationConfigData GetReplicationConfig(); - bool GetNamespaceConfig(const std::string &nsName, NamespaceConfigData &data); + bool GetNamespaceConfig(std::string_view nsName, NamespaceConfigData &data); LongQueriesLoggingParams GetSelectLoggingParams() const noexcept { return profilingData_.longSelectLoggingParams.load(std::memory_order_relaxed); } @@ -150,8 +155,8 @@ class DBConfigProvider { private: ProfilingConfigData profilingData_; ReplicationConfigData replicationData_; - std::unordered_map namespacesData_; - std::unordered_map> handlers_; + fast_hash_map namespacesData_; + std::array, kConfigTypesTotalCount> handlers_; shared_timed_mutex mtx_; }; diff --git a/cpp_src/core/defnsconfigs.h b/cpp_src/core/defnsconfigs.h index 36bcd02b4..702604c84 100644 --- a/cpp_src/core/defnsconfigs.h +++ b/cpp_src/core/defnsconfigs.h @@ -114,7 +114,6 @@ const std::vector kSystemNsDefs = { .AddIndex("last_sec_avg_lock_time_us", "-", "int64", IndexOpts().Dense()) .AddIndex("latency_stddev", "-", "double", IndexOpts().Dense()), NamespaceDef(kNamespacesNamespace, StorageOpts()).AddIndex(kNsNameField, "hash", "string", IndexOpts().PK()), - NamespaceDef(kPerfStatsNamespace, StorageOpts()).AddIndex(kNsNameField, "hash", "string", IndexOpts().PK()), NamespaceDef(kMemStatsNamespace, StorageOpts()) .AddIndex(kNsNameField, "hash", "string", IndexOpts().PK()) .AddIndex("items_count", "-", "int64", IndexOpts().Dense()) diff --git a/cpp_src/core/expressiontree.cc b/cpp_src/core/expressiontree.cc deleted file mode 100644 index 651ba26ff..000000000 --- a/cpp_src/core/expressiontree.cc +++ /dev/null @@ -1,11 +0,0 @@ -#include "expressiontree.h" -#include "core/nsselecter/selectiterator.h" - -namespace reindexer { - -template <> -bool ExpressionTree::Node::operator==(const Node&) const { - throw std::runtime_error("Cannot compare"); -} - -} // namespace reindexer diff --git a/cpp_src/core/expressiontree.h b/cpp_src/core/expressiontree.h index 6f4147452..f71302b1e 100644 --- a/cpp_src/core/expressiontree.h +++ b/cpp_src/core/expressiontree.h @@ -1,8 +1,8 @@ #pragma once -#include #include #include "estl/h_vector.h" +#include "estl/overloaded.h" #include "tools/errors.h" namespace reindexer { @@ -29,8 +29,21 @@ class Bracket { size_t size_ = 1; }; -template -struct Skip {}; +template +class Skip : private Skip { +public: + using Skip::operator(); + void operator()(const T&) const noexcept {} +}; + +template +class Skip { +public: + void operator()(const T&) const noexcept {} +}; + +template