Update to version v4.17.1

CMakeLists.txt

@@ -1,5 +1,7 @@
+cmake_minimum_required(VERSION 3.10..3.13)
+
 project(reindexer)
-cmake_minimum_required(VERSION 3.10)
+
 enable_testing()
 set(CMAKE_DISABLE_IN_SOURCE_BUILD ON)
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)

bindings/consts.go

@@ -2,7 +2,7 @@ package bindings
 
 const CInt32Max = int(^uint32(0) >> 1)
 
-const ReindexerVersion = "v4.17.0"
+const ReindexerVersion = "v4.17.1"
 
 // public go consts from type_consts.h and reindexer_ctypes.h
 const (

changelog.md

@@ -1,3 +1,27 @@
+# Version 4.17.1 *beta* (05.09.2024)
+## Core
+- [fix] Fixed pagination for `MERGE` fulltext queries. Previously those queries could return duplicates on different pages due to missing ordering guarantees
+- [fix] Fixed fields filters serialization for `DISTINCT` aggregations (affects SQL logging only)
+- [fix] Temporary disabled default values logic for indexed fields from v4.16.0 - this logic may cause stability issues and will be reworked in further releases
+- [fix] Add extra check for composites indexes substitution
+- [fix] Fix composites substitution after indexes update
+
+## Reindexer tool
+- [fix] Fixed crash in `\quit`-command after previous errors
+- [fix] Fixed crash in `git bash` on `Windows` platforms
+
+## Face
+- [fea] Added RU language in UI
+- [fea] Changed BM25 settings layout
+- [fea] Added new documentation links
+- [fix] Fixed database deletion
+- [fix] Fixed default value of the stop words in fulltext config
+- [fix] Fixed default value of the rtree-indexes in item creation tab
+- [fix] Fixed last namespace displaying
+- [fix] Fixed the issue with removing of 'Statistics -> Replication' table on the page reloading
+- [fix] Renamed 'Stat(ms)' field on Replication page
+- [fix] Fixed XSS vulnerability in table view
+
 # Version 4.17.0 *beta* (16.08.2024)
 ## Core
 - [fea] Updated [logging library](https://github.com/gabime/spdlog) to v1.14.1 and [formatting library](https://github.com/fmtlib/fmt) to v11.0.2

cjson/decoder.go

@@ -313,6 +313,7 @@ func (dec *Decoder) decodeSlice(pl *payloadIface, rdser *Serializer, v *reflect.
 	var ptr unsafe.Pointer
 
 	k := v.Kind()
+
 	offset := 0
 	switch k {
 	case reflect.Slice:
@@ -331,7 +332,11 @@ func (dec *Decoder) decodeSlice(pl *payloadIface, rdser *Serializer, v *reflect.
 		// offset is 0
 		// No concatenation for the fixed size arrays
 	default:
-		panic(fmt.Errorf("can not convert '%s' to 'array'", v.Type().Kind().String()))
+		if count == 0 { // Allows empty slice for any scalar type (using default value)
+			return
+		} else {
+			panic(fmt.Errorf("can not convert '%s' to 'array'", v.Type().Kind().String()))
+		}
 	}
 
 	if subtag != TAG_OBJECT {
@@ -583,6 +588,7 @@ func (dec *Decoder) decodeValue(pl *payloadIface, rdser *Serializer, v reflect.V
 	ctagName := ctag.Name()
 
 	k := v.Kind()
+	initialV := v
 	if k == reflect.Ptr {
 		if v.IsNil() {
 			v.Set(reflect.New(v.Type().Elem()))
@@ -641,6 +647,7 @@ func (dec *Decoder) decodeValue(pl *payloadIface, rdser *Serializer, v reflect.V
 		} else {
 			panic(fmt.Errorf("err: intf=%s, name='%s' %s", v.Type().Name(), dec.state.tagsMatcher.tag2name(ctagName), ctag.Dump()))
 		}
+		initialV = v
 		k = v.Kind()
 		if k == reflect.Ptr {
 			if v.IsNil() {
@@ -659,8 +666,12 @@ func (dec *Decoder) decodeValue(pl *payloadIface, rdser *Serializer, v reflect.V
 		switch ctagType {
 		case TAG_ARRAY:
 			count := int(rdser.GetVarUInt())
-			pl.getArray(int(ctagField), *cnt, count, v)
-			*cnt += count
+			if k == reflect.Slice || k == reflect.Array || count != 0 { // Allows empty slice for any scalar type (using default value)
+				pl.getArray(int(ctagField), *cnt, count, v)
+				*cnt += count
+			} else {
+				initialV.Set(reflect.Zero(initialV.Type())) // Set nil to scalar pointers, intialized with empty arrays
+			}
 		default:
 			pl.getValue(int(ctagField), *cnt, v)
 			(*cnt)++
@@ -670,6 +681,9 @@ func (dec *Decoder) decodeValue(pl *payloadIface, rdser *Serializer, v reflect.V
 		switch ctagType {
 		case TAG_ARRAY:
 			dec.decodeSlice(pl, rdser, &v, fieldsoutcnt, cctagsPath)
+			if v.Kind() != reflect.Array && v.Kind() != reflect.Slice && v.Kind() != reflect.Interface {
+				initialV.Set(reflect.Zero(initialV.Type())) // Set nil to scalar pointers, intialized with empty arrays
+			}
 		case TAG_OBJECT:
 			for dec.decodeValue(pl, rdser, v, fieldsoutcnt, cctagsPath) {
 			}

cpp_src/CMakeLists.txt

@@ -45,7 +45,7 @@ else()
   option(LINK_RESOURCES "Link web resources as binary data" ON)
 endif()
 
-set (REINDEXER_VERSION_DEFAULT "4.17.0")
+set (REINDEXER_VERSION_DEFAULT "4.17.1")
 
 if(NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE "RelWithDebInfo")
@@ -297,7 +297,11 @@ else()
 endif()
 
 # Static LevelDB v1.23 is built with -fno-rtti by default. To inherit our logger from leveldb's logger, this file must be built with -fno-rtti to
-set_source_files_properties (${REINDEXER_SOURCE_PATH}/core/storage/leveldblogger.cc PROPERTIES COMPILE_FLAGS "-fno-rtti")
+if(MSVC)
+  set_source_files_properties (${REINDEXER_SOURCE_PATH}/core/storage/leveldblogger.cc PROPERTIES COMPILE_FLAGS "/GR-")
+else()
+  set_source_files_properties (${REINDEXER_SOURCE_PATH}/core/storage/leveldblogger.cc PROPERTIES COMPILE_FLAGS "-fno-rtti")
+endif()
 
 list(APPEND REINDEXER_LIBRARIES reindexer)
 add_library(${TARGET} STATIC ${HDRS} ${SRCS} ${VENDORS})
@@ -357,6 +361,7 @@ else()
     GIT_TAG "1.1.8"
     CMAKE_ARGS -DSNAPPY_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_BINARY_DIR}
     -DCMAKE_INSTALL_LIBDIR=${CMAKE_CURRENT_BINARY_DIR}
+    -DCMAKE_POSITION_INDEPENDENT_CODE=ON
     )
   include_directories(${CMAKE_CURRENT_BINARY_DIR}/include)
   link_directories(${CMAKE_CURRENT_BINARY_DIR})
@@ -443,9 +448,10 @@ if(NOT LevelDB_LIBRARY OR NOT LevelDB_INCLUDE_DIR OR WITH_TSAN)
     GIT_TAG "master"
     CMAKE_ARGS -DLEVELDB_BUILD_TESTS=OFF -DLEVELDB_BUILD_BENCHMARKS=OFF
     -DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_BINARY_DIR}
-    "-DCMAKE_CXX_FLAGS=-fPIC -I${CMAKE_CURRENT_BINARY_DIR}/include"
+    "-DCMAKE_CXX_FLAGS=-I${CMAKE_CURRENT_BINARY_DIR}/include"
     -DCMAKE_EXE_LINKER_FLAGS=-L${CMAKE_CURRENT_BINARY_DIR}
     -DCMAKE_INSTALL_LIBDIR=${CMAKE_CURRENT_BINARY_DIR}
+    -DCMAKE_POSITION_INDEPENDENT_CODE=ON
   )
   if(NOT SNAPPY_FOUND)
     add_dependencies(leveldb_lib snappy_lib)

cpp_src/cmd/reindexer_tool/commandsexecutor.cc

@@ -59,11 +59,14 @@ Error CommandsExecutor<DBInterface>::GetSuggestions(const std::string& input, st
 
 template <typename DBInterface>
 Error CommandsExecutor<DBInterface>::Stop() {
-	GenericCommand cmd([this] {
-		stop(true);
-		return Error{};
-	});
-	auto err = execCommand(cmd);
+	Error err;
+	if (GetStatus().running) {
+		GenericCommand cmd([this] {
+			stop(true);
+			return Error{};
+		});
+		err = execCommand(cmd);
+	}
 	if (err.ok() && executorThr_.joinable()) {
 		executorThr_.join();
 	}

cpp_src/cmd/reindexer_tool/commandsprocessor.cc

@@ -177,10 +177,7 @@ bool CommandsProcessor<DBInterface>::Run(const std::string& command, const std::
 
 template <typename DBInterface>
 Error CommandsProcessor<DBInterface>::stop() {
-	if (executor_.GetStatus().running) {
-		return executor_.Stop();
-	}
-	return Error();
+	return executor_.Stop();
 }
 
 template class CommandsProcessor<reindexer::client::CoroReindexer>;

cpp_src/core/ft/ft_fast/selecter.cc

@@ -120,8 +120,8 @@ void Selecter<IdCont>::prepareVariants(std::vector<FtVariantEntry>& variants, RV
 // RX_NO_INLINE just for build test purpose. Do not expect any effect here
 template <typename IdCont>
 template <FtUseExternStatuses useExternSt>
-RX_NO_INLINE MergeData Selecter<IdCont>::Process(FtDSLQuery&& dsl, bool inTransaction, FtMergeStatuses::Statuses&& mergeStatuses,
-												 const RdxContext& rdxCtx) {
+RX_NO_INLINE MergeData Selecter<IdCont>::Process(FtDSLQuery&& dsl, bool inTransaction, FtSortType ftSortType,
+												 FtMergeStatuses::Statuses&& mergeStatuses, const RdxContext& rdxCtx) {
 	FtSelectContext ctx;
 	ctx.rawResults.reserve(dsl.size());
 	// STEP 2: Search dsl terms for each variant
@@ -213,11 +213,11 @@ RX_NO_INLINE MergeData Selecter<IdCont>::Process(FtDSLQuery&& dsl, bool inTransa
 	const auto maxMergedSize = std::min(size_t(holder_.cfg_->mergeLimit), ctx.totalORVids);
 
 	if (maxMergedSize < 0xFFFF) {
-		return mergeResultsBmType<uint16_t>(std::move(results), ctx.totalORVids, synonymsBounds, inTransaction, std::move(mergeStatuses),
-											rdxCtx);
+		return mergeResultsBmType<uint16_t>(std::move(results), ctx.totalORVids, synonymsBounds, inTransaction, ftSortType,
+											std::move(mergeStatuses), rdxCtx);
 	} else if (maxMergedSize < 0xFFFFFFFF) {
-		return mergeResultsBmType<uint32_t>(std::move(results), ctx.totalORVids, synonymsBounds, inTransaction, std::move(mergeStatuses),
-											rdxCtx);
+		return mergeResultsBmType<uint32_t>(std::move(results), ctx.totalORVids, synonymsBounds, inTransaction, ftSortType,
+											std::move(mergeStatuses), rdxCtx);
 	} else {
 		assertrx_throw(false);
 	}
@@ -227,17 +227,17 @@ RX_NO_INLINE MergeData Selecter<IdCont>::Process(FtDSLQuery&& dsl, bool inTransa
 template <typename IdCont>
 template <typename MergedOffsetT>
 MergeData Selecter<IdCont>::mergeResultsBmType(std::vector<TextSearchResults>&& results, size_t totalORVids,
-											   const std::vector<size_t>& synonymsBounds, bool inTransaction,
+											   const std::vector<size_t>& synonymsBounds, bool inTransaction, FtSortType ftSortType,
 											   FtMergeStatuses::Statuses&& mergeStatuses, const RdxContext& rdxCtx) {
 	switch (holder_.cfg_->bm25Config.bm25Type) {
 		case FtFastConfig::Bm25Config::Bm25Type::rx:
-			return mergeResults<Bm25Rx, MergedOffsetT>(std::move(results), totalORVids, synonymsBounds, inTransaction,
+			return mergeResults<Bm25Rx, MergedOffsetT>(std::move(results), totalORVids, synonymsBounds, inTransaction, ftSortType,
 													   std::move(mergeStatuses), rdxCtx);
 		case FtFastConfig::Bm25Config::Bm25Type::classic:
-			return mergeResults<Bm25Classic, MergedOffsetT>(std::move(results), totalORVids, synonymsBounds, inTransaction,
+			return mergeResults<Bm25Classic, MergedOffsetT>(std::move(results), totalORVids, synonymsBounds, inTransaction, ftSortType,
 															std::move(mergeStatuses), rdxCtx);
 		case FtFastConfig::Bm25Config::Bm25Type::wordCount:
-			return mergeResults<TermCount, MergedOffsetT>(std::move(results), totalORVids, synonymsBounds, inTransaction,
+			return mergeResults<TermCount, MergedOffsetT>(std::move(results), totalORVids, synonymsBounds, inTransaction, ftSortType,
 														  std::move(mergeStatuses), rdxCtx);
 	}
 	assertrx_throw(false);
@@ -1284,7 +1284,7 @@ bool Selecter<IdCont>::TyposHandler::isWordFitMaxLettPerm(const std::string_view
 template <typename IdCont>
 template <typename Bm25T, typename MergedOffsetT>
 MergeData Selecter<IdCont>::mergeResults(std::vector<TextSearchResults>&& rawResults, size_t maxMergedSize,
-										 const std::vector<size_t>& synonymsBounds, bool inTransaction,
+										 const std::vector<size_t>& synonymsBounds, bool inTransaction, FtSortType ftSortType,
 										 FtMergeStatuses::Statuses&& mergeStatuses, const RdxContext& rdxCtx) {
 	const auto& vdocs = holder_.vdocs_;
 	MergeData merged;
@@ -1388,9 +1388,19 @@ MergeData Selecter<IdCont>::mergeResults(std::vector<TextSearchResults>&& rawRes
 			merged.maxRank = m.proc;
 		}
 	}
-
-	boost::sort::pdqsort_branchless(merged.begin(), merged.end(),
-									[](const MergeInfo& lhs, const MergeInfo& rhs) noexcept { return lhs.proc > rhs.proc; });
+	switch (ftSortType) {
+		case FtSortType::RankOnly: {
+			boost::sort::pdqsort_branchless(merged.begin(), merged.end(),
+											[](const MergeInfo& lhs, const MergeInfo& rhs) noexcept { return lhs.proc > rhs.proc; });
+			return merged;
+		}
+		case FtSortType::RankAndID: {
+			return merged;
+		}
+		case FtSortType::ExternalExpression:
+			throw Error(errLogic, "FtSortType::ExternalExpression not implemented.");
+			break;
+	}
 	return merged;
 }
 
@@ -1431,13 +1441,14 @@ void Selecter<IdCont>::printVariants(const FtSelectContext& ctx, const TextSearc
 }
 
 template class Selecter<PackedIdRelVec>;
-template MergeData Selecter<PackedIdRelVec>::Process<FtUseExternStatuses::No>(FtDSLQuery&&, bool, FtMergeStatuses::Statuses&&,
+template MergeData Selecter<PackedIdRelVec>::Process<FtUseExternStatuses::No>(FtDSLQuery&&, bool, FtSortType, FtMergeStatuses::Statuses&&,
 																			  const RdxContext&);
-template MergeData Selecter<PackedIdRelVec>::Process<FtUseExternStatuses::Yes>(FtDSLQuery&&, bool, FtMergeStatuses::Statuses&&,
+template MergeData Selecter<PackedIdRelVec>::Process<FtUseExternStatuses::Yes>(FtDSLQuery&&, bool, FtSortType, FtMergeStatuses::Statuses&&,
 																			   const RdxContext&);
 template class Selecter<IdRelVec>;
-template MergeData Selecter<IdRelVec>::Process<FtUseExternStatuses::No>(FtDSLQuery&&, bool, FtMergeStatuses::Statuses&&, const RdxContext&);
-template MergeData Selecter<IdRelVec>::Process<FtUseExternStatuses::Yes>(FtDSLQuery&&, bool, FtMergeStatuses::Statuses&&,
+template MergeData Selecter<IdRelVec>::Process<FtUseExternStatuses::No>(FtDSLQuery&&, bool, FtSortType, FtMergeStatuses::Statuses&&,
+																		const RdxContext&);
+template MergeData Selecter<IdRelVec>::Process<FtUseExternStatuses::Yes>(FtDSLQuery&&, bool, FtSortType, FtMergeStatuses::Statuses&&,
 																		 const RdxContext&);
 
 }  // namespace reindexer

cpp_src/core/ft/ft_fast/selecter.h

@@ -1,6 +1,7 @@
 #pragma once
 #include "core/ft/ftdsl.h"
 #include "core/ft/idrelset.h"
+#include "core/selectfunc/ctx/ftctx.h"
 #include "dataholder.h"
 
 namespace reindexer {
@@ -53,7 +54,8 @@ class Selecter {
 	};
 
 	template <FtUseExternStatuses>
-	MergeData Process(FtDSLQuery&& dsl, bool inTransaction, FtMergeStatuses::Statuses&& mergeStatuses, const RdxContext&);
+	MergeData Process(FtDSLQuery&& dsl, bool inTransaction, FtSortType ftSortType, FtMergeStatuses::Statuses&& mergeStatuses,
+					  const RdxContext&);
 
 private:
 	struct TextSearchResult {
@@ -178,7 +180,7 @@ class Selecter {
 
 	template <typename Bm25Type, typename MergedOffsetT>
 	MergeData mergeResults(std::vector<TextSearchResults>&& rawResults, size_t maxMergedSize, const std::vector<size_t>& synonymsBounds,
-						   bool inTransaction, FtMergeStatuses::Statuses&& mergeStatuses, const RdxContext&);
+						   bool inTransaction, FtSortType ftSortType, FtMergeStatuses::Statuses&& mergeStatuses, const RdxContext&);
 
 	template <typename Bm25Type, typename MergedOffsetT>
 	void mergeIteration(TextSearchResults& rawRes, index_t rawResIndex, FtMergeStatuses::Statuses& mergeStatuses, MergeData& merged,
@@ -244,7 +246,8 @@ class Selecter {
 
 	template <typename MergedOffsetT>
 	MergeData mergeResultsBmType(std::vector<TextSearchResults>&& results, size_t totalORVids, const std::vector<size_t>& synonymsBounds,
-								 bool inTransaction, FtMergeStatuses::Statuses&& mergeStatuses, const RdxContext& rdxCtx);
+								 bool inTransaction, FtSortType ftSortType, FtMergeStatuses::Statuses&& mergeStatuses,
+								 const RdxContext& rdxCtx);
 
 	void debugMergeStep(const char* msg, int vid, float normBm25, float normDist, int finalRank, int prevRank);
 	template <FtUseExternStatuses>

cpp_src/core/idset.h

@@ -35,6 +35,7 @@ class IdSetPlain : protected base_idset {
 	using base_idset::rend;
 	using base_idset::const_reverse_iterator;
 	using base_idset::const_iterator;
+	using base_idset::operator[];
 
 	enum EditMode {
 		Ordered,   // Keep idset ordered, and ready to select (insert is slow O(logN)+O(N))

cpp_src/core/index/index.cc

@@ -86,7 +86,7 @@ void Index::dump(S& os, std::string_view step, std::string_view offset) const {
 	payloadType_.Dump(os, step, newOffset);
 	if (IsComposite(type_)) {
 		os << ",\n" << newOffset << "fields: ";
-		fields_.Dump(os);
+		fields_.Dump(os, FieldsSet::DumpWithMask::Yes);
 	}
 	os << ",\n"
 	   << newOffset << "sortedIdxCount: " << sortedIdxCount_ << ",\n"

cpp_src/core/index/index.h

@@ -34,7 +34,8 @@ class Index {
 			  forceComparator(0),
 			  unbuiltSortOrders(0),
 			  indexesNotOptimized(0),
-			  inTransaction{0} {}
+			  inTransaction{0},
+			  ftSortType(0) {}
 		unsigned itemsCountInNamespace;
 		int maxIterations;
 		unsigned distinct : 1;
@@ -43,6 +44,7 @@ class Index {
 		unsigned unbuiltSortOrders : 1;
 		unsigned indexesNotOptimized : 1;
 		unsigned inTransaction : 1;
+		unsigned ftSortType : 2;
 	};
 	using KeyEntry = reindexer::KeyEntry<IdSet>;
 	using KeyEntryPlain = reindexer::KeyEntry<IdSetPlain>;