Update to version v3.30.0

.github/workflows/test.yml

@@ -47,8 +47,11 @@ jobs:
   build:
     strategy:
       matrix:
-        os: [ubuntu-20.04, ubuntu-22.04, macos-13]
+        os: [ubuntu-22.04, macos-13]
         include:
+          - os: ubuntu-24.04
+            cc: gcc-12
+            cxx: g++-12
           - os: ubuntu-latest
             sanitizer: ASAN
             cc: gcc-10
@@ -111,7 +114,7 @@ jobs:
   test:
     strategy:
       matrix:
-        os: [ubuntu-20.04, ubuntu-22.04, macos-13]
+        os: [ubuntu-22.04, ubuntu-24.04, macos-13]
         test: ['C++', 'GO']
         include:
           - os: ubuntu-latest
@@ -188,7 +191,7 @@ jobs:
   test-pyreindexer:
     strategy:
       matrix:
-        os: [ubuntu-20.04, ubuntu-22.04]
+        os: [ubuntu-22.04, ubuntu-24.04]
       fail-fast: false
     runs-on: ${{matrix.os}}
     needs: build

appveyor.yml

@@ -12,11 +12,11 @@ environment:
       TOOLCHAIN: x86_64-8.1.0-posix-seh-rt_v6-rev0
       APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
 
-    - BUILD_TYPE: Release
-      COMPILER: MinGW
-      PLATFORM: Win32
-      TOOLCHAIN: i686-8.1.0-posix-dwarf-rt_v6-rev0
-      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
+#    - BUILD_TYPE: Release
+#      COMPILER: MinGW
+#      PLATFORM: Win32
+#      TOOLCHAIN: i686-8.1.0-posix-dwarf-rt_v6-rev0
+#      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
 
 build_script:
   - git describe --tags

bindings/consts.go

@@ -2,7 +2,7 @@ package bindings
 
 const CInt32Max = int(^uint32(0) >> 1)
 
-const ReindexerVersion = "v3.29.0"
+const ReindexerVersion = "v3.30.0"
 
 // public go consts from type_consts.h and reindexer_ctypes.h
 const (

changelog.md

@@ -1,3 +1,42 @@
+# Version 3.30.0 (29.11.2024)
+## Core
+- [fea] Optimized memory layout for `-tuple`-index (this slightly reduces memory consumation for any `reindexer` database)
+- [fea] Added [dragonbox](https://github.com/jk-jeon/dragonbox)/[fmt](https://github.com/fmtlib/fmt) instead of [double-conversion](https://github.com/google/double-conversion) for JSON floating point number serialization (the dragonbox serialization works about 2 times faster)
+- [fix] Fixed assertion in aggregations by `composite indexes`
+
+## Fulltext
+- [fea] Added basic support for Korean symbols
+- [fea] Added basic support for Japanese Katakana symbols
+- [fea] Added alternative tokenizer based on [friso](https://github.com/lionsoul2014/friso) for Chinese language splitting. [Read more...](fulltext.md#text-splitters). Current implementation works only for chinese and english symbols and will be extended in further releases
+
+## Reindexer server
+- [fix] Fixed logical operations(`OR`/`NOT`) handling for `joined`-queries in `JSON DSL` (method `POST /api/v1/db/<db>/query`)
+- [fix] Unified `int<->double` conversion logic for `JSON`/`MgsPack`/`Protobuf`
+
+## Reindexer tool
+- [fix] Fixed `\bench` command for `ucproto`-mode
+
+## CPP client
+- [fix] Fixed `precepts` in client's transaction
+- [fix] Fixed `DWITHIN` condition serialization for select `Query`-object
+- [fix] Fixed fulltext select functions serialization for select `Query`-object
+- [fix] Fixed `SetObject` serialization for update `Query`-object
+
+## Build
+- [fix] Fixed build for `MinGW 14.1.0/14.2.0`
+
+## Deploy
+- [upd] Added deploy for `Fedora 41` packages
+- [upd] Deprecated deploy for `Fedora 39` packages
+
+## Face
+- [fea] Added disabled mode for selectors on the NS config page
+- [fea] Added `column_size` field to Memory statistics table
+- [fix] Fixed scroll position on NS menu 
+- [fix] Fixed doubled requests on Indexes and Items pages
+- [fix] Fixed wrong info message about disabled statistics when the database do not have Namespaces
+- [fix] Fixed last row position in Memory statistics table
+
 # Version 3.29.0 (08.10.2024)
 ## Core
 - [fea] Added Chinese and Japanese symbols into locale. Currently those symbols will be indexed by fulltext engine (using space-separators), but proper tokenization for the languages will be added in the separate release

cjson/creflect.go

@@ -78,7 +78,7 @@ type payloadIface struct {
 func (pl *payloadIface) ptr(field, idx, typ int) unsafe.Pointer {
 
 	if pl.p == 0 {
-		panic(fmt.Errorf("Null pointer derefernce"))
+		panic(fmt.Errorf("Null pointer dereference"))
 	}
 
 	f := &pl.t.Fields[field]
@@ -91,7 +91,7 @@ func (pl *payloadIface) ptr(field, idx, typ int) unsafe.Pointer {
 
 	if !f.IsArray {
 		if idx != 0 {
-			panic(fmt.Errorf("Trying to acces by index '%d' to non array field '%s'", idx, f.Name))
+			panic(fmt.Errorf("Trying to access by index '%d' to non array field '%s'", idx, f.Name))
 		}
 		return p
 	}
@@ -197,7 +197,7 @@ func (pl *payloadIface) getArrayLen(field int) int {
 	return int((*ArrayHeader)(p).len)
 }
 
-// get c reflect value and set to go reflect valie
+// get c reflect value and set to go reflect value
 func (pl *payloadIface) getValue(field int, idx int, v reflect.Value) {
 
 	k := v.Type().Kind()

cpp_src/CMakeLists.txt

@@ -44,7 +44,7 @@ else()
   option(LINK_RESOURCES "Link web resources as binary data" ON)
 endif()
 
-set(REINDEXER_VERSION_DEFAULT "3.29.0")
+set(REINDEXER_VERSION_DEFAULT "3.30.0")
 
 if(NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE "RelWithDebInfo")
@@ -302,6 +302,7 @@ add_library(${TARGET} STATIC ${HDRS} ${SRCS} ${VENDORS})
 add_definitions(-DREINDEX_CORE_BUILD=1)
 add_definitions(-DFMT_HEADER_ONLY=1)
 add_definitions(-DSPDLOG_FMT_EXTERNAL=1)
+add_definitions(-DFMT_USE_FULL_CACHE_DRAGONBOX=1)
 
 # add_definitions(-DREINDEX_FT_EXTRA_DEBUG=1)
 
@@ -464,6 +465,13 @@ set(THREADS_PREFER_PTHREAD_FLAG TRUE)
 find_package(Threads REQUIRED ON)
 list(APPEND REINDEXER_LIBRARIES ${CMAKE_THREAD_LIBS_INIT} )
 
+include(CMakeRC)
+
+file(GLOB CHINA_DICT LIST_DIRECTORIES false ${REINDEXER_SOURCE_PATH}/resource/china_dict/*.lex)
+cmrc_add_resource_library(friso_dict_resources WHENCE ${REINDEXER_SOURCE_PATH}/resource ${CHINA_DICT})
+list(APPEND REINDEXER_LIBRARIES friso_dict_resources)
+add_dependencies(${TARGET} friso_dict_resources)
+
 if(WITH_CPPTRACE)
   ExternalProject_Add(
     cpptrace_lib
@@ -571,7 +579,6 @@ endif()
 if(NOT REINDEXER_VERSION_CUR_H STREQUAL REINDEXER_VERSION_H)
   file(WRITE ${PROJECT_BINARY_DIR}/reindexer_version.h ${REINDEXER_VERSION_H})
 endif()
-
 include_directories(${PROJECT_BINARY_DIR})
 
 string ( REGEX REPLACE "v([0-9]+)\\.([0-9]+)\\.([0-9]+)(.*)" "\\1.\\2.\\3" REINDEXER_VERSION ${REINDEXER_VERSION_FULL})

cpp_src/client/cororeindexer.cc

@@ -79,7 +79,7 @@ Error CoroReindexer::GetSqlSuggestions(const std::string_view sqlQuery, int pos,
 Error CoroReindexer::Status() { return impl_->Status(ctx_); }
 
 CoroTransaction CoroReindexer::NewTransaction(std::string_view nsName) { return impl_->NewTransaction(nsName, ctx_); }
-Error CoroReindexer::CommitTransaction(CoroTransaction& tr) { return impl_->CommitTransaction(tr, ctx_); }
+Error CoroReindexer::CommitTransaction(CoroTransaction& tr, CoroQueryResults& result) { return impl_->CommitTransaction(tr, result, ctx_); }
 Error CoroReindexer::RollBackTransaction(CoroTransaction& tr) { return impl_->RollBackTransaction(tr, ctx_); }
 
 }  // namespace client

cpp_src/client/cororeindexer.h

@@ -32,7 +32,7 @@ class CoroReindexer {
 	typedef std::function<void(const Error& err)> Completion;
 
 	/// Create Reindexer database object
-	CoroReindexer(const ReindexerConfig& = ReindexerConfig());
+	explicit CoroReindexer(const ReindexerConfig& = ReindexerConfig());
 	/// Destroy Reindexer database object
 	~CoroReindexer();
 	CoroReindexer(const CoroReindexer&) = delete;
@@ -180,7 +180,8 @@ class CoroReindexer {
 	CoroTransaction NewTransaction(std::string_view nsName);
 	/// Commit transaction - transaction will be deleted after commit
 	/// @param tr - transaction to commit
-	Error CommitTransaction(CoroTransaction& tr);
+	/// @param result - QueryResults with IDs of items changed by tx.
+	Error CommitTransaction(CoroTransaction& tr, CoroQueryResults& result);
 	/// RollBack transaction - transaction will be deleted after rollback
 	/// @param tr - transaction to rollback
 	Error RollBackTransaction(CoroTransaction& tr);
@@ -196,6 +197,7 @@ class CoroReindexer {
 	typedef CoroQueryResults QueryResultsT;
 	typedef Item ItemT;
 	typedef ReindexerConfig ConfigT;
+	typedef CoroTransaction TransactionT;
 
 private:
 	CoroReindexer(CoroRPCClient* impl, InternalRdxContext&& ctx) : impl_(impl), owner_(false), ctx_(std::move(ctx)) {}

cpp_src/client/cororpcclient.cc

@@ -1,12 +1,10 @@
 #include "client/cororpcclient.h"
-#include <stdio.h>
 #include <functional>
 #include "client/itemimpl.h"
 #include "core/namespacedef.h"
 #include "gason/gason.h"
 #include "tools/errors.h"
 #include "tools/logger.h"
-#include "vendor/gason/gason.h"
 
 namespace reindexer {
 namespace client {
@@ -164,23 +162,25 @@ Error CoroRPCClient::modifyItem(std::string_view nsName, Item& item, int mode, s
 			}
 			CoroQueryResults qr;
 			InternalRdxContext ctxCompl = ctx.WithCompletion(nullptr);
-			auto ret = selectImpl(Query(nsName).Limit(0), qr, netTimeout, ctxCompl);
-			if (ret.code() == errTimeout) {
+			Error err = selectImpl(Query(nsName).Limit(0), qr, netTimeout, ctxCompl);
+			if (err.code() == errTimeout) {
 				return Error(errTimeout, "Request timeout");
 			}
 			if (withNetTimeout) {
 				netTimeout = netDeadline - conn_.Now();
 			}
 			auto newItem = NewItem(nsName);
 			char* endp = nullptr;
-			Error err = newItem.FromJSON(item.impl_->GetJSON(), &endp);
+			err = newItem.FromJSON(item.impl_->GetJSON(), &endp);
 			if (!err.ok()) {
 				return err;
 			}
 
 			item = std::move(newItem);
 		}
 	}
+
+	return errOK;
 }
 
 Error CoroRPCClient::subscribeImpl(bool subscribe) {
@@ -299,7 +299,6 @@ void vec2pack(const h_vector<int32_t, 4>& vec, WrSerializer& ser) {
 	for (auto v : vec) {
 		ser.PutVarUint(v);
 	}
-	return;
 }
 
 Error CoroRPCClient::selectImpl(std::string_view query, CoroQueryResults& result, seconds netTimeout, const InternalRdxContext& ctx) {
@@ -595,14 +594,29 @@ CoroTransaction CoroRPCClient::NewTransaction(std::string_view nsName, const Int
 	return CoroTransaction(std::move(err));
 }
 
-Error CoroRPCClient::CommitTransaction(CoroTransaction& tr, const InternalRdxContext& ctx) {
+Error CoroRPCClient::CommitTransaction(CoroTransaction& tr, CoroQueryResults& result, const InternalRdxContext& ctx) {
+	Error returnErr;
 	if (tr.conn_) {
-		auto ret = tr.conn_->Call(mkCommand(cproto::kCmdCommitTx, &ctx), tr.txId_).Status();
-		tr.clear();
-		return ret;
+		const int flags = result.fetchFlags_ ? result.fetchFlags_ : (kResultsWithItemID | kResultsWithPayloadTypes);
+		NsArray nsArray{getNamespace(tr.nsName_)};
+		result = CoroQueryResults(tr.conn_, std::move(nsArray), flags, config_.FetchAmount, config_.RequestTimeout);
+		auto ret = tr.conn_->Call(mkCommand(cproto::kCmdCommitTx, &ctx), tr.txId_, flags);
+		returnErr = ret.Status();
+		try {
+			if (ret.Status().ok()) {
+				auto args = ret.GetArgs(2);
+				result.Bind(p_string(args[0]), RPCQrId{int(args[1]), args.size() > 2 ? int64_t(args[2]) : -1});
+			}
+		} catch (const Error& err) {
+			returnErr = err;
+		}
+	} else {
+		returnErr = Error(errLogic, "connection is nullptr");
 	}
-	return Error(errLogic, "connection is nullptr");
+	tr.clear();
+	return returnErr;
 }
+
 Error CoroRPCClient::RollBackTransaction(CoroTransaction& tr, const InternalRdxContext& ctx) {
 	if (tr.conn_) {
 		auto ret = tr.conn_->Call(mkCommand(cproto::kCmdRollbackTx, &ctx), tr.txId_).Status();

cpp_src/client/cororpcclient.h

@@ -73,7 +73,7 @@ class CoroRPCClient {
 	Error Status(const InternalRdxContext& ctx);
 
 	CoroTransaction NewTransaction(std::string_view nsName, const InternalRdxContext& ctx);
-	Error CommitTransaction(CoroTransaction& tr, const InternalRdxContext& ctx);
+	Error CommitTransaction(CoroTransaction& tr, CoroQueryResults& result, const InternalRdxContext& ctx);
 	Error RollBackTransaction(CoroTransaction& tr, const InternalRdxContext& ctx);
 
 protected: