[feature](decimal) support decimal256

be/src/common/consts.h

@@ -30,8 +30,20 @@ const std::string ROWID_COL = "__DORIS_ROWID_COL__";
 const std::string ROW_STORE_COL = "__DORIS_ROW_STORE_COL__";
 const std::string DYNAMIC_COLUMN_NAME = "__DORIS_DYNAMIC_COL__";
 
+/// The maximum precision representable by a 4-byte decimal (Decimal4Value)
 constexpr int MAX_DECIMAL32_PRECISION = 9;
+/// The maximum precision representable by a 8-byte decimal (Decimal8Value)
 constexpr int MAX_DECIMAL64_PRECISION = 18;
+/// The maximum precision representable by a 16-byte decimal
 constexpr int MAX_DECIMAL128_PRECISION = 38;
+/// The maximum precision representable by a 32-byte decimal
+constexpr int MAX_DECIMAL256_PRECISION = 76;
+
+/// Must be kept in sync with FE's max precision/scale.
+static constexpr int MAX_DECIMALV2_PRECISION = MAX_DECIMAL128_PRECISION;
+static constexpr int MAX_DECIMALV2_SCALE = MAX_DECIMALV2_PRECISION;
+
+static constexpr int MAX_DECIMALV3_PRECISION = MAX_DECIMAL256_PRECISION;
+static constexpr int MAX_DECIMALV3_SCALE = MAX_DECIMALV3_PRECISION;
 } // namespace BeConsts
 } // namespace doris

be/src/exec/olap_common.h

@@ -55,6 +55,8 @@ std::string cast_to_string(T value, int scale) {
         return ((vectorized::Decimal<int64_t>)value).to_string(scale);
     } else if constexpr (primitive_type == TYPE_DECIMAL128I) {
         return ((vectorized::Decimal<int128_t>)value).to_string(scale);
+    } else if constexpr (primitive_type == TYPE_DECIMAL256) {
+        return ((vectorized::Decimal<Int256>)value).to_string(scale);
     } else if constexpr (primitive_type == TYPE_TINYINT) {
         return std::to_string(static_cast<int>(value));
     } else if constexpr (primitive_type == TYPE_LARGEINT) {
@@ -503,16 +505,15 @@ class OlapScanKeys {
     bool _is_convertible;
 };
 
-using ColumnValueRangeType =
-        std::variant<ColumnValueRange<TYPE_TINYINT>, ColumnValueRange<TYPE_SMALLINT>,
-                     ColumnValueRange<TYPE_INT>, ColumnValueRange<TYPE_BIGINT>,
-                     ColumnValueRange<TYPE_LARGEINT>, ColumnValueRange<TYPE_CHAR>,
-                     ColumnValueRange<TYPE_VARCHAR>, ColumnValueRange<TYPE_STRING>,
-                     ColumnValueRange<TYPE_DATE>, ColumnValueRange<TYPE_DATEV2>,
-                     ColumnValueRange<TYPE_DATETIME>, ColumnValueRange<TYPE_DATETIMEV2>,
-                     ColumnValueRange<TYPE_DECIMALV2>, ColumnValueRange<TYPE_BOOLEAN>,
-                     ColumnValueRange<TYPE_HLL>, ColumnValueRange<TYPE_DECIMAL32>,
-                     ColumnValueRange<TYPE_DECIMAL64>, ColumnValueRange<TYPE_DECIMAL128I>>;
+using ColumnValueRangeType = std::variant<
+        ColumnValueRange<TYPE_TINYINT>, ColumnValueRange<TYPE_SMALLINT>, ColumnValueRange<TYPE_INT>,
+        ColumnValueRange<TYPE_BIGINT>, ColumnValueRange<TYPE_LARGEINT>, ColumnValueRange<TYPE_CHAR>,
+        ColumnValueRange<TYPE_VARCHAR>, ColumnValueRange<TYPE_STRING>, ColumnValueRange<TYPE_DATE>,
+        ColumnValueRange<TYPE_DATEV2>, ColumnValueRange<TYPE_DATETIME>,
+        ColumnValueRange<TYPE_DATETIMEV2>, ColumnValueRange<TYPE_DECIMALV2>,
+        ColumnValueRange<TYPE_BOOLEAN>, ColumnValueRange<TYPE_HLL>,
+        ColumnValueRange<TYPE_DECIMAL32>, ColumnValueRange<TYPE_DECIMAL64>,
+        ColumnValueRange<TYPE_DECIMAL128I>, ColumnValueRange<TYPE_DECIMAL256>>;
 
 template <PrimitiveType primitive_type>
 const typename ColumnValueRange<primitive_type>::CppType

be/src/exec/schema_scanner.cpp

@@ -287,6 +287,12 @@ Status SchemaScanner::fill_dest_column_for_range(vectorized::Block* block, size_
                     reinterpret_cast<const char*>(&num), 0);
             break;
         }
+            // case TYPE_DECIMAL256: {
+            //     const vectorized::Int256 num = (reinterpret_cast<PackedInt256*>(data))->value;
+            //     reinterpret_cast<vectorized::ColumnDecimal256*>(col_ptr)->insert_data(
+            //             reinterpret_cast<const char*>(&num), 0);
+            //     break;
+            // }
 
         case TYPE_DECIMAL32: {
             const int32_t num = *reinterpret_cast<int32_t*>(data);

be/src/exec/schema_scanner/schema_columns_scanner.cpp

@@ -138,6 +138,7 @@ std::string SchemaColumnsScanner::_to_mysql_data_type_string(TColumnDesc& desc)
     case TPrimitiveType::DECIMAL32:
     case TPrimitiveType::DECIMAL64:
     case TPrimitiveType::DECIMAL128I:
+    case TPrimitiveType::DECIMAL256:
     case TPrimitiveType::DECIMALV2: {
         return "decimal";
     }
@@ -208,7 +209,8 @@ std::string SchemaColumnsScanner::_type_to_string(TColumnDesc& desc) {
     }
     case TPrimitiveType::DECIMAL32:
     case TPrimitiveType::DECIMAL64:
-    case TPrimitiveType::DECIMAL128I: {
+    case TPrimitiveType::DECIMAL128I:
+    case TPrimitiveType::DECIMAL256: {
         fmt::memory_buffer debug_string_buffer;
         fmt::format_to(
                 debug_string_buffer, "decimalv3({}, {})",

be/src/exec/table_connector.cpp

@@ -250,7 +250,8 @@ Status TableConnector::convert_column_data(const vectorized::ColumnPtr& column_p
     }
     case TYPE_DECIMAL32:
     case TYPE_DECIMAL64:
-    case TYPE_DECIMAL128I: {
+    case TYPE_DECIMAL128I:
+    case TYPE_DECIMAL256: {
         auto decimal_type = remove_nullable(type_ptr);
         auto val = decimal_type->to_string(*column, row);
         fmt::format_to(_insert_stmt_buffer, "{}", val);

be/src/exec/text_converter.cpp

@@ -41,6 +41,7 @@
 #include "vec/columns/column_struct.h"
 #include "vec/columns/column_vector.h"
 #include "vec/core/types.h"
+#include "vec/core/wide_integer.h"
 #include "vec/runtime/vdatetime_value.h"
 
 namespace doris {
@@ -290,6 +291,18 @@ bool TextConverter::_write_data(const TypeDescriptor& type_desc,
                 .resize_fill(origin_size + rows, value);
         break;
     }
+    case TYPE_DECIMAL256: {
+        StringParser::ParseResult result = StringParser::PARSE_SUCCESS;
+        wide::Int256 value = StringParser::string_to_decimal<TYPE_DECIMAL256>(
+                data, len, type_desc.precision, type_desc.scale, &result);
+        if (result != StringParser::PARSE_SUCCESS) {
+            parse_result = StringParser::PARSE_FAILURE;
+            break;
+        }
+        reinterpret_cast<vectorized::ColumnVector<wide::Int256>*>(col_ptr)->get_data().resize_fill(
+                origin_size + rows, value);
+        break;
+    }
     case TYPE_ARRAY: {
         auto col = reinterpret_cast<vectorized::ColumnArray*>(col_ptr);
 

be/src/exprs/create_predicate_function.h

@@ -34,10 +34,11 @@ class MinmaxFunctionTraits {
     using BasePtr = MinMaxFuncBase*;
     template <PrimitiveType type, size_t N>
     static BasePtr get_function() {
-        return new MinMaxNumFunc<std::conditional_t<
-                type == TYPE_DECIMAL32 || type == TYPE_DECIMAL64 || type == TYPE_DECIMAL128I,
-                vectorized::Decimal<typename PrimitiveTypeTraits<type>::CppType>,
-                typename PrimitiveTypeTraits<type>::CppType>>();
+        return new MinMaxNumFunc<
+                std::conditional_t<type == TYPE_DECIMAL32 || type == TYPE_DECIMAL64 ||
+                                           type == TYPE_DECIMAL128I || type == TYPE_DECIMAL256,
+                                   vectorized::Decimal<typename PrimitiveTypeTraits<type>::CppType>,
+                                   typename PrimitiveTypeTraits<type>::CppType>>();
     }
 };
 
@@ -106,7 +107,8 @@ class PredicateFunctionCreator {
     M(TYPE_STRING)            \
     M(TYPE_DECIMAL32)         \
     M(TYPE_DECIMAL64)         \
-    M(TYPE_DECIMAL128I)
+    M(TYPE_DECIMAL128I)       \
+    M(TYPE_DECIMAL256)
 
 template <class Traits, size_t N = 0>
 typename Traits::BasePtr create_predicate_function(PrimitiveType type) {

be/src/exprs/runtime_filter.cpp

@@ -99,6 +99,8 @@ PColumnType to_proto(PrimitiveType type) {
         return PColumnType::COLUMN_TYPE_DECIMAL64;
     case TYPE_DECIMAL128I:
         return PColumnType::COLUMN_TYPE_DECIMAL128I;
+    case TYPE_DECIMAL256:
+        return PColumnType::COLUMN_TYPE_DECIMAL256;
     case TYPE_CHAR:
         return PColumnType::COLUMN_TYPE_CHAR;
     case TYPE_VARCHAR:
@@ -148,6 +150,8 @@ PrimitiveType to_primitive_type(PColumnType type) {
         return TYPE_DECIMAL64;
     case PColumnType::COLUMN_TYPE_DECIMAL128I:
         return TYPE_DECIMAL128I;
+    case PColumnType::COLUMN_TYPE_DECIMAL256:
+        return TYPE_DECIMAL256;
     case PColumnType::COLUMN_TYPE_VARCHAR:
         return TYPE_VARCHAR;
     case PColumnType::COLUMN_TYPE_CHAR:

be/src/gutil/endian.h

@@ -35,6 +35,7 @@
 #include "gutil/int128.h"
 #include "gutil/integral_types.h"
 #include "gutil/port.h"
+#include "vec/core/wide_integer.h"
 
 inline uint64 gbswap_64(uint64 host_int) {
 #if defined(__GNUC__) && defined(__x86_64__) && !defined(__APPLE__)
@@ -59,6 +60,11 @@ inline unsigned __int128 gbswap_128(unsigned __int128 host_int) {
            (static_cast<unsigned __int128>(bswap_64(static_cast<uint64>(host_int))) << 64);
 }
 
+inline wide::UInt256 gbswap_256(wide::UInt256 host_int) {
+    wide::UInt256 result{gbswap_64(host_int.items[0]), gbswap_64(host_int.items[1]), gbswap_64(host_int.items[2]), gbswap_64(host_int.items[3])};
+    return result;
+}
+
 // Swap bytes of a 24-bit value.
 inline uint32_t bswap_24(uint32_t x) {
     return ((x & 0x0000ffULL) << 16) | ((x & 0x00ff00ULL)) | ((x & 0xff0000ULL) >> 16);
@@ -252,6 +258,9 @@ class BigEndian {
     static unsigned __int128 FromHost128(unsigned __int128 x) { return gbswap_128(x); }
     static unsigned __int128 ToHost128(unsigned __int128 x) { return gbswap_128(x); }
 
+    static wide::UInt256 FromHost256(wide::UInt256 x) { return gbswap_256(x); }
+    static wide::UInt256 ToHost256(wide::UInt256 x) { return gbswap_256(x); }
+
     static bool IsLittleEndian() { return true; }
 
 #elif defined IS_BIG_ENDIAN
@@ -271,6 +280,9 @@ class BigEndian {
     static uint128 FromHost128(uint128 x) { return x; }
     static uint128 ToHost128(uint128 x) { return x; }
 
+    static wide::UInt256 FromHost256(wide::UInt256 x) { return x; }
+    static wide::UInt256 ToHost256(wide::UInt256 x) { return x; }
+
     static bool IsLittleEndian() { return false; }
 
 #endif /* ENDIAN */

be/src/olap/delete_handler.cpp

@@ -199,6 +199,8 @@ bool DeleteHandler::is_condition_value_valid(const TabletColumn& column,
         return valid_decimal(value_str, column.precision(), column.frac());
     case FieldType::OLAP_FIELD_TYPE_DECIMAL128I:
         return valid_decimal(value_str, column.precision(), column.frac());
+    case FieldType::OLAP_FIELD_TYPE_DECIMAL256:
+        return valid_decimal(value_str, column.precision(), column.frac());
     case FieldType::OLAP_FIELD_TYPE_CHAR:
     case FieldType::OLAP_FIELD_TYPE_VARCHAR:
         return value_str.size() <= column.length();

be/src/olap/field.h

@@ -589,6 +589,8 @@ class FieldFactory {
                 [[fallthrough]];
             case FieldType::OLAP_FIELD_TYPE_DECIMAL128I:
                 [[fallthrough]];
+            case FieldType::OLAP_FIELD_TYPE_DECIMAL256:
+                [[fallthrough]];
             case FieldType::OLAP_FIELD_TYPE_DATETIMEV2: {
                 Field* field = new Field(column);
                 field->set_precision(column.precision());
@@ -647,6 +649,8 @@ class FieldFactory {
                 [[fallthrough]];
             case FieldType::OLAP_FIELD_TYPE_DECIMAL128I:
                 [[fallthrough]];
+            case FieldType::OLAP_FIELD_TYPE_DECIMAL256:
+                [[fallthrough]];
             case FieldType::OLAP_FIELD_TYPE_DATETIMEV2: {
                 Field* field = new Field(column);
                 field->set_precision(column.precision());

be/src/olap/in_list_predicate.h

@@ -97,7 +97,7 @@ class InListPredicateBase : public ColumnPredicate {
             if constexpr (Type == TYPE_STRING || Type == TYPE_CHAR) {
                 tmp = convert(*col, condition, arena);
             } else if constexpr (Type == TYPE_DECIMAL32 || Type == TYPE_DECIMAL64 ||
-                                 Type == TYPE_DECIMAL128I) {
+                                 Type == TYPE_DECIMAL128I || Type == TYPE_DECIMAL256) {
                 tmp = convert(*col, condition);
             } else {
                 tmp = convert(condition);

be/src/olap/key_coder.cpp

@@ -80,6 +80,7 @@ class KeyCoderResolver {
         add_mapping<FieldType::OLAP_FIELD_TYPE_DECIMAL32>();
         add_mapping<FieldType::OLAP_FIELD_TYPE_DECIMAL64>();
         add_mapping<FieldType::OLAP_FIELD_TYPE_DECIMAL128I>();
+        add_mapping<FieldType::OLAP_FIELD_TYPE_DECIMAL256>();
     }
 
     template <FieldType field_type>

be/src/olap/key_coder.h

@@ -85,6 +85,7 @@ class KeyCoderTraits<
         field_type,
         typename std::enable_if<
                 std::is_integral<typename CppTypeTraits<field_type>::CppType>::value ||
+                field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL256 ||
                 vectorized::IsDecimalNumber<typename CppTypeTraits<field_type>::CppType>>::type> {
 public:
     using CppType = typename CppTypeTraits<field_type>::CppType;
@@ -93,20 +94,24 @@ class KeyCoderTraits<
 private:
     // Swap value's endian from/to big endian
     static UnsignedCppType swap_big_endian(UnsignedCppType val) {
-        switch (sizeof(UnsignedCppType)) {
-        case 1:
-            return val;
-        case 2:
-            return BigEndian::FromHost16(val);
-        case 4:
-            return BigEndian::FromHost32(val);
-        case 8:
-            return BigEndian::FromHost64(val);
-        case 16:
-            return BigEndian::FromHost128(val);
-        default:
-            LOG(FATAL) << "Invalid type to big endian, type=" << int(field_type)
-                       << ", size=" << sizeof(UnsignedCppType);
+        if constexpr (field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL256) {
+            return BigEndian::FromHost256(val);
+        } else {
+            switch (sizeof(UnsignedCppType)) {
+            case 1:
+                return val;
+            case 2:
+                return BigEndian::FromHost16(val);
+            case 4:
+                return BigEndian::FromHost32(val);
+            case 8:
+                return BigEndian::FromHost64(val);
+            case 16:
+                return BigEndian::FromHost128(val);
+            default:
+                LOG(FATAL) << "Invalid type to big endian, type=" << int(field_type)
+                           << ", size=" << sizeof(UnsignedCppType);
+            }
         }
     }
 

be/src/olap/olap_common.h

@@ -143,7 +143,8 @@ enum class FieldType {
     OLAP_FIELD_TYPE_DECIMAL128I = 33,
     OLAP_FIELD_TYPE_JSONB = 34,
     OLAP_FIELD_TYPE_VARIANT = 35,
-    OLAP_FIELD_TYPE_AGG_STATE = 36
+    OLAP_FIELD_TYPE_AGG_STATE = 36,
+    OLAP_FIELD_TYPE_DECIMAL256 = 37,
 };
 
 // Define all aggregation methods supported by Field
@@ -197,6 +198,7 @@ constexpr bool field_is_numeric_type(const FieldType& field_type) {
            field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL32 ||
            field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL64 ||
            field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL128I ||
+           field_type == FieldType::OLAP_FIELD_TYPE_DECIMAL256 ||
            field_type == FieldType::OLAP_FIELD_TYPE_BOOL;
 }
 

be/src/olap/predicate_creator.h

@@ -96,8 +96,8 @@ class DecimalPredicateCreator : public PredicateCreator<ConditionType> {
     static CppType convert(const TabletColumn& column, const std::string& condition) {
         StringParser::ParseResult result = StringParser::ParseResult::PARSE_SUCCESS;
         // return CppType value cast from int128_t
-        return StringParser::string_to_decimal<Type>(condition.data(), condition.size(),
-                                                     column.precision(), column.frac(), &result);
+        return CppType(StringParser::string_to_decimal<Type>(
+                condition.data(), condition.size(), column.precision(), column.frac(), &result));
     }
 };
 
@@ -195,6 +195,9 @@ std::unique_ptr<PredicateCreator<ConditionType>> get_creator(const FieldType& ty
     case FieldType::OLAP_FIELD_TYPE_DECIMAL128I: {
         return std::make_unique<DecimalPredicateCreator<TYPE_DECIMAL128I, PT, ConditionType>>();
     }
+    case FieldType::OLAP_FIELD_TYPE_DECIMAL256: {
+        return std::make_unique<DecimalPredicateCreator<TYPE_DECIMAL256, PT, ConditionType>>();
+    }
     case FieldType::OLAP_FIELD_TYPE_CHAR: {
         return std::make_unique<StringPredicateCreator<TYPE_CHAR, PT, ConditionType>>();
     }

be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp

@@ -247,6 +247,9 @@ Status BitmapIndexWriter::create(const TypeInfo* type_info,
     case FieldType::OLAP_FIELD_TYPE_DECIMAL128I:
         res->reset(new BitmapIndexWriterImpl<FieldType::OLAP_FIELD_TYPE_DECIMAL128I>(type_info));
         break;
+    case FieldType::OLAP_FIELD_TYPE_DECIMAL256:
+        res->reset(new BitmapIndexWriterImpl<FieldType::OLAP_FIELD_TYPE_DECIMAL256>(type_info));
+        break;
     case FieldType::OLAP_FIELD_TYPE_BOOL:
         res->reset(new BitmapIndexWriterImpl<FieldType::OLAP_FIELD_TYPE_BOOL>(type_info));
         break;

be/src/olap/rowset/segment_v2/bitshuffle_page.h

@@ -267,6 +267,7 @@ inline Status parse_bit_shuffle_header(const Slice& data, size_t& num_elements,
     case 8:
     case 12:
     case 16:
+    case 32:
         break;
     default:
         return Status::InternalError("invalid size_of_elem:{}", size_of_element);

be/src/olap/rowset/segment_v2/bloom_filter_index_writer.cpp

@@ -315,6 +315,7 @@ Status BloomFilterIndexWriter::create(const BloomFilterOptions& bf_options,
         M(FieldType::OLAP_FIELD_TYPE_DECIMAL32)
         M(FieldType::OLAP_FIELD_TYPE_DECIMAL64)
         M(FieldType::OLAP_FIELD_TYPE_DECIMAL128I)
+        M(FieldType::OLAP_FIELD_TYPE_DECIMAL256)
 #undef M
     default:
         return Status::NotSupported("unsupported type for bitmap index: {}",

be/src/olap/rowset/segment_v2/encoding_info.cpp

@@ -321,6 +321,10 @@ EncodingInfoResolver::EncodingInfoResolver() {
     _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL128I, PLAIN_ENCODING>();
     _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL128I, BIT_SHUFFLE, true>();
 
+    _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL256, BIT_SHUFFLE>();
+    _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL256, PLAIN_ENCODING>();
+    _add_map<FieldType::OLAP_FIELD_TYPE_DECIMAL256, BIT_SHUFFLE, true>();
+
     _add_map<FieldType::OLAP_FIELD_TYPE_HLL, PLAIN_ENCODING>();
 
     _add_map<FieldType::OLAP_FIELD_TYPE_OBJECT, PLAIN_ENCODING>();

be/src/olap/rowset/segment_v2/inverted_index_writer.cpp

@@ -624,6 +624,12 @@ Status InvertedIndexColumnWriter::create(const Field* field,
                 field_name, segment_file_name, dir, fs, index_meta);
         break;
     }
+    case FieldType::OLAP_FIELD_TYPE_DECIMAL256: {
+        *res = std::make_unique<
+                InvertedIndexColumnWriterImpl<FieldType::OLAP_FIELD_TYPE_DECIMAL256>>(
+                field_name, segment_file_name, dir, fs, index_meta);
+        break;
+    }
     case FieldType::OLAP_FIELD_TYPE_BOOL: {
         *res = std::make_unique<InvertedIndexColumnWriterImpl<FieldType::OLAP_FIELD_TYPE_BOOL>>(
                 field_name, segment_file_name, dir, fs, index_meta);

be/src/olap/rowset/segment_v2/zone_map_index.cpp

@@ -200,7 +200,8 @@ Status ZoneMapIndexReader::_load(bool use_page_cache, bool kept_in_memory,
     M(TYPE_STRING)               \
     M(TYPE_DECIMAL32)            \
     M(TYPE_DECIMAL64)            \
-    M(TYPE_DECIMAL128I)
+    M(TYPE_DECIMAL128I)          \
+    M(TYPE_DECIMAL256)
 
 Status ZoneMapIndexWriter::create(Field* field, std::unique_ptr<ZoneMapIndexWriter>& res) {
     switch (field->type()) {