Merge branch 'branch-2.0' of github.com:apache/doris into branch-2.0-…

…fix-not-hit-partition

.gitignore

@@ -94,7 +94,6 @@ be/tags
 be/test/olap/test_data/tablet_meta_test.hdr
 be/.devcontainer/
 be/src/apache-orc/
-zoneinfo/
 
 ## tools
 tools/ssb-tools/ssb-data/

be/src/common/config.cpp

@@ -1129,10 +1129,6 @@ DEFINE_Bool(exit_on_exception, "false");
 
 DEFINE_Bool(ignore_always_true_predicate_for_segment, "true");
 
-// Dir of default timezone files
-DEFINE_String(default_tzfiles_path, "${DORIS_HOME}/zoneinfo");
-DEFINE_Bool(use_doris_tzfile, "false");
-
 // the max package bytes be thrift server can receive
 // avoid accepting error or too large package causing OOM,default 20000000(20M)
 DEFINE_Int32(be_thrift_max_pkg_bytes, "20000000");

be/src/common/config.h

@@ -1179,10 +1179,6 @@ DECLARE_mBool(exit_on_exception);
 // Remove predicate that is always true for a segment.
 DECLARE_Bool(ignore_always_true_predicate_for_segment);
 
-// Dir of default timezone files
-DECLARE_String(default_tzfiles_path);
-DECLARE_Bool(use_doris_tzfile);
-
 // the max package bytes be thrift server can receive
 // avoid accepting error or too large package causing OOM,default 20000000(20M)
 DECLARE_Int32(be_thrift_max_pkg_bytes);

be/src/exec/olap_common.h

@@ -43,6 +43,7 @@
 #include "runtime/type_limit.h"
 #include "vec/core/types.h"
 #include "vec/io/io_helper.h"
+#include "vec/runtime/time_value.h"
 #include "vec/runtime/vdatetime_value.h"
 
 namespace doris {
@@ -68,6 +69,8 @@ std::string cast_to_string(T value, int scale) {
         std::stringstream ss;
         ss << buf;
         return ss.str();
+    } else if constexpr (primitive_type == TYPE_TIMEV2) {
+        return TimeValue::to_string(value, scale);
     } else {
         return boost::lexical_cast<std::string>(value);
     }

be/src/http/action/stream_load.cpp

@@ -140,7 +140,7 @@ Status StreamLoadAction::_handle(std::shared_ptr<StreamLoadContext> ctx) {
     if (ctx->body_bytes > 0 && ctx->receive_bytes != ctx->body_bytes) {
         LOG(WARNING) << "recevie body don't equal with body bytes, body_bytes=" << ctx->body_bytes
                      << ", receive_bytes=" << ctx->receive_bytes << ", id=" << ctx->id;
-        return Status::InternalError("receive body don't equal with body bytes");
+        return Status::InternalError<false>("receive body don't equal with body bytes");
     }
     if (!ctx->use_streaming) {
         // if we use non-streaming, we need to close file first,
@@ -218,13 +218,13 @@ Status StreamLoadAction::_on_header(HttpRequest* http_req, std::shared_ptr<Strea
     // auth information
     if (!parse_basic_auth(*http_req, &ctx->auth)) {
         LOG(WARNING) << "parse basic authorization failed." << ctx->brief();
-        return Status::InternalError("no valid Basic authorization");
+        return Status::InternalError<false>("no valid Basic authorization");
     }
 
     // get format of this put
     if (!http_req->header(HTTP_COMPRESS_TYPE).empty() &&
         iequal(http_req->header(HTTP_FORMAT_KEY), "JSON")) {
-        return Status::InternalError("compress data of JSON format is not supported.");
+        return Status::InternalError<false>("compress data of JSON format is not supported.");
     }
     std::string format_str = http_req->header(HTTP_FORMAT_KEY);
     if (iequal(format_str, BeConsts::CSV_WITH_NAMES) ||
@@ -240,8 +240,8 @@ Status StreamLoadAction::_on_header(HttpRequest* http_req, std::shared_ptr<Strea
     LoadUtil::parse_format(format_str, http_req->header(HTTP_COMPRESS_TYPE), &ctx->format,
                            &ctx->compress_type);
     if (ctx->format == TFileFormatType::FORMAT_UNKNOWN) {
-        return Status::InternalError("unknown data format, format={}",
-                                     http_req->header(HTTP_FORMAT_KEY));
+        return Status::InternalError<false>("unknown data format, format={}",
+                                            http_req->header(HTTP_FORMAT_KEY));
     }
 
     // check content length
@@ -259,16 +259,16 @@ Status StreamLoadAction::_on_header(HttpRequest* http_req, std::shared_ptr<Strea
         // json max body size
         if ((ctx->format == TFileFormatType::FORMAT_JSON) &&
             (ctx->body_bytes > json_max_body_bytes) && !read_json_by_line) {
-            return Status::InternalError(
+            return Status::InternalError<false>(
                     "The size of this batch exceed the max size [{}]  of json type data "
                     " data [ {} ]. Split the file, or use 'read_json_by_line'",
                     json_max_body_bytes, ctx->body_bytes);
         }
         // csv max body size
         else if (ctx->body_bytes > csv_max_body_bytes) {
             LOG(WARNING) << "body exceed max size." << ctx->brief();
-            return Status::InternalError("body exceed max size: {}, data: {}", csv_max_body_bytes,
-                                         ctx->body_bytes);
+            return Status::InternalError<false>("body exceed max size: {}, data: {}",
+                                                csv_max_body_bytes, ctx->body_bytes);
         }
     } else {
 #ifndef BE_TEST
@@ -653,5 +653,4 @@ void StreamLoadAction::_save_stream_load_record(std::shared_ptr<StreamLoadContex
         LOG(WARNING) << "put stream_load_record rocksdb failed. stream_load_recorder is null.";
     }
 }
-
 } // namespace doris

be/src/olap/compaction.cpp

@@ -189,7 +189,7 @@ bool Compaction::is_rowset_tidy(std::string& pre_max_key, const RowsetSharedPtr&
     if (!ret) {
         return false;
     }
-    if (min_key < pre_max_key) {
+    if (min_key <= pre_max_key) {
         return false;
     }
     CHECK(rhs->max_key(&pre_max_key));

be/src/olap/rowset/segment_v2/segment_iterator.cpp

@@ -320,12 +320,18 @@ Status SegmentIterator::_init_impl(const StorageReadOptions& opts) {
         std::set<std::string> push_down_preds;
         for (auto* pred : _col_predicates) {
             if (!_check_apply_by_inverted_index(pred)) {
+                //column predicate, like column predicate etc. always need read data
+                auto cid = pred->column_id();
+                _need_read_data_indices[cid] = true;
                 continue;
             }
             push_down_preds.insert(_gen_predicate_result_sign(pred));
         }
         for (auto* pred : _col_preds_except_leafnode_of_andnode) {
             if (!_check_apply_by_inverted_index(pred)) {
+                //column predicate, like column predicate etc. always need read data
+                auto cid = pred->column_id();
+                _need_read_data_indices[cid] = true;
                 continue;
             }
             push_down_preds.insert(_gen_predicate_result_sign(pred));

be/src/runtime/exec_env_init.cpp

@@ -118,7 +118,6 @@ Status ExecEnv::_init(const std::vector<StorePath>& store_paths) {
     _frontend_client_cache = new FrontendServiceClientCache(config::max_client_cache_size_per_host);
     _broker_client_cache = new BrokerServiceClientCache(config::max_client_cache_size_per_host);
 
-    TimezoneUtils::load_timezone_names();
     TimezoneUtils::load_timezones_to_cache();
 
     ThreadPoolBuilder("SendBatchThreadPool")

be/src/runtime/runtime_predicate.cpp

@@ -31,6 +31,12 @@ namespace doris {
 
 namespace vectorized {
 
+std::string get_time_value(const Field& field) {
+    using ValueType = typename PrimitiveTypeTraits<TYPE_TIMEV2>::CppType;
+    ValueType value = field.get<ValueType>();
+    return cast_to_string<TYPE_TIMEV2, ValueType>(value, 0);
+}
+
 Status RuntimePredicate::init(const PrimitiveType type, const bool nulls_first) {
     std::unique_lock<std::shared_mutex> wlock(_rwlock);
 
@@ -98,6 +104,10 @@ Status RuntimePredicate::init(const PrimitiveType type, const bool nulls_first)
         _get_value_fn = get_datetime_value;
         break;
     }
+    case PrimitiveType::TYPE_TIMEV2: {
+        _get_value_fn = get_time_value;
+        break;
+    }
     case PrimitiveType::TYPE_DECIMAL32: {
         _get_value_fn = get_decimal32_value;
         break;

be/src/runtime/snapshot_loader.cpp

@@ -439,7 +439,6 @@ Status SnapshotLoader::remote_http_download(
     }
 
     // Step 3: Validate remote tablet snapshot paths && remote files map
-    // TODO(Drogon): Add md5sum check
     // key is remote snapshot paths, value is filelist
     // get all these use http download action
     // http://172.16.0.14:6781/api/_tablet/_download?token=e804dd27-86da-4072-af58-70724075d2a4&file=/home/ubuntu/doris_master/output/be/storage/snapshot/20230410102306.9.180//2774718/217609978/2774718.hdr