diff --git a/cloud/src/meta-service/meta_service_helper.h b/cloud/src/meta-service/meta_service_helper.h index 9e9ff38c2ecda0..8a1a53061949e0 100644 --- a/cloud/src/meta-service/meta_service_helper.h +++ b/cloud/src/meta-service/meta_service_helper.h @@ -19,7 +19,9 @@ #include #include +#include +#include #include #include #include @@ -29,12 +31,26 @@ #include "common/logging.h" #include "common/stopwatch.h" #include "common/util.h" +#include "cpp/sync_point.h" #include "meta-service/keys.h" #include "meta-service/txn_kv.h" #include "meta-service/txn_kv_error.h" #include "resource-manager/resource_manager.h" namespace doris::cloud { +inline std::string md5(const std::string& str) { + unsigned char digest[MD5_DIGEST_LENGTH]; + MD5_CTX context; + MD5_Init(&context); + MD5_Update(&context, str.c_str(), str.length()); + MD5_Final(digest, &context); + + std::ostringstream ss; + for (unsigned char i : digest) { + ss << std::setw(2) << std::setfill('0') << std::hex << (int)i; + } + return ss.str(); +} template void begin_rpc(std::string_view func_name, brpc::Controller* ctrl, const Request* req) { @@ -101,7 +117,34 @@ void finish_rpc(std::string_view func_name, brpc::Controller* ctrl, Response* re LOG(INFO) << "finish " << func_name << " from " << ctrl->remote_side() << " status=" << res->status().ShortDebugString() << " delete_bitmap_size=" << res->segment_delete_bitmaps_size(); - + } else if constexpr (std::is_same_v || + std::is_same_v) { + std::string debug_string = res->DebugString(); + // Start position for searching "sk" fields + size_t pos = 0; + // Iterate through the string and find all occurrences of "sk: " + while ((pos = debug_string.find("sk: ", pos)) != std::string::npos) { + // Find the start and end of the "sk" value (assumed to be within quotes) + // Start after the quote + size_t sk_value_start = debug_string.find('\"', pos) + 1; + // End at the next quote + size_t sk_value_end = debug_string.find('\"', sk_value_start); + + // Extract the "sk" value + std::string sk_value = + debug_string.substr(sk_value_start, sk_value_end - sk_value_start); + // Encrypt the "sk" value with MD5 + std::string encrypted_sk = "md5: " + md5(sk_value); + + // Replace the original "sk" value with the encrypted MD5 value + debug_string.replace(sk_value_start, sk_value_end - sk_value_start, encrypted_sk); + + // Move the position to the end of the current "sk" field and continue searching + pos = sk_value_end; + } + TEST_SYNC_POINT_CALLBACK("sk_finish_rpc", &debug_string); + LOG(INFO) << "finish " << func_name << " from " << ctrl->remote_side() + << " response=" << debug_string; } else { LOG(INFO) << "finish " << func_name << " from " << ctrl->remote_side() << " response=" << res->ShortDebugString(); diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp index 92020005c3a5d0..cc459c090bfd28 100644 --- a/cloud/src/meta-service/meta_service_resource.cpp +++ b/cloud/src/meta-service/meta_service_resource.cpp @@ -203,6 +203,8 @@ void MetaServiceImpl::get_obj_store_info(google::protobuf::RpcController* contro GetObjStoreInfoResponse* response, ::google::protobuf::Closure* done) { RPC_PREPROCESS(get_obj_store_info); + TEST_SYNC_POINT_CALLBACK("obj-store-info_sk_response", &response); + TEST_SYNC_POINT_RETURN_WITH_VOID("obj-store-info_sk_response_return"); // Prepare data std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { @@ -2600,6 +2602,8 @@ void MetaServiceImpl::get_stage(google::protobuf::RpcController* controller, const GetStageRequest* request, GetStageResponse* response, ::google::protobuf::Closure* done) { RPC_PREPROCESS(get_stage); + TEST_SYNC_POINT_CALLBACK("stage_sk_response", &response); + TEST_SYNC_POINT_RETURN_WITH_VOID("stage_sk_response_return"); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; diff --git a/cloud/test/meta_service_http_test.cpp b/cloud/test/meta_service_http_test.cpp index e49628fcb3a783..d1b8fd66943a20 100644 --- a/cloud/test/meta_service_http_test.cpp +++ b/cloud/test/meta_service_http_test.cpp @@ -1456,4 +1456,83 @@ TEST(MetaServiceHttpTest, TxnLazyCommit) { } } +TEST(MetaServiceHttpTest, get_stage_response_sk) { + auto sp = SyncPoint::get_instance(); + sp->enable_processing(); + std::unique_ptr> defer((int*)0x01, + [&](...) { sp->disable_processing(); }); + + GetStageResponse res; + auto* stage = res.add_stage(); + stage->mutable_obj_info()->set_ak("stage-ak"); + stage->mutable_obj_info()->set_sk("stage-sk"); + auto foo = [res](auto args) { (*(try_any_cast(args[0])))->CopyFrom(res); }; + sp->set_call_back("stage_sk_response", foo); + sp->set_call_back("stage_sk_response_return", + [](auto&& args) { *try_any_cast(args.back()) = true; }); + + auto rate_limiter = std::make_shared(); + + auto ms = std::make_unique(nullptr, nullptr, rate_limiter); + + auto bar = [](auto args) { + std::cout << *try_any_cast(args[0]); + + EXPECT_TRUE((*try_any_cast(args[0])).find("stage-sk") == std::string::npos); + EXPECT_TRUE((*try_any_cast(args[0])) + .find("md5: f497d053066fa4b7d3b1f6564597d233") != std::string::npos); + }; + sp->set_call_back("sk_finish_rpc", bar); + + GetStageResponse res1; + GetStageRequest req1; + brpc::Controller cntl; + ms->get_stage(&cntl, &req1, &res1, nullptr); +} + +TEST(MetaServiceHttpTest, get_obj_store_info_response_sk) { + auto sp = SyncPoint::get_instance(); + sp->enable_processing(); + std::unique_ptr> defer((int*)0x01, + [&](...) { sp->disable_processing(); }); + + GetObjStoreInfoResponse res; + auto* obj_info = res.add_obj_info(); + obj_info->set_ak("obj-store-info-ak1"); + obj_info->set_sk("obj-store-info-sk1"); + obj_info = res.add_storage_vault()->mutable_obj_info(); + obj_info->set_ak("obj-store-info-ak2"); + obj_info->set_sk("obj-store-info-sk2"); + auto foo = [res](auto args) { + (*(try_any_cast(args[0])))->CopyFrom(res); + }; + sp->set_call_back("obj-store-info_sk_response", foo); + sp->set_call_back("obj-store-info_sk_response_return", + [](auto&& args) { *try_any_cast(args.back()) = true; }); + + auto rate_limiter = std::make_shared(); + + auto ms = std::make_unique(nullptr, nullptr, rate_limiter); + + auto bar = [](auto args) { + std::cout << *try_any_cast(args[0]); + + EXPECT_TRUE((*try_any_cast(args[0])).find("obj-store-info-sk1") == + std::string::npos); + EXPECT_TRUE((*try_any_cast(args[0])) + .find("md5: 35d5a637fd9d45a28207a888b751efc4") != std::string::npos); + + EXPECT_TRUE((*try_any_cast(args[0])).find("obj-store-info-sk2") == + std::string::npos); + EXPECT_TRUE((*try_any_cast(args[0])) + .find("md5: 01d7473ae201a2ecdf1f7c064eb81a95") != std::string::npos); + }; + sp->set_call_back("sk_finish_rpc", bar); + + GetObjStoreInfoResponse res1; + GetObjStoreInfoRequest req1; + brpc::Controller cntl; + ms->get_obj_store_info(&cntl, &req1, &res1, nullptr); +} + } // namespace doris::cloud diff --git a/common/cpp/sync_point.cpp b/common/cpp/sync_point.cpp index c2263415d30d5a..02bdce1ec3487a 100644 --- a/common/cpp/sync_point.cpp +++ b/common/cpp/sync_point.cpp @@ -248,8 +248,7 @@ void SyncPoint::Data::disable_processing() { bool SyncPoint::Data::has_point(const std::string& point) { std::unique_lock lock(mutex_); - auto marked_point_iter = marked_thread_id_.find(point); - return marked_point_iter != marked_thread_id_.end(); + return callbacks_.find(point) != callbacks_.end(); } bool SyncPoint::Data::get_enable() { diff --git a/fe/fe-common/pom.xml b/fe/fe-common/pom.xml index c49d13a3c03603..9322286047a8aa 100644 --- a/fe/fe-common/pom.xml +++ b/fe/fe-common/pom.xml @@ -145,6 +145,12 @@ under the License. com.esotericsoftware kryo-shaded + + commons-collections + commons-collections + ${commons-collections.version} + test + doris-fe-common diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index ccc93ed799ade2..993702c4dac57f 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -3249,4 +3249,10 @@ public static int metaServiceRpcRetryTimes() { "For testing purposes, all queries are forcibly forwarded to the master to verify" + "the behavior of forwarding queries."}) public static boolean force_forward_all_queries = false; + + @ConfField(description = {"用于禁用某些SQL，配置项为AST的class simple name列表(例如CreateRepositoryStmt," + + "CreatePolicyCommand)，用逗号间隔开", + "For disabling certain SQL queries, the configuration item is a list of simple class names of AST" + + "(for example CreateRepositoryStmt, CreatePolicyCommand), separated by commas."}) + public static String block_sql_ast_names = ""; } diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/AuthenticationConfig.java b/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/AuthenticationConfig.java index 875ae4542e1193..b580f9ecbe0582 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/AuthenticationConfig.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/AuthenticationConfig.java @@ -17,10 +17,14 @@ package org.apache.doris.common.security.authentication; +import com.google.common.base.Strings; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; public abstract class AuthenticationConfig { + private static final Logger LOG = LogManager.getLogger(AuthenticationConfig.class); public static String HADOOP_USER_NAME = "hadoop.username"; public static String HADOOP_KERBEROS_PRINCIPAL = "hadoop.kerberos.principal"; public static String HADOOP_KERBEROS_KEYTAB = "hadoop.kerberos.keytab"; @@ -42,6 +46,10 @@ public static AuthenticationConfig getKerberosConfig(Configuration conf) { return AuthenticationConfig.getKerberosConfig(conf, HADOOP_KERBEROS_PRINCIPAL, HADOOP_KERBEROS_KEYTAB); } + public static AuthenticationConfig getSimpleAuthenticationConfig(Configuration conf) { + return AuthenticationConfig.createSimpleAuthenticationConfig(conf); + } + /** * get kerberos config from hadoop conf * @param conf config @@ -54,17 +62,35 @@ public static AuthenticationConfig getKerberosConfig(Configuration conf, String krbKeytabKey) { String authentication = conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, null); if (AuthType.KERBEROS.getDesc().equals(authentication)) { - KerberosAuthenticationConfig krbConfig = new KerberosAuthenticationConfig(); - krbConfig.setKerberosPrincipal(conf.get(krbPrincipalKey)); - krbConfig.setKerberosKeytab(conf.get(krbKeytabKey)); - krbConfig.setConf(conf); - krbConfig.setPrintDebugLog(Boolean.parseBoolean(conf.get(DORIS_KRB5_DEBUG, "false"))); - return krbConfig; - } else { - // AuthType.SIMPLE - SimpleAuthenticationConfig simpleAuthenticationConfig = new SimpleAuthenticationConfig(); - simpleAuthenticationConfig.setUsername(conf.get(HADOOP_USER_NAME)); - return simpleAuthenticationConfig; + String principalKey = conf.get(krbPrincipalKey); + String keytabKey = conf.get(krbKeytabKey); + if (!Strings.isNullOrEmpty(principalKey) && !Strings.isNullOrEmpty(keytabKey)) { + KerberosAuthenticationConfig krbConfig = new KerberosAuthenticationConfig(); + krbConfig.setKerberosPrincipal(principalKey); + krbConfig.setKerberosKeytab(keytabKey); + krbConfig.setConf(conf); + krbConfig.setPrintDebugLog(Boolean.parseBoolean(conf.get(DORIS_KRB5_DEBUG, "false"))); + return krbConfig; + } else { + // Due to some historical reasons, `core-size.xml` may be stored in path:`fe/conf`, + // but this file may only contain `hadoop.security.authentication configuration`, + // and no krbPrincipalKey and krbKeytabKey, + // which will cause kerberos initialization failure. + // Now: + // if kerberos is needed, the relevant configuration can be written in the catalog properties, + // if kerberos is not needed, to prevent the influence of historical reasons, + // the following simple authentication method needs to be used. + LOG.warn("{} or {} is null or empty, fallback to simple authentication", + krbPrincipalKey, krbKeytabKey); + } } + return createSimpleAuthenticationConfig(conf); + } + + private static AuthenticationConfig createSimpleAuthenticationConfig(Configuration conf) { + // AuthType.SIMPLE + SimpleAuthenticationConfig simpleAuthenticationConfig = new SimpleAuthenticationConfig(); + simpleAuthenticationConfig.setUsername(conf.get(HADOOP_USER_NAME)); + return simpleAuthenticationConfig; } } diff --git a/fe/fe-common/src/test/java/org/apache/doris/common/security/authentication/AuthenticationTest.java b/fe/fe-common/src/test/java/org/apache/doris/common/security/authentication/AuthenticationTest.java new file mode 100644 index 00000000000000..62606a22a64fcb --- /dev/null +++ b/fe/fe-common/src/test/java/org/apache/doris/common/security/authentication/AuthenticationTest.java @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.security.authentication; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.junit.Assert; +import org.junit.Test; + +public class AuthenticationTest { + + @Test + public void testAuthConf() { + Configuration conf = new Configuration(); + + AuthenticationConfig conf1 = AuthenticationConfig.getKerberosConfig(conf); + Assert.assertEquals(SimpleAuthenticationConfig.class, conf1.getClass()); + + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + + AuthenticationConfig conf2 = AuthenticationConfig.getKerberosConfig(conf); + Assert.assertEquals(SimpleAuthenticationConfig.class, conf2.getClass()); + + conf.set(AuthenticationConfig.HADOOP_KERBEROS_PRINCIPAL, "principal"); + conf.set(AuthenticationConfig.HADOOP_KERBEROS_KEYTAB, "keytab"); + + AuthenticationConfig conf3 = AuthenticationConfig.getKerberosConfig(conf); + Assert.assertEquals(KerberosAuthenticationConfig.class, conf3.getClass()); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 4ed78ab66c9484..b7a0fa5cfbc746 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -98,6 +98,7 @@ import org.apache.doris.task.AgentTaskQueue; import org.apache.doris.task.ClearAlterTask; import org.apache.doris.task.UpdateTabletMetaInfoTask; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; import org.apache.doris.thrift.TStorageFormat; import org.apache.doris.thrift.TStorageMedium; import org.apache.doris.thrift.TTaskType; @@ -2730,12 +2731,15 @@ private boolean processAddIndex(CreateIndexClause alterClause, OlapTable olapTab + " ) already exist."); } } - + boolean disableInvertedIndexV1ForVariant = olapTable.getInvertedIndexFileStorageFormat() + == TInvertedIndexFileStorageFormat.V1 && ConnectContext.get().getSessionVariable() + .getDisableInvertedIndexV1ForVaraint(); for (String col : indexDef.getColumns()) { Column column = olapTable.getColumn(col); if (column != null) { indexDef.checkColumn(column, olapTable.getKeysType(), - olapTable.getTableProperty().getEnableUniqueKeyMergeOnWrite()); + olapTable.getTableProperty().getEnableUniqueKeyMergeOnWrite(), + disableInvertedIndexV1ForVariant); indexDef.getColumnUniqueIds().add(column.getUniqueId()); } else { throw new DdlException("index column does not exist in table. invalid column: " + col); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java index b07424056906b8..90f7d6f72ede1c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java @@ -47,6 +47,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.rewrite.ExprRewriteRule; import org.apache.doris.rewrite.ExprRewriter; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; import com.google.common.base.Strings; import com.google.common.collect.ImmutableSet; @@ -598,7 +599,9 @@ public void analyze(Analyzer analyzer) throws UserException { if (CollectionUtils.isNotEmpty(indexDefs)) { Set distinct = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); Set>> distinctCol = new HashSet<>(); - + boolean disableInvertedIndexV1ForVariant = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat( + new HashMap<>(properties)) == TInvertedIndexFileStorageFormat.V1 + && ConnectContext.get().getSessionVariable().getDisableInvertedIndexV1ForVaraint(); for (IndexDef indexDef : indexDefs) { indexDef.analyze(); if (!engineName.equalsIgnoreCase(DEFAULT_ENGINE_NAME)) { @@ -608,7 +611,8 @@ public void analyze(Analyzer analyzer) throws UserException { boolean found = false; for (Column column : columns) { if (column.getName().equalsIgnoreCase(indexColName)) { - indexDef.checkColumn(column, getKeysDesc().getKeysType(), enableUniqueKeyMergeOnWrite); + indexDef.checkColumn(column, getKeysDesc().getKeysType(), + enableUniqueKeyMergeOnWrite, disableInvertedIndexV1ForVariant); found = true; break; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java index d98a3b93e45dbf..39b32662614f15 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java @@ -212,8 +212,8 @@ public boolean isInvertedIndex() { return (this.indexType == IndexType.INVERTED); } - public void checkColumn(Column column, KeysType keysType, boolean enableUniqueKeyMergeOnWrite) - throws AnalysisException { + public void checkColumn(Column column, KeysType keysType, boolean enableUniqueKeyMergeOnWrite, + boolean disableInvertedIndexV1ForVariant) throws AnalysisException { if (indexType == IndexType.BITMAP || indexType == IndexType.INVERTED || indexType == IndexType.BLOOMFILTER || indexType == IndexType.NGRAM_BF) { String indexColName = column.getName(); @@ -225,6 +225,14 @@ public void checkColumn(Column column, KeysType keysType, boolean enableUniqueKe throw new AnalysisException(colType + " is not supported in " + indexType.toString() + " index. " + "invalid index: " + indexName); } + + // In inverted index format v1, each subcolumn of a variant has its own index file, leading to high IOPS. + // when the subcolumn type changes, it may result in missing files, causing link file failure. + if (colType.isVariantType() && disableInvertedIndexV1ForVariant) { + throw new AnalysisException(colType + " is not supported in inverted index format V1," + + "Please set properties(\"inverted_index_storage_format\"= \"v2\")," + + "or upgrade to a newer version"); + } if (!column.isKey()) { if (keysType == KeysType.AGG_KEYS) { throw new AnalysisException("index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table" diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java index 60d091bf202050..35588568511a3f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java @@ -593,8 +593,13 @@ public long read(DataInputStream dis, long checksum) throws IOException { continue; } - // Step 2.2: check if there is in next db Binlogs region long dbId = binlog.getDbId(); + if (binlog.getType().getValue() >= TBinlogType.MIN_UNKNOWN.getValue()) { + LOG.warn("skip unknown binlog, type: {}, db: {}", binlog.getType().getValue(), dbId); + continue; + } + + // Step 2.2: check if there is in next db Binlogs region if (dbId != currentDbId) { // if there is in next db Binlogs region, check and update metadata Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 9e65e5c866b6a0..f67444b6cb1b7e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -246,6 +246,7 @@ import org.apache.doris.qe.JournalObservable; import org.apache.doris.qe.QueryCancelWorker; import org.apache.doris.qe.SessionVariable; +import org.apache.doris.qe.StmtExecutor; import org.apache.doris.qe.VariableMgr; import org.apache.doris.resource.AdmissionControl; import org.apache.doris.resource.Tag; @@ -1116,6 +1117,8 @@ public void initialize(String[] args) throws Exception { notifyNewFETypeTransfer(FrontendNodeType.MASTER); } queryCancelWorker.start(); + + StmtExecutor.initBlockSqlAstNames(); } // wait until FE is ready. diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java index f8805bd0d4fb9a..be53ffde2e095b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java @@ -43,6 +43,7 @@ public class S3FileSystem extends ObjFileSystem { private static final Logger LOG = LogManager.getLogger(S3FileSystem.class); + private HadoopAuthenticator authenticator = null; public S3FileSystem(Map properties) { super(StorageBackend.StorageType.S3.name(), StorageBackend.StorageType.S3, new S3ObjStorage(properties)); @@ -77,7 +78,9 @@ protected FileSystem nativeFileSystem(String remotePath) throws UserException { PropertyConverter.convertToHadoopFSProperties(properties).entrySet().stream() .filter(entry -> entry.getKey() != null && entry.getValue() != null) .forEach(entry -> conf.set(entry.getKey(), entry.getValue())); - AuthenticationConfig authConfig = AuthenticationConfig.getKerberosConfig(conf); + // S3 does not support Kerberos authentication, + // so here we create a simple authentication + AuthenticationConfig authConfig = AuthenticationConfig.getSimpleAuthenticationConfig(conf); HadoopAuthenticator authenticator = HadoopAuthenticator.getHadoopAuthenticator(authConfig); try { dfsFileSystem = authenticator.doAs(() -> { @@ -87,6 +90,7 @@ protected FileSystem nativeFileSystem(String remotePath) throws UserException { throw new RuntimeException(e); } }); + this.authenticator = authenticator; RemoteFSPhantomManager.registerPhantomReference(this); } catch (Exception e) { throw new UserException("Failed to get S3 FileSystem for " + e.getMessage(), e); @@ -134,4 +138,9 @@ public Status globList(String remotePath, List result, boolean fileN } return Status.OK; } + + @VisibleForTesting + public HadoopAuthenticator getAuthenticator() { + return authenticator; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java index 2146472aec7b21..89f4af2817ec05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java @@ -489,4 +489,9 @@ public Status makeDir(String remotePath) { } return Status.OK; } + + @VisibleForTesting + public HadoopAuthenticator getAuthenticator() { + return authenticator; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java index 325e676fc046a0..a9acfeb2d6095b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java @@ -694,12 +694,17 @@ private static Expression checkOutputBoundary(Literal input) { return input; } + private static Expression castDecimalV3Literal(DecimalV3Literal literal, int precision) { + return new DecimalV3Literal(DecimalV3Type.createDecimalV3Type(precision, literal.getValue().scale()), + literal.getValue()); + } + /** * round */ @ExecFunction(name = "round") public static Expression round(DecimalV3Literal first) { - return first.round(0); + return castDecimalV3Literal(first.round(0), first.getValue().precision()); } /** @@ -707,7 +712,7 @@ public static Expression round(DecimalV3Literal first) { */ @ExecFunction(name = "round") public static Expression round(DecimalV3Literal first, IntegerLiteral second) { - return first.round(second.getValue()); + return castDecimalV3Literal(first.round(second.getValue()), first.getValue().precision()); } /** @@ -733,7 +738,7 @@ public static Expression round(DoubleLiteral first, IntegerLiteral second) { */ @ExecFunction(name = "ceil") public static Expression ceil(DecimalV3Literal first) { - return first.roundCeiling(0); + return castDecimalV3Literal(first.roundCeiling(0), first.getValue().precision()); } /** @@ -741,7 +746,7 @@ public static Expression ceil(DecimalV3Literal first) { */ @ExecFunction(name = "ceil") public static Expression ceil(DecimalV3Literal first, IntegerLiteral second) { - return first.roundCeiling(second.getValue()); + return castDecimalV3Literal(first.roundCeiling(second.getValue()), first.getValue().precision()); } /** @@ -767,7 +772,7 @@ public static Expression ceil(DoubleLiteral first, IntegerLiteral second) { */ @ExecFunction(name = "floor") public static Expression floor(DecimalV3Literal first) { - return first.roundFloor(0); + return castDecimalV3Literal(first.roundFloor(0), first.getValue().precision()); } /** @@ -775,7 +780,7 @@ public static Expression floor(DecimalV3Literal first) { */ @ExecFunction(name = "floor") public static Expression floor(DecimalV3Literal first, IntegerLiteral second) { - return first.roundFloor(second.getValue()); + return castDecimalV3Literal(first.roundFloor(second.getValue()), first.getValue().precision()); } /** @@ -1136,21 +1141,10 @@ public static Expression mathE() { public static Expression truncate(DecimalV3Literal first, IntegerLiteral second) { if (first.getValue().compareTo(BigDecimal.ZERO) == 0) { return first; + } else if (first.getValue().compareTo(BigDecimal.ZERO) < 0) { + return castDecimalV3Literal(first.roundCeiling(second.getValue()), first.getValue().precision()); } else { - if (first.getValue().scale() < second.getValue()) { - return first; - } - if (second.getValue() < 0) { - double factor = Math.pow(10, Math.abs(second.getValue())); - return new DecimalV3Literal( - DecimalV3Type.createDecimalV3Type(first.getValue().precision(), 0), - BigDecimal.valueOf(Math.floor(first.getDouble() / factor) * factor)); - } - if (first.getValue().compareTo(BigDecimal.ZERO) == -1) { - return first.roundCeiling(second.getValue()); - } else { - return first.roundFloor(second.getValue()); - } + return castDecimalV3Literal(first.roundFloor(second.getValue()), first.getValue().precision()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index 3365f380832e02..e45e71bd1f997d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -74,6 +74,7 @@ import org.apache.doris.nereids.util.TypeCoercionUtils; import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; import com.google.common.base.Preconditions; import com.google.common.base.Strings; @@ -607,6 +608,14 @@ public void validate(ConnectContext ctx) { if (!indexes.isEmpty()) { Set distinct = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); Set>> distinctCol = new HashSet<>(); + boolean disableInvertedIndexV1ForVariant = false; + try { + disableInvertedIndexV1ForVariant = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat( + new HashMap<>(properties)) == TInvertedIndexFileStorageFormat.V1 + && ConnectContext.get().getSessionVariable().getDisableInvertedIndexV1ForVaraint(); + } catch (Exception e) { + throw new AnalysisException(e.getMessage(), e.getCause()); + } for (IndexDefinition indexDef : indexes) { indexDef.validate(); @@ -618,7 +627,8 @@ public void validate(ConnectContext ctx) { boolean found = false; for (ColumnDefinition column : columns) { if (column.getName().equalsIgnoreCase(indexColName)) { - indexDef.checkColumn(column, keysType, isEnableMergeOnWrite); + indexDef.checkColumn(column, keysType, isEnableMergeOnWrite, + disableInvertedIndexV1ForVariant); found = true; break; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java index 61f2c874fd7ebc..4d22e5af51c0ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java @@ -93,7 +93,7 @@ public IndexDefinition(String name, List cols, String indexTypeName, * checkColumn */ public void checkColumn(ColumnDefinition column, KeysType keysType, - boolean enableUniqueKeyMergeOnWrite) throws AnalysisException { + boolean enableUniqueKeyMergeOnWrite, boolean disableInvertedIndexV1ForVariant) throws AnalysisException { if (indexType == IndexType.BITMAP || indexType == IndexType.INVERTED || indexType == IndexType.BLOOMFILTER || indexType == IndexType.NGRAM_BF) { String indexColName = column.getName(); @@ -106,6 +106,14 @@ public void checkColumn(ColumnDefinition column, KeysType keysType, throw new AnalysisException(colType + " is not supported in " + indexType.toString() + " index. " + "invalid index: " + name); } + + // In inverted index format v1, each subcolumn of a variant has its own index file, leading to high IOPS. + // when the subcolumn type changes, it may result in missing files, causing link file failure. + if (colType.isVariantType() && disableInvertedIndexV1ForVariant) { + throw new AnalysisException(colType + " is not supported in inverted index format V1," + + "Please set properties(\"inverted_index_storage_format\"= \"v2\")," + + "or upgrade to a newer version"); + } if (!column.isKey()) { if (keysType == KeysType.AGG_KEYS) { throw new AnalysisException("index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table" diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java index a14aea25463ad4..d718089fcabfee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java @@ -199,7 +199,11 @@ public long getNumReceivedRows() { @Override public long getJobId() { - return coordinatorContext.asLoadProcessor().jobId; + JobProcessor jobProcessor = coordinatorContext.getJobProcessor(); + if (jobProcessor instanceof LoadProcessor) { + return ((LoadProcessor) jobProcessor).jobId; + } + return -1L; } /* diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 99043902819b56..e4588844422057 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -665,6 +665,8 @@ public class SessionVariable implements Serializable, Writable { public static final String MAX_FETCH_REMOTE_TABLET_COUNT = "max_fetch_remote_schema_tablet_count"; + public static final String DISABLE_INVERTED_INDEX_V1_FOR_VARIANT = "disable_inverted_index_v1_for_variant"; + // CLOUD_VARIABLES_BEGIN public static final String CLOUD_CLUSTER = "cloud_cluster"; public static final String DISABLE_EMPTY_PARTITION_PRUNE = "disable_empty_partition_prune"; @@ -1224,6 +1226,9 @@ public enum IgnoreSplitType { @VariableMgr.VarAttr(name = WAIT_FULL_BLOCK_SCHEDULE_TIMES) public int waitFullBlockScheduleTimes = 2; + @VariableMgr.VarAttr(name = DISABLE_INVERTED_INDEX_V1_FOR_VARIANT) + private boolean disableInvertedIndexV1ForVaraint = true; + public int getBeNumberForTest() { return beNumberForTest; } @@ -4561,4 +4566,12 @@ public boolean isEnableCooldownReplicaAffinity() { public boolean isUseSerialExchange() { return useSerialExchange && getEnableLocalExchange(); } + + public void setDisableInvertedIndexV1ForVaraint(boolean disableInvertedIndexV1ForVaraint) { + this.disableInvertedIndexV1ForVaraint = disableInvertedIndexV1ForVaraint; + } + + public boolean getDisableInvertedIndexV1ForVaraint() { + return disableInvertedIndexV1ForVaraint; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 80c7ad0912ddbf..8b76933e9e0238 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -259,6 +259,8 @@ public class StmtExecutor { private static final AtomicLong STMT_ID_GENERATOR = new AtomicLong(0); public static final int MAX_DATA_TO_SEND_FOR_TXN = 100; public static final String NULL_VALUE_FOR_LOAD = "\\N"; + private static Set blockSqlAstNames = Sets.newHashSet(); + private Pattern beIpPattern = Pattern.compile("\\[(\\d+):"); private ConnectContext context; private final StatementContext statementContext; @@ -694,6 +696,7 @@ private void executeByNereids(TUniqueId queryId) throws Exception { "Nereids only process LogicalPlanAdapter, but parsedStmt is " + parsedStmt.getClass().getName()); context.getState().setNereids(true); LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan(); + checkSqlBlocked(logicalPlan.getClass()); if (context.getCommand() == MysqlCommand.COM_STMT_PREPARE) { if (isForwardToMaster()) { throw new UserException("Forward master command is not supported for prepare statement"); @@ -834,6 +837,23 @@ private void executeByNereids(TUniqueId queryId) throws Exception { } } + public static void initBlockSqlAstNames() { + blockSqlAstNames.clear(); + blockSqlAstNames = Pattern.compile(",") + .splitAsStream(Config.block_sql_ast_names) + .map(String::trim) + .collect(Collectors.toSet()); + if (blockSqlAstNames.isEmpty() && !Config.block_sql_ast_names.isEmpty()) { + blockSqlAstNames.add(Config.block_sql_ast_names); + } + } + + public void checkSqlBlocked(Class clazz) throws UserException { + if (blockSqlAstNames.contains(clazz.getSimpleName())) { + throw new UserException("SQL is blocked with AST name: " + clazz.getSimpleName()); + } + } + private void parseByNereids() { if (parsedStmt != null) { return; @@ -981,6 +1001,7 @@ public void executeByLegacy(TUniqueId queryId) throws Exception { try { // parsedStmt maybe null here, we parse it. Or the predicate will not work. parseByLegacy(); + checkSqlBlocked(parsedStmt.getClass()); if (context.isTxnModel() && !(parsedStmt instanceof InsertStmt) && !(parsedStmt instanceof TransactionStmt)) { throw new TException("This is in a transaction, only insert, update, delete, " diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java index a4eeec679b24d1..ccabece832d023 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java @@ -17,6 +17,9 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.common.AnalysisException; import com.google.common.collect.Lists; @@ -60,6 +63,16 @@ public void testAnalyzeExpection() throws AnalysisException { } catch (AnalysisException e) { Assert.assertTrue(e instanceof AnalysisException); } + try { + def = new IndexDef("variant_index", false, Lists.newArrayList("col1"), + IndexDef.IndexType.INVERTED, null, "comment"); + boolean isIndexFormatV1 = true; + def.checkColumn(new Column("col1", PrimitiveType.VARIANT), KeysType.UNIQUE_KEYS, true, isIndexFormatV1); + Assert.fail("No exception throws."); + } catch (AnalysisException e) { + Assert.assertTrue(e instanceof AnalysisException); + Assert.assertTrue(e.getMessage().contains("not supported in inverted index format V1")); + } } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/fs/remote/RemoteFileSystemTest.java b/fe/fe-core/src/test/java/org/apache/doris/fs/remote/RemoteFileSystemTest.java new file mode 100644 index 00000000000000..3fc15ab8e374fa --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/fs/remote/RemoteFileSystemTest.java @@ -0,0 +1,158 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.fs.remote; + +import org.apache.doris.common.Pair; +import org.apache.doris.common.UserException; +import org.apache.doris.common.security.authentication.AuthenticationConfig; +import org.apache.doris.common.security.authentication.HadoopAuthenticator; +import org.apache.doris.common.security.authentication.HadoopKerberosAuthenticator; +import org.apache.doris.common.security.authentication.HadoopSimpleAuthenticator; +import org.apache.doris.common.util.LocationPath; +import org.apache.doris.fs.FileSystemCache; +import org.apache.doris.fs.FileSystemType; +import org.apache.doris.fs.remote.dfs.DFSFileSystem; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; +import mockit.Mock; +import mockit.MockUp; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.security.UserGroupInformation; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.net.URI; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.Map; + +public class RemoteFileSystemTest { + + @Test + public void testFilesystemAndAuthType() throws UserException { + + // These paths should use s3 filesystem, and use simple auth + ArrayList s3Paths = new ArrayList<>(); + s3Paths.add("s3://a/b/c"); + s3Paths.add("s3a://a/b/c"); + s3Paths.add("s3n://a/b/c"); + s3Paths.add("oss://a/b/c"); // default use s3 filesystem + s3Paths.add("gs://a/b/c"); + s3Paths.add("bos://a/b/c"); + s3Paths.add("cos://a/b/c"); + s3Paths.add("cosn://a/b/c"); + s3Paths.add("lakefs://a/b/c"); + s3Paths.add("obs://a/b/c"); + + // These paths should use dfs filesystem, and auth will be changed by configure + ArrayList dfsPaths = new ArrayList<>(); + dfsPaths.add("ofs://a/b/c"); + dfsPaths.add("gfs://a/b/c"); + dfsPaths.add("hdfs://a/b/c"); + dfsPaths.add("oss://a/b/c"); // if endpoint contains 'oss-dls.aliyuncs', will use dfs filesystem + + new MockUp(UserGroupInformation.class) { + @Mock + public T doAs(PrivilegedExceptionAction action) throws IOException, InterruptedException { + return (T) new LocalFileSystem(); + } + }; + + new MockUp(HadoopKerberosAuthenticator.class) { + @Mock + public synchronized UserGroupInformation getUGI() throws IOException { + return UserGroupInformation.getCurrentUser(); + } + }; + + Configuration confWithoutKerberos = new Configuration(); + + Configuration confWithKerberosIncomplete = new Configuration(); + confWithKerberosIncomplete.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + + Configuration confWithKerberos = new Configuration(); + confWithKerberos.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + confWithKerberos.set(AuthenticationConfig.HADOOP_KERBEROS_PRINCIPAL, "principal"); + confWithKerberos.set(AuthenticationConfig.HADOOP_KERBEROS_KEYTAB, "keytab"); + + ImmutableMap s3props = ImmutableMap.of("s3.endpoint", "http://127.0.0.1"); + s3props.forEach(confWithKerberos::set); + s3props.forEach(confWithoutKerberos::set); + s3props.forEach(confWithKerberosIncomplete::set); + + for (String path : s3Paths) { + checkS3Filesystem(path, confWithKerberos, s3props); + } + for (String path : s3Paths) { + checkS3Filesystem(path, confWithKerberosIncomplete, s3props); + } + for (String path : s3Paths) { + checkS3Filesystem(path, confWithoutKerberos, s3props); + } + + s3props = ImmutableMap.of("s3.endpoint", "oss://xx-oss-dls.aliyuncs/abc"); + System.setProperty("java.security.krb5.realm", "realm"); + System.setProperty("java.security.krb5.kdc", "kdc"); + + for (String path : dfsPaths) { + checkDFSFilesystem(path, confWithKerberos, HadoopKerberosAuthenticator.class.getName(), s3props); + } + for (String path : dfsPaths) { + checkDFSFilesystem(path, confWithKerberosIncomplete, HadoopSimpleAuthenticator.class.getName(), s3props); + } + for (String path : dfsPaths) { + checkDFSFilesystem(path, confWithoutKerberos, HadoopSimpleAuthenticator.class.getName(), s3props); + } + + } + + private void checkS3Filesystem(String path, Configuration conf, Map m) throws UserException { + RemoteFileSystem fs = createFs(path, conf, m); + Assert.assertTrue(fs instanceof S3FileSystem); + HadoopAuthenticator authenticator = ((S3FileSystem) fs).getAuthenticator(); + Assert.assertTrue(authenticator instanceof HadoopSimpleAuthenticator); + } + + private void checkDFSFilesystem(String path, Configuration conf, String authClass, Map m) throws UserException { + RemoteFileSystem fs = createFs(path, conf, m); + Assert.assertTrue(fs instanceof DFSFileSystem); + HadoopAuthenticator authenticator = ((DFSFileSystem) fs).getAuthenticator(); + Assert.assertEquals(authClass, authenticator.getClass().getName()); + } + + private RemoteFileSystem createFs(String path, Configuration conf, Map m) throws UserException { + LocationPath locationPath = new LocationPath(path, m); + FileSystemType fileSystemType = locationPath.getFileSystemType(); + URI uri = locationPath.getPath().toUri(); + String fsIdent = Strings.nullToEmpty(uri.getScheme()) + "://" + Strings.nullToEmpty(uri.getAuthority()); + FileSystemCache fileSystemCache = new FileSystemCache(); + RemoteFileSystem fs = fileSystemCache.getRemoteFileSystem( + new FileSystemCache.FileSystemCacheKey( + Pair.of(fileSystemType, fsIdent), + ImmutableMap.of(), + null, + conf)); + fs.nativeFileSystem(path); + return fs; + } + +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java index ff9e81f2bf3cfb..9a46b810586eec 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java @@ -43,6 +43,8 @@ import org.apache.doris.nereids.types.DateType; import org.apache.doris.nereids.types.DecimalV2Type; import org.apache.doris.nereids.types.DecimalV3Type; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.StmtExecutor; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -704,4 +706,38 @@ public void testQualify() { nereidsParser.parseSingle(sql); } } + + @Test + public void testBlockSqlAst() { + String sql = "plan replayer dump select `AD``D` from t1 where a = 1"; + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle(sql); + + Config.block_sql_ast_names = "ReplayCommand"; + StmtExecutor.initBlockSqlAstNames(); + StmtExecutor stmtExecutor = new StmtExecutor(new ConnectContext(), ""); + try { + stmtExecutor.checkSqlBlocked(logicalPlan.getClass()); + Assertions.fail(); + } catch (Exception ignore) { + // do nothing + } + + Config.block_sql_ast_names = "CreatePolicyCommand, ReplayCommand"; + StmtExecutor.initBlockSqlAstNames(); + try { + stmtExecutor.checkSqlBlocked(logicalPlan.getClass()); + Assertions.fail(); + } catch (Exception ignore) { + // do nothing + } + + Config.block_sql_ast_names = ""; + StmtExecutor.initBlockSqlAstNames(); + try { + stmtExecutor.checkSqlBlocked(logicalPlan.getClass()); + } catch (Exception ex) { + Assertions.fail(ex); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IndexDefinitionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IndexDefinitionTest.java new file mode 100644 index 00000000000000..0632945ee0416e --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IndexDefinitionTest.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.catalog.AggregateType; +import org.apache.doris.catalog.KeysType; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition; +import org.apache.doris.nereids.trees.plans.commands.info.IndexDefinition; +import org.apache.doris.nereids.types.VariantType; + +import com.google.common.collect.Lists; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class IndexDefinitionTest { + @Test + void testVariantIndexFormatV1() throws AnalysisException { + IndexDefinition def = new IndexDefinition("variant_index", Lists.newArrayList("col1"), "INVERTED", + null, "comment"); + try { + boolean isIndexFormatV1 = true; + def.checkColumn(new ColumnDefinition("col1", VariantType.INSTANCE, false, AggregateType.NONE, true, + null, "comment"), KeysType.UNIQUE_KEYS, true, isIndexFormatV1); + Assertions.fail("No exception throws."); + } catch (AnalysisException e) { + Assertions.assertTrue(e instanceof AnalysisException); + Assertions.assertTrue(e.getMessage().contains("not supported in inverted index format V1")); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java index 07d39e52180858..8ab187315d8681 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java @@ -19,6 +19,8 @@ import org.apache.doris.analysis.AccessTestUtil; import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.CreateFileStmt; +import org.apache.doris.analysis.CreateFunctionStmt; import org.apache.doris.analysis.DdlStmt; import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.KillStmt; @@ -31,6 +33,7 @@ import org.apache.doris.analysis.StatementBase; import org.apache.doris.analysis.UseStmt; import org.apache.doris.catalog.Env; +import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.common.profile.Profile; @@ -801,4 +804,91 @@ public void testUseWithCatalogFail(@Mocked UseStmt useStmt, @Mocked SqlParser pa Assert.assertEquals(QueryState.MysqlStateType.ERR, state.getStateType()); } + + @Test + public void testBlockSqlAst(@Mocked UseStmt useStmt, @Mocked CreateFileStmt createFileStmt, + @Mocked CreateFunctionStmt createFunctionStmt, @Mocked SqlParser parser) throws Exception { + new Expectations() { + { + useStmt.analyze((Analyzer) any); + minTimes = 0; + + useStmt.getDatabase(); + minTimes = 0; + result = "testDb"; + + useStmt.getRedirectStatus(); + minTimes = 0; + result = RedirectStatus.NO_FORWARD; + + useStmt.getCatalogName(); + minTimes = 0; + result = InternalCatalog.INTERNAL_CATALOG_NAME; + + Symbol symbol = new Symbol(0, Lists.newArrayList(createFileStmt)); + parser.parse(); + minTimes = 0; + result = symbol; + } + }; + + Config.block_sql_ast_names = "CreateFileStmt"; + StmtExecutor.initBlockSqlAstNames(); + StmtExecutor executor = new StmtExecutor(ctx, ""); + try { + executor.execute(); + } catch (Exception ignore) { + // do nothing + } + Assert.assertEquals(QueryState.MysqlStateType.ERR, state.getStateType()); + Assert.assertTrue(state.getErrorMessage().contains("SQL is blocked with AST name: CreateFileStmt")); + + Config.block_sql_ast_names = "AlterStmt, CreateFileStmt"; + StmtExecutor.initBlockSqlAstNames(); + executor = new StmtExecutor(ctx, ""); + try { + executor.execute(); + } catch (Exception ignore) { + // do nothing + } + Assert.assertEquals(QueryState.MysqlStateType.ERR, state.getStateType()); + Assert.assertTrue(state.getErrorMessage().contains("SQL is blocked with AST name: CreateFileStmt")); + + new Expectations() { + { + Symbol symbol = new Symbol(0, Lists.newArrayList(createFunctionStmt)); + parser.parse(); + minTimes = 0; + result = symbol; + } + }; + Config.block_sql_ast_names = "CreateFunctionStmt, CreateFileStmt"; + StmtExecutor.initBlockSqlAstNames(); + executor = new StmtExecutor(ctx, ""); + try { + executor.execute(); + } catch (Exception ignore) { + // do nothing + } + Assert.assertEquals(QueryState.MysqlStateType.ERR, state.getStateType()); + Assert.assertTrue(state.getErrorMessage().contains("SQL is blocked with AST name: CreateFunctionStmt")); + + new Expectations() { + { + Symbol symbol = new Symbol(0, Lists.newArrayList(useStmt)); + parser.parse(); + minTimes = 0; + result = symbol; + } + }; + executor = new StmtExecutor(ctx, ""); + executor.execute(); + Assert.assertEquals(QueryState.MysqlStateType.OK, state.getStateType()); + + Config.block_sql_ast_names = ""; + StmtExecutor.initBlockSqlAstNames(); + executor = new StmtExecutor(ctx, ""); + executor.execute(); + Assert.assertEquals(QueryState.MysqlStateType.OK, state.getStateType()); + } } diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index bb8cb8c8f2a7ba..fbbe8254e8aff9 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -1189,6 +1189,122 @@ enum TBinlogType { RENAME_TABLE = 14, RENAME_COLUMN = 15, MODIFY_COMMENT = 16, + + // Keep some IDs for allocation so that when new binlog types are added in the + // future, the changes can be picked back to the old versions without breaking + // compatibility. + // + // The code will check the IDs of binlog types, any binlog types whose IDs are + // greater than or equal to MIN_UNKNOWN will be ignored. + // + // For example, before you adding new binlog type MODIFY_XXX: + // MIN_UNKNOWN = 17, + // UNKNOWN_2 = 18, + // UNKNOWN_3 = 19, + // After adding binlog type MODIFY_XXX: + // MODIFY_XXX = 17, + // MIN_UNKNOWN = 18, + // UNKNOWN_3 = 19, + MIN_UNKNOWN = 17, + UNKNOWN_2 = 18, + UNKNOWN_3 = 19, + UNKNOWN_4 = 20, + UNKNOWN_5 = 21, + UNKNOWN_6 = 22, + UNKNOWN_7 = 23, + UNKNOWN_8 = 24, + UNKNOWN_9 = 25, + UNKNOWN_10 = 26, + UNKNOWN_11 = 27, + UNKNOWN_12 = 28, + UNKNOWN_13 = 29, + UNKNOWN_14 = 30, + UNKNOWN_15 = 31, + UNKNOWN_16 = 32, + UNKNOWN_17 = 33, + UNKNOWN_18 = 34, + UNKNOWN_19 = 35, + UNKNOWN_20 = 36, + UNKNOWN_21 = 37, + UNKNOWN_22 = 38, + UNKNOWN_23 = 39, + UNKNOWN_24 = 40, + UNKNOWN_25 = 41, + UNKNOWN_26 = 42, + UNKNOWN_27 = 43, + UNKNOWN_28 = 44, + UNKNOWN_29 = 45, + UNKNOWN_30 = 46, + UNKNOWN_31 = 47, + UNKNOWN_32 = 48, + UNKNOWN_33 = 49, + UNKNOWN_34 = 50, + UNKNOWN_35 = 51, + UNKNOWN_36 = 52, + UNKNOWN_37 = 53, + UNKNOWN_38 = 54, + UNKNOWN_39 = 55, + UNKNOWN_40 = 56, + UNKNOWN_41 = 57, + UNKNOWN_42 = 58, + UNKNOWN_43 = 59, + UNKNOWN_44 = 60, + UNKNOWN_45 = 61, + UNKNOWN_46 = 62, + UNKNOWN_47 = 63, + UNKNOWN_48 = 64, + UNKNOWN_49 = 65, + UNKNOWN_50 = 66, + UNKNOWN_51 = 67, + UNKNOWN_52 = 68, + UNKNOWN_53 = 69, + UNKNOWN_54 = 70, + UNKNOWN_55 = 71, + UNKNOWN_56 = 72, + UNKNOWN_57 = 73, + UNKNOWN_58 = 74, + UNKNOWN_59 = 75, + UNKNOWN_60 = 76, + UNKNOWN_61 = 77, + UNKNOWN_62 = 78, + UNKNOWN_63 = 79, + UNKNOWN_64 = 80, + UNKNOWN_65 = 81, + UNKNOWN_66 = 82, + UNKNOWN_67 = 83, + UNKNOWN_68 = 84, + UNKNOWN_69 = 85, + UNKNOWN_70 = 86, + UNKNOWN_71 = 87, + UNKNOWN_72 = 88, + UNKNOWN_73 = 89, + UNKNOWN_74 = 90, + UNKNOWN_75 = 91, + UNKNOWN_76 = 92, + UNKNOWN_77 = 93, + UNKNOWN_78 = 94, + UNKNOWN_79 = 95, + UNKNOWN_80 = 96, + UNKNOWN_81 = 97, + UNKNOWN_82 = 98, + UNKNOWN_83 = 99, + UNKNOWN_84 = 100, + UNKNOWN_85 = 101, + UNKNOWN_86 = 102, + UNKNOWN_87 = 103, + UNKNOWN_88 = 104, + UNKNOWN_89 = 105, + UNKNOWN_90 = 106, + UNKNOWN_91 = 107, + UNKNOWN_92 = 108, + UNKNOWN_93 = 109, + UNKNOWN_94 = 110, + UNKNOWN_95 = 111, + UNKNOWN_96 = 112, + UNKNOWN_97 = 113, + UNKNOWN_98 = 114, + UNKNOWN_99 = 115, + UNKNOWN_100 = 116, } struct TBinlog { diff --git a/regression-test/pipeline/common/github-utils.sh b/regression-test/pipeline/common/github-utils.sh index 242b77b832d6c4..079101b35b9496 100644 --- a/regression-test/pipeline/common/github-utils.sh +++ b/regression-test/pipeline/common/github-utils.sh @@ -262,6 +262,7 @@ file_changed_be_ut() { if [[ -z ${all_files} ]]; then echo "return need" && return 0; fi for af in ${all_files}; do if [[ "${af}" == 'be'* ]] || + [[ "${af}" == 'common/cpp'* ]] || [[ "${af}" == 'contrib'* ]] || [[ "${af}" == 'thirdparty'* ]] || [[ "${af}" == 'bin/start_be.sh' ]] || @@ -280,6 +281,7 @@ file_changed_cloud_ut() { if [[ -z ${all_files} ]]; then echo "return need" && return 0; fi for af in ${all_files}; do if [[ "${af}" == 'cloud/src/'* ]] || + [[ "${af}" == 'common/cpp'* ]] || [[ "${af}" == 'cloud/test/'* ]]; then echo "cloud-ut related file changed, return need" && return 0 fi diff --git a/regression-test/suites/compaction/test_single_compaction_with_variant_inverted_index.groovy b/regression-test/suites/compaction/test_single_compaction_with_variant_inverted_index.groovy index 79c6322204585d..d450df1868c5a8 100644 --- a/regression-test/suites/compaction/test_single_compaction_with_variant_inverted_index.groovy +++ b/regression-test/suites/compaction/test_single_compaction_with_variant_inverted_index.groovy @@ -132,6 +132,7 @@ suite("test_single_compaction_with_variant_inverted", "p2") { sql """ DROP TABLE IF EXISTS ${tableName}; """ + sql """ set disable_inverted_index_v1_for_variant = false """ sql """ CREATE TABLE ${tableName} ( `id` int(11) NULL, @@ -150,6 +151,7 @@ suite("test_single_compaction_with_variant_inverted", "p2") { "compaction_policy" = "time_series" ); """ + sql """ set disable_inverted_index_v1_for_variant = true """ def tablets = sql_return_maparray """ show tablets from ${tableName}; """ diff --git a/regression-test/suites/inverted_index_p0/load/test_insert.groovy b/regression-test/suites/inverted_index_p0/load/test_insert.groovy index 97b3ca07937f49..03e7a3ccb26404 100644 --- a/regression-test/suites/inverted_index_p0/load/test_insert.groovy +++ b/regression-test/suites/inverted_index_p0/load/test_insert.groovy @@ -72,10 +72,14 @@ suite("test_insert_with_index", "p0") { } set_be_config("inverted_index_ram_dir_enable", "true") + sql """ set disable_inverted_index_v1_for_variant = false """ test.call("V1") + sql """ set disable_inverted_index_v1_for_variant = true """ test.call("V2") set_be_config("inverted_index_ram_dir_enable", "false") + sql """ set disable_inverted_index_v1_for_variant = false """ test.call("V1") + sql """ set disable_inverted_index_v1_for_variant = true """ test.call("V2") set_be_config("inverted_index_ram_dir_enable", "true") } \ No newline at end of file diff --git a/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy b/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy index 53ee5d546c2709..1912fdcfc50c4c 100644 --- a/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy +++ b/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy @@ -105,10 +105,14 @@ suite("test_stream_load_with_inverted_index_p0", "p0") { } set_be_config("inverted_index_ram_dir_enable", "true") + sql """ set disable_inverted_index_v1_for_variant = false """ test.call("V1") + sql """ set disable_inverted_index_v1_for_variant = true """ test.call("V2") set_be_config("inverted_index_ram_dir_enable", "false") + sql """ set disable_inverted_index_v1_for_variant = false """ test.call("V1") + sql """ set disable_inverted_index_v1_for_variant = true """ test.call("V2") set_be_config("inverted_index_ram_dir_enable", "true") } \ No newline at end of file diff --git a/regression-test/suites/inverted_index_p0/test_show_nested_index_file_http_action_with_variant.groovy b/regression-test/suites/inverted_index_p0/test_show_nested_index_file_http_action_with_variant.groovy index 71831140a996a7..a7718e3927a4a3 100644 --- a/regression-test/suites/inverted_index_p0/test_show_nested_index_file_http_action_with_variant.groovy +++ b/regression-test/suites/inverted_index_p0/test_show_nested_index_file_http_action_with_variant.groovy @@ -64,6 +64,7 @@ suite("test_show_nested_index_file_http_action_with_variant", "nonConcurrent,p0" def tableName = "test_show_nested_index_file_http_action_with_variant_" + format sql "DROP TABLE IF EXISTS ${tableName}" + sql """ set disable_inverted_index_v1_for_variant = false """ sql """ CREATE TABLE IF NOT EXISTS ${tableName} ( k bigint, @@ -74,6 +75,7 @@ suite("test_show_nested_index_file_http_action_with_variant", "nonConcurrent,p0" DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "${format}"); """ + sql """ set disable_inverted_index_v1_for_variant = true """ load_json_data.call(tableName, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""") load_json_data.call(tableName, """${getS3Url() + '/regression/gharchive.m/2015-01-01-1.json'}""") diff --git a/regression-test/suites/inverted_index_p2/test_variant_index_format_v1.groovy b/regression-test/suites/inverted_index_p2/test_variant_index_format_v1.groovy index cc44ac2bb38234..5f262c8bf07775 100644 --- a/regression-test/suites/inverted_index_p2/test_variant_index_format_v1.groovy +++ b/regression-test/suites/inverted_index_p2/test_variant_index_format_v1.groovy @@ -60,6 +60,7 @@ suite("test_variant_index_format_v1", "p2") { def table_name = "github_events" sql """DROP TABLE IF EXISTS ${table_name}""" + sql """ set disable_inverted_index_v1_for_variant = false """ sql """ CREATE TABLE IF NOT EXISTS ${table_name} ( k bigint, @@ -70,7 +71,7 @@ suite("test_variant_index_format_v1", "p2") { DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1"); """ - + sql """ set disable_inverted_index_v1_for_variant = true """ set_be_config.call("memory_limitation_per_thread_for_schema_change_bytes", "6294967296") load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""") load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2015-01-01-1.json'}""") diff --git a/regression-test/suites/load_p2/broker_load/test_broker_load.groovy b/regression-test/suites/load_p2/broker_load/test_broker_load.groovy index 0009d26269f093..f8a0043842c094 100644 --- a/regression-test/suites/load_p2/broker_load/test_broker_load.groovy +++ b/regression-test/suites/load_p2/broker_load/test_broker_load.groovy @@ -19,247 +19,361 @@ suite("test_broker_load_p2", "p2") { def s3BucketName = getS3BucketName() def s3Endpoint = getS3Endpoint() def s3Region = getS3Region() - def tables = ["part", - "upper_case", - "reverse", - "set1", - "set2", - "set3", - "set4", - "set5", - "set6", - "set7", - "null_default", - "filter", - "path_column", - "parquet_s3_case1", // col1 not in file but in table, will load default value for it. - "parquet_s3_case2", // x1 not in file, not in table, will throw "col not found" error. - "parquet_s3_case3", // p_comment not in table but in file, load normally. - "parquet_s3_case4", // all columns are in table but not in file, will fill default values. - "parquet_s3_case5", // x1 not in file, not in table, will throw "col not found" error. - "parquet_s3_case6", // normal - "parquet_s3_case7", // col5 will be ignored, load normally - "parquet_s3_case8", // first column in table is not specified, will load default value for it. - "orc_s3_case1", // table column capitalize firsrt - "orc_s3_case2", // table column lowercase * load column lowercase * orc file lowercase - "orc_s3_case3", // table column lowercase * load column uppercase * orc file lowercase - "orc_s3_case4", // table column lowercase * load column lowercase * orc file uppercase - "orc_s3_case5", // table column lowercase * load column uppercase * orc file uppercase - "orc_s3_case6", // table column uppercase * load column uppercase * orc file lowercase - "orc_s3_case7", // table column uppercase * load column lowercase * orc file lowercase - "orc_s3_case8", // table column uppercase * load column uppercase * orc file uppercase - "orc_s3_case9", // table column uppercase * load column lowercase * orc file uppercase - "csv_s3_case_line_delimiter" // csv format table with special line delimiter + def tables = ["part", // case 0 + "upper_case", // case 1 + "reverse", // case 2 + "set1", // case 3 + "set2", // case 4 + "set3", // case 5 + "set4", // case 6 + "set5", // case 7 + "set6", // case 8 + "set7", // case 9 + "null_default", // case 10 + "filter", // case 11 + "path_column", // case 12 + "parquet_s3_case1", // case 13, col1 not in file but in table, will load default value for it. + "parquet_s3_case2", // case 14, x1 not in file, not in table, will throw "col not found" error. + "parquet_s3_case3", // case 15, p_comment not in table but in file, load normally. + "parquet_s3_case4", // case 16, all columns are in table but not in file, will fill default values. + "parquet_s3_case5", // case 17, x1 not in file, not in table, will throw "col not found" error. + "parquet_s3_case6", // case 18, normal + "parquet_s3_case7", // case 19, col5 will be ignored, load normally + "parquet_s3_case8", // case 20, first column in table is not specified, will load default value for it. + "orc_s3_case1", // case 21, table column capitalize * load column lowercase * orc file lowercase + "orc_s3_case2", // case 22, table column lowercase * load column lowercase * orc file lowercase + "orc_s3_case3", // case 23, table column lowercase * load column uppercase * orc file lowercase + "orc_s3_case4", // case 24, table column lowercase * load column lowercase * orc file uppercase + "orc_s3_case5", // case 25, table column lowercase * load column uppercase * orc file uppercase + "orc_s3_case6", // case 26, table column uppercase * load column uppercase * orc file lowercase + "orc_s3_case7", // case 27, table column uppercase * load column lowercase * orc file lowercase + "orc_s3_case8", // case 28, table column uppercase * load column uppercase * orc file uppercase + "orc_s3_case9", // case 29, table column uppercase * load column lowercase * orc file uppercase + "csv_s3_case_line_delimiter" // case 30, csv format table with special line delimiter ] - def paths = ["s3://${s3BucketName}/regression/load/data/part*", - "s3://${s3BucketName}/regression/load/data/part*", - "s3://${s3BucketName}/regression/load/data/part*", - "s3://${s3BucketName}/regression/load/data/part*", - "s3://${s3BucketName}/regression/load/data/part*", - "s3://${s3BucketName}/regression/load/data/part*", - "s3://${s3BucketName}/regression/load/data/part*", - "s3://${s3BucketName}/regression/load/data/part*", - "s3://${s3BucketName}/regression/load/data/part*", - "s3://${s3BucketName}/regression/load/data/part*", - "s3://${s3BucketName}/regression/load/data/part*", - "s3://${s3BucketName}/regression/load/data/part*", - "s3://${s3BucketName}/regression/load/data/path/*/part*", - "s3://${s3BucketName}/regression/load/data/part*", - "s3://${s3BucketName}/regression/load/data/part*", - "s3://${s3BucketName}/regression/load/data/part*", - "s3://${s3BucketName}/regression/load/data/part*", - "s3://${s3BucketName}/regression/load/data/part*", - "s3://${s3BucketName}/regression/load/data/part*", - "s3://${s3BucketName}/regression/load/data/part*", - "s3://${s3BucketName}/regression/load/data/part*", - "s3://${s3BucketName}/regression/load/data/orc/hits_100k_rows.orc", - "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc", - "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc", - "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc", - "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc", - "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc", - "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc", - "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc", - "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc", - "s3://${s3BucketName}/regression/line_delimiter/lineitem_0x7.csv.gz" + def paths = ["s3://${s3BucketName}/regression/load/data/part*", // case 0 + "s3://${s3BucketName}/regression/load/data/part*", // case 1 + "s3://${s3BucketName}/regression/load/data/part*", // case 2 + "s3://${s3BucketName}/regression/load/data/part*", // case 3 + "s3://${s3BucketName}/regression/load/data/part*", // case 4 + "s3://${s3BucketName}/regression/load/data/part*", // case 5 + "s3://${s3BucketName}/regression/load/data/part*", // case 6 + "s3://${s3BucketName}/regression/load/data/part*", // case 7 + "s3://${s3BucketName}/regression/load/data/part*", // case 8 + "s3://${s3BucketName}/regression/load/data/part*", // case 9 + "s3://${s3BucketName}/regression/load/data/part*", // case 10 + "s3://${s3BucketName}/regression/load/data/part*", // case 11 + "s3://${s3BucketName}/regression/load/data/path/*/part*", // case 12 + "s3://${s3BucketName}/regression/load/data/part*", // case 13 + "s3://${s3BucketName}/regression/load/data/part*", // case 14 + "s3://${s3BucketName}/regression/load/data/part*", // case 15 + "s3://${s3BucketName}/regression/load/data/part*", // case 16 + "s3://${s3BucketName}/regression/load/data/part*", // case 17 + "s3://${s3BucketName}/regression/load/data/part*", // case 18 + "s3://${s3BucketName}/regression/load/data/part*", // case 19 + "s3://${s3BucketName}/regression/load/data/part*", // case 20 + "s3://${s3BucketName}/regression/load/data/orc/hits_100k_rows.orc", // case 21 + "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc", // case 22 + "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc", // case 23 + "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc", // case 24 + "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc", // case 25 + "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc", // case 26 + "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc", // case 27 + "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc", // case 28 + "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc", // case 29 + "s3://${s3BucketName}/regression/line_delimiter/lineitem_0x7.csv.gz" // case 30 ] - def columns_list = ["""p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""", - """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""", - """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""", - """p_partkey, p_name, p_size""", - """p_partkey""", - """p_partkey""", - """p_partkey, p_size""", - """p_partkey""", - """p_partkey, p_size""", - """p_partkey, p_size""", - """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""", - """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""", - """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""", - """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment, col1""", - """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment, x1""", - """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""", - """col1, col2, col3, col4""", - """p_partkey, p_name, p_mfgr, x1""", - """p_partkey, p_name, p_mfgr, p_brand""", - """p_partkey, p_name, p_mfgr, p_brand""", - """p_name, p_mfgr""", - """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""", - //TODO: comment blow 8 rows after jibing fix - """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""", - """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""", - """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""", - """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""", - """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""", - """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""", - """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""", - """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""", - """""" - //TODO: uncomment blow 8 rows after jibing fix - // """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""", - // """WATCHID,JAVAENABLE,TITLE,GOODEVENT,EVENTTIME,EVENTDATE,COUNTERID,CLIENTIP,REGIONID,USERID,COUNTERCLASS,OS,USERAGENT,URL,REFERER,ISREFRESH,REFERERCATEGORYID,REFERERREGIONID,URLCATEGORYID,URLREGIONID,RESOLUTIONWIDTH,RESOLUTIONHEIGHT,RESOLUTIONDEPTH,FLASHMAJOR,FLASHMINOR,FLASHMINOR2,NETMAJOR,NETMINOR,USERAGENTMAJOR,USERAGENTMINOR,COOKIEENABLE,JAVASCRIPTENABLE,ISMOBILE,MOBILEPHONE,MOBILEPHONEMODEL,PARAMS,IPNETWORKID,TRAFICSOURCEID,SEARCHENGINEID,SEARCHPHRASE,ADVENGINEID,ISARTIFICAL,WINDOWCLIENTWIDTH,WINDOWCLIENTHEIGHT,CLIENTTIMEZONE,CLIENTEVENTTIME,SILVERLIGHTVERSION1,SILVERLIGHTVERSION2,SILVERLIGHTVERSION3,SILVERLIGHTVERSION4,PAGECHARSET,CODEVERSION,ISLINK,ISDOWNLOAD,ISNOTBOUNCE,FUNIQID,ORIGINALURL,HID,ISOLDCOUNTER,ISEVENT,ISPARAMETER,DONTCOUNTHITS,WITHHASH,HITCOLOR,LOCALEVENTTIME,AGE,SEX,INCOME,INTERESTS,ROBOTNESS,REMOTEIP,WINDOWNAME,OPENERNAME,HISTORYLENGTH,BROWSERLANGUAGE,BROWSERCOUNTRY,SOCIALNETWORK,SOCIALACTION,HTTPERROR,SENDTIMING,DNSTIMING,CONNECTTIMING,RESPONSESTARTTIMING,RESPONSEENDTIMING,FETCHTIMING,SOCIALSOURCENETWORKID,SOCIALSOURCEPAGE,PARAMPRICE,PARAMORDERID,PARAMCURRENCY,PARAMCURRENCYID,OPENSTATSERVICENAME,OPENSTATCAMPAIGNID,OPENSTATADID,OPENSTATSOURCEID,UTMSOURCE,UTMMEDIUM,UTMCAMPAIGN,UTMCONTENT,UTMTERM,FROMTAG,HASGCLID,REFERERHASH,URLHASH,CLID""", - // """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""", - // """WATCHID,JAVAENABLE,TITLE,GOODEVENT,EVENTTIME,EVENTDATE,COUNTERID,CLIENTIP,REGIONID,USERID,COUNTERCLASS,OS,USERAGENT,URL,REFERER,ISREFRESH,REFERERCATEGORYID,REFERERREGIONID,URLCATEGORYID,URLREGIONID,RESOLUTIONWIDTH,RESOLUTIONHEIGHT,RESOLUTIONDEPTH,FLASHMAJOR,FLASHMINOR,FLASHMINOR2,NETMAJOR,NETMINOR,USERAGENTMAJOR,USERAGENTMINOR,COOKIEENABLE,JAVASCRIPTENABLE,ISMOBILE,MOBILEPHONE,MOBILEPHONEMODEL,PARAMS,IPNETWORKID,TRAFICSOURCEID,SEARCHENGINEID,SEARCHPHRASE,ADVENGINEID,ISARTIFICAL,WINDOWCLIENTWIDTH,WINDOWCLIENTHEIGHT,CLIENTTIMEZONE,CLIENTEVENTTIME,SILVERLIGHTVERSION1,SILVERLIGHTVERSION2,SILVERLIGHTVERSION3,SILVERLIGHTVERSION4,PAGECHARSET,CODEVERSION,ISLINK,ISDOWNLOAD,ISNOTBOUNCE,FUNIQID,ORIGINALURL,HID,ISOLDCOUNTER,ISEVENT,ISPARAMETER,DONTCOUNTHITS,WITHHASH,HITCOLOR,LOCALEVENTTIME,AGE,SEX,INCOME,INTERESTS,ROBOTNESS,REMOTEIP,WINDOWNAME,OPENERNAME,HISTORYLENGTH,BROWSERLANGUAGE,BROWSERCOUNTRY,SOCIALNETWORK,SOCIALACTION,HTTPERROR,SENDTIMING,DNSTIMING,CONNECTTIMING,RESPONSESTARTTIMING,RESPONSEENDTIMING,FETCHTIMING,SOCIALSOURCENETWORKID,SOCIALSOURCEPAGE,PARAMPRICE,PARAMORDERID,PARAMCURRENCY,PARAMCURRENCYID,OPENSTATSERVICENAME,OPENSTATCAMPAIGNID,OPENSTATADID,OPENSTATSOURCEID,UTMSOURCE,UTMMEDIUM,UTMCAMPAIGN,UTMCONTENT,UTMTERM,FROMTAG,HASGCLID,REFERERHASH,URLHASH,CLID""", - // """WATCHID,JAVAENABLE,TITLE,GOODEVENT,EVENTTIME,EVENTDATE,COUNTERID,CLIENTIP,REGIONID,USERID,COUNTERCLASS,OS,USERAGENT,URL,REFERER,ISREFRESH,REFERERCATEGORYID,REFERERREGIONID,URLCATEGORYID,URLREGIONID,RESOLUTIONWIDTH,RESOLUTIONHEIGHT,RESOLUTIONDEPTH,FLASHMAJOR,FLASHMINOR,FLASHMINOR2,NETMAJOR,NETMINOR,USERAGENTMAJOR,USERAGENTMINOR,COOKIEENABLE,JAVASCRIPTENABLE,ISMOBILE,MOBILEPHONE,MOBILEPHONEMODEL,PARAMS,IPNETWORKID,TRAFICSOURCEID,SEARCHENGINEID,SEARCHPHRASE,ADVENGINEID,ISARTIFICAL,WINDOWCLIENTWIDTH,WINDOWCLIENTHEIGHT,CLIENTTIMEZONE,CLIENTEVENTTIME,SILVERLIGHTVERSION1,SILVERLIGHTVERSION2,SILVERLIGHTVERSION3,SILVERLIGHTVERSION4,PAGECHARSET,CODEVERSION,ISLINK,ISDOWNLOAD,ISNOTBOUNCE,FUNIQID,ORIGINALURL,HID,ISOLDCOUNTER,ISEVENT,ISPARAMETER,DONTCOUNTHITS,WITHHASH,HITCOLOR,LOCALEVENTTIME,AGE,SEX,INCOME,INTERESTS,ROBOTNESS,REMOTEIP,WINDOWNAME,OPENERNAME,HISTORYLENGTH,BROWSERLANGUAGE,BROWSERCOUNTRY,SOCIALNETWORK,SOCIALACTION,HTTPERROR,SENDTIMING,DNSTIMING,CONNECTTIMING,RESPONSESTARTTIMING,RESPONSEENDTIMING,FETCHTIMING,SOCIALSOURCENETWORKID,SOCIALSOURCEPAGE,PARAMPRICE,PARAMORDERID,PARAMCURRENCY,PARAMCURRENCYID,OPENSTATSERVICENAME,OPENSTATCAMPAIGNID,OPENSTATADID,OPENSTATSOURCEID,UTMSOURCE,UTMMEDIUM,UTMCAMPAIGN,UTMCONTENT,UTMTERM,FROMTAG,HASGCLID,REFERERHASH,URLHASH,CLID""", - // """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""", - // """WATCHID,JAVAENABLE,TITLE,GOODEVENT,EVENTTIME,EVENTDATE,COUNTERID,CLIENTIP,REGIONID,USERID,COUNTERCLASS,OS,USERAGENT,URL,REFERER,ISREFRESH,REFERERCATEGORYID,REFERERREGIONID,URLCATEGORYID,URLREGIONID,RESOLUTIONWIDTH,RESOLUTIONHEIGHT,RESOLUTIONDEPTH,FLASHMAJOR,FLASHMINOR,FLASHMINOR2,NETMAJOR,NETMINOR,USERAGENTMAJOR,USERAGENTMINOR,COOKIEENABLE,JAVASCRIPTENABLE,ISMOBILE,MOBILEPHONE,MOBILEPHONEMODEL,PARAMS,IPNETWORKID,TRAFICSOURCEID,SEARCHENGINEID,SEARCHPHRASE,ADVENGINEID,ISARTIFICAL,WINDOWCLIENTWIDTH,WINDOWCLIENTHEIGHT,CLIENTTIMEZONE,CLIENTEVENTTIME,SILVERLIGHTVERSION1,SILVERLIGHTVERSION2,SILVERLIGHTVERSION3,SILVERLIGHTVERSION4,PAGECHARSET,CODEVERSION,ISLINK,ISDOWNLOAD,ISNOTBOUNCE,FUNIQID,ORIGINALURL,HID,ISOLDCOUNTER,ISEVENT,ISPARAMETER,DONTCOUNTHITS,WITHHASH,HITCOLOR,LOCALEVENTTIME,AGE,SEX,INCOME,INTERESTS,ROBOTNESS,REMOTEIP,WINDOWNAME,OPENERNAME,HISTORYLENGTH,BROWSERLANGUAGE,BROWSERCOUNTRY,SOCIALNETWORK,SOCIALACTION,HTTPERROR,SENDTIMING,DNSTIMING,CONNECTTIMING,RESPONSESTARTTIMING,RESPONSEENDTIMING,FETCHTIMING,SOCIALSOURCENETWORKID,SOCIALSOURCEPAGE,PARAMPRICE,PARAMORDERID,PARAMCURRENCY,PARAMCURRENCYID,OPENSTATSERVICENAME,OPENSTATCAMPAIGNID,OPENSTATADID,OPENSTATSOURCEID,UTMSOURCE,UTMMEDIUM,UTMCAMPAIGN,UTMCONTENT,UTMTERM,FROMTAG,HASGCLID,REFERERHASH,URLHASH,CLID""", - // """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""", + def columns_list = ["""p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""", // case 0 + """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""", // case 1 + """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""", // case 2 + """p_partkey, p_name, p_size""", // case 3 + """p_partkey""", // case 4 + """p_partkey""", // case 5 + """p_partkey, p_size""", // case 6 + """p_partkey""", // case 7 + """p_partkey, p_size""", // case 8 + """p_partkey, p_size""", // case 9 + """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""", // case 10 + """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""", // case 11 + """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""", // case 12 + """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment, col1""", // case 13 + """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment, x1""", // case 14 + """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""", // case 15 + """col1, col2, col3, col4""", // case 16 + """p_partkey, p_name, p_mfgr, x1""", // case 17 + """p_partkey, p_name, p_mfgr, p_brand""", // case 18 + """p_partkey, p_name, p_mfgr, p_brand""", // case 19 + """p_name, p_mfgr""", // case 20 + """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""", // case 21 + """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""", // case 22 + """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""", // case 23 + """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""", // case 24 + """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""", // case 25 + """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""", // case 26 + """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""", // case 27 + """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""", // case 28 + """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""", // case 29 + """""" // case 30 ] - def column_in_paths = ["", "", "", "", "", "", "", "", "", "", "", "", "COLUMNS FROM PATH AS (city)", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""] - def preceding_filters = ["", "", "", "", "", "", "", "", "", "", "", "preceding filter p_size < 10", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""] - def set_values = ["", - "", - "SET(comment=p_comment, retailprice=p_retailprice, container=p_container, size=p_size, type=p_type, brand=p_brand, mfgr=p_mfgr, name=p_name, partkey=p_partkey)", - "set(p_name=upper(p_name),p_greatest=greatest(cast(p_partkey as int), cast(p_size as int)))", - "set(p_partkey = p_partkey + 100)", - "set(partkey = p_partkey + 100)", - "set(partkey = p_partkey + p_size)", - "set(tmpk = p_partkey + 1, partkey = tmpk*2)", - "set(partkey = p_partkey + 1, partsize = p_size*2)", - "set(partsize = p_partkey + p_size)", - "", - "", - "", - "", - "", - "", - "", - "set(col4 = x1)", - "set(col4 = p_brand)", - "set(col5 = p_brand)", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "" + def column_in_paths = ["", // case 0 + "", // case 1 + "", // case 2 + "", // case 3 + "", // case 4 + "", // case 5 + "", // case 6 + "", // case 7 + "", // case 8 + "", // case 9 + "", // case 10 + "", // case 11 + "COLUMNS FROM PATH AS (city)", // case 12 + "", // case 13 + "", // case 14 + "", // case 15 + "", // case 16 + "", // case 17 + "", // case 18 + "", // case 19 + "", // case 20 + "", // case 21 + "", // case 22 + "", // case 23 + "", // case 24 + "", // case 25 + "", // case 26 + "", // case 27 + "", // case 28 + "", // case 29 + "" // case 30 + ] + def preceding_filters = ["", // case 0 + "", // case 1 + "", // case 2 + "", // case 3 + "", // case 4 + "", // case 5 + "", // case 6 + "", // case 7 + "", // case 8 + "", // case 9 + "", // case 10 + "preceding filter p_size < 10", // case 11 + "", // case 12 + "", // case 13 + "", // case 14 + "", // case 15 + "", // case 16 + "", // case 17 + "", // case 18 + "", // case 19 + "", // case 20 + "", // case 21 + "", // case 22 + "", // case 23 + "", // case 24 + "", // case 25 + "", // case 26 + "", // case 27 + "", // case 28 + "", // case 29 + "" // case 30 + ] + def set_values = ["", // case 0 + "", // case 1 + "SET(comment=p_comment, retailprice=p_retailprice, container=p_container, size=p_size, type=p_type, brand=p_brand, mfgr=p_mfgr, name=p_name, partkey=p_partkey)", // case 2 + "set(p_name=upper(p_name),p_greatest=greatest(cast(p_partkey as int), cast(p_size as int)))", // case 3 + "set(p_partkey = p_partkey + 100)", // case 4 + "set(partkey = p_partkey + 100)", // case 5 + "set(partkey = p_partkey + p_size)", // case 6 + "set(tmpk = p_partkey + 1, partkey = tmpk*2)", // case 7 + "set(partkey = p_partkey + 1, partsize = p_size*2)", // case 8 + "set(partsize = p_partkey + p_size)", // case 9 + "", // case 10 + "", // case 11 + "", // case 12 + "", // case 13 + "", // case 14 + "", // case 15 + "", // case 16 + "set(col4 = x1)", // case 17 + "set(col4 = p_brand)", // case 18 + "set(col5 = p_brand)", // case 19 + "", // case 20 + "", // case 21 + "", // case 22 + "", // case 23 + "", // case 24 + "", // case 25 + "", // case 26 + "", // case 27 + "", // case 28 + "", // case 29 + "" // case 30 ] - def where_exprs = ["", "", "", "", "", "", "", "", "", "", "", "where p_partkey>10", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""] + def where_exprs = ["", // case 0 + "", // case 1 + "", // case 2 + "", // case 3 + "", // case 4 + "", // case 5 + "", // case 6 + "", // case 7 + "", // case 8 + "", // case 9 + "", // case 10 + "where p_partkey>10", // case 11 + "", // case 12 + "", // case 13 + "", // case 14 + "", // case 15 + "", // case 16 + "", // case 17 + "", // case 18 + "", // case 19 + "", // case 20 + "", // case 21 + "", // case 22 + "", // case 23 + "", // case 24 + "", // case 25 + "", // case 26 + "", // case 27 + "", // case 28 + "", // case 29 + "" // case 30 + ] - def line_delimiters = ["", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "\u0007"] + def line_delimiters = ["", // case 0 + "", // case 1 + "", // case 2 + "", // case 3 + "", // case 4 + "", // case 5 + "", // case 6 + "", // case 7 + "", // case 8 + "", // case 9 + "", // case 10 + "", // case 11 + "", // case 12 + "", // case 13 + "", // case 14 + "", // case 15 + "", // case 16 + "", // case 17 + "", // case 18 + "", // case 19 + "", // case 20 + "", // case 21 + "", // case 22 + "", // case 23 + "", // case 24 + "", // case 25 + "", // case 26 + "", // case 27 + "", // case 28 + "", // case 29 + "\u0007" // case 30 + ] - def etl_info = ["unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", - "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", - "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", - "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", - "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", - "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", - "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", - "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", - "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", - "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", - "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", - "unselected.rows=163706; dpp.abnorm.ALL=0; dpp.norm.ALL=36294", - "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", - "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", - "\\N", - "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", - "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", - "\\N", - "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", - "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", - "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", - "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=100000", - "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000", - "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000", - "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000", - "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000", - "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000", - "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000", - "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000", - "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000", - "\\N" + def etl_info = ["unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", // case 0 + "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", // case 1 + "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", // case 2 + "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", // case 3 + "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", // case 4 + "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", // case 5 + "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", // case 6 + "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", // case 7 + "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", // case 8 + "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", // case 9 + "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", // case 10 + "unselected.rows=163706; dpp.abnorm.ALL=0; dpp.norm.ALL=36294", // case 11 + "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", // case 12 + "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", // case 13 + "\\N", // case 14 + "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", // case 15 + "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", // case 16 + "\\N", // case 17 + "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", // case 18 + "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", // case 19 + "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000", // case 20 + "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=100000", // case 21 + "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000", // case 22 + "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000", // case 23 + "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000", // case 24 + "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000", // case 25 + "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000", // case 26 + "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000", // case 27 + "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000", // case 28 + "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000", // case 29 + "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000" // case 30 ] - def task_info = ["cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0" + def task_info = ["cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 0 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 1 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 2 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 3 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 4 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 5 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 6 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 7 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 8 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 9 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 10 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 11 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 12 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 13 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 14 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 15 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 16 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 17 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 18 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 19 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 20 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 21 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 22 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 23 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 24 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 25 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 26 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 27 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 28 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 29 + "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0" // case 30 ] - def error_msg = ["", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "failed to find default value expr for slot: x1", - "", - "", - "failed to find default value expr for slot: x1", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "", - "" + def error_msg = ["", // case 0 + "", // case 1 + "", // case 2 + "", // case 3 + "", // case 4 + "", // case 5 + "", // case 6 + "", // case 7 + "", // case 8 + "", // case 9 + "", // case 10 + "", // case 11 + "", // case 12 + "", // case 13 + "failed to find default value expr for slot: x1", // case 14 + "", // case 15 + "failed to find default value expr for slot: x1", // case 16 + "", // case 17 + "", // case 18 + "", // case 19 + "", // case 20 + "", // case 21 + "", // case 22 + "", // case 23 + "", // case 24 + "", // case 25 + "", // case 26 + "", // case 27 + "", // case 28 + "", // case 29 + "", // case 30 ] String ak = getS3AK() @@ -347,7 +461,7 @@ suite("test_broker_load_p2", "p2") { if (result[0][2].equals("CANCELLED")) { logger.info("Load result: " + result[0]) assertTrue(result[0][6].contains(task_info[i])) - assertTrue(result[0][9].contains(error_msg[i]), "expected: " + error_msg[i] + ", actual: " + result[0][7] + ", label: $label") + assertTrue(result[0][7].contains(error_msg[i]), "expected: " + error_msg[i] + ", actual: " + result[0][7] + ", label: $label") break; } Thread.sleep(1000) diff --git a/regression-test/suites/load_p2/test_single_replica_load.groovy b/regression-test/suites/load_p2/test_single_replica_load.groovy index c8ea2c578eb8e4..cb4f970ac66d1f 100644 --- a/regression-test/suites/load_p2/test_single_replica_load.groovy +++ b/regression-test/suites/load_p2/test_single_replica_load.groovy @@ -51,6 +51,7 @@ suite("test_single_replica_load", "p2") { def tableName = "test_single_replica_load" sql "DROP TABLE IF EXISTS ${tableName}" + sql """ set disable_inverted_index_v1_for_variant = false """ sql """ CREATE TABLE IF NOT EXISTS ${tableName} ( k bigint, @@ -61,6 +62,7 @@ suite("test_single_replica_load", "p2") { DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num" = "2", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1"); """ + sql """ set disable_inverted_index_v1_for_variant = true """ load_json_data.call(tableName, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""") load_json_data.call(tableName, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""") load_json_data.call(tableName, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""") diff --git a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_numeric_arithmatic.groovy b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_numeric_arithmatic.groovy index 5f728651267e82..dbfd3fad7bf913 100644 --- a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_numeric_arithmatic.groovy +++ b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_numeric_arithmatic.groovy @@ -409,4 +409,29 @@ test { //Additional cases for Xor, Conv, and other mathematical functions testFoldConst("SELECT CONV(-10, 10, 2) AS conv_invalid_base") //Conv with negative input (may be undefined) + + // fix floor/ceil/round function return type with DecimalV3 input + testFoldConst("with cte as (select floor(300.343) order by 1 limit 1) select * from cte") + testFoldConst("with cte as (select round(300.343) order by 1 limit 1) select * from cte") + testFoldConst("with cte as (select ceil(300.343) order by 1 limit 1) select * from cte") + + testFoldConst("with cte as (select floor(300.343, 2) order by 1 limit 1) select * from cte") + testFoldConst("with cte as (select round(300.343, 2) order by 1 limit 1) select * from cte") + testFoldConst("with cte as (select ceil(300.343, 2) order by 1 limit 1) select * from cte") + testFoldConst("with cte as (select truncate(300.343, 2) order by 1 limit 1) select * from cte") + + testFoldConst("with cte as (select floor(300.343, 0) order by 1 limit 1) select * from cte") + testFoldConst("with cte as (select round(300.343, 0) order by 1 limit 1) select * from cte") + testFoldConst("with cte as (select ceil(300.343, 0) order by 1 limit 1) select * from cte") + testFoldConst("with cte as (select truncate(300.343, 0) order by 1 limit 1) select * from cte") + + testFoldConst("with cte as (select floor(300.343, -1) order by 1 limit 1) select * from cte") + testFoldConst("with cte as (select round(300.343, -1) order by 1 limit 1) select * from cte") + testFoldConst("with cte as (select ceil(300.343, -1) order by 1 limit 1) select * from cte") + testFoldConst("with cte as (select truncate(300.343, -1) order by 1 limit 1) select * from cte") + + testFoldConst("with cte as (select floor(300.343, -4) order by 1 limit 1) select * from cte") + testFoldConst("with cte as (select round(300.343, -4) order by 1 limit 1) select * from cte") + testFoldConst("with cte as (select ceil(300.343, -4) order by 1 limit 1) select * from cte") + testFoldConst("with cte as (select truncate(300.343, -4) order by 1 limit 1) select * from cte") } diff --git a/regression-test/suites/variant_p0/load.groovy b/regression-test/suites/variant_p0/load.groovy index e79a58f8a7a149..cd5e9ee523d922 100644 --- a/regression-test/suites/variant_p0/load.groovy +++ b/regression-test/suites/variant_p0/load.groovy @@ -372,7 +372,7 @@ suite("regression_test_variant", "p0"){ "is_being_synced" = "false", "storage_medium" = "hdd", "storage_format" = "V2", - "inverted_index_storage_format" = "V1", + "inverted_index_storage_format" = "V2", "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "store_row_column" = "true", diff --git a/regression-test/suites/variant_p0/with_index/var_index.groovy b/regression-test/suites/variant_p0/with_index/var_index.groovy index 4d69bf0a6ebeeb..2a302b96b6a22b 100644 --- a/regression-test/suites/variant_p0/with_index/var_index.groovy +++ b/regression-test/suites/variant_p0/with_index/var_index.groovy @@ -46,4 +46,64 @@ suite("regression_test_variant_var_index", "p0"){ sql """insert into var_index values(9, '{"timestamp": 17.0}'),(10, '{"timestamp": "17.0"}')""" sql """insert into var_index values(11, '{"nested": [{"a" : 1}]}'),(11, '{"nested": [{"b" : "1024"}]}')""" qt_sql "select * from var_index order by k limit 15" + + sql """ set disable_inverted_index_v1_for_variant = true """ + sql "DROP TABLE IF EXISTS var_index" + try { + sql """ + CREATE TABLE IF NOT EXISTS var_index ( + k bigint, + v variant, + INDEX idx_var(v) USING INVERTED PROPERTIES("parser" = "english") COMMENT '' + ) + DUPLICATE KEY(`k`) + DISTRIBUTED BY HASH(k) BUCKETS 1 + properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1"); + """ + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("not supported in inverted index format V1")) + } + + sql """ + CREATE TABLE IF NOT EXISTS var_index ( + k bigint, + v variant + ) + DUPLICATE KEY(`k`) + DISTRIBUTED BY HASH(k) BUCKETS 1 + properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1"); + """ + + try { + sql """ALTER TABLE var_index ADD INDEX idx_var(v) USING INVERTED""" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("not supported in inverted index format V1")) + } + + sql """ set disable_inverted_index_v1_for_variant = false """ + sql """ + CREATE TABLE IF NOT EXISTS var_index ( + k bigint, + v variant, + INDEX idx_var(v) USING INVERTED PROPERTIES("parser" = "english") COMMENT '' + ) + DUPLICATE KEY(`k`) + DISTRIBUTED BY HASH(k) BUCKETS 1 + properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1"); + """ + + sql "DROP TABLE IF EXISTS var_index" + sql """ + CREATE TABLE IF NOT EXISTS var_index ( + k bigint, + v variant + ) + DUPLICATE KEY(`k`) + DISTRIBUTED BY HASH(k) BUCKETS 1 + properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1"); + """ + sql """ALTER TABLE var_index ADD INDEX idx_var(v) USING INVERTED""" + sql """ set disable_inverted_index_v1_for_variant = true """ } \ No newline at end of file