From 3fffc6178f266b612b15687a361b22d101b63ed6 Mon Sep 17 00:00:00 2001
From: LiBinfeng <libinfeng@selectdb.com>
Date: Tue, 12 Nov 2024 20:18:51 +0800
Subject: [PATCH 01/10] [Fix](Nereids) fix floor/round/ceil/truncate functions
 type compute precision problem (#43422)

- Problem
function like ```select floor(300.343, 2)``` precision should be 5 and
scale should be 2, but now is (6, 2) after compute precision, but after
folding const on fe, it changed to (5, 2) but upper level of plan still
expect the output of child to be (6, 2). So it would rise an exception
when executing.

- How it was fixed
fix folding constant precision of floor/round/ceil/truncate functions
from (5, 2) to (6, 2) in upper case

- Notion
when second value is negative and it absolute value >= precision -
value, it can not be expressed in fe which result is zero with decimal
type (3, 0). like 000. So just let it go back and no using folding
constant by fe.

- Related PR: #40744

- Release note
    Fix floor/round/ceil functions precision problem in folding constant
---
 .../executable/NumericArithmetic.java         | 34 ++++++++-----------
 .../fold_constant_numeric_arithmatic.groovy   | 25 ++++++++++++++
 2 files changed, 39 insertions(+), 20 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java
index 325e676fc046a0..a9acfeb2d6095b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java
@@ -694,12 +694,17 @@ private static Expression checkOutputBoundary(Literal input) {
         return input;
     }
 
+    private static Expression castDecimalV3Literal(DecimalV3Literal literal, int precision) {
+        return new DecimalV3Literal(DecimalV3Type.createDecimalV3Type(precision, literal.getValue().scale()),
+                literal.getValue());
+    }
+
     /**
      * round
      */
     @ExecFunction(name = "round")
     public static Expression round(DecimalV3Literal first) {
-        return first.round(0);
+        return castDecimalV3Literal(first.round(0), first.getValue().precision());
     }
 
     /**
@@ -707,7 +712,7 @@ public static Expression round(DecimalV3Literal first) {
      */
     @ExecFunction(name = "round")
     public static Expression round(DecimalV3Literal first, IntegerLiteral second) {
-        return first.round(second.getValue());
+        return castDecimalV3Literal(first.round(second.getValue()), first.getValue().precision());
     }
 
     /**
@@ -733,7 +738,7 @@ public static Expression round(DoubleLiteral first, IntegerLiteral second) {
      */
     @ExecFunction(name = "ceil")
     public static Expression ceil(DecimalV3Literal first) {
-        return first.roundCeiling(0);
+        return castDecimalV3Literal(first.roundCeiling(0), first.getValue().precision());
     }
 
     /**
@@ -741,7 +746,7 @@ public static Expression ceil(DecimalV3Literal first) {
      */
     @ExecFunction(name = "ceil")
     public static Expression ceil(DecimalV3Literal first, IntegerLiteral second) {
-        return first.roundCeiling(second.getValue());
+        return castDecimalV3Literal(first.roundCeiling(second.getValue()), first.getValue().precision());
     }
 
     /**
@@ -767,7 +772,7 @@ public static Expression ceil(DoubleLiteral first, IntegerLiteral second) {
      */
     @ExecFunction(name = "floor")
     public static Expression floor(DecimalV3Literal first) {
-        return first.roundFloor(0);
+        return castDecimalV3Literal(first.roundFloor(0), first.getValue().precision());
     }
 
     /**
@@ -775,7 +780,7 @@ public static Expression floor(DecimalV3Literal first) {
      */
     @ExecFunction(name = "floor")
     public static Expression floor(DecimalV3Literal first, IntegerLiteral second) {
-        return first.roundFloor(second.getValue());
+        return castDecimalV3Literal(first.roundFloor(second.getValue()), first.getValue().precision());
     }
 
     /**
@@ -1136,21 +1141,10 @@ public static Expression mathE() {
     public static Expression truncate(DecimalV3Literal first, IntegerLiteral second) {
         if (first.getValue().compareTo(BigDecimal.ZERO) == 0) {
             return first;
+        } else if (first.getValue().compareTo(BigDecimal.ZERO) < 0) {
+            return castDecimalV3Literal(first.roundCeiling(second.getValue()), first.getValue().precision());
         } else {
-            if (first.getValue().scale() < second.getValue()) {
-                return first;
-            }
-            if (second.getValue() < 0) {
-                double factor = Math.pow(10, Math.abs(second.getValue()));
-                return new DecimalV3Literal(
-                    DecimalV3Type.createDecimalV3Type(first.getValue().precision(), 0),
-                    BigDecimal.valueOf(Math.floor(first.getDouble() / factor) * factor));
-            }
-            if (first.getValue().compareTo(BigDecimal.ZERO) == -1) {
-                return first.roundCeiling(second.getValue());
-            } else {
-                return first.roundFloor(second.getValue());
-            }
+            return castDecimalV3Literal(first.roundFloor(second.getValue()), first.getValue().precision());
         }
     }
 
diff --git a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_numeric_arithmatic.groovy b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_numeric_arithmatic.groovy
index 5f728651267e82..dbfd3fad7bf913 100644
--- a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_numeric_arithmatic.groovy
+++ b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_numeric_arithmatic.groovy
@@ -409,4 +409,29 @@ test {
 
 //Additional cases for Xor, Conv, and other mathematical functions
     testFoldConst("SELECT CONV(-10, 10, 2) AS conv_invalid_base") //Conv with negative input (may be undefined)
+
+    // fix floor/ceil/round function return type with DecimalV3 input
+    testFoldConst("with cte as (select floor(300.343) order by 1 limit 1) select * from cte")
+    testFoldConst("with cte as (select round(300.343) order by 1 limit 1) select * from cte")
+    testFoldConst("with cte as (select ceil(300.343) order by 1 limit 1) select * from cte")
+
+    testFoldConst("with cte as (select floor(300.343, 2) order by 1 limit 1) select * from cte")
+    testFoldConst("with cte as (select round(300.343, 2) order by 1 limit 1) select * from cte")
+    testFoldConst("with cte as (select ceil(300.343, 2) order by 1 limit 1) select * from cte")
+    testFoldConst("with cte as (select truncate(300.343, 2) order by 1 limit 1) select * from cte")
+
+    testFoldConst("with cte as (select floor(300.343, 0) order by 1 limit 1) select * from cte")
+    testFoldConst("with cte as (select round(300.343, 0) order by 1 limit 1) select * from cte")
+    testFoldConst("with cte as (select ceil(300.343, 0) order by 1 limit 1) select * from cte")
+    testFoldConst("with cte as (select truncate(300.343, 0) order by 1 limit 1) select * from cte")
+
+    testFoldConst("with cte as (select floor(300.343, -1) order by 1 limit 1) select * from cte")
+    testFoldConst("with cte as (select round(300.343, -1) order by 1 limit 1) select * from cte")
+    testFoldConst("with cte as (select ceil(300.343, -1) order by 1 limit 1) select * from cte")
+    testFoldConst("with cte as (select truncate(300.343, -1) order by 1 limit 1) select * from cte")
+
+    testFoldConst("with cte as (select floor(300.343, -4) order by 1 limit 1) select * from cte")
+    testFoldConst("with cte as (select round(300.343, -4) order by 1 limit 1) select * from cte")
+    testFoldConst("with cte as (select ceil(300.343, -4) order by 1 limit 1) select * from cte")
+    testFoldConst("with cte as (select truncate(300.343, -4) order by 1 limit 1) select * from cte")
 }

From 127253a4588c74dcde29efeead8560a23f1e861f Mon Sep 17 00:00:00 2001
From: Sun Chenyang <sunchenyang@selectdb.com>
Date: Tue, 12 Nov 2024 21:00:39 +0800
Subject: [PATCH 02/10] [fix] (inverted index) Disallow variant columns from
 using inverted index format v1 (#43599)

### What problem does this PR solve?

Problem Summary:
1. When the inverted index of a variant column uses storage format v1,
schema changes can cause some segments to lack corresponding index
files.

2. By using storage format v2 for inverted indexes, all indexes
correspond to a single file, and the corresponding files will always
exist regardless of whether the variant includes subcolumn indexes.

### Release note

When creating an inverted index for a variant column, file format v1 is
not supported
---
 .../doris/alter/SchemaChangeHandler.java      |  8 ++-
 .../doris/analysis/CreateTableStmt.java       |  8 ++-
 .../org/apache/doris/analysis/IndexDef.java   | 12 +++-
 .../plans/commands/info/CreateTableInfo.java  | 12 +++-
 .../plans/commands/info/IndexDefinition.java  | 10 +++-
 .../org/apache/doris/qe/SessionVariable.java  | 13 ++++
 .../apache/doris/analysis/IndexDefTest.java   | 13 ++++
 .../plans/commands/IndexDefinitionTest.java   | 46 ++++++++++++++
 ...paction_with_variant_inverted_index.groovy |  2 +
 .../inverted_index_p0/load/test_insert.groovy |  4 ++
 .../load/test_stream_load.groovy              |  4 ++
 ...index_file_http_action_with_variant.groovy |  2 +
 .../test_variant_index_format_v1.groovy       |  3 +-
 .../load_p2/test_single_replica_load.groovy   |  2 +
 regression-test/suites/variant_p0/load.groovy |  2 +-
 .../variant_p0/with_index/var_index.groovy    | 60 +++++++++++++++++++
 16 files changed, 191 insertions(+), 10 deletions(-)
 create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IndexDefinitionTest.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index 4ed78ab66c9484..b7a0fa5cfbc746 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -98,6 +98,7 @@
 import org.apache.doris.task.AgentTaskQueue;
 import org.apache.doris.task.ClearAlterTask;
 import org.apache.doris.task.UpdateTabletMetaInfoTask;
+import org.apache.doris.thrift.TInvertedIndexFileStorageFormat;
 import org.apache.doris.thrift.TStorageFormat;
 import org.apache.doris.thrift.TStorageMedium;
 import org.apache.doris.thrift.TTaskType;
@@ -2730,12 +2731,15 @@ private boolean processAddIndex(CreateIndexClause alterClause, OlapTable olapTab
                                 + " ) already exist.");
             }
         }
-
+        boolean disableInvertedIndexV1ForVariant = olapTable.getInvertedIndexFileStorageFormat()
+                        == TInvertedIndexFileStorageFormat.V1 && ConnectContext.get().getSessionVariable()
+                                                                        .getDisableInvertedIndexV1ForVaraint();
         for (String col : indexDef.getColumns()) {
             Column column = olapTable.getColumn(col);
             if (column != null) {
                 indexDef.checkColumn(column, olapTable.getKeysType(),
-                        olapTable.getTableProperty().getEnableUniqueKeyMergeOnWrite());
+                        olapTable.getTableProperty().getEnableUniqueKeyMergeOnWrite(),
+                                                                        disableInvertedIndexV1ForVariant);
                 indexDef.getColumnUniqueIds().add(column.getUniqueId());
             } else {
                 throw new DdlException("index column does not exist in table. invalid column: " + col);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
index b07424056906b8..90f7d6f72ede1c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
@@ -47,6 +47,7 @@
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.rewrite.ExprRewriteRule;
 import org.apache.doris.rewrite.ExprRewriter;
+import org.apache.doris.thrift.TInvertedIndexFileStorageFormat;
 
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableSet;
@@ -598,7 +599,9 @@ public void analyze(Analyzer analyzer) throws UserException {
         if (CollectionUtils.isNotEmpty(indexDefs)) {
             Set<String> distinct = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
             Set<Pair<IndexType, List<String>>> distinctCol = new HashSet<>();
-
+            boolean disableInvertedIndexV1ForVariant = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(
+                        new HashMap<>(properties)) == TInvertedIndexFileStorageFormat.V1
+                            && ConnectContext.get().getSessionVariable().getDisableInvertedIndexV1ForVaraint();
             for (IndexDef indexDef : indexDefs) {
                 indexDef.analyze();
                 if (!engineName.equalsIgnoreCase(DEFAULT_ENGINE_NAME)) {
@@ -608,7 +611,8 @@ public void analyze(Analyzer analyzer) throws UserException {
                     boolean found = false;
                     for (Column column : columns) {
                         if (column.getName().equalsIgnoreCase(indexColName)) {
-                            indexDef.checkColumn(column, getKeysDesc().getKeysType(), enableUniqueKeyMergeOnWrite);
+                            indexDef.checkColumn(column, getKeysDesc().getKeysType(),
+                                                    enableUniqueKeyMergeOnWrite, disableInvertedIndexV1ForVariant);
                             found = true;
                             break;
                         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
index d98a3b93e45dbf..39b32662614f15 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
@@ -212,8 +212,8 @@ public boolean isInvertedIndex() {
         return (this.indexType == IndexType.INVERTED);
     }
 
-    public void checkColumn(Column column, KeysType keysType, boolean enableUniqueKeyMergeOnWrite)
-            throws AnalysisException {
+    public void checkColumn(Column column, KeysType keysType, boolean enableUniqueKeyMergeOnWrite,
+                boolean disableInvertedIndexV1ForVariant) throws AnalysisException {
         if (indexType == IndexType.BITMAP || indexType == IndexType.INVERTED || indexType == IndexType.BLOOMFILTER
                 || indexType == IndexType.NGRAM_BF) {
             String indexColName = column.getName();
@@ -225,6 +225,14 @@ public void checkColumn(Column column, KeysType keysType, boolean enableUniqueKe
                 throw new AnalysisException(colType + " is not supported in " + indexType.toString() + " index. "
                         + "invalid index: " + indexName);
             }
+
+            // In inverted index format v1, each subcolumn of a variant has its own index file, leading to high IOPS.
+            // when the subcolumn type changes, it may result in missing files, causing link file failure.
+            if (colType.isVariantType() && disableInvertedIndexV1ForVariant) {
+                throw new AnalysisException(colType + " is not supported in inverted index format V1,"
+                        + "Please set properties(\"inverted_index_storage_format\"= \"v2\"),"
+                        + "or upgrade to a newer version");
+            }
             if (!column.isKey()) {
                 if (keysType == KeysType.AGG_KEYS) {
                     throw new AnalysisException("index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table"
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
index 3365f380832e02..e45e71bd1f997d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
@@ -74,6 +74,7 @@
 import org.apache.doris.nereids.util.TypeCoercionUtils;
 import org.apache.doris.nereids.util.Utils;
 import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.thrift.TInvertedIndexFileStorageFormat;
 
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
@@ -607,6 +608,14 @@ public void validate(ConnectContext ctx) {
         if (!indexes.isEmpty()) {
             Set<String> distinct = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
             Set<Pair<IndexDef.IndexType, List<String>>> distinctCol = new HashSet<>();
+            boolean disableInvertedIndexV1ForVariant = false;
+            try {
+                disableInvertedIndexV1ForVariant = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(
+                            new HashMap<>(properties)) == TInvertedIndexFileStorageFormat.V1
+                                && ConnectContext.get().getSessionVariable().getDisableInvertedIndexV1ForVaraint();
+            } catch (Exception e) {
+                throw new AnalysisException(e.getMessage(), e.getCause());
+            }
 
             for (IndexDefinition indexDef : indexes) {
                 indexDef.validate();
@@ -618,7 +627,8 @@ public void validate(ConnectContext ctx) {
                     boolean found = false;
                     for (ColumnDefinition column : columns) {
                         if (column.getName().equalsIgnoreCase(indexColName)) {
-                            indexDef.checkColumn(column, keysType, isEnableMergeOnWrite);
+                            indexDef.checkColumn(column, keysType, isEnableMergeOnWrite,
+                                                                disableInvertedIndexV1ForVariant);
                             found = true;
                             break;
                         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java
index 61f2c874fd7ebc..4d22e5af51c0ee 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java
@@ -93,7 +93,7 @@ public IndexDefinition(String name, List<String> cols, String indexTypeName,
      * checkColumn
      */
     public void checkColumn(ColumnDefinition column, KeysType keysType,
-            boolean enableUniqueKeyMergeOnWrite) throws AnalysisException {
+            boolean enableUniqueKeyMergeOnWrite, boolean disableInvertedIndexV1ForVariant) throws AnalysisException {
         if (indexType == IndexType.BITMAP || indexType == IndexType.INVERTED
                 || indexType == IndexType.BLOOMFILTER || indexType == IndexType.NGRAM_BF) {
             String indexColName = column.getName();
@@ -106,6 +106,14 @@ public void checkColumn(ColumnDefinition column, KeysType keysType,
                 throw new AnalysisException(colType + " is not supported in " + indexType.toString()
                         + " index. " + "invalid index: " + name);
             }
+
+            // In inverted index format v1, each subcolumn of a variant has its own index file, leading to high IOPS.
+            // when the subcolumn type changes, it may result in missing files, causing link file failure.
+            if (colType.isVariantType() && disableInvertedIndexV1ForVariant) {
+                throw new AnalysisException(colType + " is not supported in inverted index format V1,"
+                        + "Please set properties(\"inverted_index_storage_format\"= \"v2\"),"
+                        + "or upgrade to a newer version");
+            }
             if (!column.isKey()) {
                 if (keysType == KeysType.AGG_KEYS) {
                     throw new AnalysisException("index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table"
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 99043902819b56..e4588844422057 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -665,6 +665,8 @@ public class SessionVariable implements Serializable, Writable {
 
     public static final String MAX_FETCH_REMOTE_TABLET_COUNT = "max_fetch_remote_schema_tablet_count";
 
+    public static final String DISABLE_INVERTED_INDEX_V1_FOR_VARIANT = "disable_inverted_index_v1_for_variant";
+
     // CLOUD_VARIABLES_BEGIN
     public static final String CLOUD_CLUSTER = "cloud_cluster";
     public static final String DISABLE_EMPTY_PARTITION_PRUNE = "disable_empty_partition_prune";
@@ -1224,6 +1226,9 @@ public enum IgnoreSplitType {
     @VariableMgr.VarAttr(name = WAIT_FULL_BLOCK_SCHEDULE_TIMES)
     public int waitFullBlockScheduleTimes = 2;
 
+    @VariableMgr.VarAttr(name = DISABLE_INVERTED_INDEX_V1_FOR_VARIANT)
+    private boolean disableInvertedIndexV1ForVaraint = true;
+
     public int getBeNumberForTest() {
         return beNumberForTest;
     }
@@ -4561,4 +4566,12 @@ public boolean isEnableCooldownReplicaAffinity() {
     public boolean isUseSerialExchange() {
         return useSerialExchange && getEnableLocalExchange();
     }
+
+    public void setDisableInvertedIndexV1ForVaraint(boolean disableInvertedIndexV1ForVaraint) {
+        this.disableInvertedIndexV1ForVaraint = disableInvertedIndexV1ForVaraint;
+    }
+
+    public boolean getDisableInvertedIndexV1ForVaraint() {
+        return disableInvertedIndexV1ForVaraint;
+    }
 }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java
index a4eeec679b24d1..ccabece832d023 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java
@@ -17,6 +17,9 @@
 
 package org.apache.doris.analysis;
 
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.KeysType;
+import org.apache.doris.catalog.PrimitiveType;
 import org.apache.doris.common.AnalysisException;
 
 import com.google.common.collect.Lists;
@@ -60,6 +63,16 @@ public void testAnalyzeExpection() throws AnalysisException {
         } catch (AnalysisException e) {
             Assert.assertTrue(e instanceof AnalysisException);
         }
+        try {
+            def = new IndexDef("variant_index", false, Lists.newArrayList("col1"),
+                                    IndexDef.IndexType.INVERTED, null, "comment");
+            boolean isIndexFormatV1 = true;
+            def.checkColumn(new Column("col1", PrimitiveType.VARIANT), KeysType.UNIQUE_KEYS, true, isIndexFormatV1);
+            Assert.fail("No exception throws.");
+        } catch (AnalysisException e) {
+            Assert.assertTrue(e instanceof AnalysisException);
+            Assert.assertTrue(e.getMessage().contains("not supported in inverted index format V1"));
+        }
     }
 
     @Test
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IndexDefinitionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IndexDefinitionTest.java
new file mode 100644
index 00000000000000..0632945ee0416e
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/IndexDefinitionTest.java
@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.AggregateType;
+import org.apache.doris.catalog.KeysType;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition;
+import org.apache.doris.nereids.trees.plans.commands.info.IndexDefinition;
+import org.apache.doris.nereids.types.VariantType;
+
+import com.google.common.collect.Lists;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class IndexDefinitionTest {
+    @Test
+    void testVariantIndexFormatV1() throws AnalysisException {
+        IndexDefinition def = new IndexDefinition("variant_index", Lists.newArrayList("col1"), "INVERTED",
+                                        null, "comment");
+        try {
+            boolean isIndexFormatV1 = true;
+            def.checkColumn(new ColumnDefinition("col1", VariantType.INSTANCE, false, AggregateType.NONE, true,
+                                    null, "comment"), KeysType.UNIQUE_KEYS, true, isIndexFormatV1);
+            Assertions.fail("No exception throws.");
+        } catch (AnalysisException e) {
+            Assertions.assertTrue(e instanceof AnalysisException);
+            Assertions.assertTrue(e.getMessage().contains("not supported in inverted index format V1"));
+        }
+    }
+}
diff --git a/regression-test/suites/compaction/test_single_compaction_with_variant_inverted_index.groovy b/regression-test/suites/compaction/test_single_compaction_with_variant_inverted_index.groovy
index 79c6322204585d..d450df1868c5a8 100644
--- a/regression-test/suites/compaction/test_single_compaction_with_variant_inverted_index.groovy
+++ b/regression-test/suites/compaction/test_single_compaction_with_variant_inverted_index.groovy
@@ -132,6 +132,7 @@ suite("test_single_compaction_with_variant_inverted", "p2") {
 
 
     sql """ DROP TABLE IF EXISTS ${tableName}; """
+    sql """ set disable_inverted_index_v1_for_variant = false """
     sql """
         CREATE TABLE ${tableName} (
             `id` int(11) NULL,
@@ -150,6 +151,7 @@ suite("test_single_compaction_with_variant_inverted", "p2") {
             "compaction_policy" = "time_series"
         );
     """
+    sql """ set disable_inverted_index_v1_for_variant = true """
 
     def tablets = sql_return_maparray """ show tablets from ${tableName}; """
 
diff --git a/regression-test/suites/inverted_index_p0/load/test_insert.groovy b/regression-test/suites/inverted_index_p0/load/test_insert.groovy
index 97b3ca07937f49..03e7a3ccb26404 100644
--- a/regression-test/suites/inverted_index_p0/load/test_insert.groovy
+++ b/regression-test/suites/inverted_index_p0/load/test_insert.groovy
@@ -72,10 +72,14 @@ suite("test_insert_with_index", "p0") {
     }
 
     set_be_config("inverted_index_ram_dir_enable", "true")
+    sql """ set disable_inverted_index_v1_for_variant = false """
     test.call("V1")
+    sql """ set disable_inverted_index_v1_for_variant = true """
     test.call("V2")
     set_be_config("inverted_index_ram_dir_enable", "false")
+    sql """ set disable_inverted_index_v1_for_variant = false """
     test.call("V1")
+    sql """ set disable_inverted_index_v1_for_variant = true """
     test.call("V2")
     set_be_config("inverted_index_ram_dir_enable", "true")
 }
\ No newline at end of file
diff --git a/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy b/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy
index 53ee5d546c2709..1912fdcfc50c4c 100644
--- a/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy
+++ b/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy
@@ -105,10 +105,14 @@ suite("test_stream_load_with_inverted_index_p0", "p0") {
     }
 
     set_be_config("inverted_index_ram_dir_enable", "true")
+    sql """ set disable_inverted_index_v1_for_variant = false """
     test.call("V1")
+    sql """ set disable_inverted_index_v1_for_variant = true """
     test.call("V2")
     set_be_config("inverted_index_ram_dir_enable", "false")
+    sql """ set disable_inverted_index_v1_for_variant = false """
     test.call("V1")
+    sql """ set disable_inverted_index_v1_for_variant = true """
     test.call("V2")
     set_be_config("inverted_index_ram_dir_enable", "true")
 }
\ No newline at end of file
diff --git a/regression-test/suites/inverted_index_p0/test_show_nested_index_file_http_action_with_variant.groovy b/regression-test/suites/inverted_index_p0/test_show_nested_index_file_http_action_with_variant.groovy
index 71831140a996a7..a7718e3927a4a3 100644
--- a/regression-test/suites/inverted_index_p0/test_show_nested_index_file_http_action_with_variant.groovy
+++ b/regression-test/suites/inverted_index_p0/test_show_nested_index_file_http_action_with_variant.groovy
@@ -64,6 +64,7 @@ suite("test_show_nested_index_file_http_action_with_variant", "nonConcurrent,p0"
         def tableName = "test_show_nested_index_file_http_action_with_variant_" + format
 
         sql "DROP TABLE IF EXISTS ${tableName}"
+        sql """ set disable_inverted_index_v1_for_variant = false """
         sql """
             CREATE TABLE IF NOT EXISTS ${tableName} (
                 k bigint,
@@ -74,6 +75,7 @@ suite("test_show_nested_index_file_http_action_with_variant", "nonConcurrent,p0"
             DISTRIBUTED BY HASH(k) BUCKETS 1
             properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "${format}");
         """
+        sql """ set disable_inverted_index_v1_for_variant = true """
         load_json_data.call(tableName, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""")
         load_json_data.call(tableName, """${getS3Url() + '/regression/gharchive.m/2015-01-01-1.json'}""")
 
diff --git a/regression-test/suites/inverted_index_p2/test_variant_index_format_v1.groovy b/regression-test/suites/inverted_index_p2/test_variant_index_format_v1.groovy
index cc44ac2bb38234..5f262c8bf07775 100644
--- a/regression-test/suites/inverted_index_p2/test_variant_index_format_v1.groovy
+++ b/regression-test/suites/inverted_index_p2/test_variant_index_format_v1.groovy
@@ -60,6 +60,7 @@ suite("test_variant_index_format_v1", "p2") {
 
     def table_name = "github_events"
     sql """DROP TABLE IF EXISTS ${table_name}"""
+    sql """ set disable_inverted_index_v1_for_variant = false """
     sql """
         CREATE TABLE IF NOT EXISTS ${table_name} (
             k bigint,
@@ -70,7 +71,7 @@ suite("test_variant_index_format_v1", "p2") {
         DISTRIBUTED BY HASH(k) BUCKETS 1
         properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1");
     """
-
+    sql """ set disable_inverted_index_v1_for_variant = true """
     set_be_config.call("memory_limitation_per_thread_for_schema_change_bytes", "6294967296")
     load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""")
     load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2015-01-01-1.json'}""")
diff --git a/regression-test/suites/load_p2/test_single_replica_load.groovy b/regression-test/suites/load_p2/test_single_replica_load.groovy
index c8ea2c578eb8e4..cb4f970ac66d1f 100644
--- a/regression-test/suites/load_p2/test_single_replica_load.groovy
+++ b/regression-test/suites/load_p2/test_single_replica_load.groovy
@@ -51,6 +51,7 @@ suite("test_single_replica_load", "p2") {
     def tableName = "test_single_replica_load"
 
     sql "DROP TABLE IF EXISTS ${tableName}"
+    sql """ set disable_inverted_index_v1_for_variant = false """
     sql """
         CREATE TABLE IF NOT EXISTS ${tableName} (
             k bigint,
@@ -61,6 +62,7 @@ suite("test_single_replica_load", "p2") {
         DISTRIBUTED BY HASH(k) BUCKETS 1
         properties("replication_num" = "2", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1");
     """
+    sql """ set disable_inverted_index_v1_for_variant = true """
     load_json_data.call(tableName, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""")
     load_json_data.call(tableName, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""")
     load_json_data.call(tableName, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""")
diff --git a/regression-test/suites/variant_p0/load.groovy b/regression-test/suites/variant_p0/load.groovy
index e79a58f8a7a149..cd5e9ee523d922 100644
--- a/regression-test/suites/variant_p0/load.groovy
+++ b/regression-test/suites/variant_p0/load.groovy
@@ -372,7 +372,7 @@ suite("regression_test_variant", "p0"){
                 "is_being_synced" = "false",
                 "storage_medium" = "hdd",
                 "storage_format" = "V2",
-                "inverted_index_storage_format" = "V1",
+                "inverted_index_storage_format" = "V2",
                 "enable_unique_key_merge_on_write" = "true",
                 "light_schema_change" = "true",
                 "store_row_column" = "true",
diff --git a/regression-test/suites/variant_p0/with_index/var_index.groovy b/regression-test/suites/variant_p0/with_index/var_index.groovy
index 4d69bf0a6ebeeb..2a302b96b6a22b 100644
--- a/regression-test/suites/variant_p0/with_index/var_index.groovy
+++ b/regression-test/suites/variant_p0/with_index/var_index.groovy
@@ -46,4 +46,64 @@ suite("regression_test_variant_var_index", "p0"){
     sql """insert into var_index values(9, '{"timestamp": 17.0}'),(10, '{"timestamp": "17.0"}')"""
     sql """insert into var_index values(11, '{"nested": [{"a" : 1}]}'),(11, '{"nested": [{"b" : "1024"}]}')"""
     qt_sql "select * from var_index order by k limit 15"
+
+    sql """ set disable_inverted_index_v1_for_variant = true """
+    sql "DROP TABLE IF EXISTS var_index"
+    try {
+        sql """
+            CREATE TABLE IF NOT EXISTS var_index (
+                k bigint,
+                v variant,
+                INDEX idx_var(v) USING INVERTED  PROPERTIES("parser" = "english") COMMENT ''
+            )
+            DUPLICATE KEY(`k`)
+            DISTRIBUTED BY HASH(k) BUCKETS 1 
+            properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1");
+        """
+    } catch (Exception e) {
+        log.info(e.getMessage())
+        assertTrue(e.getMessage().contains("not supported in inverted index format V1"))
+    }
+
+    sql """
+        CREATE TABLE IF NOT EXISTS var_index (
+            k bigint,
+            v variant
+        )
+        DUPLICATE KEY(`k`)
+        DISTRIBUTED BY HASH(k) BUCKETS 1 
+        properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1");
+    """
+
+    try {
+        sql """ALTER TABLE var_index ADD INDEX idx_var(v) USING INVERTED"""
+    } catch (Exception e) {
+        log.info(e.getMessage())
+        assertTrue(e.getMessage().contains("not supported in inverted index format V1"))
+    }
+
+    sql """ set disable_inverted_index_v1_for_variant = false """
+    sql """
+        CREATE TABLE IF NOT EXISTS var_index (
+            k bigint,
+            v variant,
+            INDEX idx_var(v) USING INVERTED  PROPERTIES("parser" = "english") COMMENT ''
+        )
+        DUPLICATE KEY(`k`)
+        DISTRIBUTED BY HASH(k) BUCKETS 1 
+        properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1");
+    """
+
+    sql "DROP TABLE IF EXISTS var_index"
+    sql """
+        CREATE TABLE IF NOT EXISTS var_index (
+            k bigint,
+            v variant
+        )
+        DUPLICATE KEY(`k`)
+        DISTRIBUTED BY HASH(k) BUCKETS 1 
+        properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1");
+    """
+    sql """ALTER TABLE var_index ADD INDEX idx_var(v) USING INVERTED"""
+    sql """ set disable_inverted_index_v1_for_variant = true """
 }
\ No newline at end of file

From 2eae8bb2de5625415374638c15d444d5a3d6e1cf Mon Sep 17 00:00:00 2001
From: 924060929 <lanhuajian@selectdb.com>
Date: Tue, 12 Nov 2024 21:05:11 +0800
Subject: [PATCH 03/10] [fix](Nereids) fix QueryProcessor cannot be cast to
 class LoadProcessor (#43763)

fix QueryProcessor cannot be cast to class LoadProcessor, introduced by
#41730

Problem Summary:

sql: any select statement

it only meet when open debug log, so I can not write a test
```
2024-11-12 08:15:52,266 WARN (mysql-nio-pool-0|206) [ConnectProcessor.handleQueryException():480] Process one query failed because unknown reason:
java.lang.ClassCastException: class org.apache.doris.qe.runtime.QueryProcessor cannot be cast to class org.apache.doris.qe.runtime.LoadProcessor (org.apache.doris.qe.runtime.QueryProcessor and org.apache.doris.qe.runtime.LoadProcessor are in unnamed module of loader 'app')
	at org.apache.doris.qe.CoordinatorContext.asLoadProcessor(CoordinatorContext.java:262) ~[doris-fe.jar:1.2-SNAPSHOT]
	at org.apache.doris.qe.NereidsCoordinator.getJobId(NereidsCoordinator.java:202) ~[doris-fe.jar:1.2-SNAPSHOT]
	at org.apache.doris.qe.QeProcessorImpl.registerQuery(QeProcessorImpl.java:116) ~[doris-fe.jar:1.2-SNAPSHOT]
	at org.apache.doris.qe.StmtExecutor.executeAndSendResult(StmtExecutor.java:1925) ~[doris-fe.jar:1.2-SNAPSHOT]
	at org.apache.doris.qe.StmtExecutor.handleQueryStmt(StmtExecutor.java:1897) ~[doris-fe.jar:1.2-SNAPSHOT]
	at org.apache.doris.qe.StmtExecutor.handleQueryWithRetry(StmtExecutor.java:901) ~[doris-fe.jar:1.2-SNAPSHOT]
	at org.apache.doris.qe.StmtExecutor.executeByNereids(StmtExecutor.java:833) ~[doris-fe.jar:1.2-SNAPSHOT]
	at org.apache.doris.qe.StmtExecutor.execute(StmtExecutor.java:605) ~[doris-fe.jar:1.2-SNAPSHOT]
	at org.apache.doris.qe.StmtExecutor.queryRetry(StmtExecutor.java:568) ~[doris-fe.jar:1.2-SNAPSHOT]
	at org.apache.doris.qe.StmtExecutor.execute(StmtExecutor.java:558) ~[doris-fe.jar:1.2-SNAPSHOT]
	at org.apache.doris.qe.ConnectProcessor.executeQuery(ConnectProcessor.java:340) ~[doris-fe.jar:1.2-SNAPSHOT]
	at org.apache.doris.qe.ConnectProcessor.handleQuery(ConnectProcessor.java:243) ~[doris-fe.jar:1.2-SNAPSHOT]
	at org.apache.doris.qe.MysqlConnectProcessor.handleQuery(MysqlConnectProcessor.java:209) ~[doris-fe.jar:1.2-SNAPSHOT]
	at org.apache.doris.qe.MysqlConnectProcessor.dispatch(MysqlConnectProcessor.java:237) ~[doris-fe.jar:1.2-SNAPSHOT]
	at org.apache.doris.qe.MysqlConnectProcessor.processOnce(MysqlConnectProcessor.java:414) ~[doris-fe.jar:1.2-SNAPSHOT]
	at org.apache.doris.mysql.ReadListener.lambda$handleEvent$0(ReadListener.java:52) ~[doris-fe.jar:1.2-SNAPSHOT]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
```
---
 .../main/java/org/apache/doris/qe/NereidsCoordinator.java   | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java
index a14aea25463ad4..d718089fcabfee 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java
@@ -199,7 +199,11 @@ public long getNumReceivedRows() {
 
     @Override
     public long getJobId() {
-        return coordinatorContext.asLoadProcessor().jobId;
+        JobProcessor jobProcessor = coordinatorContext.getJobProcessor();
+        if (jobProcessor instanceof LoadProcessor) {
+            return ((LoadProcessor) jobProcessor).jobId;
+        }
+        return -1L;
     }
 
     /*

From de856351f9b37f06d62840a71af083cf994587aa Mon Sep 17 00:00:00 2001
From: Gavin Chou <gavin@selectdb.com>
Date: Tue, 12 Nov 2024 21:29:10 +0800
Subject: [PATCH 04/10] [fix](syncpoint) Fix incorrect syncpoint 'has_point()'
 intorduced by (#43461) (#43784)

Related PR: #43461
---
 common/cpp/sync_point.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/common/cpp/sync_point.cpp b/common/cpp/sync_point.cpp
index c2263415d30d5a..02bdce1ec3487a 100644
--- a/common/cpp/sync_point.cpp
+++ b/common/cpp/sync_point.cpp
@@ -248,8 +248,7 @@ void SyncPoint::Data::disable_processing() {
 
 bool SyncPoint::Data::has_point(const std::string& point) {
   std::unique_lock lock(mutex_);
-  auto marked_point_iter = marked_thread_id_.find(point);
-  return marked_point_iter != marked_thread_id_.end();
+  return callbacks_.find(point) != callbacks_.end();
 }
 
 bool SyncPoint::Data::get_enable() {

From 3d7785de2aacbd2152a3655059559267c5bff199 Mon Sep 17 00:00:00 2001
From: walter <w41ter.l@gmail.com>
Date: Tue, 12 Nov 2024 21:33:55 +0800
Subject: [PATCH 05/10] [improve](binlog) Reserve ids for binlog type
 allocation (#43769)

### What problem does this PR solve?

Problem Summary:

Keep some IDs for allocation so that when new binlog types are added in
the future, the changes can be picked back to the old versions without
breaking compatibility.

The code will check the IDs of binlog types. Any binlog types whose IDs
are greater than or equal to MIN_UNKNOWN will be ignored.
---
 .../apache/doris/binlog/BinlogManager.java    |   7 +-
 gensrc/thrift/FrontendService.thrift          | 116 ++++++++++++++++++
 2 files changed, 122 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java
index 60d091bf202050..35588568511a3f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java
@@ -593,8 +593,13 @@ public long read(DataInputStream dis, long checksum) throws IOException {
                     continue;
                 }
 
-                // Step 2.2: check if there is in next db Binlogs region
                 long dbId = binlog.getDbId();
+                if (binlog.getType().getValue() >= TBinlogType.MIN_UNKNOWN.getValue()) {
+                    LOG.warn("skip unknown binlog, type: {}, db: {}", binlog.getType().getValue(), dbId);
+                    continue;
+                }
+
+                // Step 2.2: check if there is in next db Binlogs region
                 if (dbId != currentDbId) {
                     // if there is in next db Binlogs region, check and update metadata
                     Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId);
diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift
index bb8cb8c8f2a7ba..fbbe8254e8aff9 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -1189,6 +1189,122 @@ enum TBinlogType {
   RENAME_TABLE = 14,
   RENAME_COLUMN = 15,
   MODIFY_COMMENT = 16,
+
+  // Keep some IDs for allocation so that when new binlog types are added in the
+  // future, the changes can be picked back to the old versions without breaking
+  // compatibility.
+  //
+  // The code will check the IDs of binlog types, any binlog types whose IDs are
+  // greater than or equal to MIN_UNKNOWN will be ignored.
+  //
+  // For example, before you adding new binlog type MODIFY_XXX:
+  //    MIN_UNKNOWN = 17,
+  //    UNKNOWN_2 = 18,
+  //    UNKNOWN_3 = 19,
+  // After adding binlog type MODIFY_XXX:
+  //    MODIFY_XXX = 17,
+  //    MIN_UNKNOWN = 18,
+  //    UNKNOWN_3 = 19,
+  MIN_UNKNOWN = 17,
+  UNKNOWN_2 = 18,
+  UNKNOWN_3 = 19,
+  UNKNOWN_4 = 20,
+  UNKNOWN_5 = 21,
+  UNKNOWN_6 = 22,
+  UNKNOWN_7 = 23,
+  UNKNOWN_8 = 24,
+  UNKNOWN_9 = 25,
+  UNKNOWN_10 = 26,
+  UNKNOWN_11 = 27,
+  UNKNOWN_12 = 28,
+  UNKNOWN_13 = 29,
+  UNKNOWN_14 = 30,
+  UNKNOWN_15 = 31,
+  UNKNOWN_16 = 32,
+  UNKNOWN_17 = 33,
+  UNKNOWN_18 = 34,
+  UNKNOWN_19 = 35,
+  UNKNOWN_20 = 36,
+  UNKNOWN_21 = 37,
+  UNKNOWN_22 = 38,
+  UNKNOWN_23 = 39,
+  UNKNOWN_24 = 40,
+  UNKNOWN_25 = 41,
+  UNKNOWN_26 = 42,
+  UNKNOWN_27 = 43,
+  UNKNOWN_28 = 44,
+  UNKNOWN_29 = 45,
+  UNKNOWN_30 = 46,
+  UNKNOWN_31 = 47,
+  UNKNOWN_32 = 48,
+  UNKNOWN_33 = 49,
+  UNKNOWN_34 = 50,
+  UNKNOWN_35 = 51,
+  UNKNOWN_36 = 52,
+  UNKNOWN_37 = 53,
+  UNKNOWN_38 = 54,
+  UNKNOWN_39 = 55,
+  UNKNOWN_40 = 56,
+  UNKNOWN_41 = 57,
+  UNKNOWN_42 = 58,
+  UNKNOWN_43 = 59,
+  UNKNOWN_44 = 60,
+  UNKNOWN_45 = 61,
+  UNKNOWN_46 = 62,
+  UNKNOWN_47 = 63,
+  UNKNOWN_48 = 64,
+  UNKNOWN_49 = 65,
+  UNKNOWN_50 = 66,
+  UNKNOWN_51 = 67,
+  UNKNOWN_52 = 68,
+  UNKNOWN_53 = 69,
+  UNKNOWN_54 = 70,
+  UNKNOWN_55 = 71,
+  UNKNOWN_56 = 72,
+  UNKNOWN_57 = 73,
+  UNKNOWN_58 = 74,
+  UNKNOWN_59 = 75,
+  UNKNOWN_60 = 76,
+  UNKNOWN_61 = 77,
+  UNKNOWN_62 = 78,
+  UNKNOWN_63 = 79,
+  UNKNOWN_64 = 80,
+  UNKNOWN_65 = 81,
+  UNKNOWN_66 = 82,
+  UNKNOWN_67 = 83,
+  UNKNOWN_68 = 84,
+  UNKNOWN_69 = 85,
+  UNKNOWN_70 = 86,
+  UNKNOWN_71 = 87,
+  UNKNOWN_72 = 88,
+  UNKNOWN_73 = 89,
+  UNKNOWN_74 = 90,
+  UNKNOWN_75 = 91,
+  UNKNOWN_76 = 92,
+  UNKNOWN_77 = 93,
+  UNKNOWN_78 = 94,
+  UNKNOWN_79 = 95,
+  UNKNOWN_80 = 96,
+  UNKNOWN_81 = 97,
+  UNKNOWN_82 = 98,
+  UNKNOWN_83 = 99,
+  UNKNOWN_84 = 100,
+  UNKNOWN_85 = 101,
+  UNKNOWN_86 = 102,
+  UNKNOWN_87 = 103,
+  UNKNOWN_88 = 104,
+  UNKNOWN_89 = 105,
+  UNKNOWN_90 = 106,
+  UNKNOWN_91 = 107,
+  UNKNOWN_92 = 108,
+  UNKNOWN_93 = 109,
+  UNKNOWN_94 = 110,
+  UNKNOWN_95 = 111,
+  UNKNOWN_96 = 112,
+  UNKNOWN_97 = 113,
+  UNKNOWN_98 = 114,
+  UNKNOWN_99 = 115,
+  UNKNOWN_100 = 116,
 }
 
 struct TBinlog {

From 954661c53f9019cf9a3b6f703ad048f372272653 Mon Sep 17 00:00:00 2001
From: abmdocrt <lianyukang@selectdb.com>
Date: Tue, 12 Nov 2024 23:18:48 +0800
Subject: [PATCH 06/10] [Fix](sk) All sk in log should be encrypted (#43544)

---
 cloud/src/meta-service/meta_service_helper.h  | 45 ++++++++++-
 .../meta-service/meta_service_resource.cpp    |  4 +
 cloud/test/meta_service_http_test.cpp         | 79 +++++++++++++++++++
 3 files changed, 127 insertions(+), 1 deletion(-)

diff --git a/cloud/src/meta-service/meta_service_helper.h b/cloud/src/meta-service/meta_service_helper.h
index 9e9ff38c2ecda0..8a1a53061949e0 100644
--- a/cloud/src/meta-service/meta_service_helper.h
+++ b/cloud/src/meta-service/meta_service_helper.h
@@ -19,7 +19,9 @@
 
 #include <brpc/controller.h>
 #include <gen_cpp/cloud.pb.h>
+#include <openssl/md5.h>
 
+#include <iomanip>
 #include <memory>
 #include <string>
 #include <string_view>
@@ -29,12 +31,26 @@
 #include "common/logging.h"
 #include "common/stopwatch.h"
 #include "common/util.h"
+#include "cpp/sync_point.h"
 #include "meta-service/keys.h"
 #include "meta-service/txn_kv.h"
 #include "meta-service/txn_kv_error.h"
 #include "resource-manager/resource_manager.h"
 
 namespace doris::cloud {
+inline std::string md5(const std::string& str) {
+    unsigned char digest[MD5_DIGEST_LENGTH];
+    MD5_CTX context;
+    MD5_Init(&context);
+    MD5_Update(&context, str.c_str(), str.length());
+    MD5_Final(digest, &context);
+
+    std::ostringstream ss;
+    for (unsigned char i : digest) {
+        ss << std::setw(2) << std::setfill('0') << std::hex << (int)i;
+    }
+    return ss.str();
+}
 
 template <class Request>
 void begin_rpc(std::string_view func_name, brpc::Controller* ctrl, const Request* req) {
@@ -101,7 +117,34 @@ void finish_rpc(std::string_view func_name, brpc::Controller* ctrl, Response* re
         LOG(INFO) << "finish " << func_name << " from " << ctrl->remote_side()
                   << " status=" << res->status().ShortDebugString()
                   << " delete_bitmap_size=" << res->segment_delete_bitmaps_size();
-
+    } else if constexpr (std::is_same_v<Response, GetObjStoreInfoResponse> ||
+                         std::is_same_v<Response, GetStageResponse>) {
+        std::string debug_string = res->DebugString();
+        // Start position for searching "sk" fields
+        size_t pos = 0;
+        // Iterate through the string and find all occurrences of "sk: "
+        while ((pos = debug_string.find("sk: ", pos)) != std::string::npos) {
+            // Find the start and end of the "sk" value (assumed to be within quotes)
+            // Start after the quote
+            size_t sk_value_start = debug_string.find('\"', pos) + 1;
+            // End at the next quote
+            size_t sk_value_end = debug_string.find('\"', sk_value_start);
+
+            // Extract the "sk" value
+            std::string sk_value =
+                    debug_string.substr(sk_value_start, sk_value_end - sk_value_start);
+            // Encrypt the "sk" value with MD5
+            std::string encrypted_sk = "md5: " + md5(sk_value);
+
+            // Replace the original "sk" value with the encrypted MD5 value
+            debug_string.replace(sk_value_start, sk_value_end - sk_value_start, encrypted_sk);
+
+            // Move the position to the end of the current "sk" field and continue searching
+            pos = sk_value_end;
+        }
+        TEST_SYNC_POINT_CALLBACK("sk_finish_rpc", &debug_string);
+        LOG(INFO) << "finish " << func_name << " from " << ctrl->remote_side()
+                  << " response=" << debug_string;
     } else {
         LOG(INFO) << "finish " << func_name << " from " << ctrl->remote_side()
                   << " response=" << res->ShortDebugString();
diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp
index 92020005c3a5d0..cc459c090bfd28 100644
--- a/cloud/src/meta-service/meta_service_resource.cpp
+++ b/cloud/src/meta-service/meta_service_resource.cpp
@@ -203,6 +203,8 @@ void MetaServiceImpl::get_obj_store_info(google::protobuf::RpcController* contro
                                          GetObjStoreInfoResponse* response,
                                          ::google::protobuf::Closure* done) {
     RPC_PREPROCESS(get_obj_store_info);
+    TEST_SYNC_POINT_CALLBACK("obj-store-info_sk_response", &response);
+    TEST_SYNC_POINT_RETURN_WITH_VOID("obj-store-info_sk_response_return");
     // Prepare data
     std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : "";
     if (cloud_unique_id.empty()) {
@@ -2600,6 +2602,8 @@ void MetaServiceImpl::get_stage(google::protobuf::RpcController* controller,
                                 const GetStageRequest* request, GetStageResponse* response,
                                 ::google::protobuf::Closure* done) {
     RPC_PREPROCESS(get_stage);
+    TEST_SYNC_POINT_CALLBACK("stage_sk_response", &response);
+    TEST_SYNC_POINT_RETURN_WITH_VOID("stage_sk_response_return");
     std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : "";
     if (cloud_unique_id.empty()) {
         code = MetaServiceCode::INVALID_ARGUMENT;
diff --git a/cloud/test/meta_service_http_test.cpp b/cloud/test/meta_service_http_test.cpp
index e49628fcb3a783..d1b8fd66943a20 100644
--- a/cloud/test/meta_service_http_test.cpp
+++ b/cloud/test/meta_service_http_test.cpp
@@ -1456,4 +1456,83 @@ TEST(MetaServiceHttpTest, TxnLazyCommit) {
     }
 }
 
+TEST(MetaServiceHttpTest, get_stage_response_sk) {
+    auto sp = SyncPoint::get_instance();
+    sp->enable_processing();
+    std::unique_ptr<int, std::function<void(int*)>> defer((int*)0x01,
+                                                          [&](...) { sp->disable_processing(); });
+
+    GetStageResponse res;
+    auto* stage = res.add_stage();
+    stage->mutable_obj_info()->set_ak("stage-ak");
+    stage->mutable_obj_info()->set_sk("stage-sk");
+    auto foo = [res](auto args) { (*(try_any_cast<GetStageResponse**>(args[0])))->CopyFrom(res); };
+    sp->set_call_back("stage_sk_response", foo);
+    sp->set_call_back("stage_sk_response_return",
+                      [](auto&& args) { *try_any_cast<bool*>(args.back()) = true; });
+
+    auto rate_limiter = std::make_shared<cloud::RateLimiter>();
+
+    auto ms = std::make_unique<cloud::MetaServiceImpl>(nullptr, nullptr, rate_limiter);
+
+    auto bar = [](auto args) {
+        std::cout << *try_any_cast<std::string*>(args[0]);
+
+        EXPECT_TRUE((*try_any_cast<std::string*>(args[0])).find("stage-sk") == std::string::npos);
+        EXPECT_TRUE((*try_any_cast<std::string*>(args[0]))
+                            .find("md5: f497d053066fa4b7d3b1f6564597d233") != std::string::npos);
+    };
+    sp->set_call_back("sk_finish_rpc", bar);
+
+    GetStageResponse res1;
+    GetStageRequest req1;
+    brpc::Controller cntl;
+    ms->get_stage(&cntl, &req1, &res1, nullptr);
+}
+
+TEST(MetaServiceHttpTest, get_obj_store_info_response_sk) {
+    auto sp = SyncPoint::get_instance();
+    sp->enable_processing();
+    std::unique_ptr<int, std::function<void(int*)>> defer((int*)0x01,
+                                                          [&](...) { sp->disable_processing(); });
+
+    GetObjStoreInfoResponse res;
+    auto* obj_info = res.add_obj_info();
+    obj_info->set_ak("obj-store-info-ak1");
+    obj_info->set_sk("obj-store-info-sk1");
+    obj_info = res.add_storage_vault()->mutable_obj_info();
+    obj_info->set_ak("obj-store-info-ak2");
+    obj_info->set_sk("obj-store-info-sk2");
+    auto foo = [res](auto args) {
+        (*(try_any_cast<GetObjStoreInfoResponse**>(args[0])))->CopyFrom(res);
+    };
+    sp->set_call_back("obj-store-info_sk_response", foo);
+    sp->set_call_back("obj-store-info_sk_response_return",
+                      [](auto&& args) { *try_any_cast<bool*>(args.back()) = true; });
+
+    auto rate_limiter = std::make_shared<cloud::RateLimiter>();
+
+    auto ms = std::make_unique<cloud::MetaServiceImpl>(nullptr, nullptr, rate_limiter);
+
+    auto bar = [](auto args) {
+        std::cout << *try_any_cast<std::string*>(args[0]);
+
+        EXPECT_TRUE((*try_any_cast<std::string*>(args[0])).find("obj-store-info-sk1") ==
+                    std::string::npos);
+        EXPECT_TRUE((*try_any_cast<std::string*>(args[0]))
+                            .find("md5: 35d5a637fd9d45a28207a888b751efc4") != std::string::npos);
+
+        EXPECT_TRUE((*try_any_cast<std::string*>(args[0])).find("obj-store-info-sk2") ==
+                    std::string::npos);
+        EXPECT_TRUE((*try_any_cast<std::string*>(args[0]))
+                            .find("md5: 01d7473ae201a2ecdf1f7c064eb81a95") != std::string::npos);
+    };
+    sp->set_call_back("sk_finish_rpc", bar);
+
+    GetObjStoreInfoResponse res1;
+    GetObjStoreInfoRequest req1;
+    brpc::Controller cntl;
+    ms->get_obj_store_info(&cntl, &req1, &res1, nullptr);
+}
+
 } // namespace doris::cloud

From 8d6659cb2e46cb764918416f1d44cc26b4e4ec5f Mon Sep 17 00:00:00 2001
From: Siyang Tang <tangsiyang@selectdb.com>
Date: Tue, 12 Nov 2024 23:51:57 +0800
Subject: [PATCH 07/10] [feature](security) Support block specific query with
 AST names (#43533)

Support block specific query with AST names when necessary for security
reasons, configure the name list in fe.conf, for example:
```
block_sql_ast_names="CreateFileStmt, CreateFunctionStmt"
```
---
 .../java/org/apache/doris/common/Config.java  |  6 ++
 .../java/org/apache/doris/catalog/Env.java    |  3 +
 .../org/apache/doris/qe/StmtExecutor.java     | 21 +++++
 .../nereids/parser/NereidsParserTest.java     | 36 ++++++++
 .../org/apache/doris/qe/StmtExecutorTest.java | 90 +++++++++++++++++++
 5 files changed, 156 insertions(+)

diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index ccc93ed799ade2..993702c4dac57f 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -3249,4 +3249,10 @@ public static int metaServiceRpcRetryTimes() {
             "For testing purposes, all queries are forcibly forwarded to the master to verify"
                     + "the behavior of forwarding queries."})
     public static boolean force_forward_all_queries = false;
+
+    @ConfField(description = {"用于禁用某些SQL，配置项为AST的class simple name列表(例如CreateRepositoryStmt,"
+            + "CreatePolicyCommand)，用逗号间隔开",
+            "For disabling certain SQL queries, the configuration item is a list of simple class names of AST"
+                    + "(for example CreateRepositoryStmt, CreatePolicyCommand), separated by commas."})
+    public static String block_sql_ast_names = "";
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 9e65e5c866b6a0..f67444b6cb1b7e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -246,6 +246,7 @@
 import org.apache.doris.qe.JournalObservable;
 import org.apache.doris.qe.QueryCancelWorker;
 import org.apache.doris.qe.SessionVariable;
+import org.apache.doris.qe.StmtExecutor;
 import org.apache.doris.qe.VariableMgr;
 import org.apache.doris.resource.AdmissionControl;
 import org.apache.doris.resource.Tag;
@@ -1116,6 +1117,8 @@ public void initialize(String[] args) throws Exception {
             notifyNewFETypeTransfer(FrontendNodeType.MASTER);
         }
         queryCancelWorker.start();
+
+        StmtExecutor.initBlockSqlAstNames();
     }
 
     // wait until FE is ready.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index 80c7ad0912ddbf..8b76933e9e0238 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -259,6 +259,8 @@ public class StmtExecutor {
     private static final AtomicLong STMT_ID_GENERATOR = new AtomicLong(0);
     public static final int MAX_DATA_TO_SEND_FOR_TXN = 100;
     public static final String NULL_VALUE_FOR_LOAD = "\\N";
+    private static Set<String> blockSqlAstNames = Sets.newHashSet();
+
     private Pattern beIpPattern = Pattern.compile("\\[(\\d+):");
     private ConnectContext context;
     private final StatementContext statementContext;
@@ -694,6 +696,7 @@ private void executeByNereids(TUniqueId queryId) throws Exception {
                 "Nereids only process LogicalPlanAdapter, but parsedStmt is " + parsedStmt.getClass().getName());
         context.getState().setNereids(true);
         LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan();
+        checkSqlBlocked(logicalPlan.getClass());
         if (context.getCommand() == MysqlCommand.COM_STMT_PREPARE) {
             if (isForwardToMaster()) {
                 throw new UserException("Forward master command is not supported for prepare statement");
@@ -834,6 +837,23 @@ private void executeByNereids(TUniqueId queryId) throws Exception {
         }
     }
 
+    public static void initBlockSqlAstNames() {
+        blockSqlAstNames.clear();
+        blockSqlAstNames = Pattern.compile(",")
+                .splitAsStream(Config.block_sql_ast_names)
+                .map(String::trim)
+                .collect(Collectors.toSet());
+        if (blockSqlAstNames.isEmpty() && !Config.block_sql_ast_names.isEmpty()) {
+            blockSqlAstNames.add(Config.block_sql_ast_names);
+        }
+    }
+
+    public void checkSqlBlocked(Class<?> clazz) throws UserException {
+        if (blockSqlAstNames.contains(clazz.getSimpleName())) {
+            throw new UserException("SQL is blocked with AST name: " + clazz.getSimpleName());
+        }
+    }
+
     private void parseByNereids() {
         if (parsedStmt != null) {
             return;
@@ -981,6 +1001,7 @@ public void executeByLegacy(TUniqueId queryId) throws Exception {
         try {
             // parsedStmt maybe null here, we parse it. Or the predicate will not work.
             parseByLegacy();
+            checkSqlBlocked(parsedStmt.getClass());
             if (context.isTxnModel() && !(parsedStmt instanceof InsertStmt)
                     && !(parsedStmt instanceof TransactionStmt)) {
                 throw new TException("This is in a transaction, only insert, update, delete, "
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
index ff9e81f2bf3cfb..9a46b810586eec 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
@@ -43,6 +43,8 @@
 import org.apache.doris.nereids.types.DateType;
 import org.apache.doris.nereids.types.DecimalV2Type;
 import org.apache.doris.nereids.types.DecimalV3Type;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
 
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
@@ -704,4 +706,38 @@ public void testQualify() {
             nereidsParser.parseSingle(sql);
         }
     }
+
+    @Test
+    public void testBlockSqlAst() {
+        String sql = "plan replayer dump select `AD``D` from t1 where a = 1";
+        NereidsParser nereidsParser = new NereidsParser();
+        LogicalPlan logicalPlan = nereidsParser.parseSingle(sql);
+
+        Config.block_sql_ast_names = "ReplayCommand";
+        StmtExecutor.initBlockSqlAstNames();
+        StmtExecutor stmtExecutor = new StmtExecutor(new ConnectContext(), "");
+        try {
+            stmtExecutor.checkSqlBlocked(logicalPlan.getClass());
+            Assertions.fail();
+        } catch (Exception ignore) {
+            // do nothing
+        }
+
+        Config.block_sql_ast_names = "CreatePolicyCommand, ReplayCommand";
+        StmtExecutor.initBlockSqlAstNames();
+        try {
+            stmtExecutor.checkSqlBlocked(logicalPlan.getClass());
+            Assertions.fail();
+        } catch (Exception ignore) {
+            // do nothing
+        }
+
+        Config.block_sql_ast_names = "";
+        StmtExecutor.initBlockSqlAstNames();
+        try {
+            stmtExecutor.checkSqlBlocked(logicalPlan.getClass());
+        } catch (Exception ex) {
+            Assertions.fail(ex);
+        }
+    }
 }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java
index 07d39e52180858..8ab187315d8681 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java
@@ -19,6 +19,8 @@
 
 import org.apache.doris.analysis.AccessTestUtil;
 import org.apache.doris.analysis.Analyzer;
+import org.apache.doris.analysis.CreateFileStmt;
+import org.apache.doris.analysis.CreateFunctionStmt;
 import org.apache.doris.analysis.DdlStmt;
 import org.apache.doris.analysis.Expr;
 import org.apache.doris.analysis.KillStmt;
@@ -31,6 +33,7 @@
 import org.apache.doris.analysis.StatementBase;
 import org.apache.doris.analysis.UseStmt;
 import org.apache.doris.catalog.Env;
+import org.apache.doris.common.Config;
 import org.apache.doris.common.DdlException;
 import org.apache.doris.common.jmockit.Deencapsulation;
 import org.apache.doris.common.profile.Profile;
@@ -801,4 +804,91 @@ public void testUseWithCatalogFail(@Mocked UseStmt useStmt, @Mocked SqlParser pa
 
         Assert.assertEquals(QueryState.MysqlStateType.ERR, state.getStateType());
     }
+
+    @Test
+    public void testBlockSqlAst(@Mocked UseStmt useStmt, @Mocked CreateFileStmt createFileStmt,
+            @Mocked CreateFunctionStmt createFunctionStmt, @Mocked SqlParser parser) throws Exception {
+        new Expectations() {
+            {
+                useStmt.analyze((Analyzer) any);
+                minTimes = 0;
+
+                useStmt.getDatabase();
+                minTimes = 0;
+                result = "testDb";
+
+                useStmt.getRedirectStatus();
+                minTimes = 0;
+                result = RedirectStatus.NO_FORWARD;
+
+                useStmt.getCatalogName();
+                minTimes = 0;
+                result = InternalCatalog.INTERNAL_CATALOG_NAME;
+
+                Symbol symbol = new Symbol(0, Lists.newArrayList(createFileStmt));
+                parser.parse();
+                minTimes = 0;
+                result = symbol;
+            }
+        };
+
+        Config.block_sql_ast_names = "CreateFileStmt";
+        StmtExecutor.initBlockSqlAstNames();
+        StmtExecutor executor = new StmtExecutor(ctx, "");
+        try {
+            executor.execute();
+        } catch (Exception ignore) {
+            // do nothing
+        }
+        Assert.assertEquals(QueryState.MysqlStateType.ERR, state.getStateType());
+        Assert.assertTrue(state.getErrorMessage().contains("SQL is blocked with AST name: CreateFileStmt"));
+
+        Config.block_sql_ast_names = "AlterStmt, CreateFileStmt";
+        StmtExecutor.initBlockSqlAstNames();
+        executor = new StmtExecutor(ctx, "");
+        try {
+            executor.execute();
+        } catch (Exception ignore) {
+            // do nothing
+        }
+        Assert.assertEquals(QueryState.MysqlStateType.ERR, state.getStateType());
+        Assert.assertTrue(state.getErrorMessage().contains("SQL is blocked with AST name: CreateFileStmt"));
+
+        new Expectations() {
+            {
+                Symbol symbol = new Symbol(0, Lists.newArrayList(createFunctionStmt));
+                parser.parse();
+                minTimes = 0;
+                result = symbol;
+            }
+        };
+        Config.block_sql_ast_names = "CreateFunctionStmt, CreateFileStmt";
+        StmtExecutor.initBlockSqlAstNames();
+        executor = new StmtExecutor(ctx, "");
+        try {
+            executor.execute();
+        } catch (Exception ignore) {
+            // do nothing
+        }
+        Assert.assertEquals(QueryState.MysqlStateType.ERR, state.getStateType());
+        Assert.assertTrue(state.getErrorMessage().contains("SQL is blocked with AST name: CreateFunctionStmt"));
+
+        new Expectations() {
+            {
+                Symbol symbol = new Symbol(0, Lists.newArrayList(useStmt));
+                parser.parse();
+                minTimes = 0;
+                result = symbol;
+            }
+        };
+        executor = new StmtExecutor(ctx, "");
+        executor.execute();
+        Assert.assertEquals(QueryState.MysqlStateType.OK, state.getStateType());
+
+        Config.block_sql_ast_names = "";
+        StmtExecutor.initBlockSqlAstNames();
+        executor = new StmtExecutor(ctx, "");
+        executor.execute();
+        Assert.assertEquals(QueryState.MysqlStateType.OK, state.getStateType());
+    }
 }

From ffe6650439481fbf9e73616de795396b3c82b35c Mon Sep 17 00:00:00 2001
From: wuwenchi <wuwenchi@selectdb.com>
Date: Wed, 13 Nov 2024 00:15:06 +0800
Subject: [PATCH 08/10] [fix](filesystem)Use simple authentication directly in
 S3FileSystem (#43636)

### What problem does this PR solve?

Related PR: #43049

1. S3 does not support Kerberos authentication, so here we create a
simple authentication.
2. When generating Kerberos authentication information, add
configuration information integrity check.
---
 fe/fe-common/pom.xml                          |   6 +
 .../authentication/AuthenticationConfig.java  |  48 ++++--
 .../authentication/AuthenticationTest.java    |  45 +++++
 .../apache/doris/fs/remote/S3FileSystem.java  |  11 +-
 .../doris/fs/remote/dfs/DFSFileSystem.java    |   5 +
 .../doris/fs/remote/RemoteFileSystemTest.java | 158 ++++++++++++++++++
 6 files changed, 261 insertions(+), 12 deletions(-)
 create mode 100644 fe/fe-common/src/test/java/org/apache/doris/common/security/authentication/AuthenticationTest.java
 create mode 100644 fe/fe-core/src/test/java/org/apache/doris/fs/remote/RemoteFileSystemTest.java

diff --git a/fe/fe-common/pom.xml b/fe/fe-common/pom.xml
index c49d13a3c03603..9322286047a8aa 100644
--- a/fe/fe-common/pom.xml
+++ b/fe/fe-common/pom.xml
@@ -145,6 +145,12 @@ under the License.
             <groupId>com.esotericsoftware</groupId>
             <artifactId>kryo-shaded</artifactId>
         </dependency>
+        <dependency>
+            <groupId>commons-collections</groupId>
+            <artifactId>commons-collections</artifactId>
+            <version>${commons-collections.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
     <build>
         <finalName>doris-fe-common</finalName>
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/AuthenticationConfig.java b/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/AuthenticationConfig.java
index 875ae4542e1193..b580f9ecbe0582 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/AuthenticationConfig.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/AuthenticationConfig.java
@@ -17,10 +17,14 @@
 
 package org.apache.doris.common.security.authentication;
 
+import com.google.common.base.Strings;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 public abstract class AuthenticationConfig {
+    private static final Logger LOG = LogManager.getLogger(AuthenticationConfig.class);
     public static String HADOOP_USER_NAME = "hadoop.username";
     public static String HADOOP_KERBEROS_PRINCIPAL = "hadoop.kerberos.principal";
     public static String HADOOP_KERBEROS_KEYTAB = "hadoop.kerberos.keytab";
@@ -42,6 +46,10 @@ public static AuthenticationConfig getKerberosConfig(Configuration conf) {
         return AuthenticationConfig.getKerberosConfig(conf, HADOOP_KERBEROS_PRINCIPAL, HADOOP_KERBEROS_KEYTAB);
     }
 
+    public static AuthenticationConfig getSimpleAuthenticationConfig(Configuration conf) {
+        return AuthenticationConfig.createSimpleAuthenticationConfig(conf);
+    }
+
     /**
      * get kerberos config from hadoop conf
      * @param conf config
@@ -54,17 +62,35 @@ public static AuthenticationConfig getKerberosConfig(Configuration conf,
                                                          String krbKeytabKey) {
         String authentication = conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, null);
         if (AuthType.KERBEROS.getDesc().equals(authentication)) {
-            KerberosAuthenticationConfig krbConfig = new KerberosAuthenticationConfig();
-            krbConfig.setKerberosPrincipal(conf.get(krbPrincipalKey));
-            krbConfig.setKerberosKeytab(conf.get(krbKeytabKey));
-            krbConfig.setConf(conf);
-            krbConfig.setPrintDebugLog(Boolean.parseBoolean(conf.get(DORIS_KRB5_DEBUG, "false")));
-            return krbConfig;
-        } else {
-            // AuthType.SIMPLE
-            SimpleAuthenticationConfig simpleAuthenticationConfig = new SimpleAuthenticationConfig();
-            simpleAuthenticationConfig.setUsername(conf.get(HADOOP_USER_NAME));
-            return simpleAuthenticationConfig;
+            String principalKey = conf.get(krbPrincipalKey);
+            String keytabKey = conf.get(krbKeytabKey);
+            if (!Strings.isNullOrEmpty(principalKey) && !Strings.isNullOrEmpty(keytabKey)) {
+                KerberosAuthenticationConfig krbConfig = new KerberosAuthenticationConfig();
+                krbConfig.setKerberosPrincipal(principalKey);
+                krbConfig.setKerberosKeytab(keytabKey);
+                krbConfig.setConf(conf);
+                krbConfig.setPrintDebugLog(Boolean.parseBoolean(conf.get(DORIS_KRB5_DEBUG, "false")));
+                return krbConfig;
+            } else {
+                // Due to some historical reasons, `core-size.xml` may be stored in path:`fe/conf`,
+                // but this file may only contain `hadoop.security.authentication configuration`,
+                // and no krbPrincipalKey and krbKeytabKey,
+                // which will cause kerberos initialization failure.
+                // Now:
+                //   if kerberos is needed, the relevant configuration can be written in the catalog properties,
+                //   if kerberos is not needed, to prevent the influence of historical reasons,
+                //      the following simple authentication method needs to be used.
+                LOG.warn("{} or {} is null or empty, fallback to simple authentication",
+                        krbPrincipalKey, krbKeytabKey);
+            }
         }
+        return createSimpleAuthenticationConfig(conf);
+    }
+
+    private static AuthenticationConfig createSimpleAuthenticationConfig(Configuration conf) {
+        // AuthType.SIMPLE
+        SimpleAuthenticationConfig simpleAuthenticationConfig = new SimpleAuthenticationConfig();
+        simpleAuthenticationConfig.setUsername(conf.get(HADOOP_USER_NAME));
+        return simpleAuthenticationConfig;
     }
 }
diff --git a/fe/fe-common/src/test/java/org/apache/doris/common/security/authentication/AuthenticationTest.java b/fe/fe-common/src/test/java/org/apache/doris/common/security/authentication/AuthenticationTest.java
new file mode 100644
index 00000000000000..62606a22a64fcb
--- /dev/null
+++ b/fe/fe-common/src/test/java/org/apache/doris/common/security/authentication/AuthenticationTest.java
@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.common.security.authentication;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class AuthenticationTest {
+
+    @Test
+    public void testAuthConf() {
+        Configuration conf = new Configuration();
+
+        AuthenticationConfig conf1 = AuthenticationConfig.getKerberosConfig(conf);
+        Assert.assertEquals(SimpleAuthenticationConfig.class, conf1.getClass());
+
+        conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+
+        AuthenticationConfig conf2 = AuthenticationConfig.getKerberosConfig(conf);
+        Assert.assertEquals(SimpleAuthenticationConfig.class, conf2.getClass());
+
+        conf.set(AuthenticationConfig.HADOOP_KERBEROS_PRINCIPAL, "principal");
+        conf.set(AuthenticationConfig.HADOOP_KERBEROS_KEYTAB, "keytab");
+
+        AuthenticationConfig conf3 = AuthenticationConfig.getKerberosConfig(conf);
+        Assert.assertEquals(KerberosAuthenticationConfig.class, conf3.getClass());
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java
index f8805bd0d4fb9a..be53ffde2e095b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java
@@ -43,6 +43,7 @@
 public class S3FileSystem extends ObjFileSystem {
 
     private static final Logger LOG = LogManager.getLogger(S3FileSystem.class);
+    private HadoopAuthenticator authenticator = null;
 
     public S3FileSystem(Map<String, String> properties) {
         super(StorageBackend.StorageType.S3.name(), StorageBackend.StorageType.S3, new S3ObjStorage(properties));
@@ -77,7 +78,9 @@ protected FileSystem nativeFileSystem(String remotePath) throws UserException {
                     PropertyConverter.convertToHadoopFSProperties(properties).entrySet().stream()
                             .filter(entry -> entry.getKey() != null && entry.getValue() != null)
                             .forEach(entry -> conf.set(entry.getKey(), entry.getValue()));
-                    AuthenticationConfig authConfig = AuthenticationConfig.getKerberosConfig(conf);
+                    // S3 does not support Kerberos authentication,
+                    // so here we create a simple authentication
+                    AuthenticationConfig authConfig = AuthenticationConfig.getSimpleAuthenticationConfig(conf);
                     HadoopAuthenticator authenticator = HadoopAuthenticator.getHadoopAuthenticator(authConfig);
                     try {
                         dfsFileSystem = authenticator.doAs(() -> {
@@ -87,6 +90,7 @@ protected FileSystem nativeFileSystem(String remotePath) throws UserException {
                                 throw new RuntimeException(e);
                             }
                         });
+                        this.authenticator = authenticator;
                         RemoteFSPhantomManager.registerPhantomReference(this);
                     } catch (Exception e) {
                         throw new UserException("Failed to get S3 FileSystem for " + e.getMessage(), e);
@@ -134,4 +138,9 @@ public Status globList(String remotePath, List<RemoteFile> result, boolean fileN
         }
         return Status.OK;
     }
+
+    @VisibleForTesting
+    public HadoopAuthenticator getAuthenticator() {
+        return authenticator;
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java
index 2146472aec7b21..89f4af2817ec05 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java
@@ -489,4 +489,9 @@ public Status makeDir(String remotePath) {
         }
         return Status.OK;
     }
+
+    @VisibleForTesting
+    public HadoopAuthenticator getAuthenticator() {
+        return authenticator;
+    }
 }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/fs/remote/RemoteFileSystemTest.java b/fe/fe-core/src/test/java/org/apache/doris/fs/remote/RemoteFileSystemTest.java
new file mode 100644
index 00000000000000..3fc15ab8e374fa
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/fs/remote/RemoteFileSystemTest.java
@@ -0,0 +1,158 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.fs.remote;
+
+import org.apache.doris.common.Pair;
+import org.apache.doris.common.UserException;
+import org.apache.doris.common.security.authentication.AuthenticationConfig;
+import org.apache.doris.common.security.authentication.HadoopAuthenticator;
+import org.apache.doris.common.security.authentication.HadoopKerberosAuthenticator;
+import org.apache.doris.common.security.authentication.HadoopSimpleAuthenticator;
+import org.apache.doris.common.util.LocationPath;
+import org.apache.doris.fs.FileSystemCache;
+import org.apache.doris.fs.FileSystemType;
+import org.apache.doris.fs.remote.dfs.DFSFileSystem;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableMap;
+import mockit.Mock;
+import mockit.MockUp;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.URI;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.Map;
+
+public class RemoteFileSystemTest {
+
+    @Test
+    public void testFilesystemAndAuthType() throws UserException {
+
+        // These paths should use s3 filesystem, and use simple auth
+        ArrayList<String> s3Paths = new ArrayList<>();
+        s3Paths.add("s3://a/b/c");
+        s3Paths.add("s3a://a/b/c");
+        s3Paths.add("s3n://a/b/c");
+        s3Paths.add("oss://a/b/c");  // default use s3 filesystem
+        s3Paths.add("gs://a/b/c");
+        s3Paths.add("bos://a/b/c");
+        s3Paths.add("cos://a/b/c");
+        s3Paths.add("cosn://a/b/c");
+        s3Paths.add("lakefs://a/b/c");
+        s3Paths.add("obs://a/b/c");
+
+        // These paths should use dfs filesystem, and auth will be changed by configure
+        ArrayList<String> dfsPaths = new ArrayList<>();
+        dfsPaths.add("ofs://a/b/c");
+        dfsPaths.add("gfs://a/b/c");
+        dfsPaths.add("hdfs://a/b/c");
+        dfsPaths.add("oss://a/b/c");  // if endpoint contains 'oss-dls.aliyuncs', will use dfs filesystem
+
+        new MockUp<UserGroupInformation>(UserGroupInformation.class) {
+            @Mock
+            public <T> T doAs(PrivilegedExceptionAction<T> action) throws IOException, InterruptedException {
+                return (T) new LocalFileSystem();
+            }
+        };
+
+        new MockUp<HadoopKerberosAuthenticator>(HadoopKerberosAuthenticator.class) {
+            @Mock
+            public synchronized UserGroupInformation getUGI() throws IOException {
+                return UserGroupInformation.getCurrentUser();
+            }
+        };
+
+        Configuration confWithoutKerberos = new Configuration();
+
+        Configuration confWithKerberosIncomplete = new Configuration();
+        confWithKerberosIncomplete.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+
+        Configuration confWithKerberos = new Configuration();
+        confWithKerberos.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+        confWithKerberos.set(AuthenticationConfig.HADOOP_KERBEROS_PRINCIPAL, "principal");
+        confWithKerberos.set(AuthenticationConfig.HADOOP_KERBEROS_KEYTAB, "keytab");
+
+        ImmutableMap<String, String> s3props = ImmutableMap.of("s3.endpoint", "http://127.0.0.1");
+        s3props.forEach(confWithKerberos::set);
+        s3props.forEach(confWithoutKerberos::set);
+        s3props.forEach(confWithKerberosIncomplete::set);
+
+        for (String path : s3Paths) {
+            checkS3Filesystem(path, confWithKerberos, s3props);
+        }
+        for (String path : s3Paths) {
+            checkS3Filesystem(path, confWithKerberosIncomplete, s3props);
+        }
+        for (String path : s3Paths) {
+            checkS3Filesystem(path, confWithoutKerberos, s3props);
+        }
+
+        s3props = ImmutableMap.of("s3.endpoint", "oss://xx-oss-dls.aliyuncs/abc");
+        System.setProperty("java.security.krb5.realm", "realm");
+        System.setProperty("java.security.krb5.kdc", "kdc");
+
+        for (String path : dfsPaths) {
+            checkDFSFilesystem(path, confWithKerberos, HadoopKerberosAuthenticator.class.getName(), s3props);
+        }
+        for (String path : dfsPaths) {
+            checkDFSFilesystem(path, confWithKerberosIncomplete, HadoopSimpleAuthenticator.class.getName(), s3props);
+        }
+        for (String path : dfsPaths) {
+            checkDFSFilesystem(path, confWithoutKerberos, HadoopSimpleAuthenticator.class.getName(), s3props);
+        }
+
+    }
+
+    private void checkS3Filesystem(String path, Configuration conf, Map<String, String> m) throws UserException {
+        RemoteFileSystem fs = createFs(path, conf, m);
+        Assert.assertTrue(fs instanceof S3FileSystem);
+        HadoopAuthenticator authenticator = ((S3FileSystem) fs).getAuthenticator();
+        Assert.assertTrue(authenticator instanceof HadoopSimpleAuthenticator);
+    }
+
+    private void checkDFSFilesystem(String path, Configuration conf, String authClass, Map<String, String> m) throws UserException {
+        RemoteFileSystem fs = createFs(path, conf, m);
+        Assert.assertTrue(fs instanceof DFSFileSystem);
+        HadoopAuthenticator authenticator = ((DFSFileSystem) fs).getAuthenticator();
+        Assert.assertEquals(authClass, authenticator.getClass().getName());
+    }
+
+    private RemoteFileSystem createFs(String path, Configuration conf, Map<String, String> m) throws UserException {
+        LocationPath locationPath = new LocationPath(path, m);
+        FileSystemType fileSystemType = locationPath.getFileSystemType();
+        URI uri = locationPath.getPath().toUri();
+        String fsIdent = Strings.nullToEmpty(uri.getScheme()) + "://" + Strings.nullToEmpty(uri.getAuthority());
+        FileSystemCache fileSystemCache = new FileSystemCache();
+        RemoteFileSystem fs = fileSystemCache.getRemoteFileSystem(
+            new FileSystemCache.FileSystemCacheKey(
+                Pair.of(fileSystemType, fsIdent),
+                ImmutableMap.of(),
+                null,
+                conf));
+        fs.nativeFileSystem(path);
+        return fs;
+    }
+
+}

From 860857ab637322c579a99ec544027afcf7b88298 Mon Sep 17 00:00:00 2001
From: Dongyang Li <lidongyang@selectdb.com>
Date: Wed, 13 Nov 2024 01:25:25 +0800
Subject: [PATCH 09/10] [chore](ci) trigger be ut and cloud ut if dir
 common/cpp changed (#43783)

---
 regression-test/pipeline/common/github-utils.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/regression-test/pipeline/common/github-utils.sh b/regression-test/pipeline/common/github-utils.sh
index 242b77b832d6c4..079101b35b9496 100644
--- a/regression-test/pipeline/common/github-utils.sh
+++ b/regression-test/pipeline/common/github-utils.sh
@@ -262,6 +262,7 @@ file_changed_be_ut() {
     if [[ -z ${all_files} ]]; then echo "return need" && return 0; fi
     for af in ${all_files}; do
         if [[ "${af}" == 'be'* ]] ||
+            [[ "${af}" == 'common/cpp'* ]] ||
             [[ "${af}" == 'contrib'* ]] ||
             [[ "${af}" == 'thirdparty'* ]] ||
             [[ "${af}" == 'bin/start_be.sh' ]] ||
@@ -280,6 +281,7 @@ file_changed_cloud_ut() {
     if [[ -z ${all_files} ]]; then echo "return need" && return 0; fi
     for af in ${all_files}; do
         if [[ "${af}" == 'cloud/src/'* ]] ||
+            [[ "${af}" == 'common/cpp'* ]] ||
             [[ "${af}" == 'cloud/test/'* ]]; then
             echo "cloud-ut related file changed, return need" && return 0
         fi

From d6476ddc51a5ab1e3ba77b16aa0bd5be5d858443 Mon Sep 17 00:00:00 2001
From: Xin Liao <liaoxin@selectdb.com>
Date: Wed, 13 Nov 2024 09:43:16 +0800
Subject: [PATCH 10/10] [fix](test) fix regression test case in
 test_broker_load_p2 (#43638)

---
 .../broker_load/test_broker_load.groovy       | 578 +++++++++++-------
 1 file changed, 346 insertions(+), 232 deletions(-)

diff --git a/regression-test/suites/load_p2/broker_load/test_broker_load.groovy b/regression-test/suites/load_p2/broker_load/test_broker_load.groovy
index 0009d26269f093..f8a0043842c094 100644
--- a/regression-test/suites/load_p2/broker_load/test_broker_load.groovy
+++ b/regression-test/suites/load_p2/broker_load/test_broker_load.groovy
@@ -19,247 +19,361 @@ suite("test_broker_load_p2", "p2") {
     def s3BucketName = getS3BucketName()
     def s3Endpoint = getS3Endpoint()
     def s3Region = getS3Region()
-    def tables = ["part",
-                  "upper_case",
-                  "reverse",
-                  "set1",
-                  "set2",
-                  "set3",
-                  "set4",
-                  "set5",
-                  "set6",
-                  "set7",
-                  "null_default",
-                  "filter",
-                  "path_column",
-                  "parquet_s3_case1", // col1 not in file but in table, will load default value for it.
-                  "parquet_s3_case2", // x1 not in file, not in table, will throw "col not found" error.
-                  "parquet_s3_case3", // p_comment not in table but in file, load normally.
-                  "parquet_s3_case4", // all columns are in table but not in file, will fill default values.
-                  "parquet_s3_case5", // x1 not in file, not in table, will throw "col not found" error.
-                  "parquet_s3_case6", // normal
-                  "parquet_s3_case7", // col5 will be ignored, load normally
-                  "parquet_s3_case8", // first column in table is not specified, will load default value for it.
-                  "orc_s3_case1", // table column capitalize firsrt
-                  "orc_s3_case2", // table column lowercase * load column lowercase * orc file lowercase
-                  "orc_s3_case3", // table column lowercase * load column uppercase * orc file lowercase
-                  "orc_s3_case4", // table column lowercase * load column lowercase * orc file uppercase
-                  "orc_s3_case5", // table column lowercase * load column uppercase * orc file uppercase
-                  "orc_s3_case6", // table column uppercase * load column uppercase * orc file lowercase
-                  "orc_s3_case7", // table column uppercase * load column lowercase * orc file lowercase
-                  "orc_s3_case8", // table column uppercase * load column uppercase * orc file uppercase
-                  "orc_s3_case9", // table column uppercase * load column lowercase * orc file uppercase
-                  "csv_s3_case_line_delimiter" // csv format table with special line delimiter
+    def tables = ["part",             // case 0
+                  "upper_case",       // case 1
+                  "reverse",          // case 2
+                  "set1",             // case 3
+                  "set2",             // case 4
+                  "set3",             // case 5
+                  "set4",             // case 6
+                  "set5",             // case 7
+                  "set6",             // case 8
+                  "set7",             // case 9
+                  "null_default",     // case 10
+                  "filter",           // case 11
+                  "path_column",      // case 12
+                  "parquet_s3_case1", // case 13, col1 not in file but in table, will load default value for it.
+                  "parquet_s3_case2", // case 14, x1 not in file, not in table, will throw "col not found" error.
+                  "parquet_s3_case3", // case 15, p_comment not in table but in file, load normally.
+                  "parquet_s3_case4", // case 16, all columns are in table but not in file, will fill default values.
+                  "parquet_s3_case5", // case 17, x1 not in file, not in table, will throw "col not found" error.
+                  "parquet_s3_case6", // case 18, normal
+                  "parquet_s3_case7", // case 19, col5 will be ignored, load normally
+                  "parquet_s3_case8", // case 20, first column in table is not specified, will load default value for it.
+                  "orc_s3_case1", // case 21, table column capitalize * load column lowercase * orc file lowercase
+                  "orc_s3_case2", // case 22, table column lowercase * load column lowercase * orc file lowercase
+                  "orc_s3_case3", // case 23, table column lowercase * load column uppercase * orc file lowercase
+                  "orc_s3_case4", // case 24, table column lowercase * load column lowercase * orc file uppercase
+                  "orc_s3_case5", // case 25, table column lowercase * load column uppercase * orc file uppercase
+                  "orc_s3_case6", // case 26, table column uppercase * load column uppercase * orc file lowercase
+                  "orc_s3_case7", // case 27, table column uppercase * load column lowercase * orc file lowercase
+                  "orc_s3_case8", // case 28, table column uppercase * load column uppercase * orc file uppercase
+                  "orc_s3_case9", // case 29, table column uppercase * load column lowercase * orc file uppercase
+                  "csv_s3_case_line_delimiter" // case 30, csv format table with special line delimiter
                  ]
-    def paths = ["s3://${s3BucketName}/regression/load/data/part*",
-                 "s3://${s3BucketName}/regression/load/data/part*",
-                 "s3://${s3BucketName}/regression/load/data/part*",
-                 "s3://${s3BucketName}/regression/load/data/part*",
-                 "s3://${s3BucketName}/regression/load/data/part*",
-                 "s3://${s3BucketName}/regression/load/data/part*",
-                 "s3://${s3BucketName}/regression/load/data/part*",
-                 "s3://${s3BucketName}/regression/load/data/part*",
-                 "s3://${s3BucketName}/regression/load/data/part*",
-                 "s3://${s3BucketName}/regression/load/data/part*",
-                 "s3://${s3BucketName}/regression/load/data/part*",
-                 "s3://${s3BucketName}/regression/load/data/part*",
-                 "s3://${s3BucketName}/regression/load/data/path/*/part*",
-                 "s3://${s3BucketName}/regression/load/data/part*",
-                 "s3://${s3BucketName}/regression/load/data/part*",
-                 "s3://${s3BucketName}/regression/load/data/part*",
-                 "s3://${s3BucketName}/regression/load/data/part*",
-                 "s3://${s3BucketName}/regression/load/data/part*",
-                 "s3://${s3BucketName}/regression/load/data/part*",
-                 "s3://${s3BucketName}/regression/load/data/part*",
-                 "s3://${s3BucketName}/regression/load/data/part*",
-                 "s3://${s3BucketName}/regression/load/data/orc/hits_100k_rows.orc",
-                 "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc",
-                 "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc",
-                 "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc",
-                 "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc",
-                 "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc",
-                 "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc",
-                 "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc",
-                 "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc",
-                 "s3://${s3BucketName}/regression/line_delimiter/lineitem_0x7.csv.gz"
+    def paths = ["s3://${s3BucketName}/regression/load/data/part*",                              // case 0
+                 "s3://${s3BucketName}/regression/load/data/part*",                              // case 1
+                 "s3://${s3BucketName}/regression/load/data/part*",                              // case 2
+                 "s3://${s3BucketName}/regression/load/data/part*",                              // case 3
+                 "s3://${s3BucketName}/regression/load/data/part*",                              // case 4
+                 "s3://${s3BucketName}/regression/load/data/part*",                              // case 5
+                 "s3://${s3BucketName}/regression/load/data/part*",                              // case 6
+                 "s3://${s3BucketName}/regression/load/data/part*",                              // case 7
+                 "s3://${s3BucketName}/regression/load/data/part*",                              // case 8
+                 "s3://${s3BucketName}/regression/load/data/part*",                              // case 9
+                 "s3://${s3BucketName}/regression/load/data/part*",                              // case 10
+                 "s3://${s3BucketName}/regression/load/data/part*",                              // case 11
+                 "s3://${s3BucketName}/regression/load/data/path/*/part*",                       // case 12
+                 "s3://${s3BucketName}/regression/load/data/part*",                              // case 13
+                 "s3://${s3BucketName}/regression/load/data/part*",                              // case 14
+                 "s3://${s3BucketName}/regression/load/data/part*",                              // case 15
+                 "s3://${s3BucketName}/regression/load/data/part*",                              // case 16
+                 "s3://${s3BucketName}/regression/load/data/part*",                              // case 17
+                 "s3://${s3BucketName}/regression/load/data/part*",                              // case 18
+                 "s3://${s3BucketName}/regression/load/data/part*",                              // case 19
+                 "s3://${s3BucketName}/regression/load/data/part*",                              // case 20
+                 "s3://${s3BucketName}/regression/load/data/orc/hits_100k_rows.orc",             // case 21
+                 "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc",    // case 22
+                 "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc",    // case 23
+                 "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc",    // case 24
+                 "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc",    // case 25
+                 "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc",    // case 26
+                 "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc",    // case 27
+                 "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc",    // case 28
+                 "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc",    // case 29
+                 "s3://${s3BucketName}/regression/line_delimiter/lineitem_0x7.csv.gz"            // case 30
     ]
-    def columns_list = ["""p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""",
-                   """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""",
-                   """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""",
-                   """p_partkey, p_name, p_size""",
-                   """p_partkey""",
-                   """p_partkey""",
-                   """p_partkey,  p_size""",
-                   """p_partkey""",
-                   """p_partkey,  p_size""",
-                   """p_partkey,  p_size""",
-                   """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""",
-                   """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""",
-                   """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""",
-                   """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment, col1""",
-                   """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment, x1""",
-                   """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""",
-                   """col1, col2, col3, col4""",
-                   """p_partkey, p_name, p_mfgr, x1""",
-                   """p_partkey, p_name, p_mfgr, p_brand""",
-                   """p_partkey, p_name, p_mfgr, p_brand""",
-                   """p_name, p_mfgr""",
-                   """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""",
-                //TODO: comment blow 8 rows after jibing fix
-                   """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""",
-                   """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""",
-                   """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""",
-                   """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""",
-                   """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""",
-                   """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""",
-                   """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""",
-                   """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""",
-                   """"""
-                //TODO: uncomment blow 8 rows after jibing fix
-                //    """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""",
-                //    """WATCHID,JAVAENABLE,TITLE,GOODEVENT,EVENTTIME,EVENTDATE,COUNTERID,CLIENTIP,REGIONID,USERID,COUNTERCLASS,OS,USERAGENT,URL,REFERER,ISREFRESH,REFERERCATEGORYID,REFERERREGIONID,URLCATEGORYID,URLREGIONID,RESOLUTIONWIDTH,RESOLUTIONHEIGHT,RESOLUTIONDEPTH,FLASHMAJOR,FLASHMINOR,FLASHMINOR2,NETMAJOR,NETMINOR,USERAGENTMAJOR,USERAGENTMINOR,COOKIEENABLE,JAVASCRIPTENABLE,ISMOBILE,MOBILEPHONE,MOBILEPHONEMODEL,PARAMS,IPNETWORKID,TRAFICSOURCEID,SEARCHENGINEID,SEARCHPHRASE,ADVENGINEID,ISARTIFICAL,WINDOWCLIENTWIDTH,WINDOWCLIENTHEIGHT,CLIENTTIMEZONE,CLIENTEVENTTIME,SILVERLIGHTVERSION1,SILVERLIGHTVERSION2,SILVERLIGHTVERSION3,SILVERLIGHTVERSION4,PAGECHARSET,CODEVERSION,ISLINK,ISDOWNLOAD,ISNOTBOUNCE,FUNIQID,ORIGINALURL,HID,ISOLDCOUNTER,ISEVENT,ISPARAMETER,DONTCOUNTHITS,WITHHASH,HITCOLOR,LOCALEVENTTIME,AGE,SEX,INCOME,INTERESTS,ROBOTNESS,REMOTEIP,WINDOWNAME,OPENERNAME,HISTORYLENGTH,BROWSERLANGUAGE,BROWSERCOUNTRY,SOCIALNETWORK,SOCIALACTION,HTTPERROR,SENDTIMING,DNSTIMING,CONNECTTIMING,RESPONSESTARTTIMING,RESPONSEENDTIMING,FETCHTIMING,SOCIALSOURCENETWORKID,SOCIALSOURCEPAGE,PARAMPRICE,PARAMORDERID,PARAMCURRENCY,PARAMCURRENCYID,OPENSTATSERVICENAME,OPENSTATCAMPAIGNID,OPENSTATADID,OPENSTATSOURCEID,UTMSOURCE,UTMMEDIUM,UTMCAMPAIGN,UTMCONTENT,UTMTERM,FROMTAG,HASGCLID,REFERERHASH,URLHASH,CLID""",
-                //    """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""",
-                //    """WATCHID,JAVAENABLE,TITLE,GOODEVENT,EVENTTIME,EVENTDATE,COUNTERID,CLIENTIP,REGIONID,USERID,COUNTERCLASS,OS,USERAGENT,URL,REFERER,ISREFRESH,REFERERCATEGORYID,REFERERREGIONID,URLCATEGORYID,URLREGIONID,RESOLUTIONWIDTH,RESOLUTIONHEIGHT,RESOLUTIONDEPTH,FLASHMAJOR,FLASHMINOR,FLASHMINOR2,NETMAJOR,NETMINOR,USERAGENTMAJOR,USERAGENTMINOR,COOKIEENABLE,JAVASCRIPTENABLE,ISMOBILE,MOBILEPHONE,MOBILEPHONEMODEL,PARAMS,IPNETWORKID,TRAFICSOURCEID,SEARCHENGINEID,SEARCHPHRASE,ADVENGINEID,ISARTIFICAL,WINDOWCLIENTWIDTH,WINDOWCLIENTHEIGHT,CLIENTTIMEZONE,CLIENTEVENTTIME,SILVERLIGHTVERSION1,SILVERLIGHTVERSION2,SILVERLIGHTVERSION3,SILVERLIGHTVERSION4,PAGECHARSET,CODEVERSION,ISLINK,ISDOWNLOAD,ISNOTBOUNCE,FUNIQID,ORIGINALURL,HID,ISOLDCOUNTER,ISEVENT,ISPARAMETER,DONTCOUNTHITS,WITHHASH,HITCOLOR,LOCALEVENTTIME,AGE,SEX,INCOME,INTERESTS,ROBOTNESS,REMOTEIP,WINDOWNAME,OPENERNAME,HISTORYLENGTH,BROWSERLANGUAGE,BROWSERCOUNTRY,SOCIALNETWORK,SOCIALACTION,HTTPERROR,SENDTIMING,DNSTIMING,CONNECTTIMING,RESPONSESTARTTIMING,RESPONSEENDTIMING,FETCHTIMING,SOCIALSOURCENETWORKID,SOCIALSOURCEPAGE,PARAMPRICE,PARAMORDERID,PARAMCURRENCY,PARAMCURRENCYID,OPENSTATSERVICENAME,OPENSTATCAMPAIGNID,OPENSTATADID,OPENSTATSOURCEID,UTMSOURCE,UTMMEDIUM,UTMCAMPAIGN,UTMCONTENT,UTMTERM,FROMTAG,HASGCLID,REFERERHASH,URLHASH,CLID""",
-                //    """WATCHID,JAVAENABLE,TITLE,GOODEVENT,EVENTTIME,EVENTDATE,COUNTERID,CLIENTIP,REGIONID,USERID,COUNTERCLASS,OS,USERAGENT,URL,REFERER,ISREFRESH,REFERERCATEGORYID,REFERERREGIONID,URLCATEGORYID,URLREGIONID,RESOLUTIONWIDTH,RESOLUTIONHEIGHT,RESOLUTIONDEPTH,FLASHMAJOR,FLASHMINOR,FLASHMINOR2,NETMAJOR,NETMINOR,USERAGENTMAJOR,USERAGENTMINOR,COOKIEENABLE,JAVASCRIPTENABLE,ISMOBILE,MOBILEPHONE,MOBILEPHONEMODEL,PARAMS,IPNETWORKID,TRAFICSOURCEID,SEARCHENGINEID,SEARCHPHRASE,ADVENGINEID,ISARTIFICAL,WINDOWCLIENTWIDTH,WINDOWCLIENTHEIGHT,CLIENTTIMEZONE,CLIENTEVENTTIME,SILVERLIGHTVERSION1,SILVERLIGHTVERSION2,SILVERLIGHTVERSION3,SILVERLIGHTVERSION4,PAGECHARSET,CODEVERSION,ISLINK,ISDOWNLOAD,ISNOTBOUNCE,FUNIQID,ORIGINALURL,HID,ISOLDCOUNTER,ISEVENT,ISPARAMETER,DONTCOUNTHITS,WITHHASH,HITCOLOR,LOCALEVENTTIME,AGE,SEX,INCOME,INTERESTS,ROBOTNESS,REMOTEIP,WINDOWNAME,OPENERNAME,HISTORYLENGTH,BROWSERLANGUAGE,BROWSERCOUNTRY,SOCIALNETWORK,SOCIALACTION,HTTPERROR,SENDTIMING,DNSTIMING,CONNECTTIMING,RESPONSESTARTTIMING,RESPONSEENDTIMING,FETCHTIMING,SOCIALSOURCENETWORKID,SOCIALSOURCEPAGE,PARAMPRICE,PARAMORDERID,PARAMCURRENCY,PARAMCURRENCYID,OPENSTATSERVICENAME,OPENSTATCAMPAIGNID,OPENSTATADID,OPENSTATSOURCEID,UTMSOURCE,UTMMEDIUM,UTMCAMPAIGN,UTMCONTENT,UTMTERM,FROMTAG,HASGCLID,REFERERHASH,URLHASH,CLID""",
-                //    """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""",
-                //    """WATCHID,JAVAENABLE,TITLE,GOODEVENT,EVENTTIME,EVENTDATE,COUNTERID,CLIENTIP,REGIONID,USERID,COUNTERCLASS,OS,USERAGENT,URL,REFERER,ISREFRESH,REFERERCATEGORYID,REFERERREGIONID,URLCATEGORYID,URLREGIONID,RESOLUTIONWIDTH,RESOLUTIONHEIGHT,RESOLUTIONDEPTH,FLASHMAJOR,FLASHMINOR,FLASHMINOR2,NETMAJOR,NETMINOR,USERAGENTMAJOR,USERAGENTMINOR,COOKIEENABLE,JAVASCRIPTENABLE,ISMOBILE,MOBILEPHONE,MOBILEPHONEMODEL,PARAMS,IPNETWORKID,TRAFICSOURCEID,SEARCHENGINEID,SEARCHPHRASE,ADVENGINEID,ISARTIFICAL,WINDOWCLIENTWIDTH,WINDOWCLIENTHEIGHT,CLIENTTIMEZONE,CLIENTEVENTTIME,SILVERLIGHTVERSION1,SILVERLIGHTVERSION2,SILVERLIGHTVERSION3,SILVERLIGHTVERSION4,PAGECHARSET,CODEVERSION,ISLINK,ISDOWNLOAD,ISNOTBOUNCE,FUNIQID,ORIGINALURL,HID,ISOLDCOUNTER,ISEVENT,ISPARAMETER,DONTCOUNTHITS,WITHHASH,HITCOLOR,LOCALEVENTTIME,AGE,SEX,INCOME,INTERESTS,ROBOTNESS,REMOTEIP,WINDOWNAME,OPENERNAME,HISTORYLENGTH,BROWSERLANGUAGE,BROWSERCOUNTRY,SOCIALNETWORK,SOCIALACTION,HTTPERROR,SENDTIMING,DNSTIMING,CONNECTTIMING,RESPONSESTARTTIMING,RESPONSEENDTIMING,FETCHTIMING,SOCIALSOURCENETWORKID,SOCIALSOURCEPAGE,PARAMPRICE,PARAMORDERID,PARAMCURRENCY,PARAMCURRENCYID,OPENSTATSERVICENAME,OPENSTATCAMPAIGNID,OPENSTATADID,OPENSTATSOURCEID,UTMSOURCE,UTMMEDIUM,UTMCAMPAIGN,UTMCONTENT,UTMTERM,FROMTAG,HASGCLID,REFERERHASH,URLHASH,CLID""",
-                //    """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""",
+    def columns_list = ["""p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""",  // case 0
+                   """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""",       // case 1
+                   """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""",       // case 2
+                   """p_partkey, p_name, p_size""",                                                                       // case 3
+                   """p_partkey""",                                                                                       // case 4
+                   """p_partkey""",                                                                                       // case 5
+                   """p_partkey,  p_size""",                                                                              // case 6
+                   """p_partkey""",                                                                                       // case 7
+                   """p_partkey,  p_size""",                                                                              // case 8
+                   """p_partkey,  p_size""",                                                                              // case 9
+                   """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""",       // case 10
+                   """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""",       // case 11
+                   """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""",       // case 12
+                   """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment, col1""", // case 13
+                   """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment, x1""",   // case 14
+                   """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""",       // case 15
+                   """col1, col2, col3, col4""",                                                                          // case 16
+                   """p_partkey, p_name, p_mfgr, x1""",                                                                   // case 17
+                   """p_partkey, p_name, p_mfgr, p_brand""",                                                              // case 18
+                   """p_partkey, p_name, p_mfgr, p_brand""",                                                              // case 19
+                   """p_name, p_mfgr""",                                                                                  // case 20
+                   """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""",  // case 21
+                   """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""",  // case 22
+                   """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""",  // case 23
+                   """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""",  // case 24
+                   """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""",  // case 25
+                   """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""",  // case 26
+                   """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""",  // case 27
+                   """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""",  // case 28
+                   """watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid""",  // case 29
+                   """""" // case 30
                    ]
-    def column_in_paths = ["", "", "", "", "", "", "", "", "", "", "", "", "COLUMNS FROM PATH AS (city)", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""]
-    def preceding_filters = ["", "", "", "", "", "", "", "", "", "", "", "preceding filter p_size < 10", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""]
-    def set_values = ["",
-                      "",
-                      "SET(comment=p_comment, retailprice=p_retailprice, container=p_container, size=p_size, type=p_type, brand=p_brand, mfgr=p_mfgr, name=p_name, partkey=p_partkey)",
-                      "set(p_name=upper(p_name),p_greatest=greatest(cast(p_partkey as int), cast(p_size as int)))",
-                      "set(p_partkey = p_partkey + 100)",
-                      "set(partkey = p_partkey + 100)",
-                      "set(partkey = p_partkey + p_size)",
-                      "set(tmpk = p_partkey + 1, partkey = tmpk*2)",
-                      "set(partkey = p_partkey + 1, partsize = p_size*2)",
-                      "set(partsize = p_partkey + p_size)",
-                      "",
-                      "",
-                      "",
-                      "",
-                      "",
-                      "",
-                      "",
-                      "set(col4 = x1)",
-                      "set(col4 = p_brand)",
-                      "set(col5 = p_brand)",
-                      "",
-                      "",
-                      "",
-                      "",
-                      "",
-                      "",
-                      "",
-                      "",
-                      "",
-                      "",
-                      ""
+    def column_in_paths = ["",   // case 0
+                           "",   // case 1
+                           "",   // case 2
+                           "",   // case 3  
+                           "",   // case 4
+                           "",   // case 5
+                           "",   // case 6
+                           "",   // case 7
+                           "",   // case 8
+                           "",   // case 9
+                           "",   // case 10
+                           "",   // case 11
+                           "COLUMNS FROM PATH AS (city)",   // case 12
+                           "",   // case 13
+                           "",   // case 14
+                           "",   // case 15
+                           "",   // case 16
+                           "",   // case 17
+                           "",   // case 18
+                           "",   // case 19
+                           "",   // case 20
+                           "",   // case 21
+                           "",   // case 22
+                           "",   // case 23
+                           "",   // case 24
+                           "",   // case 25
+                           "",   // case 26
+                           "",   // case 27
+                           "",   // case 28
+                           "",   // case 29
+                           ""   // case 30
+                          ]
+    def preceding_filters = ["",   // case 0
+                            "",   // case 1
+                            "",   // case 2
+                            "",   // case 3
+                            "",   // case 4
+                            "",   // case 5
+                            "",   // case 6
+                            "",   // case 7
+                            "",   // case 8
+                            "",   // case 9
+                            "",   // case 10
+                            "preceding filter p_size < 10",   // case 11
+                            "",   // case 12
+                            "",   // case 13
+                            "",   // case 14
+                            "",   // case 15
+                            "",   // case 16
+                            "",   // case 17
+                            "",   // case 18
+                            "",   // case 19
+                            "",   // case 20
+                            "",   // case 21
+                            "",   // case 22
+                            "",   // case 23
+                            "",   // case 24
+                            "",   // case 25
+                            "",   // case 26
+                            "",   // case 27
+                            "",   // case 28
+                            "",   // case 29
+                            ""    // case 30
+                           ]
+    def set_values = ["",     // case 0
+                      "",     // case 1
+                      "SET(comment=p_comment, retailprice=p_retailprice, container=p_container, size=p_size, type=p_type, brand=p_brand, mfgr=p_mfgr, name=p_name, partkey=p_partkey)", // case 2
+                      "set(p_name=upper(p_name),p_greatest=greatest(cast(p_partkey as int), cast(p_size as int)))", // case 3
+                      "set(p_partkey = p_partkey + 100)", // case 4
+                      "set(partkey = p_partkey + 100)", // case 5
+                      "set(partkey = p_partkey + p_size)", // case 6
+                      "set(tmpk = p_partkey + 1, partkey = tmpk*2)", // case 7
+                      "set(partkey = p_partkey + 1, partsize = p_size*2)", // case 8
+                      "set(partsize = p_partkey + p_size)", // case 9
+                      "", // case 10
+                      "", // case 11
+                      "", // case 12
+                      "", // case 13
+                      "", // case 14
+                      "", // case 15
+                      "", // case 16
+                      "set(col4 = x1)", // case 17
+                      "set(col4 = p_brand)", // case 18
+                      "set(col5 = p_brand)", // case 19
+                      "", // case 20
+                      "", // case 21
+                      "", // case 22
+                      "", // case 23
+                      "", // case 24
+                      "", // case 25
+                      "", // case 26
+                      "", // case 27
+                      "", // case 28
+                      "", // case 29
+                      ""  // case 30
     ]
-    def where_exprs = ["", "", "", "", "", "", "", "", "", "", "", "where p_partkey>10", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""]
+    def where_exprs = ["",   // case 0
+                       "",   // case 1
+                       "",   // case 2
+                       "",   // case 3
+                       "",   // case 4
+                       "",   // case 5
+                       "",   // case 6
+                       "",   // case 7
+                       "",   // case 8
+                       "",   // case 9
+                       "",   // case 10
+                       "where p_partkey>10",   // case 11
+                       "",   // case 12
+                       "",   // case 13
+                       "",   // case 14
+                       "",   // case 15
+                       "",   // case 16
+                       "",   // case 17
+                       "",   // case 18
+                       "",   // case 19
+                       "",   // case 20
+                       "",   // case 21
+                       "",   // case 22
+                       "",   // case 23
+                       "",   // case 24
+                       "",   // case 25
+                       "",   // case 26
+                       "",   // case 27
+                       "",   // case 28
+                       "",   // case 29
+                       ""    // case 30
+                      ]
 
-    def line_delimiters = ["", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "\u0007"]
+    def line_delimiters = ["",   // case 0
+                           "",   // case 1
+                           "",   // case 2
+                           "",   // case 3
+                           "",   // case 4
+                           "",   // case 5
+                           "",   // case 6
+                           "",   // case 7
+                           "",   // case 8
+                           "",   // case 9
+                           "",   // case 10
+                           "",   // case 11
+                           "",   // case 12
+                           "",   // case 13
+                           "",   // case 14
+                           "",   // case 15
+                           "",   // case 16
+                           "",   // case 17
+                           "",   // case 18
+                           "",   // case 19
+                           "",   // case 20
+                           "",   // case 21
+                           "",   // case 22
+                           "",   // case 23
+                           "",   // case 24
+                           "",   // case 25
+                           "",   // case 26
+                           "",   // case 27
+                           "",   // case 28
+                           "",   // case 29
+                           "\u0007"   // case 30
+                          ]
 
-    def etl_info = ["unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",
-                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",
-                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",
-                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",
-                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",
-                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",
-                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",
-                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",
-                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",
-                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",
-                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",
-                    "unselected.rows=163706; dpp.abnorm.ALL=0; dpp.norm.ALL=36294",
-                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",
-                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",
-                    "\\N",
-                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",
-                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",
-                    "\\N",
-                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",
-                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",
-                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",
-                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=100000",
-                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000",
-                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000",
-                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000",
-                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000",
-                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000",
-                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000",
-                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000",
-                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000",
-                    "\\N"
+    def etl_info = ["unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",  // case 0
+                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",  // case 1
+                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",  // case 2
+                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",  // case 3
+                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",  // case 4
+                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",  // case 5
+                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",  // case 6
+                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",  // case 7
+                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",  // case 8
+                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",  // case 9
+                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",  // case 10
+                    "unselected.rows=163706; dpp.abnorm.ALL=0; dpp.norm.ALL=36294",  // case 11
+                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",  // case 12
+                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",  // case 13
+                    "\\N",  // case 14
+                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",  // case 15
+                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",  // case 16
+                    "\\N",  // case 17
+                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",  // case 18
+                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",  // case 19
+                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000",  // case 20
+                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=100000",  // case 21
+                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000",   // case 22
+                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000",   // case 23
+                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000",   // case 24
+                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000",   // case 25
+                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000",   // case 26
+                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000",   // case 27
+                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000",   // case 28
+                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000",   // case 29
+                    "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000"    // case 30
                     ]
 
-    def task_info = ["cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
-                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0"
+    def task_info = ["cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 0
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 1
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 2
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 3
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 4
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 5
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 6
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 7
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 8
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 9
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 10
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 11
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 12
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 13
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 14
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 15
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 16
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 17
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 18
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 19
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 20
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 21
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 22
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 23
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 24
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 25
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 26
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 27
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 28
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",  // case 29
+                     "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0"   // case 30
     ]
 
-    def error_msg = ["",
-                    "",
-                    "",
-                    "",
-                    "",
-                    "",
-                    "",
-                    "",
-                    "",
-                    "",
-                    "",
-                    "",
-                    "",
-                    "",
-                    "failed to find default value expr for slot: x1",
-                    "",
-                    "",
-                    "failed to find default value expr for slot: x1",
-                    "",
-                    "",
-                    "",
-                    "",
-                    "",
-                    "",
-                    "",
-                    "",
-                    "",
-                    "",
-                    "",
-                    "",
-                    ""
+    def error_msg = ["",                                                         // case 0
+                    "",                                                          // case 1
+                    "",                                                          // case 2
+                    "",                                                          // case 3
+                    "",                                                          // case 4
+                    "",                                                          // case 5
+                    "",                                                          // case 6
+                    "",                                                          // case 7
+                    "",                                                          // case 8
+                    "",                                                          // case 9
+                    "",                                                          // case 10
+                    "",                                                          // case 11
+                    "",                                                          // case 12
+                    "",                                                          // case 13
+                    "failed to find default value expr for slot: x1",            // case 14
+                    "",                                                          // case 15
+                    "failed to find default value expr for slot: x1",            // case 16
+                    "",                                                          // case 17
+                    "",                                                          // case 18
+                    "",                                                          // case 19
+                    "",                                                          // case 20
+                    "",                                                          // case 21
+                    "",                                                          // case 22
+                    "",                                                          // case 23
+                    "",                                                          // case 24
+                    "",                                                          // case 25
+                    "",                                                          // case 26
+                    "",                                                          // case 27
+                    "",                                                          // case 28
+                    "",                                                          // case 29
+                    "",                                                          // case 30
                     ]
 
     String ak = getS3AK()
@@ -347,7 +461,7 @@ suite("test_broker_load_p2", "p2") {
                     if (result[0][2].equals("CANCELLED")) {
                         logger.info("Load result: " + result[0])
                         assertTrue(result[0][6].contains(task_info[i]))
-                        assertTrue(result[0][9].contains(error_msg[i]), "expected: " + error_msg[i] + ", actual: " + result[0][7] + ", label: $label")
+                        assertTrue(result[0][7].contains(error_msg[i]), "expected: " + error_msg[i] + ", actual: " + result[0][7] + ", label: $label")
                         break;
                     }
                     Thread.sleep(1000)