add regression test cases

gnehil · Sep 18, 2024 · 0ec0d24 · 0ec0d24
1 parent bc7d7e7
commit 0ec0d24
Show file tree

Hide file tree

Showing 17 changed files with 1,176 additions and 10 deletions.
diff --git a/be/src/vec/exec/scan/scanner_context.h b/be/src/vec/exec/scan/scanner_context.h
@@ -105,9 +105,8 @@ class ScannerContext : public std::enable_shared_from_this<ScannerContext>,
                    const TupleDescriptor* output_tuple_desc,
                    const RowDescriptor* output_row_descriptor,
                    const std::list<std::shared_ptr<vectorized::ScannerDelegate>>& scanners,
-                   int64_t limit_, int64_t max_bytes_in_blocks_queue,
-                   std::shared_ptr<pipeline::Dependency> dependency,
-                   const int num_parallel_instances);
+                   int64_t limit_, std::shared_ptr<pipeline::Dependency> dependency,
+                   bool ignore_data_distribution);
 
     ~ScannerContext() override {
         SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_query_thread_context.query_mem_tracker);
@@ -208,7 +207,7 @@ class ScannerContext : public std::enable_shared_from_this<ScannerContext>,
     int64_t limit;
 
     int32_t _max_thread_num = 0;
-    int64_t _max_bytes_in_queue;
+    int64_t _max_bytes_in_queue = 0;
     doris::vectorized::ScannerScheduler* _scanner_scheduler;
     SimplifiedScanScheduler* _simple_scan_scheduler = nullptr;
     SimplifiedScanScheduler* _remote_scan_task_scheduler = nullptr;
@@ -218,7 +217,6 @@ class ScannerContext : public std::enable_shared_from_this<ScannerContext>,
     int32_t _num_running_scanners = 0;
     // weak pointer for _scanners, used in stop function
     std::vector<std::weak_ptr<ScannerDelegate>> _all_scanners;
-    const int _num_parallel_instances;
     std::shared_ptr<RuntimeProfile> _scanner_profile;
     RuntimeProfile::Counter* _scanner_sched_counter = nullptr;
     RuntimeProfile::Counter* _newly_create_free_blocks_num = nullptr;
@@ -228,6 +226,7 @@ class ScannerContext : public std::enable_shared_from_this<ScannerContext>,
     RuntimeProfile::Counter* _scale_up_scanners_counter = nullptr;
     QueryThreadContext _query_thread_context;
     std::shared_ptr<pipeline::Dependency> _dependency = nullptr;
+    bool _ignore_data_distribution = false;
 
     // for scaling up the running scanners
     size_t _estimated_block_size = 0;

diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -684,7 +684,10 @@ public class SessionVariable implements Serializable, Writable {
     @VariableMgr.VarAttr(name = EXEC_MEM_LIMIT)
     public long maxExecMemByte = 2147483648L;
 
-    @VariableMgr.VarAttr(name = SCAN_QUEUE_MEM_LIMIT)
+    @VariableMgr.VarAttr(name = SCAN_QUEUE_MEM_LIMIT,
+            description = {"每个 Scan Instance 的 block queue 能够保存多少字节的 block",
+                    "How many bytes of block can be saved in the block queue of each Scan Instance"})
+    // 100MB
     public long maxScanQueueMemByte = 2147483648L / 20;
 
     @VariableMgr.VarAttr(name = NUM_SCANNER_THREADS, needForward = true, description = {

diff --git a/regression-test/data/load_p0/ingestion_load/data1.parquet b/regression-test/data/load_p0/ingestion_load/data1.parquet
diff --git a/regression-test/data/load_p0/ingestion_load/data2-0.parquet b/regression-test/data/load_p0/ingestion_load/data2-0.parquet
diff --git a/regression-test/data/load_p0/ingestion_load/data2-1.parquet b/regression-test/data/load_p0/ingestion_load/data2-1.parquet
diff --git a/regression-test/data/load_p0/ingestion_load/data2-2.parquet b/regression-test/data/load_p0/ingestion_load/data2-2.parquet
diff --git a/regression-test/data/load_p0/ingestion_load/data2-3.parquet b/regression-test/data/load_p0/ingestion_load/data2-3.parquet
diff --git a/regression-test/data/load_p0/ingestion_load/test_ingestion_load.out b/regression-test/data/load_p0/ingestion_load/test_ingestion_load.out
@@ -11,3 +11,27 @@
 1145092610	xWJUDWAV8Nllo0F	dnZ9RMVdoqxh4kGBvy55zQdChNTVYdlvRZP4aWIkXyErUbM1XmFGQ9vuCD113JKKCyx4crDoY	false	115	-22832	-7242855305248390982	-4240353246453053617	-9074.909	-2.51212400295869E8	-502.410	618.820	2024-06-12	2024-04-18	2023-11-04T09:55:17	2023-11-13T16:30:23
 1736373707	UU14wnLhPkBid41	pmuNqYfOc3JCscf9meT5dYB2i28Pt9iaeXK4QqjVZJdoKFOeZI5bG9RKm1zInTdDMW1N0PKI5Y	true	-105	-20276	360048259532857165	-4602633478165721463	-13230.296	-1.708246954394742E9	757.147	-533.800	2024-01-05	2023-09-08	2023-11-27T05:21:33	2024-02-11T21:35:03
 
+-- !select --
+-2067215761	41V4fgCKI3Hkioe	DZGaq24Yqh7SwmbPT6IX23jfC5NKqG7gE9JT4GLwiaQtoO8l6EjhGWQP9X7NHmjdqMbIN5kNeDkffOrlS6roIwj2wXpJ	true	123	-4524	1014223769452772206	-6589390612399245616	29103.387	-4.09385322835896E8	886.394	-699.960	2024-02-22	2024-03-04	2024-01-01T22:13:54	2023-10-02T19:35:10
+-1982022740	fcZ1o6ZXG8UOFh5	iw4Ziys42GRRTFNkVPeQEA9I5EQtBD04xfefDsPCWN0vr1	true	-3	-25143	7122614823242399351	7391807634038366248	23160.604	-9.20283206353984E8	829.880	-387.403	2023-10-16	2024-05-13	2024-07-16T18:27:45	2023-11-03T05:30:21
+-1228926576	1TISaGtB01BiqVt	kq4u	false	-123	15962	3590007830423934951	-1478759439092857810	-7813.757	-6.98785793100899E8	930.743	402.350	2024-07-23	2023-07-30	2023-11-27T17:48:50	2024-03-11T21:09:58
+-575060895	rRfatUJiOO5dq9Y	ETjqrUNUnI5kSmkafjWfRTG8HIp98pLGXagNpXZHqOIZZDRkoGeahOwk9	false	16	-767	6623730208927396375	-3055706367894284822	12540.839	-1.047911096098831E9	-752.454	-241.620	2024-04-10	2024-05-16	2023-12-07T23:38:05	2023-12-11T05:48:36
+-76042627	PcVaKC43qmIzuxY	U3aGxaZumFpqcUsLI	true	44	31151	9085406701767055602	-5846138572199996843	-16845.29	2.44522690225531E8	-784.720	-467.133	2023-10-31	2023-08-29	2023-09-12T10:12:46	2023-10-19T17:02:51
+121048200	KPLWjhhbGXqflJi	rzqOYQH9ySHPwCm5K4GdeuI28G8LLmnpqLmsLMLfyRIvcfrlubQI47wUa8QILhuS38MBkjL	true	42	13182	-6601530758880565531	5619594098883737912	-2782.1506	3.86698722676211E8	478.420	-330.289	2024-06-17	2023-12-26	2024-04-28T03:29:04	2023-08-18T21:05:32
+262860291	m3XgmlbIHYNH1qS	BTJRzVrpM78zJAsHMEGhkF5BiDoc3yJuoV0s209sFcqElZsheBgolBGlFl9X4EfauD64FcFF2Mi4V0dKZfpDgaLLRPfG1SALV7	false	-42	5990	-7504815416577235660	1535659792778122944	1171.9619	1.28834143701229E8	626.721	682.828	2023-11-24	2023-11-18	2024-03-21T11:50:17	2024-03-31T12:59:27
+579428879	KsOC6WGrieGlo7B	SzeA6tRbsiGWJTBDvBQdBjCqjSE6Y	false	-111	32758	4029182463831656671	-3546198025044093789	20338.55	-2.015222388533773E9	61.981	720.310	2023-11-13	2024-07-04	2024-07-19T12:42:28	2024-01-04T10:32:53
+1145092610	xWJUDWAV8Nllo0F	dnZ9RMVdoqxh4kGBvy55zQdChNTVYdlvRZP4aWIkXyErUbM1XmFGQ9vuCD113JKKCyx4crDoY	false	115	-22832	-7242855305248390982	-4240353246453053617	-9074.909	-2.51212400295869E8	-502.410	618.820	2024-06-12	2024-04-18	2023-11-04T09:55:17	2023-11-13T16:30:23
+1736373707	UU14wnLhPkBid41	pmuNqYfOc3JCscf9meT5dYB2i28Pt9iaeXK4QqjVZJdoKFOeZI5bG9RKm1zInTdDMW1N0PKI5Y	true	-105	-20276	360048259532857165	-4602633478165721463	-13230.296	-1.708246954394742E9	757.147	-533.800	2024-01-05	2023-09-08	2023-11-27T05:21:33	2024-02-11T21:35:03
+
+-- !select --
+-9022291871392468311	2023-11-02	mOWPGHmiZ	10784	-128	2023-11-12T18:06:21	4218616419351308798	1993977685	-1857678846
+-6045452612961149194	2024-06-23	G1j	28468	-55	2024-06-09T00:12:11	-6456263257174124469	-727277974	144696403
+-1537906159489906139	2024-04-04	MRMRE18bVh49RD	32763	98	2024-01-20T00:54:03	-1289145371043997006	128181215	-1295829474
+-1510882223779118241	2024-07-24	PCwFn7r21MZr	22960	-79	2024-02-07T18:15:07	-8437284610883885859	472729036	-39626304
+-1185467471318572316	2023-11-08	ieed5Msw8X6be4HGS	16555	-79	2024-07-28T23:08:29	3263664376405334754	-809360772	-1229995615
+-234810200663664160	2024-06-07	s7GIrN805aU3cs2EM	-7555	-124	2023-12-28T18:59:15	-3600712745035417587	2035647886	126756427
+4461660295430359180	2024-04-23	K	25428	6	2023-11-15T18:38:20	-4503242152141666001	-1093190312	1511443278
+6742880469957921530	2024-05-02	cJJrvRJfpCuGh	27232	64	2024-08-18T09:46:50	-2607385663861429432	-1390108377	1758263623
+7252685688720766402	2024-03-13	891C2	-9774	-1	2023-10-12T19:45:28	-3210623791036109982	-915986651	-1794344594
+8278077411585505009	2023-11-17	gBesLQnYpjK7iDUUcIi	-26656	-50	2023-12-11T14:29:52	-8301529943262026214	-1555756888	-1318983102
+
diff --git a/regression-test/data/load_p0/ingestion_load/test_ingestion_load_multi_table.out b/regression-test/data/load_p0/ingestion_load/test_ingestion_load_multi_table.out
@@ -0,0 +1,25 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select --
+-2067215761	41V4fgCKI3Hkioe	DZGaq24Yqh7SwmbPT6IX23jfC5NKqG7gE9JT4GLwiaQtoO8l6EjhGWQP9X7NHmjdqMbIN5kNeDkffOrlS6roIwj2wXpJ	true	123	-4524	1014223769452772206	-6589390612399245616	29103.387	-4.09385322835896E8	886.394	-699.960	2024-02-22	2024-03-04	2024-01-01T22:13:54	2023-10-02T19:35:10
+-1982022740	fcZ1o6ZXG8UOFh5	iw4Ziys42GRRTFNkVPeQEA9I5EQtBD04xfefDsPCWN0vr1	true	-3	-25143	7122614823242399351	7391807634038366248	23160.604	-9.20283206353984E8	829.880	-387.403	2023-10-16	2024-05-13	2024-07-16T18:27:45	2023-11-03T05:30:21
+-1228926576	1TISaGtB01BiqVt	kq4u	false	-123	15962	3590007830423934951	-1478759439092857810	-7813.757	-6.98785793100899E8	930.743	402.350	2024-07-23	2023-07-30	2023-11-27T17:48:50	2024-03-11T21:09:58
+-575060895	rRfatUJiOO5dq9Y	ETjqrUNUnI5kSmkafjWfRTG8HIp98pLGXagNpXZHqOIZZDRkoGeahOwk9	false	16	-767	6623730208927396375	-3055706367894284822	12540.839	-1.047911096098831E9	-752.454	-241.620	2024-04-10	2024-05-16	2023-12-07T23:38:05	2023-12-11T05:48:36
+-76042627	PcVaKC43qmIzuxY	U3aGxaZumFpqcUsLI	true	44	31151	9085406701767055602	-5846138572199996843	-16845.29	2.44522690225531E8	-784.720	-467.133	2023-10-31	2023-08-29	2023-09-12T10:12:46	2023-10-19T17:02:51
+121048200	KPLWjhhbGXqflJi	rzqOYQH9ySHPwCm5K4GdeuI28G8LLmnpqLmsLMLfyRIvcfrlubQI47wUa8QILhuS38MBkjL	true	42	13182	-6601530758880565531	5619594098883737912	-2782.1506	3.86698722676211E8	478.420	-330.289	2024-06-17	2023-12-26	2024-04-28T03:29:04	2023-08-18T21:05:32
+262860291	m3XgmlbIHYNH1qS	BTJRzVrpM78zJAsHMEGhkF5BiDoc3yJuoV0s209sFcqElZsheBgolBGlFl9X4EfauD64FcFF2Mi4V0dKZfpDgaLLRPfG1SALV7	false	-42	5990	-7504815416577235660	1535659792778122944	1171.9619	1.28834143701229E8	626.721	682.828	2023-11-24	2023-11-18	2024-03-21T11:50:17	2024-03-31T12:59:27
+579428879	KsOC6WGrieGlo7B	SzeA6tRbsiGWJTBDvBQdBjCqjSE6Y	false	-111	32758	4029182463831656671	-3546198025044093789	20338.55	-2.015222388533773E9	61.981	720.310	2023-11-13	2024-07-04	2024-07-19T12:42:28	2024-01-04T10:32:53
+1145092610	xWJUDWAV8Nllo0F	dnZ9RMVdoqxh4kGBvy55zQdChNTVYdlvRZP4aWIkXyErUbM1XmFGQ9vuCD113JKKCyx4crDoY	false	115	-22832	-7242855305248390982	-4240353246453053617	-9074.909	-2.51212400295869E8	-502.410	618.820	2024-06-12	2024-04-18	2023-11-04T09:55:17	2023-11-13T16:30:23
+1736373707	UU14wnLhPkBid41	pmuNqYfOc3JCscf9meT5dYB2i28Pt9iaeXK4QqjVZJdoKFOeZI5bG9RKm1zInTdDMW1N0PKI5Y	true	-105	-20276	360048259532857165	-4602633478165721463	-13230.296	-1.708246954394742E9	757.147	-533.800	2024-01-05	2023-09-08	2023-11-27T05:21:33	2024-02-11T21:35:03
+
+-- !select --
+-2067215761	41V4fgCKI3Hkioe	DZGaq24Yqh7SwmbPT6IX23jfC5NKqG7gE9JT4GLwiaQtoO8l6EjhGWQP9X7NHmjdqMbIN5kNeDkffOrlS6roIwj2wXpJ	true	123	-4524	1014223769452772206	-6589390612399245616	29103.387	-4.09385322835896E8	886.394	-699.960	2024-02-22	2024-03-04	2024-01-01T22:13:54	2023-10-02T19:35:10
+-1982022740	fcZ1o6ZXG8UOFh5	iw4Ziys42GRRTFNkVPeQEA9I5EQtBD04xfefDsPCWN0vr1	true	-3	-25143	7122614823242399351	7391807634038366248	23160.604	-9.20283206353984E8	829.880	-387.403	2023-10-16	2024-05-13	2024-07-16T18:27:45	2023-11-03T05:30:21
+-1228926576	1TISaGtB01BiqVt	kq4u	false	-123	15962	3590007830423934951	-1478759439092857810	-7813.757	-6.98785793100899E8	930.743	402.350	2024-07-23	2023-07-30	2023-11-27T17:48:50	2024-03-11T21:09:58
+-575060895	rRfatUJiOO5dq9Y	ETjqrUNUnI5kSmkafjWfRTG8HIp98pLGXagNpXZHqOIZZDRkoGeahOwk9	false	16	-767	6623730208927396375	-3055706367894284822	12540.839	-1.047911096098831E9	-752.454	-241.620	2024-04-10	2024-05-16	2023-12-07T23:38:05	2023-12-11T05:48:36
+-76042627	PcVaKC43qmIzuxY	U3aGxaZumFpqcUsLI	true	44	31151	9085406701767055602	-5846138572199996843	-16845.29	2.44522690225531E8	-784.720	-467.133	2023-10-31	2023-08-29	2023-09-12T10:12:46	2023-10-19T17:02:51
+121048200	KPLWjhhbGXqflJi	rzqOYQH9ySHPwCm5K4GdeuI28G8LLmnpqLmsLMLfyRIvcfrlubQI47wUa8QILhuS38MBkjL	true	42	13182	-6601530758880565531	5619594098883737912	-2782.1506	3.86698722676211E8	478.420	-330.289	2024-06-17	2023-12-26	2024-04-28T03:29:04	2023-08-18T21:05:32
+262860291	m3XgmlbIHYNH1qS	BTJRzVrpM78zJAsHMEGhkF5BiDoc3yJuoV0s209sFcqElZsheBgolBGlFl9X4EfauD64FcFF2Mi4V0dKZfpDgaLLRPfG1SALV7	false	-42	5990	-7504815416577235660	1535659792778122944	1171.9619	1.28834143701229E8	626.721	682.828	2023-11-24	2023-11-18	2024-03-21T11:50:17	2024-03-31T12:59:27
+579428879	KsOC6WGrieGlo7B	SzeA6tRbsiGWJTBDvBQdBjCqjSE6Y	false	-111	32758	4029182463831656671	-3546198025044093789	20338.55	-2.015222388533773E9	61.981	720.310	2023-11-13	2024-07-04	2024-07-19T12:42:28	2024-01-04T10:32:53
+1145092610	xWJUDWAV8Nllo0F	dnZ9RMVdoqxh4kGBvy55zQdChNTVYdlvRZP4aWIkXyErUbM1XmFGQ9vuCD113JKKCyx4crDoY	false	115	-22832	-7242855305248390982	-4240353246453053617	-9074.909	-2.51212400295869E8	-502.410	618.820	2024-06-12	2024-04-18	2023-11-04T09:55:17	2023-11-13T16:30:23
+1736373707	UU14wnLhPkBid41	pmuNqYfOc3JCscf9meT5dYB2i28Pt9iaeXK4QqjVZJdoKFOeZI5bG9RKm1zInTdDMW1N0PKI5Y	true	-105	-20276	360048259532857165	-4602633478165721463	-13230.296	-1.708246954394742E9	757.147	-533.800	2024-01-05	2023-09-08	2023-11-27T05:21:33	2024-02-11T21:35:03
+
diff --git a/regression-test/suites/load_p0/ingestion_load/test_ingestion_load.groovy b/regression-test/suites/load_p0/ingestion_load/test_ingestion_load.groovy
@@ -45,7 +45,7 @@ suite('test_ingestion_load', 'p0') {
 
         httpTest {
             endpoint context.config.feHttpAddress
-            uri "/api/ingestion_load/${context.dbName}/_create"
+            uri "/api/ingestion_load/internal/${context.dbName}/_create"
             op "post"
             basicAuthorization context.config.feHttpUser, context.config.feHttpPassword
             body reqBody
@@ -92,7 +92,7 @@ suite('test_ingestion_load', 'p0') {
 
         httpTest {
             endpoint context.config.feHttpAddress
-            uri "/api/ingestion_load/${context.dbName}/_update"
+            uri "/api/ingestion_load/internal/${context.dbName}/_update"
             op "post"
             basicAuthorization context.config.feHttpUser, context.config.feHttpPassword
             body updateStatusReqBody
@@ -106,12 +106,12 @@ suite('test_ingestion_load', 'p0') {
             }
         }
 
-        max_try_milli_secs = 60000
+        max_try_milli_secs = 120000
         while (max_try_milli_secs) {
             result = sql "show load where label = '${loadLabel}'"
             if (result[0][2] == "FINISHED") {
                 sql "sync"
-                qt_select "select * from ${testTable} order by c_int"
+                qt_select "select * from ${testTable} order by 1"
                 break
             } else {
                 sleep(5000) // wait 1 second every time
@@ -147,6 +147,7 @@ suite('test_ingestion_load', 'p0') {
                 c_datetime datetime NULL,
                 c_datetimev2 datetime NULL
             )
+            DUPLICATE KEY(c_int)
             DISTRIBUTED BY HASH(c_int) BUCKETS 1
             PROPERTIES (
             "replication_num" = "1"
@@ -157,6 +158,65 @@ suite('test_ingestion_load', 'p0') {
 
         testIngestLoadJob.call(tableName, label, context.config.dataPath + '/load_p0/ingestion_load/data.parquet')
 
+        tableName = 'tbl_test_spark_load_unique_mor'
+
+        sql """
+            CREATE TABLE IF NOT EXISTS ${tableName} (
+                c_int int(11) NULL,
+                c_char char(15) NULL,
+                c_varchar varchar(100) NULL,
+                c_bool boolean NULL,
+                c_tinyint tinyint(4) NULL,
+                c_smallint smallint(6) NULL,
+                c_bigint bigint(20) NULL,
+                c_largeint largeint(40) NULL,
+                c_float float NULL,
+                c_double double NULL,
+                c_decimal decimal(6, 3) NULL,
+                c_decimalv3 decimal(6, 3) NULL,
+                c_date date NULL,
+                c_datev2 date NULL,
+                c_datetime datetime NULL,
+                c_datetimev2 datetime NULL
+            )
+            UNIQUE KEY(c_int)
+            DISTRIBUTED BY HASH(c_int) BUCKETS 1
+            PROPERTIES (
+            "replication_num" = "1",
+            "enable_unique_key_merge_on_write" = "false"
+            )
+            """
+
+        label = "test_ingestion_load_unique_mor"
+
+        testIngestLoadJob.call(tableName, label, context.config.dataPath + '/load_p0/ingestion_load/data.parquet')
+
+        tableName = 'tbl_test_spark_load_agg'
+
+        sql """
+            CREATE TABLE IF NOT EXISTS ${tableName}
+            (
+                `user_id` LARGEINT NOT NULL COMMENT "user id",
+                `date` DATE NOT NULL COMMENT "data import time",
+                `city` VARCHAR(20) COMMENT "city",
+                `age` SMALLINT COMMENT "age",
+                `sex` TINYINT COMMENT "gender",
+                `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "last visit date time",
+                `cost` BIGINT SUM DEFAULT "0" COMMENT "user total cost",
+                `max_dwell_time` INT MAX DEFAULT "0" COMMENT "user max dwell time",
+                `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "user min dwell time"
+            )
+            AGGREGATE KEY(`user_id`, `date`, `city`, `age`, `sex`)
+            DISTRIBUTED BY HASH(`user_id`) BUCKETS 1
+            PROPERTIES (
+            "replication_allocation" = "tag.location.default: 1"
+            );
+        """
+
+        label = "test_ingestion_load_agg"
+
+        testIngestLoadJob.call(tableName, label, context.config.dataPath + '/load_p0/ingestion_load/data1.parquet')
+
     }
 
 }