add regression test cases

gnehil · Sep 18, 2024 · d33628b · d33628b
1 parent 7a2718f
commit d33628b
Show file tree

Hide file tree

Showing 15 changed files with 1,123 additions and 4 deletions.
diff --git a/regression-test/data/load_p0/ingestion_load/data1.parquet b/regression-test/data/load_p0/ingestion_load/data1.parquet
diff --git a/regression-test/data/load_p0/ingestion_load/data2-0.parquet b/regression-test/data/load_p0/ingestion_load/data2-0.parquet
diff --git a/regression-test/data/load_p0/ingestion_load/data2-1.parquet b/regression-test/data/load_p0/ingestion_load/data2-1.parquet
diff --git a/regression-test/data/load_p0/ingestion_load/data2-2.parquet b/regression-test/data/load_p0/ingestion_load/data2-2.parquet
diff --git a/regression-test/data/load_p0/ingestion_load/data2-3.parquet b/regression-test/data/load_p0/ingestion_load/data2-3.parquet
diff --git a/regression-test/data/load_p0/ingestion_load/test_ingestion_load.out b/regression-test/data/load_p0/ingestion_load/test_ingestion_load.out
@@ -11,3 +11,27 @@
 1145092610	xWJUDWAV8Nllo0F	dnZ9RMVdoqxh4kGBvy55zQdChNTVYdlvRZP4aWIkXyErUbM1XmFGQ9vuCD113JKKCyx4crDoY	false	115	-22832	-7242855305248390982	-4240353246453053617	-9074.909	-2.51212400295869E8	-502.410	618.820	2024-06-12	2024-04-18	2023-11-04T09:55:17	2023-11-13T16:30:23
 1736373707	UU14wnLhPkBid41	pmuNqYfOc3JCscf9meT5dYB2i28Pt9iaeXK4QqjVZJdoKFOeZI5bG9RKm1zInTdDMW1N0PKI5Y	true	-105	-20276	360048259532857165	-4602633478165721463	-13230.296	-1.708246954394742E9	757.147	-533.800	2024-01-05	2023-09-08	2023-11-27T05:21:33	2024-02-11T21:35:03
 
+-- !select --
+-2067215761	41V4fgCKI3Hkioe	DZGaq24Yqh7SwmbPT6IX23jfC5NKqG7gE9JT4GLwiaQtoO8l6EjhGWQP9X7NHmjdqMbIN5kNeDkffOrlS6roIwj2wXpJ	true	123	-4524	1014223769452772206	-6589390612399245616	29103.387	-4.09385322835896E8	886.394	-699.960	2024-02-22	2024-03-04	2024-01-01T22:13:54	2023-10-02T19:35:10
+-1982022740	fcZ1o6ZXG8UOFh5	iw4Ziys42GRRTFNkVPeQEA9I5EQtBD04xfefDsPCWN0vr1	true	-3	-25143	7122614823242399351	7391807634038366248	23160.604	-9.20283206353984E8	829.880	-387.403	2023-10-16	2024-05-13	2024-07-16T18:27:45	2023-11-03T05:30:21
+-1228926576	1TISaGtB01BiqVt	kq4u	false	-123	15962	3590007830423934951	-1478759439092857810	-7813.757	-6.98785793100899E8	930.743	402.350	2024-07-23	2023-07-30	2023-11-27T17:48:50	2024-03-11T21:09:58
+-575060895	rRfatUJiOO5dq9Y	ETjqrUNUnI5kSmkafjWfRTG8HIp98pLGXagNpXZHqOIZZDRkoGeahOwk9	false	16	-767	6623730208927396375	-3055706367894284822	12540.839	-1.047911096098831E9	-752.454	-241.620	2024-04-10	2024-05-16	2023-12-07T23:38:05	2023-12-11T05:48:36
+-76042627	PcVaKC43qmIzuxY	U3aGxaZumFpqcUsLI	true	44	31151	9085406701767055602	-5846138572199996843	-16845.29	2.44522690225531E8	-784.720	-467.133	2023-10-31	2023-08-29	2023-09-12T10:12:46	2023-10-19T17:02:51
+121048200	KPLWjhhbGXqflJi	rzqOYQH9ySHPwCm5K4GdeuI28G8LLmnpqLmsLMLfyRIvcfrlubQI47wUa8QILhuS38MBkjL	true	42	13182	-6601530758880565531	5619594098883737912	-2782.1506	3.86698722676211E8	478.420	-330.289	2024-06-17	2023-12-26	2024-04-28T03:29:04	2023-08-18T21:05:32
+262860291	m3XgmlbIHYNH1qS	BTJRzVrpM78zJAsHMEGhkF5BiDoc3yJuoV0s209sFcqElZsheBgolBGlFl9X4EfauD64FcFF2Mi4V0dKZfpDgaLLRPfG1SALV7	false	-42	5990	-7504815416577235660	1535659792778122944	1171.9619	1.28834143701229E8	626.721	682.828	2023-11-24	2023-11-18	2024-03-21T11:50:17	2024-03-31T12:59:27
+579428879	KsOC6WGrieGlo7B	SzeA6tRbsiGWJTBDvBQdBjCqjSE6Y	false	-111	32758	4029182463831656671	-3546198025044093789	20338.55	-2.015222388533773E9	61.981	720.310	2023-11-13	2024-07-04	2024-07-19T12:42:28	2024-01-04T10:32:53
+1145092610	xWJUDWAV8Nllo0F	dnZ9RMVdoqxh4kGBvy55zQdChNTVYdlvRZP4aWIkXyErUbM1XmFGQ9vuCD113JKKCyx4crDoY	false	115	-22832	-7242855305248390982	-4240353246453053617	-9074.909	-2.51212400295869E8	-502.410	618.820	2024-06-12	2024-04-18	2023-11-04T09:55:17	2023-11-13T16:30:23
+1736373707	UU14wnLhPkBid41	pmuNqYfOc3JCscf9meT5dYB2i28Pt9iaeXK4QqjVZJdoKFOeZI5bG9RKm1zInTdDMW1N0PKI5Y	true	-105	-20276	360048259532857165	-4602633478165721463	-13230.296	-1.708246954394742E9	757.147	-533.800	2024-01-05	2023-09-08	2023-11-27T05:21:33	2024-02-11T21:35:03
+
+-- !select --
+-9022291871392468311	2023-11-02	mOWPGHmiZ	10784	-128	2023-11-12T18:06:21	4218616419351308798	1993977685	-1857678846
+-6045452612961149194	2024-06-23	G1j	28468	-55	2024-06-09T00:12:11	-6456263257174124469	-727277974	144696403
+-1537906159489906139	2024-04-04	MRMRE18bVh49RD	32763	98	2024-01-20T00:54:03	-1289145371043997006	128181215	-1295829474
+-1510882223779118241	2024-07-24	PCwFn7r21MZr	22960	-79	2024-02-07T18:15:07	-8437284610883885859	472729036	-39626304
+-1185467471318572316	2023-11-08	ieed5Msw8X6be4HGS	16555	-79	2024-07-28T23:08:29	3263664376405334754	-809360772	-1229995615
+-234810200663664160	2024-06-07	s7GIrN805aU3cs2EM	-7555	-124	2023-12-28T18:59:15	-3600712745035417587	2035647886	126756427
+4461660295430359180	2024-04-23	K	25428	6	2023-11-15T18:38:20	-4503242152141666001	-1093190312	1511443278
+6742880469957921530	2024-05-02	cJJrvRJfpCuGh	27232	64	2024-08-18T09:46:50	-2607385663861429432	-1390108377	1758263623
+7252685688720766402	2024-03-13	891C2	-9774	-1	2023-10-12T19:45:28	-3210623791036109982	-915986651	-1794344594
+8278077411585505009	2023-11-17	gBesLQnYpjK7iDUUcIi	-26656	-50	2023-12-11T14:29:52	-8301529943262026214	-1555756888	-1318983102
+
diff --git a/regression-test/data/load_p0/ingestion_load/test_ingestion_load_alter_partition.out b/regression-test/data/load_p0/ingestion_load/test_ingestion_load_alter_partition.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select --
+2024-09-01	5
+2024-09-02	1
+2024-09-03	1
+2024-09-04	3
+
diff --git a/regression-test/data/load_p0/ingestion_load/test_ingestion_load_multi_table.out b/regression-test/data/load_p0/ingestion_load/test_ingestion_load_multi_table.out
@@ -0,0 +1,25 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select --
+-2067215761	41V4fgCKI3Hkioe	DZGaq24Yqh7SwmbPT6IX23jfC5NKqG7gE9JT4GLwiaQtoO8l6EjhGWQP9X7NHmjdqMbIN5kNeDkffOrlS6roIwj2wXpJ	true	123	-4524	1014223769452772206	-6589390612399245616	29103.387	-4.09385322835896E8	886.394	-699.960	2024-02-22	2024-03-04	2024-01-01T22:13:54	2023-10-02T19:35:10
+-1982022740	fcZ1o6ZXG8UOFh5	iw4Ziys42GRRTFNkVPeQEA9I5EQtBD04xfefDsPCWN0vr1	true	-3	-25143	7122614823242399351	7391807634038366248	23160.604	-9.20283206353984E8	829.880	-387.403	2023-10-16	2024-05-13	2024-07-16T18:27:45	2023-11-03T05:30:21
+-1228926576	1TISaGtB01BiqVt	kq4u	false	-123	15962	3590007830423934951	-1478759439092857810	-7813.757	-6.98785793100899E8	930.743	402.350	2024-07-23	2023-07-30	2023-11-27T17:48:50	2024-03-11T21:09:58
+-575060895	rRfatUJiOO5dq9Y	ETjqrUNUnI5kSmkafjWfRTG8HIp98pLGXagNpXZHqOIZZDRkoGeahOwk9	false	16	-767	6623730208927396375	-3055706367894284822	12540.839	-1.047911096098831E9	-752.454	-241.620	2024-04-10	2024-05-16	2023-12-07T23:38:05	2023-12-11T05:48:36
+-76042627	PcVaKC43qmIzuxY	U3aGxaZumFpqcUsLI	true	44	31151	9085406701767055602	-5846138572199996843	-16845.29	2.44522690225531E8	-784.720	-467.133	2023-10-31	2023-08-29	2023-09-12T10:12:46	2023-10-19T17:02:51
+121048200	KPLWjhhbGXqflJi	rzqOYQH9ySHPwCm5K4GdeuI28G8LLmnpqLmsLMLfyRIvcfrlubQI47wUa8QILhuS38MBkjL	true	42	13182	-6601530758880565531	5619594098883737912	-2782.1506	3.86698722676211E8	478.420	-330.289	2024-06-17	2023-12-26	2024-04-28T03:29:04	2023-08-18T21:05:32
+262860291	m3XgmlbIHYNH1qS	BTJRzVrpM78zJAsHMEGhkF5BiDoc3yJuoV0s209sFcqElZsheBgolBGlFl9X4EfauD64FcFF2Mi4V0dKZfpDgaLLRPfG1SALV7	false	-42	5990	-7504815416577235660	1535659792778122944	1171.9619	1.28834143701229E8	626.721	682.828	2023-11-24	2023-11-18	2024-03-21T11:50:17	2024-03-31T12:59:27
+579428879	KsOC6WGrieGlo7B	SzeA6tRbsiGWJTBDvBQdBjCqjSE6Y	false	-111	32758	4029182463831656671	-3546198025044093789	20338.55	-2.015222388533773E9	61.981	720.310	2023-11-13	2024-07-04	2024-07-19T12:42:28	2024-01-04T10:32:53
+1145092610	xWJUDWAV8Nllo0F	dnZ9RMVdoqxh4kGBvy55zQdChNTVYdlvRZP4aWIkXyErUbM1XmFGQ9vuCD113JKKCyx4crDoY	false	115	-22832	-7242855305248390982	-4240353246453053617	-9074.909	-2.51212400295869E8	-502.410	618.820	2024-06-12	2024-04-18	2023-11-04T09:55:17	2023-11-13T16:30:23
+1736373707	UU14wnLhPkBid41	pmuNqYfOc3JCscf9meT5dYB2i28Pt9iaeXK4QqjVZJdoKFOeZI5bG9RKm1zInTdDMW1N0PKI5Y	true	-105	-20276	360048259532857165	-4602633478165721463	-13230.296	-1.708246954394742E9	757.147	-533.800	2024-01-05	2023-09-08	2023-11-27T05:21:33	2024-02-11T21:35:03
+
+-- !select --
+-2067215761	41V4fgCKI3Hkioe	DZGaq24Yqh7SwmbPT6IX23jfC5NKqG7gE9JT4GLwiaQtoO8l6EjhGWQP9X7NHmjdqMbIN5kNeDkffOrlS6roIwj2wXpJ	true	123	-4524	1014223769452772206	-6589390612399245616	29103.387	-4.09385322835896E8	886.394	-699.960	2024-02-22	2024-03-04	2024-01-01T22:13:54	2023-10-02T19:35:10
+-1982022740	fcZ1o6ZXG8UOFh5	iw4Ziys42GRRTFNkVPeQEA9I5EQtBD04xfefDsPCWN0vr1	true	-3	-25143	7122614823242399351	7391807634038366248	23160.604	-9.20283206353984E8	829.880	-387.403	2023-10-16	2024-05-13	2024-07-16T18:27:45	2023-11-03T05:30:21
+-1228926576	1TISaGtB01BiqVt	kq4u	false	-123	15962	3590007830423934951	-1478759439092857810	-7813.757	-6.98785793100899E8	930.743	402.350	2024-07-23	2023-07-30	2023-11-27T17:48:50	2024-03-11T21:09:58
+-575060895	rRfatUJiOO5dq9Y	ETjqrUNUnI5kSmkafjWfRTG8HIp98pLGXagNpXZHqOIZZDRkoGeahOwk9	false	16	-767	6623730208927396375	-3055706367894284822	12540.839	-1.047911096098831E9	-752.454	-241.620	2024-04-10	2024-05-16	2023-12-07T23:38:05	2023-12-11T05:48:36
+-76042627	PcVaKC43qmIzuxY	U3aGxaZumFpqcUsLI	true	44	31151	9085406701767055602	-5846138572199996843	-16845.29	2.44522690225531E8	-784.720	-467.133	2023-10-31	2023-08-29	2023-09-12T10:12:46	2023-10-19T17:02:51
+121048200	KPLWjhhbGXqflJi	rzqOYQH9ySHPwCm5K4GdeuI28G8LLmnpqLmsLMLfyRIvcfrlubQI47wUa8QILhuS38MBkjL	true	42	13182	-6601530758880565531	5619594098883737912	-2782.1506	3.86698722676211E8	478.420	-330.289	2024-06-17	2023-12-26	2024-04-28T03:29:04	2023-08-18T21:05:32
+262860291	m3XgmlbIHYNH1qS	BTJRzVrpM78zJAsHMEGhkF5BiDoc3yJuoV0s209sFcqElZsheBgolBGlFl9X4EfauD64FcFF2Mi4V0dKZfpDgaLLRPfG1SALV7	false	-42	5990	-7504815416577235660	1535659792778122944	1171.9619	1.28834143701229E8	626.721	682.828	2023-11-24	2023-11-18	2024-03-21T11:50:17	2024-03-31T12:59:27
+579428879	KsOC6WGrieGlo7B	SzeA6tRbsiGWJTBDvBQdBjCqjSE6Y	false	-111	32758	4029182463831656671	-3546198025044093789	20338.55	-2.015222388533773E9	61.981	720.310	2023-11-13	2024-07-04	2024-07-19T12:42:28	2024-01-04T10:32:53
+1145092610	xWJUDWAV8Nllo0F	dnZ9RMVdoqxh4kGBvy55zQdChNTVYdlvRZP4aWIkXyErUbM1XmFGQ9vuCD113JKKCyx4crDoY	false	115	-22832	-7242855305248390982	-4240353246453053617	-9074.909	-2.51212400295869E8	-502.410	618.820	2024-06-12	2024-04-18	2023-11-04T09:55:17	2023-11-13T16:30:23
+1736373707	UU14wnLhPkBid41	pmuNqYfOc3JCscf9meT5dYB2i28Pt9iaeXK4QqjVZJdoKFOeZI5bG9RKm1zInTdDMW1N0PKI5Y	true	-105	-20276	360048259532857165	-4602633478165721463	-13230.296	-1.708246954394742E9	757.147	-533.800	2024-01-05	2023-09-08	2023-11-27T05:21:33	2024-02-11T21:35:03
+
diff --git a/regression-test/data/load_p0/ingestion_load/test_ingestion_load_with_partition.out b/regression-test/data/load_p0/ingestion_load/test_ingestion_load_with_partition.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select --
+2024-09-01	5
+2024-09-02	1
+2024-09-03	1
+2024-09-04	3
+
diff --git a/regression-test/suites/load_p0/ingestion_load/test_ingestion_load.groovy b/regression-test/suites/load_p0/ingestion_load/test_ingestion_load.groovy
@@ -45,7 +45,7 @@ suite('test_ingestion_load', 'p0') {
 
         httpTest {
             endpoint context.config.feHttpAddress
-            uri "/api/ingestion_load/${context.dbName}/_create"
+            uri "/api/ingestion_load/internal/${context.dbName}/_create"
             op "post"
             basicAuthorization context.config.feHttpUser, context.config.feHttpPassword
             body reqBody
@@ -92,7 +92,7 @@ suite('test_ingestion_load', 'p0') {
 
         httpTest {
             endpoint context.config.feHttpAddress
-            uri "/api/ingestion_load/${context.dbName}/_update"
+            uri "/api/ingestion_load/internal/${context.dbName}/_update"
             op "post"
             basicAuthorization context.config.feHttpUser, context.config.feHttpPassword
             body updateStatusReqBody
@@ -106,12 +106,12 @@ suite('test_ingestion_load', 'p0') {
             }
         }
 
-        max_try_milli_secs = 60000
+        max_try_milli_secs = 120000
         while (max_try_milli_secs) {
             result = sql "show load where label = '${loadLabel}'"
             if (result[0][2] == "FINISHED") {
                 sql "sync"
-                qt_select "select * from ${testTable} order by c_int"
+                qt_select "select * from ${testTable} order by 1"
                 break
             } else {
                 sleep(5000) // wait 1 second every time
@@ -147,6 +147,7 @@ suite('test_ingestion_load', 'p0') {
                 c_datetime datetime NULL,
                 c_datetimev2 datetime NULL
             )
+            DUPLICATE KEY(c_int)
             DISTRIBUTED BY HASH(c_int) BUCKETS 1
             PROPERTIES (
             "replication_num" = "1"
@@ -157,6 +158,65 @@ suite('test_ingestion_load', 'p0') {
 
         testIngestLoadJob.call(tableName, label, context.config.dataPath + '/load_p0/ingestion_load/data.parquet')
 
+        tableName = 'tbl_test_spark_load_unique_mor'
+
+        sql """
+            CREATE TABLE IF NOT EXISTS ${tableName} (
+                c_int int(11) NULL,
+                c_char char(15) NULL,
+                c_varchar varchar(100) NULL,
+                c_bool boolean NULL,
+                c_tinyint tinyint(4) NULL,
+                c_smallint smallint(6) NULL,
+                c_bigint bigint(20) NULL,
+                c_largeint largeint(40) NULL,
+                c_float float NULL,
+                c_double double NULL,
+                c_decimal decimal(6, 3) NULL,
+                c_decimalv3 decimal(6, 3) NULL,
+                c_date date NULL,
+                c_datev2 date NULL,
+                c_datetime datetime NULL,
+                c_datetimev2 datetime NULL
+            )
+            UNIQUE KEY(c_int)
+            DISTRIBUTED BY HASH(c_int) BUCKETS 1
+            PROPERTIES (
+            "replication_num" = "1",
+            "enable_unique_key_merge_on_write" = "false"
+            )
+            """
+
+        label = "test_ingestion_load_unique_mor"
+
+        testIngestLoadJob.call(tableName, label, context.config.dataPath + '/load_p0/ingestion_load/data.parquet')
+
+        tableName = 'tbl_test_spark_load_agg'
+
+        sql """
+            CREATE TABLE IF NOT EXISTS ${tableName}
+            (
+                `user_id` LARGEINT NOT NULL COMMENT "user id",
+                `date` DATE NOT NULL COMMENT "data import time",
+                `city` VARCHAR(20) COMMENT "city",
+                `age` SMALLINT COMMENT "age",
+                `sex` TINYINT COMMENT "gender",
+                `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "last visit date time",
+                `cost` BIGINT SUM DEFAULT "0" COMMENT "user total cost",
+                `max_dwell_time` INT MAX DEFAULT "0" COMMENT "user max dwell time",
+                `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "user min dwell time"
+            )
+            AGGREGATE KEY(`user_id`, `date`, `city`, `age`, `sex`)
+            DISTRIBUTED BY HASH(`user_id`) BUCKETS 1
+            PROPERTIES (
+            "replication_allocation" = "tag.location.default: 1"
+            );
+        """
+
+        label = "test_ingestion_load_agg"
+
+        testIngestLoadJob.call(tableName, label, context.config.dataPath + '/load_p0/ingestion_load/data1.parquet')
+
     }
 
 }