[Feature](Row Store) support partial columns row store in cloud mode

eldenmoon · Jun 21, 2024 · 7126c8e · 7126c8e
1 parent 54cc796
commit 7126c8e
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 4 deletions.
diff --git a/be/src/cloud/pb_convert.cpp b/be/src/cloud/pb_convert.cpp
@@ -275,6 +275,7 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, const TabletSchemaPB
     out->set_skip_write_index_on_load(in.skip_write_index_on_load());
     out->mutable_cluster_key_idxes()->CopyFrom(in.cluster_key_idxes());
     out->set_is_dynamic_schema(in.is_dynamic_schema());
+    out->mutable_row_store_column_unique_ids()->CopyFrom(in.row_store_column_unique_ids());
 }
 
 void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, TabletSchemaPB&& in) {
@@ -299,6 +300,7 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, TabletSchemaPB&& in)
     out->set_skip_write_index_on_load(in.skip_write_index_on_load());
     out->mutable_cluster_key_idxes()->Swap(in.mutable_cluster_key_idxes());
     out->set_is_dynamic_schema(in.is_dynamic_schema());
+    out->mutable_row_store_column_unique_ids()->Swap(in.mutable_row_store_column_unique_ids());
 }
 
 TabletSchemaPB cloud_tablet_schema_to_doris(const TabletSchemaCloudPB& in) {
@@ -336,6 +338,7 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out, const TabletSchemaCloudPB
     out->set_skip_write_index_on_load(in.skip_write_index_on_load());
     out->mutable_cluster_key_idxes()->CopyFrom(in.cluster_key_idxes());
     out->set_is_dynamic_schema(in.is_dynamic_schema());
+    out->mutable_row_store_column_unique_ids()->CopyFrom(in.row_store_column_unique_ids());
 }
 
 void cloud_tablet_schema_to_doris(TabletSchemaPB* out, TabletSchemaCloudPB&& in) {
@@ -361,6 +364,7 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out, TabletSchemaCloudPB&& in)
     out->set_skip_write_index_on_load(in.skip_write_index_on_load());
     out->mutable_cluster_key_idxes()->Swap(in.mutable_cluster_key_idxes());
     out->set_is_dynamic_schema(in.is_dynamic_schema());
+    out->mutable_row_store_column_unique_ids()->Swap(in.mutable_row_store_column_unique_ids());
 }
 
 TabletMetaCloudPB doris_tablet_meta_to_cloud(const TabletMetaPB& in) {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java
@@ -189,6 +189,8 @@ private void createRollupReplicaForPartition(OlapTable tbl) throws Exception {
             MaterializedIndex rollupIndex = entry.getValue();
             Cloud.CreateTabletsRequest.Builder requestBuilder =
                     Cloud.CreateTabletsRequest.newBuilder();
+            List<String> rowStoreColumns =
+                                        tbl.getTableProperty().getCopiedRowStoreColumns();
             for (Tablet rollupTablet : rollupIndex.getTablets()) {
                 OlapFile.TabletMetaCloudPB.Builder builder =
                         ((CloudInternalCatalog) Env.getCurrentInternalCatalog())
@@ -206,7 +208,8 @@ private void createRollupReplicaForPartition(OlapTable tbl) throws Exception {
                             tbl.getTimeSeriesCompactionTimeThresholdSeconds(),
                             tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(),
                             tbl.getTimeSeriesCompactionLevelThreshold(),
-                            tbl.disableAutoCompaction());
+                            tbl.disableAutoCompaction(),
+                            tbl.getRowStoreColumnsUniqueIds(rowStoreColumns));
                 requestBuilder.addTabletMetas(builder);
             } // end for rollupTablets
             ((CloudInternalCatalog) Env.getCurrentInternalCatalog())

diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java
@@ -214,7 +214,8 @@ private void createShadowIndexReplicaForPartition(OlapTable tbl) throws Exceptio
                                 tbl.getTimeSeriesCompactionTimeThresholdSeconds(),
                                 tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(),
                                 tbl.getTimeSeriesCompactionLevelThreshold(),
-                                tbl.disableAutoCompaction());
+                                tbl.disableAutoCompaction(),
+                                tbl.getRowStoreColumnsUniqueIds(rowStoreColumns));
                     requestBuilder.addTabletMetas(builder);
                 } // end for rollupTablets
                 ((CloudInternalCatalog) Env.getCurrentInternalCatalog())

diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
@@ -159,6 +159,8 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa
                 indexes = Lists.newArrayList();
             }
             Cloud.CreateTabletsRequest.Builder requestBuilder = Cloud.CreateTabletsRequest.newBuilder();
+            List<String> rowStoreColumns =
+                                                tbl.getTableProperty().getCopiedRowStoreColumns();
             for (Tablet tablet : index.getTablets()) {
                 OlapFile.TabletMetaCloudPB.Builder builder = createTabletMetaBuilder(tbl.getId(), indexId,
                         partitionId, tablet, tabletType, schemaHash, keysType, shortKeyColumnCount,
@@ -170,7 +172,8 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa
                         tbl.getTimeSeriesCompactionTimeThresholdSeconds(),
                         tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(),
                         tbl.getTimeSeriesCompactionLevelThreshold(),
-                        tbl.disableAutoCompaction());
+                        tbl.disableAutoCompaction(),
+                        tbl.getRowStoreColumnsUniqueIds(rowStoreColumns));
                 requestBuilder.addTabletMetas(builder);
             }
             if (!storageVaultIdSet && ((CloudEnv) Env.getCurrentEnv()).getEnableStorageVault()) {
@@ -216,7 +219,8 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId,
             boolean storeRowColumn, int schemaVersion, String compactionPolicy,
             Long timeSeriesCompactionGoalSizeMbytes, Long timeSeriesCompactionFileCountThreshold,
             Long timeSeriesCompactionTimeThresholdSeconds, Long timeSeriesCompactionEmptyRowsetsThreshold,
-            Long timeSeriesCompactionLevelThreshold, boolean disableAutoCompaction) throws DdlException {
+            Long timeSeriesCompactionLevelThreshold, boolean disableAutoCompaction,
+            List<Integer> rowStoreColumnUniqueIds) throws DdlException {
         OlapFile.TabletMetaCloudPB.Builder builder = OlapFile.TabletMetaCloudPB.newBuilder();
         builder.setTableId(tableId);
         builder.setIndexId(indexId);
@@ -329,6 +333,9 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId,
                 schemaBuilder.addIndex(index.toPb(schemaColumns));
             }
         }
+        if (rowStoreColumnUniqueIds != null) {
+            schemaBuilder.addAllRowStoreColumnUniqueIds(rowStoreColumnUniqueIds);
+        }
         schemaBuilder.setDisableAutoCompaction(disableAutoCompaction);
 
         OlapFile.TabletSchemaCloudPB schema = schemaBuilder.build();