forked from StarRocks/starrocks
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[BugFix] Fix the bug of direct schema change (StarRocks#44854)
Signed-off-by: trueeyu <[email protected]>
- Loading branch information
Showing
11 changed files
with
746 additions
and
427 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,91 @@ | ||
| // Copyright 2021-present StarRocks, Inc. All rights reserved. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // https://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| #include "testutil/schema_test_helper.h" | ||
|
|
||
| namespace starrocks { | ||
| TabletSchemaPB SchemaTestHelper::gen_schema_pb_of_dup(TabletSchema::SchemaId schema_id, size_t num_cols, | ||
| size_t num_key_cols) { | ||
| TabletSchemaPB schema_pb; | ||
|
|
||
| schema_pb.set_keys_type(DUP_KEYS); | ||
| schema_pb.set_num_short_key_columns(num_key_cols); | ||
| schema_pb.set_id(schema_id); | ||
|
|
||
| for (size_t i = 0; i < num_cols; i++) { | ||
| auto c0 = schema_pb.add_column(); | ||
| c0->set_unique_id(i); | ||
| c0->set_name("c0"); | ||
| c0->set_type("INT"); | ||
| c0->set_is_nullable(true); | ||
| c0->set_index_length(4); | ||
| if (i < num_key_cols) { | ||
| c0->set_is_key(true); | ||
| } | ||
| } | ||
|
|
||
| return schema_pb; | ||
| } | ||
|
|
||
| TColumn SchemaTestHelper::gen_key_column(const std::string& col_name, TPrimitiveType::type type) { | ||
| TColumnType col_type; | ||
| col_type.type = type; | ||
|
|
||
| TColumn col; | ||
| col.__set_column_name(col_name); | ||
| col.__set_column_type(col_type); | ||
| col.__set_is_key(true); | ||
|
|
||
| return col; | ||
| } | ||
|
|
||
| TColumn SchemaTestHelper::gen_value_column_for_dup_table(const std::string& col_name, TPrimitiveType::type type) { | ||
| TColumnType col_type; | ||
| col_type.type = type; | ||
|
|
||
| TColumn col; | ||
| col.__set_column_name(col_name); | ||
| col.__set_column_type(col_type); | ||
| col.__set_is_key(false); | ||
|
|
||
| return col; | ||
| } | ||
|
|
||
| TColumn SchemaTestHelper::gen_value_column_for_agg_table(const std::string& col_name, TPrimitiveType::type type) { | ||
| TColumnType col_type; | ||
| col_type.type = type; | ||
|
|
||
| TColumn col; | ||
| col.__set_column_name(col_name); | ||
| col.__set_column_type(col_type); | ||
| col.__set_is_key(false); | ||
| col.__set_aggregation_type(TAggregationType::SUM); | ||
|
|
||
| return col; | ||
| } | ||
|
|
||
| void SchemaTestHelper::add_column_pb_to_tablet_schema(TabletSchemaPB* tablet_schema_pb, const std::string& name, | ||
| const std::string& type, const std::string& agg, | ||
| uint32_t length) { | ||
| ColumnPB* column = tablet_schema_pb->add_column(); | ||
| column->set_unique_id(0); | ||
| column->set_name(name); | ||
| column->set_type(type); | ||
| column->set_is_key(false); | ||
| column->set_is_nullable(false); | ||
| column->set_length(length); | ||
| column->set_aggregation(agg); | ||
| } | ||
|
|
||
| } // namespace starrocks |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| // Copyright 2021-present StarRocks, Inc. All rights reserved. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // https://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| #pragma once | ||
|
|
||
| #include "gen_cpp/tablet_schema.pb.h" | ||
| #include "storage/tablet_schema.h" | ||
|
|
||
| namespace starrocks { | ||
| class SchemaTestHelper { | ||
| public: | ||
| static TabletSchemaPB gen_schema_pb_of_dup(TabletSchema::SchemaId schema_id, size_t num_cols, size_t num_key_cols); | ||
| static TColumn gen_key_column(const std::string& col_name, TPrimitiveType::type type); | ||
| static TColumn gen_value_column_for_dup_table(const std::string& col_name, TPrimitiveType::type type); | ||
| static TColumn gen_value_column_for_agg_table(const std::string& col_name, TPrimitiveType::type type); | ||
| static void add_column_pb_to_tablet_schema(TabletSchemaPB* tablet_schema_pb, const std::string& name, | ||
| const std::string& type, const std::string& agg, uint32_t length); | ||
| }; | ||
| } // namespace starrocks |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,94 @@ | ||
| // Copyright 2021-present StarRocks, Inc. All rights reserved. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // https://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| #include "testutil/tablet_test_helper.h" | ||
|
|
||
| #include "storage/chunk_helper.h" | ||
| #include "storage/rowset/rowset_factory.h" | ||
| #include "storage/tablet.h" | ||
|
|
||
| namespace starrocks { | ||
| TCreateTabletReq TabletTestHelper::gen_create_tablet_req(TTabletId tablet_id, TKeysType::type type, | ||
| TStorageType::type storage_type) { | ||
| TCreateTabletReq req; | ||
| req.tablet_id = tablet_id; | ||
| req.__set_version(1); | ||
| req.__set_version_hash(0); | ||
| req.tablet_schema.schema_hash = 0; | ||
| req.tablet_schema.short_key_column_count = 2; | ||
| req.tablet_schema.keys_type = type; | ||
| req.tablet_schema.storage_type = storage_type; | ||
| return req; | ||
| } | ||
|
|
||
| std::shared_ptr<RowsetWriter> TabletTestHelper::create_rowset_writer(const Tablet& tablet, RowsetId rowset_id, | ||
| Version version) { | ||
| RowsetWriterContext writer_context; | ||
| writer_context.rowset_id = rowset_id; | ||
| writer_context.tablet_uid = tablet.tablet_uid(); | ||
| writer_context.tablet_id = tablet.tablet_id(); | ||
| writer_context.tablet_schema_hash = tablet.schema_hash(); | ||
| writer_context.rowset_path_prefix = tablet.schema_hash_path(); | ||
| writer_context.tablet_schema = tablet.tablet_schema(); | ||
| writer_context.rowset_state = VISIBLE; | ||
| writer_context.version = version; | ||
| std::unique_ptr<RowsetWriter> rowset_writer; | ||
| auto st = RowsetFactory::create_rowset_writer(writer_context, &rowset_writer); | ||
| CHECK(st.ok()); | ||
| return rowset_writer; | ||
| } | ||
|
|
||
| std::shared_ptr<TabletReader> TabletTestHelper::create_rowset_reader(const TabletSharedPtr& tablet, | ||
| const Schema& schema, Version version) { | ||
| TabletReaderParams read_params; | ||
| read_params.reader_type = ReaderType::READER_ALTER_TABLE; | ||
| read_params.skip_aggregation = false; | ||
| read_params.chunk_size = config::vector_chunk_size; | ||
| auto tablet_reader = std::make_unique<TabletReader>(tablet, version, schema); | ||
|
|
||
| CHECK(tablet_reader != nullptr); | ||
| CHECK(tablet_reader->prepare().ok()); | ||
| CHECK(tablet_reader->open(read_params).ok()); | ||
|
|
||
| return tablet_reader; | ||
| } | ||
|
|
||
| std::shared_ptr<DeltaWriter> TabletTestHelper::create_delta_writer(TTabletId tablet_id, | ||
| const std::vector<SlotDescriptor*>& slots, | ||
| MemTracker* mem_tracker) { | ||
| DeltaWriterOptions options; | ||
| options.tablet_id = tablet_id; | ||
| options.slots = &slots; | ||
| options.txn_id = 1; | ||
| options.partition_id = 1; | ||
| options.replica_state = ReplicaState::Primary; | ||
| auto writer = DeltaWriter::open(options, mem_tracker); | ||
| CHECK(writer.ok()); | ||
| return std::move(writer.value()); | ||
| } | ||
|
|
||
| std::vector<ChunkIteratorPtr> TabletTestHelper::create_segment_iterators(const Tablet& tablet, Version version, | ||
| OlapReaderStatistics* _stats) { | ||
| auto new_rowset = tablet.get_rowset_by_version(version); | ||
| CHECK(new_rowset != nullptr); | ||
|
|
||
| std::vector<ChunkIteratorPtr> seg_iters; | ||
| RowsetReadOptions rowset_opts; | ||
| rowset_opts.version = version.second; | ||
| rowset_opts.stats = _stats; | ||
| auto st = new_rowset->get_segment_iterators(*tablet.tablet_schema()->schema(), rowset_opts, &seg_iters); | ||
| CHECK(st.ok()); | ||
| return seg_iters; | ||
| } | ||
| } // namespace starrocks |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| // Copyright 2021-present StarRocks, Inc. All rights reserved. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // https://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| #pragma once | ||
|
|
||
| #include "gen_cpp/AgentService_types.h" | ||
| #include "storage/delta_writer.h" | ||
| #include "storage/rowset/rowset_writer.h" | ||
| #include "storage/tablet_reader.h" | ||
|
|
||
| namespace starrocks { | ||
| class TabletTestHelper { | ||
| public: | ||
| static TCreateTabletReq gen_create_tablet_req(TTabletId tablet_id, TKeysType::type type, | ||
| TStorageType::type storage_type); | ||
| static std::shared_ptr<RowsetWriter> create_rowset_writer(const Tablet& tablet, RowsetId rowset_id, | ||
| Version version); | ||
| static std::shared_ptr<TabletReader> create_rowset_reader(const TabletSharedPtr& tablet, const Schema& schema, | ||
| Version version); | ||
| static std::shared_ptr<DeltaWriter> create_delta_writer(TTabletId tablet_id, | ||
| const std::vector<SlotDescriptor*>& slots, | ||
| MemTracker* mem_tracker); | ||
| static std::vector<ChunkIteratorPtr> create_segment_iterators(const Tablet& tablet, Version version, | ||
| OlapReaderStatistics* stats); | ||
| }; | ||
| } // namespace starrocks |
Oops, something went wrong.