From 6a2bf46971230aff31d40c3793f78cc66903a9d5 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Mon, 1 Apr 2024 13:50:03 -0700 Subject: [PATCH] Move all TDP parsing to use direct-to-table. DynamicMessage now eagerly creates the tc_table doing proper crosslinking of TDP tables. Remove all kTvDefault cases from TcParser. PiperOrigin-RevId: 620943275 --- .../compiler/cpp/parse_function_generator.cc | 1 - src/google/protobuf/dynamic_message.cc | 43 ++-- .../protobuf/generated_message_reflection.cc | 34 ++- .../protobuf/generated_message_tctable_gen.cc | 26 +-- .../protobuf/generated_message_tctable_gen.h | 1 - .../protobuf/generated_message_tctable_impl.h | 24 +- .../generated_message_tctable_lite.cc | 208 +++++------------- src/google/protobuf/message.h | 23 +- src/google/protobuf/message_lite.h | 2 + 9 files changed, 138 insertions(+), 224 deletions(-) diff --git a/src/google/protobuf/compiler/cpp/parse_function_generator.cc b/src/google/protobuf/compiler/cpp/parse_function_generator.cc index 14d049139677..1ba1b139224b 100644 --- a/src/google/protobuf/compiler/cpp/parse_function_generator.cc +++ b/src/google/protobuf/compiler/cpp/parse_function_generator.cc @@ -72,7 +72,6 @@ class ParseFunctionGenerator::GeneratedOptionProvider final verify_flag(), IsStringInlined(field, gen_->options_), IsImplicitWeakField(field, gen_->options_, gen_->scc_analyzer_), - /* use_direct_tcparser_table */ true, ShouldSplit(field, gen_->options_), }; } diff --git a/src/google/protobuf/dynamic_message.cc b/src/google/protobuf/dynamic_message.cc index 385608288903..bed6846ca88a 100644 --- a/src/google/protobuf/dynamic_message.cc +++ b/src/google/protobuf/dynamic_message.cc @@ -46,11 +46,13 @@ #include #include +#include "absl/log/absl_check.h" #include "google/protobuf/arenastring.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/descriptor.pb.h" #include "google/protobuf/extension_set.h" #include "google/protobuf/generated_message_reflection.h" +#include "google/protobuf/generated_message_tctable_decl.h" #include "google/protobuf/generated_message_util.h" #include "google/protobuf/map_field.h" #include "google/protobuf/map_field_inl.h" @@ -283,6 +285,8 @@ struct DynamicMessageFactory::TypeInfo { ~TypeInfo() { delete prototype; delete class_data.reflection; + ::operator delete( + const_cast(class_data.tc_table)); auto* type = class_data.descriptor; @@ -415,9 +419,9 @@ void DynamicMessage::SharedCtor(bool lock_factory) { new (field_ptr) Message*(nullptr); } else { if (IsMapFieldInApi(field)) { - // We need to lock in most cases to avoid data racing. Only not lock - // when the constructor is called inside GetPrototype(), in which - // case we have already locked the factory. + // We need to lock in most cases to avoid data racing. + // When building the prototype via GetPrototypeNoLock, we construct + // this during crosslinking. if (lock_factory) { if (arena != nullptr) { new (field_ptr) DynamicMapField( @@ -427,17 +431,6 @@ void DynamicMessage::SharedCtor(bool lock_factory) { new (field_ptr) DynamicMapField( type_info_->factory->GetPrototype(field->message_type())); } - } else { - if (arena != nullptr) { - new (field_ptr) - DynamicMapField(type_info_->factory->GetPrototypeNoLock( - field->message_type()), - arena); - } else { - new (field_ptr) - DynamicMapField(type_info_->factory->GetPrototypeNoLock( - field->message_type())); - } } } else { new (field_ptr) RepeatedPtrField(arena); @@ -583,6 +576,18 @@ void DynamicMessage::CrossLinkPrototypes() { factory->GetPrototypeNoLock(field->message_type()); } } + + // Construct the map fields. + // We need to delay this until we can do cross references. + for (int i = 0; i < descriptor->field_count(); i++) { + const FieldDescriptor* field = descriptor->field(i); + if (field->cpp_type() == field->CPPTYPE_MESSAGE && field->is_repeated() && + IsMapFieldInApi(field)) { + void* field_ptr = MutableRaw(i); + new (field_ptr) DynamicMapField( + type_info_->factory->GetPrototypeNoLock(field->message_type())); + } + } } Message* DynamicMessage::New(Arena* arena) const { @@ -774,6 +779,16 @@ const Message* DynamicMessageFactory::GetPrototypeNoLock( type_info->class_data.reflection = new Reflection( type_info->class_data.descriptor, schema, type_info->pool, this); + type_info->class_data.reflection->CreateTcParseTable( + type_info->class_data.tc_table, [&](const auto* field) { + // The prototype we get might not necessarily be a dynamic message, so + // use GetClassData to fetch the table. + auto* table = + GetPrototypeNoLock(field->message_type())->GetTcParseTable(); + ABSL_DCHECK(table != nullptr); + return table; + }); + // Cross link prototypes. prototype->CrossLinkPrototypes(); diff --git a/src/google/protobuf/generated_message_reflection.cc b/src/google/protobuf/generated_message_reflection.cc index 1443fb5ae1e1..3419e5ac6692 100644 --- a/src/google/protobuf/generated_message_reflection.cc +++ b/src/google/protobuf/generated_message_reflection.cc @@ -25,6 +25,7 @@ #include "absl/base/call_once.h" #include "absl/base/const_init.h" #include "absl/container/flat_hash_set.h" +#include "absl/functional/function_ref.h" #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" #include "absl/strings/str_format.h" @@ -3318,6 +3319,8 @@ void Reflection::PopulateTcParseEntries( void Reflection::PopulateTcParseFieldAux( const internal::TailCallTableInfo& table_info, + absl::FunctionRef + fetch_subtable, TcParseTableBase::FieldAux* field_aux) const { for (const auto& aux_entry : table_info.aux_entries) { switch (aux_entry.type) { @@ -3335,6 +3338,8 @@ void Reflection::PopulateTcParseFieldAux( field_aux++->offset = schema_.SizeofSplit(); break; case internal::TailCallTableInfo::kSubTable: + field_aux++->table = fetch_subtable(aux_entry.field); + break; case internal::TailCallTableInfo::kSubMessageWeak: case internal::TailCallTableInfo::kCreateInArena: case internal::TailCallTableInfo::kMessageVerifyFunc: @@ -3367,7 +3372,19 @@ void Reflection::PopulateTcParseFieldAux( } -const internal::TcParseTableBase* Reflection::CreateTcParseTable() const { +const internal::TcParseTableBase* Reflection::GetTcParseTable() const { + absl::call_once(tcparse_table_once_, [&] { + CreateTcParseTable(tcparse_table_, [&](const auto* field) { + return GetDefaultMessageInstance(field)->GetTcParseTable(); + }); + }); + return tcparse_table_; +} + +void Reflection::CreateTcParseTable( + const TcParseTableBase*& out, + absl::FunctionRef + fetch_subtable) const { using TcParseTableBase = internal::TcParseTableBase; std::vector fields; @@ -3411,11 +3428,6 @@ const internal::TcParseTableBase* Reflection::CreateTcParseTable() const { // Only LITE can be implicitly weak. /* is_implicitly_weak */ false, - - // We could change this to use direct table. - // Might be easier to do when all messages support TDP. - /* use_direct_tcparser_table */ false, - ref_.schema_.IsSplit(field), // }; } @@ -3472,6 +3484,12 @@ const internal::TcParseTableBase* Reflection::CreateTcParseTable() const { nullptr #endif // PROTOBUF_PREFETCH_PARSE_TABLE }; + + // Set the `out` pointer first. `fetch_subtable` is reentrant and we need to + // update the cache tables before the reentrancy. That way we can do cross + // referencing while building the tables. + out = res; + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE // We'll prefetch `to_prefetch->to_prefetch` unconditionally to avoid // branches. Here we don't know which field is the hottest, so set the pointer @@ -3486,7 +3504,7 @@ const internal::TcParseTableBase* Reflection::CreateTcParseTable() const { PopulateTcParseEntries(table_info, res->field_entries_begin()); - PopulateTcParseFieldAux(table_info, res->field_aux(0u)); + PopulateTcParseFieldAux(table_info, fetch_subtable, res->field_aux(0u)); // Copy the name data. if (!table_info.field_name_data.empty()) { @@ -3497,8 +3515,6 @@ const internal::TcParseTableBase* Reflection::CreateTcParseTable() const { ABSL_CHECK_EQ(res->name_data() + table_info.field_name_data.size() - reinterpret_cast(res), byte_size); - - return res; } namespace { diff --git a/src/google/protobuf/generated_message_tctable_gen.cc b/src/google/protobuf/generated_message_tctable_gen.cc index 55bca3016b0b..e0dc36da7d90 100644 --- a/src/google/protobuf/generated_message_tctable_gen.cc +++ b/src/google/protobuf/generated_message_tctable_gen.cc @@ -204,16 +204,12 @@ TailCallTableInfo::FastFieldInfo::Field MakeFastFieldEntry( } break; case FieldDescriptor::TYPE_MESSAGE: - picked = - (HasLazyRep(field, options) ? PROTOBUF_PICK_SINGLE_FUNCTION(kFastMl) - : options.use_direct_tcparser_table - ? PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastMt) - : PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastMd)); + picked = HasLazyRep(field, options) + ? PROTOBUF_PICK_SINGLE_FUNCTION(kFastMl) + : PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastMt); break; case FieldDescriptor::TYPE_GROUP: - picked = (options.use_direct_tcparser_table - ? PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastGt) - : PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastGd)); + picked = PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastGt); break; } @@ -666,10 +662,8 @@ uint16_t MakeTypeCardForField( type_card |= 0 | fl::kMessage | fl::kRepGroup; if (options.is_implicitly_weak) { type_card |= fl::kTvWeakPtr; - } else if (options.use_direct_tcparser_table) { - type_card |= fl::kTvTable; } else { - type_card |= fl::kTvDefault; + type_card |= fl::kTvTable; } break; case FieldDescriptor::TYPE_MESSAGE: @@ -684,10 +678,8 @@ uint16_t MakeTypeCardForField( } else { if (options.is_implicitly_weak) { type_card |= fl::kTvWeakPtr; - } else if (options.use_direct_tcparser_table) { - type_card |= fl::kTvTable; } else { - type_card |= fl::kTvDefault; + type_card |= fl::kTvTable; } } } @@ -821,7 +813,7 @@ TailCallTableInfo::TailCallTableInfo( field->type() == FieldDescriptor::TYPE_GROUP) && !field->is_map() && !field->options().weak() && !HasLazyRep(field, options) && !options.is_implicitly_weak && - options.use_direct_tcparser_table && is_non_cold(options); + is_non_cold(options); }; for (const FieldDescriptor* field : ordered_fields) { if (is_non_cold_subtable(field)) { @@ -878,9 +870,7 @@ TailCallTableInfo::TailCallTableInfo( entry.aux_idx = TcParseTableBase::FieldEntry::kNoAuxIdx; } } else { - AuxType type = options.is_implicitly_weak ? kSubMessageWeak - : options.use_direct_tcparser_table ? kSubTable - : kSubMessage; + AuxType type = options.is_implicitly_weak ? kSubMessageWeak : kSubTable; if (message_options.should_profile_driven_cluster_aux_subtable && type == kSubTable && is_non_cold(options)) { aux_entries[subtable_aux_idx] = {type, {field}}; diff --git a/src/google/protobuf/generated_message_tctable_gen.h b/src/google/protobuf/generated_message_tctable_gen.h index 667758519b00..572bbd6bcb11 100644 --- a/src/google/protobuf/generated_message_tctable_gen.h +++ b/src/google/protobuf/generated_message_tctable_gen.h @@ -48,7 +48,6 @@ struct PROTOBUF_EXPORT TailCallTableInfo { field_layout::TransformValidation lazy_opt; bool is_string_inlined; bool is_implicitly_weak; - bool use_direct_tcparser_table; bool should_split; }; class OptionProvider { diff --git a/src/google/protobuf/generated_message_tctable_impl.h b/src/google/protobuf/generated_message_tctable_impl.h index 838e748501ab..10b08f65ac41 100644 --- a/src/google/protobuf/generated_message_tctable_impl.h +++ b/src/google/protobuf/generated_message_tctable_impl.h @@ -153,9 +153,8 @@ enum TransformValidation : uint16_t { kTvUtf8 = 2 << kTvShift, // proto3 // Message fields: - kTvDefault = 1 << kTvShift, // Aux has default_instance* - kTvTable = 2 << kTvShift, // Aux has TcParseTableBase* - kTvWeakPtr = 3 << kTvShift, // Aux has default_instance** (for weak) + kTvTable = 1 << kTvShift, // Aux has TcParseTableBase* + kTvWeakPtr = 2 << kTvShift, // Aux has default_instance** (for weak) // Lazy message fields: kTvEager = 1 << kTvShift, @@ -351,9 +350,7 @@ inline void AlignFail(std::integral_constant, PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastBc) \ PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastSc) \ PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastUc) \ - PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastGd) \ PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastGt) \ - PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastMd) \ PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastMt) \ PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastMl) \ PROTOBUF_TC_PARSE_FUNCTION_LIST_END_GROUP() \ @@ -572,22 +569,14 @@ class PROTOBUF_EXPORT TcParser final { // Functions referenced by generated fast tables (message types): // M: message G: group - // d: default* t: TcParseTable* (the contents of aux) l: lazy + // t: TcParseTable* (the contents of aux) l: lazy // S: singular R: repeated // 1/2: tag length (bytes) - PROTOBUF_NOINLINE static const char* FastMdS1(PROTOBUF_TC_PARAM_DECL); - PROTOBUF_NOINLINE static const char* FastMdS2(PROTOBUF_TC_PARAM_DECL); - PROTOBUF_NOINLINE static const char* FastGdS1(PROTOBUF_TC_PARAM_DECL); - PROTOBUF_NOINLINE static const char* FastGdS2(PROTOBUF_TC_PARAM_DECL); PROTOBUF_NOINLINE static const char* FastMtS1(PROTOBUF_TC_PARAM_DECL); PROTOBUF_NOINLINE static const char* FastMtS2(PROTOBUF_TC_PARAM_DECL); PROTOBUF_NOINLINE static const char* FastGtS1(PROTOBUF_TC_PARAM_DECL); PROTOBUF_NOINLINE static const char* FastGtS2(PROTOBUF_TC_PARAM_DECL); - PROTOBUF_NOINLINE static const char* FastMdR1(PROTOBUF_TC_PARAM_DECL); - PROTOBUF_NOINLINE static const char* FastMdR2(PROTOBUF_TC_PARAM_DECL); - PROTOBUF_NOINLINE static const char* FastGdR1(PROTOBUF_TC_PARAM_DECL); - PROTOBUF_NOINLINE static const char* FastGdR2(PROTOBUF_TC_PARAM_DECL); PROTOBUF_NOINLINE static const char* FastMtR1(PROTOBUF_TC_PARAM_DECL); PROTOBUF_NOINLINE static const char* FastMtR2(PROTOBUF_TC_PARAM_DECL); PROTOBUF_NOINLINE static const char* FastGtR1(PROTOBUF_TC_PARAM_DECL); @@ -730,9 +719,9 @@ class PROTOBUF_EXPORT TcParser final { template static const char* MiniParse(PROTOBUF_TC_PARAM_DECL); - template + template static inline const char* SingularParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL); - template + template static inline const char* RepeatedParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL); template static inline const char* LazyMessage(PROTOBUF_TC_PARAM_DECL); @@ -911,6 +900,9 @@ class PROTOBUF_EXPORT TcParser final { uint32_t field_num, ParseContext* ctx, MessageLite* msg); + static const TcParseTableBase* GetTableForAux(TcParseTableBase::FieldAux aux, + uint16_t type_card); + // UTF-8 validation: static void ReportFastUtf8Error(uint32_t decoded_tag, const TcParseTableBase* table); diff --git a/src/google/protobuf/generated_message_tctable_lite.cc b/src/google/protobuf/generated_message_tctable_lite.cc index f8b0d8794283..35fb6ed5569e 100644 --- a/src/google/protobuf/generated_message_tctable_lite.cc +++ b/src/google/protobuf/generated_message_tctable_lite.cc @@ -351,7 +351,7 @@ PROTOBUF_NOINLINE const char* TcParser::FastEndG2(PROTOBUF_TC_PARAM_DECL) { // Message fields ////////////////////////////////////////////////////////////////////////////// -template +template inline PROTOBUF_ALWAYS_INLINE const char* TcParser::SingularParseMessageAuxImpl( PROTOBUF_TC_PARAM_DECL) { if (PROTOBUF_PREDICT_FALSE(data.coded_tag() != 0)) { @@ -363,67 +363,32 @@ inline PROTOBUF_ALWAYS_INLINE const char* TcParser::SingularParseMessageAuxImpl( SyncHasbits(msg, hasbits, table); auto& field = RefAt(msg, data.offset()); - if (aux_is_table) { - const auto* inner_table = table->field_aux(data.aux_idx())->table; - if (field == nullptr) { - field = inner_table->default_instance->New(msg->GetArena()); - } - const auto inner_loop = [&](const char* ptr) { - return ParseLoopInlined(field, ptr, ctx, inner_table); - }; - return group_coding ? ctx->ParseGroupInlined(ptr, FastDecodeTag(saved_tag), - inner_loop) - : ctx->ParseLengthDelimitedInlined(ptr, inner_loop); - } else { - if (field == nullptr) { - const MessageLite* default_instance = - table->field_aux(data.aux_idx())->message_default(); - field = default_instance->New(msg->GetArena()); - } - if (group_coding) { - return ctx->ParseGroup(field, ptr, FastDecodeTag(saved_tag)); - } - return ctx->ParseMessage(field, ptr); + const auto* inner_table = table->field_aux(data.aux_idx())->table; + if (field == nullptr) { + field = inner_table->default_instance->New(msg->GetArena()); } -} - -PROTOBUF_NOINLINE const char* TcParser::FastMdS1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( - PROTOBUF_TC_PARAM_PASS); -} - -PROTOBUF_NOINLINE const char* TcParser::FastMdS2(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( - PROTOBUF_TC_PARAM_PASS); -} - -PROTOBUF_NOINLINE const char* TcParser::FastGdS1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( - PROTOBUF_TC_PARAM_PASS); -} - -PROTOBUF_NOINLINE const char* TcParser::FastGdS2(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( - PROTOBUF_TC_PARAM_PASS); + return group_coding ? ParseGroup(field, ptr, ctx, inner_table, + FastDecodeTag(saved_tag)) + : ParseMessage(field, ptr, ctx, inner_table); } PROTOBUF_NOINLINE const char* TcParser::FastMtS1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } PROTOBUF_NOINLINE const char* TcParser::FastMtS2(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } PROTOBUF_NOINLINE const char* TcParser::FastGtS1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } PROTOBUF_NOINLINE const char* TcParser::FastGtS2(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } @@ -441,7 +406,7 @@ PROTOBUF_NOINLINE const char* TcParser::FastMlS2(PROTOBUF_TC_PARAM_DECL) { PROTOBUF_MUSTTAIL return LazyMessage(PROTOBUF_TC_PARAM_PASS); } -template +template inline PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedParseMessageAuxImpl( PROTOBUF_TC_PARAM_DECL) { if (PROTOBUF_PREDICT_FALSE(data.coded_tag() != 0)) { @@ -450,25 +415,13 @@ inline PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedParseMessageAuxImpl( const auto expected_tag = UnalignedLoad(ptr); const auto aux = *table->field_aux(data.aux_idx()); auto& field = RefAt(msg, data.offset()); - const MessageLite* const default_instance = - aux_is_table ? aux.table->default_instance : aux.message_default(); + const MessageLite* const default_instance = aux.table->default_instance; do { ptr += sizeof(TagType); MessageLite* submsg = field.AddMessage(default_instance); - if (aux_is_table) { - const auto inner_loop = [&](const char* ptr) { - return ParseLoopInlined(submsg, ptr, ctx, aux.table); - }; - ptr = group_coding ? ctx->ParseGroupInlined( - ptr, FastDecodeTag(expected_tag), inner_loop) - : ctx->ParseLengthDelimitedInlined(ptr, inner_loop); - } else { - if (group_coding) { - ptr = ctx->ParseGroup(submsg, ptr, FastDecodeTag(expected_tag)); - } else { - ptr = ctx->ParseMessage(submsg, ptr); - } - } + ptr = group_coding ? ParseGroup(submsg, ptr, ctx, aux.table, + FastDecodeTag(expected_tag)) + : ParseMessage(submsg, ptr, ctx, aux.table); if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) { PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); } @@ -480,43 +433,23 @@ inline PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedParseMessageAuxImpl( PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS); } -PROTOBUF_NOINLINE const char* TcParser::FastMdR1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( - PROTOBUF_TC_PARAM_PASS); -} - -PROTOBUF_NOINLINE const char* TcParser::FastMdR2(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( - PROTOBUF_TC_PARAM_PASS); -} - -PROTOBUF_NOINLINE const char* TcParser::FastGdR1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( - PROTOBUF_TC_PARAM_PASS); -} - -PROTOBUF_NOINLINE const char* TcParser::FastGdR2(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( - PROTOBUF_TC_PARAM_PASS); -} - PROTOBUF_NOINLINE const char* TcParser::FastMtR1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } PROTOBUF_NOINLINE const char* TcParser::FastMtR2(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } PROTOBUF_NOINLINE const char* TcParser::FastGtR1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } PROTOBUF_NOINLINE const char* TcParser::FastGtR2(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } @@ -2291,6 +2224,16 @@ PROTOBUF_NOINLINE const char* TcParser::MpRepeatedString( } +PROTOBUF_ALWAYS_INLINE inline const TcParseTableBase* TcParser::GetTableForAux( + TcParseTableBase::FieldAux aux, uint16_t type_card) { + if (ABSL_PREDICT_TRUE((type_card & field_layout::kTvMask) == + field_layout::kTvTable)) { + return aux.table; + } + ABSL_DCHECK_EQ(type_card & field_layout::kTvMask, +field_layout::kTvWeakPtr); + return aux.message_default_weak()->GetTcParseTable(); +} + template PROTOBUF_NOINLINE const char* TcParser::MpMessage(PROTOBUF_TC_PARAM_DECL) { const auto& entry = RefAt(table, data.entry_offset()); @@ -2346,33 +2289,13 @@ PROTOBUF_NOINLINE const char* TcParser::MpMessage(PROTOBUF_TC_PARAM_DECL) { void* const base = MaybeGetSplitBase(msg, is_split, table); SyncHasbits(msg, hasbits, table); MessageLite*& field = RefAt(base, entry.offset); - if ((type_card & field_layout::kTvMask) == field_layout::kTvTable) { - auto* inner_table = table->field_aux(&entry)->table; - if (need_init || field == nullptr) { - field = inner_table->default_instance->New(msg->GetArena()); - } - const auto inner_loop = [&](const char* ptr) { - return ParseLoop(field, ptr, ctx, inner_table); - }; - return is_group ? ctx->ParseGroupInlined(ptr, decoded_tag, inner_loop) - : ctx->ParseLengthDelimitedInlined(ptr, inner_loop); - } else { - if (need_init || field == nullptr) { - const MessageLite* def; - if ((type_card & field_layout::kTvMask) == field_layout::kTvDefault) { - def = table->field_aux(&entry)->message_default(); - } else { - ABSL_DCHECK_EQ(type_card & field_layout::kTvMask, - +field_layout::kTvWeakPtr); - def = table->field_aux(&entry)->message_default_weak(); - } - field = def->New(msg->GetArena()); - } - if (is_group) { - return ctx->ParseGroup(field, ptr, decoded_tag); - } - return ctx->ParseMessage(field, ptr); + + auto* inner_table = GetTableForAux(*table->field_aux(&entry), type_card); + if (need_init || field == nullptr) { + field = inner_table->default_instance->New(msg->GetArena()); } + return is_group ? ParseGroup(field, ptr, ctx, inner_table, decoded_tag) + : ParseMessage(field, ptr, ctx, inner_table); } template @@ -2403,45 +2326,19 @@ const char* TcParser::MpRepeatedMessageOrGroup(PROTOBUF_TC_PARAM_DECL) { RepeatedPtrFieldBase& field = MaybeCreateRepeatedRefAt( base, entry.offset, msg); - const auto aux = *table->field_aux(&entry); - if ((type_card & field_layout::kTvMask) == field_layout::kTvTable) { - auto* inner_table = aux.table; - const MessageLite* default_instance = inner_table->default_instance; - const char* ptr2 = ptr; - uint32_t next_tag; - do { - MessageLite* value = field.AddMessage(default_instance); - const auto inner_loop = [&](const char* ptr) { - return ParseLoop(value, ptr, ctx, inner_table); - }; - ptr = is_group ? ctx->ParseGroupInlined(ptr2, decoded_tag, inner_loop) - : ctx->ParseLengthDelimitedInlined(ptr2, inner_loop); - if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) goto error; - if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) goto parse_loop; - ptr2 = ReadTag(ptr, &next_tag); - if (PROTOBUF_PREDICT_FALSE(ptr2 == nullptr)) goto error; - } while (next_tag == decoded_tag); - } else { - const MessageLite* default_instance; - if ((type_card & field_layout::kTvMask) == field_layout::kTvDefault) { - default_instance = aux.message_default(); - } else { - ABSL_DCHECK_EQ(type_card & field_layout::kTvMask, - +field_layout::kTvWeakPtr); - default_instance = aux.message_default_weak(); - } - const char* ptr2 = ptr; - uint32_t next_tag; - do { - MessageLite* value = field.AddMessage(default_instance); - ptr = is_group ? ctx->ParseGroup(value, ptr2, decoded_tag) - : ctx->ParseMessage(value, ptr2); - if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) goto error; - if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) goto parse_loop; - ptr2 = ReadTag(ptr, &next_tag); - if (PROTOBUF_PREDICT_FALSE(ptr2 == nullptr)) goto error; - } while (next_tag == decoded_tag); - } + auto* inner_table = GetTableForAux(*table->field_aux(&entry), type_card); + const MessageLite* default_instance = inner_table->default_instance; + const char* ptr2 = ptr; + uint32_t next_tag; + do { + MessageLite* value = field.AddMessage(default_instance); + ptr = is_group ? ParseGroup(value, ptr2, ctx, inner_table, decoded_tag) + : ParseMessage(value, ptr2, ctx, inner_table); + if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) goto error; + if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) goto parse_loop; + ptr2 = ReadTag(ptr, &next_tag); + if (PROTOBUF_PREDICT_FALSE(ptr2 == nullptr)) goto error; + } while (next_tag == decoded_tag); PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS); parse_loop: PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_PASS); @@ -2852,11 +2749,10 @@ std::string TypeCardToString(uint16_t type_card) { } static constexpr const char* kXFormNames[2][4] = { - {nullptr, "Default", "Table", "WeakPtr"}, {nullptr, "Eager", "Lazy"}}; + {nullptr, "Table", "WeakPtr"}, {nullptr, "Eager", "Lazy"}}; - static_assert((fl::kTvDefault >> fl::kTvShift) == 1, ""); - static_assert((fl::kTvTable >> fl::kTvShift) == 2, ""); - static_assert((fl::kTvWeakPtr >> fl::kTvShift) == 3, ""); + static_assert((fl::kTvTable >> fl::kTvShift) == 1, ""); + static_assert((fl::kTvWeakPtr >> fl::kTvShift) == 2, ""); static_assert((fl::kTvEager >> fl::kTvShift) == 1, ""); static_assert((fl::kTvLazy >> fl::kTvShift) == 2, ""); diff --git a/src/google/protobuf/message.h b/src/google/protobuf/message.h index fb0643c3d824..08a60ed78246 100644 --- a/src/google/protobuf/message.h +++ b/src/google/protobuf/message.h @@ -1045,20 +1045,25 @@ class PROTOBUF_EXPORT Reflection final { using TcParseTableBase = internal::TcParseTableBase; mutable const TcParseTableBase* tcparse_table_ = nullptr; - const TcParseTableBase* GetTcParseTable() const { - absl::call_once(tcparse_table_once_, - [&] { tcparse_table_ = CreateTcParseTable(); }); - return tcparse_table_; - } - - const TcParseTableBase* CreateTcParseTable() const; + const TcParseTableBase* GetTcParseTable() const; + + // We use an output parameter to ensure we write it as soon as we have the + // pointer. This allows DynamicMessageFactory to cross link as it is being + // built. + void CreateTcParseTable( + const TcParseTableBase*& out, + absl::FunctionRef + fetch_subtable) const; void PopulateTcParseFastEntries( const internal::TailCallTableInfo& table_info, TcParseTableBase::FastFieldEntry* fast_entries) const; void PopulateTcParseEntries(internal::TailCallTableInfo& table_info, TcParseTableBase::FieldEntry* entries) const; - void PopulateTcParseFieldAux(const internal::TailCallTableInfo& table_info, - TcParseTableBase::FieldAux* field_aux) const; + void PopulateTcParseFieldAux( + const internal::TailCallTableInfo& table_info, + absl::FunctionRef + fetch_subtable, + TcParseTableBase::FieldAux* field_aux) const; template friend class RepeatedFieldRef; diff --git a/src/google/protobuf/message_lite.h b/src/google/protobuf/message_lite.h index 722bdf990e8e..31c1ba96848a 100644 --- a/src/google/protobuf/message_lite.h +++ b/src/google/protobuf/message_lite.h @@ -53,6 +53,7 @@ class FastReflectionStringSetter; class Reflection; class Descriptor; class AssignDescriptorsHelper; +class DynamicMessageFactory; namespace io { @@ -662,6 +663,7 @@ class PROTOBUF_EXPORT MessageLite { friend class AssignDescriptorsHelper; friend class FastReflectionStringSetter; friend class Message; + friend class DynamicMessageFactory; friend class Reflection; friend class internal::DescriptorPoolExtensionFinder; friend class internal::ExtensionSet;