Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Move all TDP parsing to use direct-to-table. #16344

Merged
merged 1 commit into from
Apr 1, 2024
Merged

Move all TDP parsing to use direct-to-table. #16344

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion src/google/protobuf/compiler/cpp/parse_function_generator.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,6 @@ class ParseFunctionGenerator::GeneratedOptionProvider final
verify_flag(),
IsStringInlined(field, gen_->options_),
IsImplicitWeakField(field, gen_->options_, gen_->scc_analyzer_),
/* use_direct_tcparser_table */ true,
ShouldSplit(field, gen_->options_),
};
}
Expand Down
43 changes: 29 additions & 14 deletions src/google/protobuf/dynamic_message.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,13 @@
#include <memory>
#include <new>

#include "absl/log/absl_check.h"
#include "google/protobuf/arenastring.h"
#include "google/protobuf/descriptor.h"
#include "google/protobuf/descriptor.pb.h"
#include "google/protobuf/extension_set.h"
#include "google/protobuf/generated_message_reflection.h"
#include "google/protobuf/generated_message_tctable_decl.h"
#include "google/protobuf/generated_message_util.h"
#include "google/protobuf/map_field.h"
#include "google/protobuf/map_field_inl.h"
Expand Down Expand Up @@ -283,6 +285,8 @@ struct DynamicMessageFactory::TypeInfo {
~TypeInfo() {
delete prototype;
delete class_data.reflection;
::operator delete(
const_cast<internal::TcParseTableBase*>(class_data.tc_table));

auto* type = class_data.descriptor;

Expand Down Expand Up @@ -415,9 +419,9 @@ void DynamicMessage::SharedCtor(bool lock_factory) {
new (field_ptr) Message*(nullptr);
} else {
if (IsMapFieldInApi(field)) {
// We need to lock in most cases to avoid data racing. Only not lock
// when the constructor is called inside GetPrototype(), in which
// case we have already locked the factory.
// We need to lock in most cases to avoid data racing.
// When building the prototype via GetPrototypeNoLock, we construct
// this during crosslinking.
if (lock_factory) {
if (arena != nullptr) {
new (field_ptr) DynamicMapField(
Expand All @@ -427,17 +431,6 @@ void DynamicMessage::SharedCtor(bool lock_factory) {
new (field_ptr) DynamicMapField(
type_info_->factory->GetPrototype(field->message_type()));
}
} else {
if (arena != nullptr) {
new (field_ptr)
DynamicMapField(type_info_->factory->GetPrototypeNoLock(
field->message_type()),
arena);
} else {
new (field_ptr)
DynamicMapField(type_info_->factory->GetPrototypeNoLock(
field->message_type()));
}
}
} else {
new (field_ptr) RepeatedPtrField<Message>(arena);
Expand Down Expand Up @@ -583,6 +576,18 @@ void DynamicMessage::CrossLinkPrototypes() {
factory->GetPrototypeNoLock(field->message_type());
}
}

// Construct the map fields.
// We need to delay this until we can do cross references.
for (int i = 0; i < descriptor->field_count(); i++) {
const FieldDescriptor* field = descriptor->field(i);
if (field->cpp_type() == field->CPPTYPE_MESSAGE && field->is_repeated() &&
IsMapFieldInApi(field)) {
void* field_ptr = MutableRaw(i);
new (field_ptr) DynamicMapField(
type_info_->factory->GetPrototypeNoLock(field->message_type()));
}
}
}

Message* DynamicMessage::New(Arena* arena) const {
Expand Down Expand Up @@ -774,6 +779,16 @@ const Message* DynamicMessageFactory::GetPrototypeNoLock(
type_info->class_data.reflection = new Reflection(
type_info->class_data.descriptor, schema, type_info->pool, this);

type_info->class_data.reflection->CreateTcParseTable(
type_info->class_data.tc_table, [&](const auto* field) {
// The prototype we get might not necessarily be a dynamic message, so
// use GetClassData to fetch the table.
auto* table =
GetPrototypeNoLock(field->message_type())->GetTcParseTable();
ABSL_DCHECK(table != nullptr);
return table;
});

// Cross link prototypes.
prototype->CrossLinkPrototypes();

Expand Down
34 changes: 25 additions & 9 deletions src/google/protobuf/generated_message_reflection.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
#include "absl/base/call_once.h"
#include "absl/base/const_init.h"
#include "absl/container/flat_hash_set.h"
#include "absl/functional/function_ref.h"
#include "absl/log/absl_check.h"
#include "absl/log/absl_log.h"
#include "absl/strings/str_format.h"
Expand Down Expand Up @@ -3318,6 +3319,8 @@ void Reflection::PopulateTcParseEntries(

void Reflection::PopulateTcParseFieldAux(
const internal::TailCallTableInfo& table_info,
absl::FunctionRef<const TcParseTableBase*(const FieldDescriptor*)>
fetch_subtable,
TcParseTableBase::FieldAux* field_aux) const {
for (const auto& aux_entry : table_info.aux_entries) {
switch (aux_entry.type) {
Expand All @@ -3335,6 +3338,8 @@ void Reflection::PopulateTcParseFieldAux(
field_aux++->offset = schema_.SizeofSplit();
break;
case internal::TailCallTableInfo::kSubTable:
field_aux++->table = fetch_subtable(aux_entry.field);
break;
case internal::TailCallTableInfo::kSubMessageWeak:
case internal::TailCallTableInfo::kCreateInArena:
case internal::TailCallTableInfo::kMessageVerifyFunc:
Expand Down Expand Up @@ -3367,7 +3372,19 @@ void Reflection::PopulateTcParseFieldAux(
}


const internal::TcParseTableBase* Reflection::CreateTcParseTable() const {
const internal::TcParseTableBase* Reflection::GetTcParseTable() const {
absl::call_once(tcparse_table_once_, [&] {
CreateTcParseTable(tcparse_table_, [&](const auto* field) {
return GetDefaultMessageInstance(field)->GetTcParseTable();
});
});
return tcparse_table_;
}

void Reflection::CreateTcParseTable(
const TcParseTableBase*& out,
absl::FunctionRef<const TcParseTableBase*(const FieldDescriptor*)>
fetch_subtable) const {
using TcParseTableBase = internal::TcParseTableBase;

std::vector<const FieldDescriptor*> fields;
Expand Down Expand Up @@ -3411,11 +3428,6 @@ const internal::TcParseTableBase* Reflection::CreateTcParseTable() const {

// Only LITE can be implicitly weak.
/* is_implicitly_weak */ false,

// We could change this to use direct table.
// Might be easier to do when all messages support TDP.
/* use_direct_tcparser_table */ false,

ref_.schema_.IsSplit(field), //
};
}
Expand Down Expand Up @@ -3472,6 +3484,12 @@ const internal::TcParseTableBase* Reflection::CreateTcParseTable() const {
nullptr
#endif // PROTOBUF_PREFETCH_PARSE_TABLE
};

// Set the `out` pointer first. `fetch_subtable` is reentrant and we need to
// update the cache tables before the reentrancy. That way we can do cross
// referencing while building the tables.
out = res;

#ifdef PROTOBUF_PREFETCH_PARSE_TABLE
// We'll prefetch `to_prefetch->to_prefetch` unconditionally to avoid
// branches. Here we don't know which field is the hottest, so set the pointer
Expand All @@ -3486,7 +3504,7 @@ const internal::TcParseTableBase* Reflection::CreateTcParseTable() const {

PopulateTcParseEntries(table_info, res->field_entries_begin());

PopulateTcParseFieldAux(table_info, res->field_aux(0u));
PopulateTcParseFieldAux(table_info, fetch_subtable, res->field_aux(0u));

// Copy the name data.
if (!table_info.field_name_data.empty()) {
Expand All @@ -3497,8 +3515,6 @@ const internal::TcParseTableBase* Reflection::CreateTcParseTable() const {
ABSL_CHECK_EQ(res->name_data() + table_info.field_name_data.size() -
reinterpret_cast<char*>(res),
byte_size);

return res;
}

namespace {
Expand Down
26 changes: 8 additions & 18 deletions src/google/protobuf/generated_message_tctable_gen.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,16 +204,12 @@ TailCallTableInfo::FastFieldInfo::Field MakeFastFieldEntry(
}
break;
case FieldDescriptor::TYPE_MESSAGE:
picked =
(HasLazyRep(field, options) ? PROTOBUF_PICK_SINGLE_FUNCTION(kFastMl)
: options.use_direct_tcparser_table
? PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastMt)
: PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastMd));
picked = HasLazyRep(field, options)
? PROTOBUF_PICK_SINGLE_FUNCTION(kFastMl)
: PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastMt);
break;
case FieldDescriptor::TYPE_GROUP:
picked = (options.use_direct_tcparser_table
? PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastGt)
: PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastGd));
picked = PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastGt);
break;
}

Expand Down Expand Up @@ -666,10 +662,8 @@ uint16_t MakeTypeCardForField(
type_card |= 0 | fl::kMessage | fl::kRepGroup;
if (options.is_implicitly_weak) {
type_card |= fl::kTvWeakPtr;
} else if (options.use_direct_tcparser_table) {
type_card |= fl::kTvTable;
} else {
type_card |= fl::kTvDefault;
type_card |= fl::kTvTable;
}
break;
case FieldDescriptor::TYPE_MESSAGE:
Expand All @@ -684,10 +678,8 @@ uint16_t MakeTypeCardForField(
} else {
if (options.is_implicitly_weak) {
type_card |= fl::kTvWeakPtr;
} else if (options.use_direct_tcparser_table) {
type_card |= fl::kTvTable;
} else {
type_card |= fl::kTvDefault;
type_card |= fl::kTvTable;
}
}
}
Expand Down Expand Up @@ -821,7 +813,7 @@ TailCallTableInfo::TailCallTableInfo(
field->type() == FieldDescriptor::TYPE_GROUP) &&
!field->is_map() && !field->options().weak() &&
!HasLazyRep(field, options) && !options.is_implicitly_weak &&
options.use_direct_tcparser_table && is_non_cold(options);
is_non_cold(options);
};
for (const FieldDescriptor* field : ordered_fields) {
if (is_non_cold_subtable(field)) {
Expand Down Expand Up @@ -878,9 +870,7 @@ TailCallTableInfo::TailCallTableInfo(
entry.aux_idx = TcParseTableBase::FieldEntry::kNoAuxIdx;
}
} else {
AuxType type = options.is_implicitly_weak ? kSubMessageWeak
: options.use_direct_tcparser_table ? kSubTable
: kSubMessage;
AuxType type = options.is_implicitly_weak ? kSubMessageWeak : kSubTable;
if (message_options.should_profile_driven_cluster_aux_subtable &&
type == kSubTable && is_non_cold(options)) {
aux_entries[subtable_aux_idx] = {type, {field}};
Expand Down
1 change: 0 additions & 1 deletion src/google/protobuf/generated_message_tctable_gen.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@ struct PROTOBUF_EXPORT TailCallTableInfo {
field_layout::TransformValidation lazy_opt;
bool is_string_inlined;
bool is_implicitly_weak;
bool use_direct_tcparser_table;
bool should_split;
};
class OptionProvider {
Expand Down
24 changes: 8 additions & 16 deletions src/google/protobuf/generated_message_tctable_impl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,9 +153,8 @@ enum TransformValidation : uint16_t {
kTvUtf8 = 2 << kTvShift, // proto3

// Message fields:
kTvDefault = 1 << kTvShift, // Aux has default_instance*
kTvTable = 2 << kTvShift, // Aux has TcParseTableBase*
kTvWeakPtr = 3 << kTvShift, // Aux has default_instance** (for weak)
kTvTable = 1 << kTvShift, // Aux has TcParseTableBase*
kTvWeakPtr = 2 << kTvShift, // Aux has default_instance** (for weak)

// Lazy message fields:
kTvEager = 1 << kTvShift,
Expand Down Expand Up @@ -351,9 +350,7 @@ inline void AlignFail(std::integral_constant<size_t, 1>,
PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastBc) \
PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastSc) \
PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastUc) \
PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastGd) \
PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastGt) \
PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastMd) \
PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastMt) \
PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastMl) \
PROTOBUF_TC_PARSE_FUNCTION_LIST_END_GROUP() \
Expand Down Expand Up @@ -572,22 +569,14 @@ class PROTOBUF_EXPORT TcParser final {

// Functions referenced by generated fast tables (message types):
// M: message G: group
// d: default* t: TcParseTable* (the contents of aux) l: lazy
// t: TcParseTable* (the contents of aux) l: lazy
// S: singular R: repeated
// 1/2: tag length (bytes)
PROTOBUF_NOINLINE static const char* FastMdS1(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastMdS2(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastGdS1(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastGdS2(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastMtS1(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastMtS2(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastGtS1(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastGtS2(PROTOBUF_TC_PARAM_DECL);

PROTOBUF_NOINLINE static const char* FastMdR1(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastMdR2(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastGdR1(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastGdR2(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastMtR1(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastMtR2(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastGtR1(PROTOBUF_TC_PARAM_DECL);
Expand Down Expand Up @@ -730,9 +719,9 @@ class PROTOBUF_EXPORT TcParser final {
template <bool export_called_function>
static const char* MiniParse(PROTOBUF_TC_PARAM_DECL);

template <typename TagType, bool group_coding, bool aux_is_table>
template <typename TagType, bool group_coding>
static inline const char* SingularParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL);
template <typename TagType, bool group_coding, bool aux_is_table>
template <typename TagType, bool group_coding>
static inline const char* RepeatedParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL);
template <typename TagType>
static inline const char* LazyMessage(PROTOBUF_TC_PARAM_DECL);
Expand Down Expand Up @@ -911,6 +900,9 @@ class PROTOBUF_EXPORT TcParser final {
uint32_t field_num, ParseContext* ctx,
MessageLite* msg);

static const TcParseTableBase* GetTableForAux(TcParseTableBase::FieldAux aux,
uint16_t type_card);

// UTF-8 validation:
static void ReportFastUtf8Error(uint32_t decoded_tag,
const TcParseTableBase* table);
Expand Down
Loading
Loading