69 #include <string_view>
70 #include <unordered_map>
71 #include <unordered_set>
74 #include "../base/helpers.h"
159 static std::string normalize_schema_encoding(std::string_view encoding);
161 static bool is_flatbuffers_schema_type(std::string_view encoding);
163 bool cache_schema_data_locked(
const SchemaData& schema);
165 #ifdef VLINK_HAS_SCHEMA_PLUGIN_PROTOBUF
168 static SchemaData build_protobuf_schema_data(
const google::protobuf::Descriptor& descriptor);
171 #ifdef VLINK_HAS_SCHEMA_PLUGIN_FLATBUFFERS
172 void import_all_flatbuffers_schema_data_locked();
174 bool import_flatbuffers_schema_data_locked(
const std::string& name);
176 const SchemaData* find_cached_flatbuffers_schema_locked(
const std::string& name)
const;
178 void clear_flatbuffers_parser_cache_locked(
const std::string& name);
181 #ifdef VLINK_HAS_SCHEMA_PLUGIN_PROTOBUF
182 google::protobuf::DynamicMessageFactory factory_;
184 std::unordered_map<std::string, ProtobufDescriptorPtr> protobuf_descriptor_map_;
185 std::unordered_map<std::string, std::vector<SchemaData>> schema_map_;
186 std::unordered_map<std::string, ProtobufMessagePtr> protobuf_message_map_;
187 std::unordered_map<std::string, std::vector<FlatbuffersParserPtr>> flatbuffers_parser_map_;
188 std::vector<FlatbuffersParserPtr> retired_flatbuffers_parsers_;
189 std::unordered_map<std::string, std::unique_ptr<Bytes>> flatbuffers_schema_snapshots_;
190 std::vector<std::unique_ptr<Bytes>> retired_flatbuffers_schema_snapshots_;
201 #ifdef VLINK_HAS_SCHEMA_PLUGIN_PROTOBUF
202 : factory_(google::protobuf::DescriptorPool::generated_pool())
209 #ifdef VLINK_HAS_SCHEMA_PLUGIN_PROTOBUF
210 for (
auto& [name, ptr] : protobuf_message_map_) {
212 delete reinterpret_cast<google::protobuf::Message*
>(ptr);
216 for (
auto& [name, ptr_list] : flatbuffers_parser_map_) {
218 #ifdef VLINK_HAS_SCHEMA_PLUGIN_FLATBUFFERS
219 for (
auto* ptr : ptr_list) {
220 delete reinterpret_cast<flatbuffers::Parser*
>(ptr);
225 #ifdef VLINK_HAS_SCHEMA_PLUGIN_FLATBUFFERS
226 for (
auto* ptr : retired_flatbuffers_parsers_) {
227 delete reinterpret_cast<flatbuffers::Parser*
>(ptr);
231 protobuf_message_map_.clear();
232 flatbuffers_parser_map_.clear();
233 retired_flatbuffers_parsers_.clear();
234 flatbuffers_schema_snapshots_.clear();
235 retired_flatbuffers_schema_snapshots_.clear();
239 std::lock_guard lock(mtx_);
241 auto iter = schema_map_.find(name);
243 if VLIKELY (iter != schema_map_.end()) {
245 for (
const auto& schema : iter->second) {
246 if (schema.schema_type == schema_type) {
259 (void)cache_schema_data_locked(schema);
264 std::vector<SchemaData> matches;
267 [[maybe_unused]]
bool has_cached_protobuf =
false;
268 [[maybe_unused]]
bool has_cached_flatbuffers =
false;
269 [[maybe_unused]]
bool has_cached_zerocopy =
false;
271 if VLIKELY (iter != schema_map_.end()) {
272 for (
const auto& cached_schema : iter->second) {
273 switch (cached_schema.schema_type) {
275 has_cached_protobuf =
true;
278 has_cached_flatbuffers =
true;
281 has_cached_zerocopy =
true;
287 matches.emplace_back(cached_schema);
291 #ifdef VLINK_HAS_SCHEMA_PLUGIN_PROTOBUF
294 if (
auto* desc_ptr = find_protobuf_descriptor_locked(name); desc_ptr !=
nullptr) {
295 auto* desc =
reinterpret_cast<google::protobuf::Descriptor*
>(desc_ptr);
296 auto proto_schema = build_protobuf_schema_data(*desc);
298 if VLIKELY (!proto_schema.encoding.empty()) {
299 (void)cache_schema_data_locked(proto_schema);
300 matches.emplace_back(std::move(proto_schema));
306 #ifdef VLINK_HAS_SCHEMA_PLUGIN_FLATBUFFERS
309 if (
const auto* cached_schema = find_cached_flatbuffers_schema_locked(name)) {
310 matches.emplace_back(*cached_schema);
317 zerocopy_schema.
name = name;
318 zerocopy_schema.
encoding =
"vlink_msg";
320 (void)cache_schema_data_locked(zerocopy_schema);
321 matches.emplace_back(std::move(zerocopy_schema));
324 if (matches.size() == 1) {
325 return matches.front();
331 #ifdef VLINK_HAS_SCHEMA_PLUGIN_PROTOBUF
334 auto* desc_ptr = find_protobuf_descriptor_locked(name);
336 if (desc_ptr !=
nullptr) {
337 auto* desc =
reinterpret_cast<google::protobuf::Descriptor*
>(desc_ptr);
338 schema = build_protobuf_schema_data(*desc);
341 (void)cache_schema_data_locked(schema);
348 #ifdef VLINK_HAS_SCHEMA_PLUGIN_FLATBUFFERS
351 if (
const auto* cached_schema = find_cached_flatbuffers_schema_locked(name)) {
352 return *cached_schema;
361 std::lock_guard lock(mtx_);
363 #ifdef VLINK_HAS_SCHEMA_PLUGIN_FLATBUFFERS
366 import_all_flatbuffers_schema_data_locked();
370 size_t schema_count = 0;
372 for (
const auto& [name, items] : schema_map_) {
374 schema_count += items.size();
377 std::vector<SchemaData> schemas;
378 schemas.reserve(schema_count);
380 for (
const auto& [name, items] : schema_map_) {
383 for (
const auto& schema : items) {
388 schemas.emplace_back(schema);
396 const std::string& name) {
397 std::lock_guard lock(mtx_);
399 #ifdef VLINK_HAS_SCHEMA_PLUGIN_PROTOBUF
400 return find_protobuf_descriptor_locked(name);
408 std::lock_guard lock(mtx_);
410 auto iter = protobuf_message_map_.find(name);
412 if VLIKELY (iter != protobuf_message_map_.end()) {
416 #ifdef VLINK_HAS_SCHEMA_PLUGIN_PROTOBUF
417 auto* desc_ptr = find_protobuf_descriptor_locked(name);
423 const auto* prototype = factory_.GetPrototype(
reinterpret_cast<google::protobuf::Descriptor*
>(desc_ptr));
430 protobuf_message_map_.emplace(name, target_ptr);
439 const std::string& name) {
440 std::lock_guard lock(mtx_);
442 #ifdef VLINK_HAS_SCHEMA_PLUGIN_FLATBUFFERS
443 const auto* schema = find_cached_flatbuffers_schema_locked(name);
451 if VUNLIKELY (!reflection::VerifySchemaBuffer(verifier)) {
455 auto snapshot_iter = flatbuffers_schema_snapshots_.find(name);
457 if VLIKELY (snapshot_iter != flatbuffers_schema_snapshots_.end() && snapshot_iter->second &&
458 snapshot_iter->second->size() == schema->data.size() &&
459 std::memcmp(snapshot_iter->second->data(), schema->data.data(), schema->data.size()) == 0) {
461 const_cast<reflection::Schema*
>(reflection::GetSchema(snapshot_iter->second->data())));
464 auto snapshot = std::make_unique<Bytes>(
Bytes::deep_copy(schema->data.data(), schema->data.size()));
470 const auto* snapshot_data = snapshot->data();
472 if VLIKELY (snapshot_iter != flatbuffers_schema_snapshots_.end()) {
473 if VLIKELY (snapshot_iter->second) {
474 retired_flatbuffers_schema_snapshots_.emplace_back(std::move(snapshot_iter->second));
477 snapshot_iter->second = std::move(snapshot);
479 flatbuffers_schema_snapshots_.emplace(name, std::move(snapshot));
482 return reinterpret_cast<FlatbuffersSchemaPtr>(
const_cast<reflection::Schema*
>(reflection::GetSchema(snapshot_data)));
490 const std::string& name) {
491 std::lock_guard lock(mtx_);
493 #ifdef VLINK_HAS_SCHEMA_PLUGIN_FLATBUFFERS
494 const auto* schema = find_cached_flatbuffers_schema_locked(name);
500 auto parser = std::make_unique<flatbuffers::Parser>();
502 if VUNLIKELY (!parser->Deserialize(
reinterpret_cast<const uint8_t*
>(schema->data.data()), schema->data.size())) {
506 if VUNLIKELY (!parser->SetRootType(name.c_str())) {
511 flatbuffers_parser_map_[name].emplace_back(target_ptr);
513 (void)parser.release();
522 inline std::string SchemaPluginBase::normalize_schema_encoding(std::string_view encoding) {
523 std::string normalized{encoding};
524 std::transform(normalized.begin(), normalized.end(), normalized.begin(),
525 [](
unsigned char c) { return static_cast<char>(std::tolower(c)); });
527 if (normalized ==
"proto") {
531 if (normalized ==
"flatbuffer" || normalized ==
"flatbuffers" || normalized ==
"fbs" || normalized ==
"bfbs") {
532 return "flatbuffers";
538 inline bool SchemaPluginBase::is_flatbuffers_schema_type(std::string_view encoding) {
539 return normalize_schema_encoding(encoding) ==
"flatbuffers";
542 inline bool SchemaPluginBase::cache_schema_data_locked(
const SchemaData& schema) {
543 if VUNLIKELY (schema.name.empty() || schema.encoding.empty()) {
547 #ifdef VLINK_HAS_SCHEMA_PLUGIN_FLATBUFFERS
549 if (is_flatbuffers_schema_type(schema.encoding)) {
552 if VUNLIKELY (!reflection::VerifySchemaBuffer(verifier)) {
556 clear_flatbuffers_parser_cache_locked(schema.name);
566 auto& cached_schemas = schema_map_[schema.name];
568 for (
auto& cached_schema : cached_schemas) {
569 if (cached_schema.schema_type == cached_schema_type) {
570 cached_schema = schema;
575 cached_schemas.emplace_back(schema);
579 #ifdef VLINK_HAS_SCHEMA_PLUGIN_PROTOBUF
581 const std::string& name) {
582 auto iter = protobuf_descriptor_map_.find(name);
584 if VLIKELY (iter != protobuf_descriptor_map_.end()) {
588 const auto* desc = google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName(name);
594 auto* target_ptr =
reinterpret_cast<ProtobufDescriptorPtr>(
const_cast<google::protobuf::Descriptor*
>(desc));
595 protobuf_descriptor_map_.emplace(name, target_ptr);
599 inline SchemaData SchemaPluginBase::build_protobuf_schema_data(
const google::protobuf::Descriptor& descriptor) {
601 schema.name = descriptor.full_name();
602 schema.encoding =
"protobuf";
606 schema.encoding.clear();
611 google::protobuf::FileDescriptorSet proto_fd_set;
612 std::queue<const google::protobuf::FileDescriptor*> to_add;
614 to_add.push(descriptor.file());
616 #if GOOGLE_PROTOBUF_VERSION >= 6030000
617 std::unordered_set<std::string_view> seen_dependencies;
619 std::unordered_set<std::string> seen_dependencies;
622 seen_dependencies.insert(descriptor.file()->name());
624 while (!to_add.empty()) {
625 const auto* next = to_add.front();
628 next->CopyTo(proto_fd_set.add_file());
630 for (
int i = 0; i < next->dependency_count(); ++i) {
632 const auto* dep = next->dependency(i);
634 if (dep ==
nullptr || seen_dependencies.find(dep->name()) != seen_dependencies.end()) {
638 seen_dependencies.insert(dep->name());
646 if VUNLIKELY (!proto_fd_set.SerializeToArray(schema.data.data(), schema.data.size())) {
647 schema.encoding.clear();
656 #ifdef VLINK_HAS_SCHEMA_PLUGIN_FLATBUFFERS
657 inline void SchemaPluginBase::import_all_flatbuffers_schema_data_locked() {
659 (void)cache_schema_data_locked(schema);
663 inline bool SchemaPluginBase::import_flatbuffers_schema_data_locked(
const std::string& name) {
666 if (!is_flatbuffers_schema_type(schema.encoding) || schema.data.empty()) {
670 return cache_schema_data_locked(schema);
673 inline const SchemaData* SchemaPluginBase::find_cached_flatbuffers_schema_locked(
const std::string& name)
const {
675 auto iter =
self->schema_map_.find(name);
677 if (iter == self->schema_map_.end()) {
678 (void)self->import_flatbuffers_schema_data_locked(name);
679 iter =
self->schema_map_.find(name);
682 if (iter == self->schema_map_.end()) {
686 for (
const auto& schema : iter->second) {
691 if (!is_flatbuffers_schema_type(schema.encoding) || schema.data.empty()) {
701 inline void SchemaPluginBase::clear_flatbuffers_parser_cache_locked(
const std::string& name) {
702 auto iter = flatbuffers_parser_map_.find(name);
704 if (iter == flatbuffers_parser_map_.end()) {
708 retired_flatbuffers_parsers_.insert(retired_flatbuffers_parsers_.end(), iter->second.begin(), iter->second.end());
709 flatbuffers_parser_map_.erase(iter);
static void init_memory_pool() noexcept
Eagerly constructs the process-wide MemoryPool that backs heap allocations.
static Bytes create(size_t size, uint8_t offset=0) noexcept
Allocates an owned buffer of the requested size with an optional header offset.
static Bytes deep_copy(uint8_t *data, size_t size, uint8_t offset=0) noexcept
Produces an owned copy of an external mutable buffer.
Reusable base implementation of SchemaPluginInterface for mixed Protobuf / FlatBuffers metadata.
Definition: schema_plugin_base.h:96
~SchemaPluginBase() override
Releases cached dynamic messages and FlatBuffers parsers owned by the plugin.
Definition: schema_plugin_base.h:208
FlatbuffersParserPtr create_flatbuffers_parser(const std::string &name) override
Returns a freshly constructed FlatBuffers parser for the named root type.
Definition: schema_plugin_base.h:489
ProtobufMessagePtr create_protobuf_message(const std::string &name) override
Returns a cached dynamic Protobuf message instance for the type.
Definition: schema_plugin_base.h:407
SchemaData search_schema(const std::string &name, SchemaType schema_type=SchemaType::kUnknown) override
Locates a schema by name within an optional family.
Definition: schema_plugin_base.h:238
FlatbuffersSchemaPtr search_flatbuffers_schema(const std::string &name) override
Returns a verified FlatBuffers BFBS reflection schema.
Definition: schema_plugin_base.h:438
SchemaPluginBase()
Initialises the schema caches and primes the VLink memory pool.
Definition: schema_plugin_base.h:200
std::vector< SchemaData > get_all_schemas(SchemaType schema_type=SchemaType::kUnknown) override
Returns every cached schema, optionally restricted to a single family.
Definition: schema_plugin_base.h:360
ProtobufDescriptorPtr search_protobuf_descriptor(const std::string &name) override
Returns the Protobuf descriptor for a fully qualified type name.
Definition: schema_plugin_base.h:395
Polymorphic contract for runtime schema lookup and dynamic message construction.
Definition: schema_plugin_interface.h:78
void * ProtobufMessagePtr
Opaque alias for a google::protobuf::Message pointer.
Definition: schema_plugin_interface.h:103
void * FlatbuffersSchemaPtr
Opaque alias for a reflection::Schema pointer (FlatBuffers BFBS).
Definition: schema_plugin_interface.h:108
void * ProtobufDescriptorPtr
Opaque alias for a google::protobuf::Descriptor pointer.
Definition: schema_plugin_interface.h:94
void * FlatbuffersParserPtr
Opaque alias for a flatbuffers::Parser pointer pre-loaded with a root type.
Definition: schema_plugin_interface.h:113
Process-local lookup table for compiled-in FlatBuffers BFBS reflection blobs.
#define VUNLIKELY(...)
Short alias for VLINK_UNLIKELY.
Definition: macros.h:289
#define VLIKELY(...)
Short alias for VLINK_LIKELY.
Definition: macros.h:284
#define VLINK_DISALLOW_COPY_AND_ASSIGN(classname)
Deletes the copy constructor and copy-assignment operator of classname.
Definition: macros.h:174
bool has_startwith(const std::string &str, const char(&target)[SizeT]) noexcept
Compile-time prefix check against a string literal.
Definition: helpers.h:392
constexpr std::string_view get() noexcept
Returns the trimmed compile-time identifier for TypeT.
Definition: name_detector.h:603
SchemaType
Coarse runtime schema family used by discovery, bag metadata and proxy routing.
Definition: types.h:207
@ kUnknown
Decoding family unknown.
@ kZeroCopy
Decode through the VLink zero-copy structs.
@ kProtobuf
Decode through the Protocol Buffers stack.
@ kFlatbuffers
Decode through the FlatBuffers stack.
Header probe that enables Protobuf-backed schema-plugin code when available.
Abstract contract for runtime Protobuf / FlatBuffers schema discovery plugins.
Definition: serializer-inl.h:142
Wire-format-neutral wrapper around one serialised schema blob.
Definition: types.h:301
SchemaType schema_type
Coarse runtime family derived from encoding.
Definition: types.h:304
static bool is_real_type(SchemaType schema_type) noexcept
Returns whether schema_type carries concrete schema metadata.
static bool is_valid_type(SchemaType schema_type) noexcept
Returns whether schema_type is within the supported enum range.
std::string encoding
Schema encoding identifier, e.g. "protobuf" or "flatbuffers".
Definition: types.h:303
std::string name
Schema subject (usually a fully qualified message or table name).
Definition: types.h:302