VLink  2.3.0
A high-performance communication middleware
bag_writer.h
Go to the documentation of this file.
1 /*
2  * Copyright (C) 2026 by Thun Lu. All rights reserved.
3  * Author: Thun Lu <thun.lu@zohomail.cn>
4  * Repo: https://github.com/thun-res/vlink
5  * _ __ __ _ __
6  * | | / / / / (_) ____ / /__
7  * | | / / / / / / / __ \ / //_/
8  * | |/ / / /___ / / / / / / / ,<
9  * |___/ /_____/ /_/ /_/ /_/ /_/|_|
10  *
11  * Licensed under the Apache License, Version 2.0 (the "License");
12  * you may not use this file except in compliance with the License.
13  * You may obtain a copy of the License at
14  *
15  * http://www.apache.org/licenses/LICENSE-2.0
16  *
17  * Unless required by applicable law or agreed to in writing, software
18  * distributed under the License is distributed on an "AS IS" BASIS,
19  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20  * See the License for the specific language governing permissions and
21  * limitations under the License.
22  */
23 
24 /**
25  * @file bag_writer.h
26  * @brief Abstract VLink bag recorder with split, compression, schema embedding and a global hook.
27  *
28  * @details
29  * @c BagWriter is the polymorphic base for VLink offline recording. It exposes a
30  * @c push() entry point, which either enqueues serialised messages onto a private
31  * @c MessageLoop or persists them synchronously according to @c Config::sync_mode.
32  * Two backends ship with VLink:
33  *
34  * - @c VDBWriter for SQLite-backed @c .vdb / @c .vdbx containers; default codec is LZAV
35  * for @c kCompressAuto and @c kCompressLzav selectors.
36  * - @c VCAPWriter for MCAP-format @c .vcap / @c .vcapx containers; @c kCompressAuto and
37  * @c kCompressZstd select Zstandard when Zstd support is compiled in.
38  *
39  * Default asynchronous writer state machine:
40  *
41  * @verbatim
42  * async_run() push() close()/dtor
43  * +---------+ ----------> +-----------+ ---------> +---------+ ----------> +---------+
44  * | Open | | Running | <--------- | Pending | | Closed |
45  * +---------+ +-----------+ ack +---------+ +---------+
46  * ^ ^
47  * | |
48  * +--- split_by_size / split_by_time -- rotate --+
49  * @endverbatim
50  *
51  * On-disk layout produced by the writers:
52  *
53  * @verbatim
54  * +---------+----------------+---------------+----------------+--------+
55  * | Header | URL index | Schema index | Message stream | Footer |
56  * +---------+----------------+---------------+----------------+--------+
57  * tag url_metas schema_data payloads finalisation
58  * app
59  * timezone
60  * @endverbatim
61  *
62  * Feature highlights:
63  * - Writer-wide asynchronous or synchronous record policy selected by @c Config::sync_mode.
64  * - File splitting by byte size and/or by wall-clock interval, with optional oldest-file eviction.
65  * - Optional WAL mode for SQLite crash resilience.
66  * - URL-level loss reporting via @c set_url_loss().
67  * - Schema embedding through @c push_schema() for offline introspection.
68  * - Process-global writer triggered by the @c VLINK_BAG_PATH environment variable.
69  *
70  * @par Example
71  * @code
72  * vlink::BagWriter::Config cfg;
73  * cfg.compress = vlink::BagWriter::kCompressAuto;
74  * cfg.split_by_size = 1024LL * 1024LL * 512; // 512 MiB per split
75  * cfg.max_split_count = 10; // retain the newest 10 splits
76  *
77  * auto writer = vlink::BagWriter::create("/data/drive_log.vdbx", cfg);
78  * writer->async_run();
79  *
80  * vlink::Frame frame;
81  * frame.timestamp = -1; // < 0 => writer auto-assigns from its clock (0 is verbatim)
82  * frame.url = "dds://camera/front";
83  * frame.ser_type = "demo.proto.Image";
84  * frame.schema_type = vlink::SchemaType::kProtobuf;
85  * frame.action_type = vlink::ActionType::kPublish;
86  * frame.data = bytes;
87  * writer->push(frame);
88  * writer->wait_for_idle();
89  * writer->quit();
90  * writer->wait_for_quit();
91  * writer->close();
92  * if (writer->fail()) { handle_recording_error(); }
93  * @endcode
94  *
95  * @par Global writer
96  * @code
97  * // Set VLINK_BAG_PATH=/data/global.vdb before process launch.
98  * if (auto* gw = vlink::BagWriter::global_get(); gw != nullptr) {
99  * vlink::Frame frame;
100  * frame.timestamp = -1; // < 0 => auto-assign (0 would be recorded verbatim)
101  * frame.url = "intra://debug";
102  * frame.ser_type = "raw";
103  * frame.schema_type = vlink::SchemaType::kRaw;
104  * frame.action_type = vlink::ActionType::kPublish;
105  * frame.data = bytes;
106  * gw->push(frame);
107  * }
108  * @endcode
109  *
110  * @note @c push() is thread-safe. @c Config::sync_mode selects synchronous writes for the
111  * writer's entire lifetime; otherwise writes are queued on the recording loop.
112  */
113 
114 #pragma once
115 
116 #include <chrono>
117 #include <cstdint>
118 #include <memory>
119 #include <mutex>
120 #include <string>
121 #include <string_view>
122 #include <unordered_map>
123 #include <unordered_set>
124 #include <vector>
125 
126 #include "../base/functional.h"
127 #include "../base/macros.h"
128 #include "../base/message_loop.h"
129 #include "../impl/types.h"
130 
131 namespace vlink {
132 
133 class BagPluginInterface;
134 class SchemaPluginInterface;
135 
136 /**
137  * @class BagWriter
138  * @brief Asynchronous VLink message recorder built on top of @c MessageLoop.
139  *
140  * @details
141  * Construct via @c create() (or directly) and call @c async_run() to start the recording
142  * thread, then push messages with @c push(). Concrete subclasses implement every virtual
143  * persistence operation; the base class owns the shared bookkeeping and the loop wiring.
144  */
146  public:
147  /**
148  * @brief Compression codec selector understood by the writer backends.
149  *
150  * | Value | Algorithm | Notes |
151  * | --------------- | --------- | ------------------------------------------------------ |
152  * | kCompressNone | none | Payloads stored as raw bytes |
153  * | kCompressAuto | backend | Uses the backend default (LZAV for VDB, Zstd for MCAP) |
154  * | kCompressZstd | Zstandard | Active for MCAP when Zstd support is available |
155  * | kCompressLz4 | LZ4 | Reserved selector; not currently used by built-ins |
156  * | kCompressLzav | LZAV | Active for SQLite-backed VDB recordings |
157  */
158  enum CompressType : uint8_t {
159  kCompressNone = 0, ///< Store payloads uncompressed.
160  kCompressAuto = 1, ///< Defer codec choice to the active backend.
161  kCompressZstd = 2, ///< Force Zstandard codec where supported.
162  kCompressLz4 = 3, ///< Reserved selector; no built-in writer emits LZ4 today.
163  kCompressLzav = 4, ///< Force LZAV codec where supported.
164  };
165 
166  /**
167  * @struct Config
168  * @brief Recording behaviour, split policy and resource budgets.
169  *
170  * @details
171  * Sizes are expressed in bytes and durations in milliseconds unless explicitly stated.
172  */
173  struct Config final {
174  std::string tag_name; ///< Optional tag stored in the bag header.
175  CompressType compress{CompressType::kCompressNone}; ///< Compression codec selector.
176  bool wal_mode{false}; ///< Enable SQLite WAL for crash resilience.
177  bool enable_limit{false}; ///< When true, evict oldest rows at the row/byte limit.
178  bool split_name_by_time{false}; ///< Append a timestamp suffix to split filenames.
179  bool sync_mode{false}; ///< Write synchronously and disable the VDB periodic cache-flush timer.
180  bool optimize_on_exit{false}; ///< Run VACUUM/OPTIMIZE while closing the file.
181  bool enable_chunk_crc{false}; ///< Compute VCAP chunk CRC; ignored by VDB.
182  int64_t max_row_count{5'000'000'000LL}; ///< SQLite row cap; either evicts or fails new writes.
183  int64_t max_bytes_size{1024LL * 1024LL * 1024LL * 512LL}; ///< SQLite byte cap; either evicts or fails new writes.
184  int64_t split_by_size{1024LL * 1024LL * 1024LL * 1LL}; ///< Split threshold in bytes (0 disables).
185  int64_t split_by_time{0}; ///< Split interval in milliseconds (0 disables).
186  int64_t max_split_count{0}; ///< Split-file retention cap (0 is unlimited).
187  int64_t begin_time{0}; ///< Anchor (ms) used by time-based splits.
188  int64_t cache_size{1024LL * 1024LL * 4}; ///< VDB commit chunk / MCAP chunk size in bytes.
189  int64_t compress_start_size{128}; ///< Minimum payload size eligible for compression.
190  int64_t compress_level{3}; ///< Codec-specific compression level.
191  int64_t max_task_depth{20000}; ///< Maximum pending writes in the loop queue.
192  int64_t max_memory_size{1024LL * 1024LL * 1024LL * 2LL}; ///< Maximum in-memory cache size in bytes.
193  int64_t start_timestamp{0}; ///< Override for the wall-clock start timestamp (ms).
194  std::unordered_set<std::string> ignore_compress_urls; ///< URLs whose payloads must never be compressed.
195 
196  Config() {} // NOLINT(modernize-use-equals-default)
197  };
198 
199  /**
200  * @brief Notification fired when the writer rotates to a new split file.
201  *
202  * @details
203  * Called with the zero-based split index and the new file path. The @c before flag of
204  * @c register_split_callback() chooses whether the hook runs before or after the
205  * rotation is committed.
206  */
207  using SplitCallback = MoveFunction<void(int split_index, const std::string& split_filename)>;
208 
209  /**
210  * @brief Schema resolver used by the writer when a previously unseen URL is recorded.
211  *
212  * @details
213  * The writer passes the requested serialisation type together with a coarse schema
214  * family hint so that families sharing a single type name (e.g. Protobuf vs Arrow) can
215  * still be disambiguated.
216  */
217  using SchemaCallback = MoveFunction<SchemaData(const std::string& ser_type, SchemaType schema_type)>;
218 
219  /**
220  * @brief System clock alias used when formatting timestamps into split file names.
221  */
222  using SystemClock = std::chrono::time_point<std::chrono::system_clock, std::chrono::milliseconds>;
223 
224  /**
225  * @brief Builds the concrete writer matching the extension of @p path.
226  *
227  * @details
228  * Suffix dispatch: @c .vdb / @c .vdbx select @c VDBWriter, @c .vcap / @c .vcapx select
229  * @c VCAPWriter; other suffixes return @c nullptr. The returned writer is open immediately;
230  * asynchronous writers need @c async_run(), while synchronous writers do not.
231  *
232  * @param path Output file path.
233  * @param config Recording configuration.
234  * @return Shared pointer to the new writer, or @c nullptr on unsupported suffix.
235  */
236  [[nodiscard]] static std::shared_ptr<BagWriter> create(const std::string& path, const Config& config = {});
237 
238  /**
239  * @brief Returns the cached writer for @p path, lazily creating and starting one.
240  *
241  * @details
242  * Looks up the process-wide writer registry. When no entry exists, a writer is built
243  * by @c create(), its loop is started with @c async_run(), and it is registered for
244  * reuse. The registry releases the entry automatically when the last shared owner
245  * goes away. Unsupported suffixes return @c nullptr and are not registered.
246  *
247  * @param path Output file path.
248  * @return Shared pointer to a started writer, or @c nullptr on unsupported suffix.
249  */
250  [[nodiscard]] static std::shared_ptr<BagWriter> filter_get(const std::string& path);
251 
252  /**
253  * @brief Returns the singleton writer driven by the @c VLINK_BAG_PATH environment variable.
254  *
255  * @details
256  * On first call, the writer is created from @c VLINK_BAG_PATH and started. Returns
257  * @c nullptr when the environment variable is absent or carries an unsupported suffix.
258  *
259  * @return Raw pointer to the global writer, or @c nullptr.
260  */
262 
263  /**
264  * @brief Constructs the base writer and opens the output file.
265  *
266  * @details
267  * The recording loop is not yet running; call @c async_run() when @c Config::sync_mode is false.
268  * A synchronous writer performs frame, schema and plugin-output writes on the calling thread and
269  * does not require a recording-loop thread.
270  *
271  * @param path Output file path.
272  * @param config Recording configuration.
273  */
274  explicit BagWriter(const std::string& path, const Config& config = {});
275 
276  /**
277  * @brief Halts the loop, flushes pending writes and closes the file.
278  */
279  virtual ~BagWriter(); // NOLINT(modernize-use-override)
280 
281  /**
282  * @brief Attaches a custom frame-forwarding plugin to this writer.
283  *
284  * @details
285  * The plugin's @c on_write() hook runs for every frame before it is persisted; it re-emits each
286  * frame through @c do_callback(), and may transcode, drop, fan out, or buffer and reorder frames by
287  * their true data-plane time (a sliding-window reorder) before they reach the bag. The writer
288  * supplies the record sink via @c BagPluginInterface::register_callback() and binds the plugin with
289  * @c BagPluginInterface::Direction::kWrite. Passing @c nullptr detaches and clears the previous
290  * plugin's sink.
291  *
292  * @param bag_interface Plugin interface instance, or @c nullptr to detach the current binding.
293  *
294  * @see clear_bag_interface() for the named equivalent of passing @c nullptr.
295  */
296  virtual void bind_bag_interface(const std::shared_ptr<BagPluginInterface>& bag_interface);
297 
298  /**
299  * @brief Detaches the currently bound plugin, if any.
300  *
301  * @details
302  * Convenience wrapper equivalent to @c bind_bag_interface(nullptr): flushes the bound plugin's
303  * pending frames, clears its record sink and drops the binding. Safe to call when no plugin is
304  * bound (in which case it is a no-op).
305  */
306  virtual void clear_bag_interface();
307 
308  /**
309  * @brief Installs a hook fired around split rotation.
310  *
311  * @param callback Receives the new split index and the new file path.
312  * @param before When true, the hook fires before the new file is opened; otherwise after.
313  */
314  virtual void register_split_callback(SplitCallback&& callback, bool before) = 0;
315 
316  /**
317  * @brief Installs the resolver invoked when an unseen serialisation type is recorded.
318  *
319  * @param callback Function mapping (ser_type, schema_type) to @c SchemaData.
320  */
321  virtual void register_schema_callback(SchemaCallback&& callback) = 0;
322 
323  /**
324  * @brief Embeds a schema descriptor into the bag for downstream introspection.
325  *
326  * The operation follows @c Config::sync_mode: synchronous writers merge on the caller's thread;
327  * asynchronous writers enqueue the merge on the recording loop.
328  *
329  * @param schema_data Schema descriptor to persist.
330  * @return @c true on success; @c false when a synchronous merge fails or an asynchronous merge
331  * task cannot be enqueued.
332  */
333  virtual bool push_schema(const SchemaData& schema_data) = 0;
334 
335  /**
336  * @brief Records a single frame to the bag.
337  *
338  * @details
339  * The write follows the mode fixed at construction: @c Config::sync_mode writes on the caller's
340  * thread; otherwise a task is enqueued on the recording loop.
341  * Once accepted, an asynchronous frame is not evicted to admit a later frame.
342  * When @c frame.timestamp is negative the writer assigns a recording-relative timestamp from its
343  * elapsed clock; a non-negative @c frame.timestamp (including @c 0) is recorded verbatim.
344  *
345  * When a plugin is bound via @c bind_bag_interface(), the frame is handed to the plugin's
346  * @c on_write() hook, which re-emits it (possibly transcoded, dropped, fanned out or reordered)
347  * through the writer's record sink into the concrete @c record() implementation. Because the
348  * plugin may emit asynchronously, the return value is then the assigned timestamp rather than a
349  * per-frame record result; a frame the plugin drops simply never reaches @c record().
350  *
351  * @param frame Frame to record. @c url must not be empty; @c timestamp < 0 requests auto-assign.
352  * @return Assigned timestamp in microseconds, or a negative value on validation/write failure or when an
353  * asynchronous write cannot be queued (for example, because a task or memory limit was reached).
354  */
355  int64_t push(const Frame& frame);
356 
357  /**
358  * @brief Streaming shorthand for @c push(frame).
359  *
360  * @details
361  * Records @p frame according to @c Config::sync_mode and returns the writer so calls can be chained,
362  * e.g. @c *writer << frame_a << frame_b. The per-frame timestamp that @c push() returns is not surfaced; instead,
363  * a negative @c push() result (e.g. an empty URL, a queue or memory-limit rejection, or a synchronous
364  * record failure forwarded by a bound plugin) latches the @c fail() state so failures are observable
365  * without inspecting every return value.
366  *
367  * @param frame Frame to record; @c url must not be empty, @c timestamp < 0 requests auto-assign.
368  * @return Reference to @c *this for chaining.
369  */
370  BagWriter& operator<<(const Frame& frame);
371 
372  /**
373  * @brief Streaming shorthand for @c push_schema(schema_data).
374  *
375  * @details
376  * Embeds @p schema_data according to @c Config::sync_mode and returns the writer for chaining,
377  * e.g. @c *writer << schema << frame. A @c false result -- the
378  * merge task could not be enqueued, or a bound backend rejected it -- latches the @c fail() state.
379  *
380  * @param schema_data Schema descriptor to persist.
381  * @return Reference to @c *this for chaining.
382  */
383  BagWriter& operator<<(const SchemaData& schema_data);
384 
385  /**
386  * @brief Returns whether a stream operation, deferred backend write or finalisation has failed.
387  *
388  * @details
389  * Latches when a stream insertion is rejected, a concrete backend cannot persist an accepted asynchronous
390  * frame or schema, or @c close() cannot finalise the bag. Callers may wait for the queue to become idle and
391  * then query this method; call @c close() first when close-time metadata, footer or manifest failures must
392  * also be observed. Synchronous callers continue to use the return value from @c push() or @c push_schema().
393  * Cleared by @c clear().
394  */
395  [[nodiscard]] bool fail() const noexcept;
396 
397  /**
398  * @brief Reports whether the streaming write state is still good (no latched failure).
399  *
400  * @details
401  * Returns @c true while no write failure has been latched, so @c if (*writer << frame) tests the
402  * post-write state.
403  */
404  explicit operator bool() const noexcept;
405 
406  /**
407  * @brief Clears a latched @c fail() state so streaming writes can resume being observed.
408  */
409  void clear() noexcept;
410 
411  /**
412  * @brief Returns the backend-specific "dump in progress" flag.
413  */
414  [[nodiscard]] virtual bool is_dumping() const = 0;
415 
416  /**
417  * @brief Returns whether split mode is currently in effect.
418  *
419  * @return @c true when the bag uses a splittable multi-file container
420  * (e.g. a @c .vdbx / @c .vcapx suffix), in which case
421  * @c split_by_size / @c split_by_time control the rotation timing;
422  * @c false otherwise, regardless of the @c split_by_* values.
423  */
424  [[nodiscard]] virtual bool is_split_mode() const = 0;
425 
426  /**
427  * @brief Returns the zero-based index of the active split file.
428  *
429  * @return Active split index, or 0 outside split mode.
430  */
431  [[nodiscard]] virtual int get_split_index() const = 0;
432 
433  /**
434  * @brief Records the expected loss ratio for @p url as bag metadata.
435  *
436  * @details
437  * Loss values feed offline diagnostics so that intentional drops can be distinguished
438  * from unexpected loss.
439  *
440  * @param url Topic URL.
441  * @param loss Loss ratio; values greater than 1.0 are normalised to -1.
442  */
443  virtual void set_url_loss(const std::string& url, double loss);
444 
445  protected:
446  virtual int64_t record(const Frame& frame, int64_t timestamp) = 0;
447 
448  virtual int64_t get_record_timestamp() const = 0;
449 
450  public:
451  /**
452  * @brief Finalizes the backend file (final commit, metadata, footer) and latches any failure.
453  *
454  * @details
455  * Idempotent; invoked automatically at destruction. Callers that must verify the close-time
456  * writes call it explicitly and then query @c fail() while the writer is still alive. It is not
457  * synchronised against the recording loop. After producers have stopped, detach any bound write plugin with
458  * @c clear_bag_interface() so its buffered tail is emitted while the loop still accepts tasks; then call
459  * @c wait_for_idle(), @c quit() and @c wait_for_quit() before calling @c close() from another thread.
460  *
461  * @note Declared after the original virtual interface to preserve its vtable slot ordering.
462  */
463  virtual void close();
464 
465  /**
466  * @brief Formats a wall-clock timestamp with millisecond precision.
467  *
468  * @param current Time point to format; @c nullptr formats the current system time.
469  * @param file_format When true, produces the file-name-safe form @c YYYY-MM-DD_hh-mm-ss-mmm shared by
470  * generated bag names; otherwise the log form @c YYYY/MM/DD hh:mm:ss:mmm.
471  * @return Formatted timestamp string.
472  */
473  static std::string get_format_date(SystemClock* current = nullptr, bool file_format = false);
474 
475  protected:
476  std::string convert_recorded_url(const std::string& url) const;
477 
478  std::vector<std::string> recorded_urls_for_origin(const std::string& url) const;
479 
480  std::string recover_recorded_url(const std::string& url) const;
481 
482  void get_url_meta(const std::string& url, const std::string& ser, int& url_index, int& ser_index) const;
483 
484  void get_url_meta(int url_index, int ser_index, std::string& url, std::string& ser) const;
485 
486  std::mutex& sample_mutex();
487 
488  std::unordered_map<std::string, double>& url_loss_map_ref();
489 
490  std::unordered_map<std::string, double>& total_url_loss_map_ref();
491 
492  static const std::string& get_default_tag_name();
493 
494  static const std::string& get_default_app_name();
495 
496  static SchemaPluginInterface* get_schema_interface();
497 
498  static int32_t get_default_timezone_diff();
499 
500  static std::string_view convert_action(ActionType type);
501 
502  void flush_plugin();
503 
504  void detach_plugin();
505 
506  bool post_persistent_task(Callback&& callback);
507 
508  /**
509  * @brief Latches the writer failure state from a concrete backend.
510  */
511  void set_fail() noexcept;
512 
513  private:
514  void learn_recorded_url(const std::string& origin_url, const std::string& recorded_url);
515 
516  struct Impl;
517  std::unique_ptr<Impl> impl_;
518 
520 };
521 
522 } // namespace vlink
#define VLINK_EXPORT
Definition: macros.h:81
#define VLINK_DISALLOW_COPY_AND_ASSIGN(classname)
Deletes the copy constructor and copy-assignment operator of classname.
Definition: macros.h:174