Binary Protocols¶

#include "quill/Backend.h"
#include "quill/BinaryDataDeferredFormatCodec.h"
#include "quill/Frontend.h"
#include "quill/LogMacros.h"
#include "quill/Logger.h"
#include "quill/Utility.h"
#include "quill/sinks/ConsoleSink.h"

#include <array>
#include <iostream>
#include <sstream>
#include <utility>

#include "sample/CancelOrder.h"
#include "sample/NewOrder.h"

/**
 * @brief Efficient binary data logging with deferred formatting
 *
 * This example demonstrates how to efficiently log variable-sized binary data such as network messages
 * or binary protocol buffers (like SBE messages) using the library's deferred formatting capabilities.
 *
 * ## Key Benefits
 *
 * - **Performance**: Only performs a memory copy on the hot path (critical logging section)
 * - **Deferred Formatting**: All parsing and formatting happen in the background logging thread
 * - **Human-Readable Output**: Formats binary data appropriately for log files
 *
 * ## How It Works
 *
 * 1. Create a tag struct to identify your binary protocol type
 * 2. Define a type alias using `quill::BinaryData<YourTag>`
 * 3. Implement a formatter for your binary data type
 * 4. Specialize `quill::Codec` to use `quill::BinaryDataDeferredFormatCodec`
 * 5. Log your binary data using the type alias
 *
 * This example uses SBE (Simple Binary Encoding) messages as a demonstration.
 *
 * SBE is a binary encoding protocol often used in financial systems.
 * (See https://github.com/aeron-io/simple-binary-encoding for more information)
 */

/**
 * Step 1: Create a tag struct to give semantic meaning to your binary protocol
 * This empty struct acts as a compile-time tag to identify the type of binary data.
 * You can create different tags for different protocols or message formats.
 */
struct TradingProtocol
{
};

/**
 * Step 2: Create a BinaryData specialization for your protocol
 * This type alias will be used when logging binary data of this specific protocol.
 */
using TradingProtocolData = quill::BinaryData<TradingProtocol>;

/**
 * Step 3: Implement a formatter for your binary data type
 * This formatter is called in the backend thread to convert binary data to a human-readable text.
 * Since the library always writes to human-readable log files, you must format binary data
 * appropriately.
 */
template <>
struct fmtquill::formatter<TradingProtocolData>
{
  constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }

  auto format(::TradingProtocolData const& bin_data, format_context& ctx) const
  {
    // Option 1: Convert binary data to hexadecimal representation
    // Useful when you don't need to parse the structure
    // return fmtquill::format_to(ctx.out(), "{}", quill::utility::to_hex(bin_data.data(), bin_data.size()));

    // Option 2: Parse the binary data into a meaningful representation
    // Get raw data pointer in the format expected by the SBE API
    char* data = reinterpret_cast<char*>(const_cast<std::byte*>(bin_data.data()));
    std::stringstream oss;

    // Parse the SBE message header to determine message type
    sbe::sample::MessageHeader header{data, bin_data.size()};

    if (header.templateId() == sbe::sample::NewOrder::sbeTemplateId())
    {
      sbe::sample::NewOrder msg;
      msg.wrapForDecode(data, sbe::sample::MessageHeader::encodedLength(), header.blockLength(),
                        header.version(), bin_data.size());
      oss << msg;
    }
    else if (header.templateId() == sbe::sample::CancelOrder::sbeTemplateId())
    {
      sbe::sample::CancelOrder msg;
      msg.wrapForDecode(data, sbe::sample::MessageHeader::encodedLength(), header.blockLength(),
                        header.version(), bin_data.size());
      oss << msg;
    }

    return fmtquill::format_to(ctx.out(), "{}", oss.str());
  }
};

/**
 * Step 4: Specialize the Codec for your binary data type
 */
template <>
struct quill::Codec<TradingProtocolData> : quill::BinaryDataDeferredFormatCodec<TradingProtocolData>
{
};

int main()
{
  // Initialize Quill backend
  quill::BackendOptions backend_options;
  quill::Backend::start(backend_options);

  // Create a console sink and logger
  auto console_sink = quill::Frontend::create_or_get_sink<quill::ConsoleSink>("sink_id_1");
  quill::Logger* logger = quill::Frontend::create_or_get_logger(
    "trading", std::move(console_sink),
    quill::PatternFormatterOptions{"[%(time)] %(message)", "%H:%M:%S.%Qns", quill::Timezone::GmtTime});

  // Sample data for our demonstration
  std::array<std::string, 9> symbols = {"AAPL", "MSFT", "AMZN", "GOOGL", "META",
                                        "TSLA", "NVDA", "PYPL", "NFLX"};

  std::array<std::string, 4> cancel_reasons = {"User requested", "Risk limit exceeded",
                                               "Price away from market", "Timeout"};

  // Buffer for encoding SBE messages
  std::array<char, 128> buffer{};

  // Example 1: Log new order messages
  for (uint32_t i = 0; i < symbols.size(); ++i)
  {
    sbe::sample::NewOrder order;
    order.wrapAndApplyHeader(buffer.data(), 0, buffer.size());
    order.orderId(i);
    order.price(10000 + i * 1000);
    order.quantity(1000 + i * 500);
    order.side(i % 2 == 0 ? sbe::sample::Side::BUY : sbe::sample::Side::SELL);
    order.putSymbol(symbols[i]);

    size_t const encoded_size = order.encodedLength() + sbe::sample::MessageHeader::encodedLength();

    // Step 5: Log the binary data using TradingProtocolData
    // Only a memcpy happens here (on the hot path)
    // The actual formatting will be deferred to the backend thread
    LOG_INFO(logger, "[SEND] {}", TradingProtocolData{reinterpret_cast<uint8_t*>(buffer.data()), encoded_size});
  }

  // Log cancel order messages for some orders
  for (uint32_t i = 0; i < 5; ++i)
  {
    sbe::sample::CancelOrder cancel;
    cancel.wrapAndApplyHeader(buffer.data(), 0, buffer.size());
    cancel.orderId(2000000 + i);
    cancel.origOrderId(i);
    cancel.cancelQuantity(500 + i * 100);
    cancel.putReason(cancel_reasons[i % cancel_reasons.size()]);

    size_t const encoded_size = cancel.encodedLength() + sbe::sample::MessageHeader::encodedLength();

    // Step 5: Log using the same pattern - formatting happens in the backend thread
    LOG_INFO(logger, "[SEND] {}", TradingProtocolData{reinterpret_cast<uint8_t*>(buffer.data()), encoded_size});
  }

  return 0;
}

#include "quill/Backend.h"
#include "quill/BinaryDataDeferredFormatCodec.h"
#include "quill/Frontend.h"
#include "quill/LogMacros.h"
#include "quill/Logger.h"
#include "quill/Utility.h"
#include "quill/sinks/ConsoleSink.h"

#include <array>
#include <iostream>
#include <sstream>
#include <utility>

#if defined(_WIN32) && defined(_MSC_VER) && !defined(__GNUC__)
  #pragma warning(push)
  #pragma warning(disable : 4996)
#endif

/**
 * @brief Efficient binary data logging with deferred formatting
 *
 * This example demonstrates how to efficiently log binary data (such as network packets,
 * protocol buffers, or market data) using Quill's deferred formatting capabilities.
 *
 * Key benefits:
 * 1. Minimal overhead on the critical path - a memory copy is performed
 * 2. All formatting is deferred to the background logger thread
 * 3. Human-readable output for binary data in log files
 * 4. Support for different message types within the same binary protocol
 */

//------------------------------------------------------------------------------
// Sample message types for demonstration purposes
// (These would be your actual protocol messages in a real application)
//------------------------------------------------------------------------------

struct Position
{
  uint32_t id;
  uint32_t width;
  uint32_t height;
};

std::ostream& operator<<(std::ostream& os, Position const& position)
{
  os << "Position {" << position.width << ", " << position.height << "}";
  return os;
}

struct StateInfo
{
  uint32_t id;
  int64_t timestamp;
  int64_t magnitude;
  bool active;
};

std::ostream& operator<<(std::ostream& os, StateInfo const& state)
{
  os << "StateInfo {" << state.timestamp << ", " << state.magnitude << ", " << state.active << "}";
  return os;
}

struct Entity
{
  uint32_t id;
  char name[24];
};

std::ostream& operator<<(std::ostream& os, Entity const& entity)
{
  os << "Entity {" << entity.name << "}";
  return os;
}

//------------------------------------------------------------------------------
// Step 1: Create a tag struct to identify your binary protocol
//------------------------------------------------------------------------------

struct TypeProvider
{
};

//------------------------------------------------------------------------------
// Step 2: Define your binary data type using the tag
//------------------------------------------------------------------------------

using BinaryTypeData = quill::BinaryData<TypeProvider>;

//------------------------------------------------------------------------------
// Step 4: Implement a formatter for your binary data type
// This formatter runs in the background logger thread
//------------------------------------------------------------------------------

template <>
struct fmtquill::formatter<BinaryTypeData>
{
  constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }

  auto format(::BinaryTypeData const& bin_data, format_context& ctx) const
  {
    // Check if we have enough data to read the message ID
    if (bin_data.size() < sizeof(uint32_t))
    {
      return fmtquill::format_to(ctx.out(), "Invalid binary data: too small");
    }

    // Extract the message ID (first 4 bytes)
    uint32_t id;
    std::memcpy(&id, bin_data.data(), sizeof(uint32_t));

    std::stringstream oss;

    // Format based on message type
    switch (id)
    {
    case 1:
      if (bin_data.size() >= sizeof(Position))
      {
        Position position;
        std::memcpy(&position, bin_data.data(), sizeof(Position));
        oss << position;
      }
      else
      {
        oss << "Incomplete Position message";
      }
      break;

    case 2:
      if (bin_data.size() >= sizeof(StateInfo))
      {
        StateInfo state;
        std::memcpy(&state, bin_data.data(), sizeof(StateInfo));
        oss << state;
      }
      else
      {
        oss << "Incomplete StateInfo message";
      }
      break;

    case 3:
      if (bin_data.size() >= sizeof(Entity))
      {
        Entity entity;
        std::memcpy(&entity, bin_data.data(), sizeof(Entity));
        oss << entity;
      }
      else
      {
        oss << "Incomplete Entity message";
      }
      break;

    default:
      oss << "Unknown message type: " << id;
      break;
    }

    // Add a hex dump of the raw data
    static constexpr size_t upper_case_hex = false;
    oss << " <" << quill::utility::to_hex(bin_data.data(), bin_data.size(), upper_case_hex) << ">";

    return fmtquill::format_to(ctx.out(), "{}", oss.str());
  }
};

//------------------------------------------------------------------------------
// Step 4: Tell Quill to use deferred formatting for your binary data type
//------------------------------------------------------------------------------

template <>
struct quill::Codec<BinaryTypeData> : quill::BinaryDataDeferredFormatCodec<BinaryTypeData>
{
};

int main()
{
  // Initialize Quill backend
  quill::BackendOptions backend_options;
  quill::Backend::start(backend_options);

  // Create a console sink and logger
  auto console_sink = quill::Frontend::create_or_get_sink<quill::ConsoleSink>("sink_id_1");
  quill::Logger* logger = quill::Frontend::create_or_get_logger(
    "trading", std::move(console_sink),
    quill::PatternFormatterOptions{"[%(time)] %(message)", "%H:%M:%S.%Qns", quill::Timezone::GmtTime});

  // Buffer for our binary messages
  std::array<uint8_t, 128> buffer{};
  uint32_t message_size{0};

  // Log different types of data in rotation
  for (uint32_t i = 0; i < 15; ++i)
  {
    // Simulate encoding into the buffer different message types
    if ((i % 3) == 0)
    {
      // encode position data
      Position position;
      position.id = 1;
      position.width = i;
      position.height = i * 10;

      std::memcpy(buffer.data(), &position, sizeof(Position));
      message_size = sizeof(Position);
    }
    else if ((i % 3) == 1)
    {
      // encode state info data
      StateInfo state;
      state.id = 2;
      state.timestamp = i * 1000;
      state.magnitude = i * 10000;
      state.active = (i % 2 == 0); // Alternate between true and false

      std::memcpy(buffer.data(), &state, sizeof(StateInfo));
      message_size = sizeof(StateInfo);
    }
    else if ((i % 3) == 2)
    {
      // encode entity data
      Entity entity;
      entity.id = 3;
      auto name = std::to_string(i);
      strcpy(&entity.name[0], name.c_str());

      std::memcpy(buffer.data(), &entity, sizeof(Entity));
      message_size = sizeof(Entity);
    }

    // Pass the buffer for logging
    LOG_INFO(logger, "[IN] {}", BinaryTypeData{buffer.data(), message_size});
  }

  return 0;
}

#if defined(_WIN32) && defined(_MSC_VER) && !defined(__GNUC__)
  #pragma warning(pop)
#endif

#include "quill/Backend.h"
#include "quill/BinaryDataDeferredFormatCodec.h"
#include "quill/Frontend.h"
#include "quill/LogMacros.h"
#include "quill/Logger.h"
#include "quill/Utility.h"
#include "quill/sinks/FileSink.h"

#include <array>
#include <fstream>
#include <iostream>
#include <sstream>
#include <utility>

#if defined(_WIN32) && defined(_MSC_VER) && !defined(__GNUC__)
  #pragma warning(push)
  #pragma warning(disable : 4996)
#endif

struct TemperatureReading
{
  uint32_t id;
  uint32_t celsius;
  uint32_t humidity;
};

std::ostream& operator<<(std::ostream& os, TemperatureReading const& temp)
{
  os << "Temperature {" << temp.celsius << "°C, " << temp.humidity << "% humidity}";
  return os;
}

struct WindData
{
  uint32_t id;
  int64_t timestamp;
  int64_t speed_kmh;
  bool storm_warning;
};

std::ostream& operator<<(std::ostream& os, WindData const& wind)
{
  os << "Wind {" << wind.timestamp << ", " << wind.speed_kmh << " km/h, storm: " << wind.storm_warning << "}";
  return os;
}

struct WeatherStation
{
  uint32_t id;
  char location[24];
};

std::ostream& operator<<(std::ostream& os, WeatherStation const& station)
{
  os << "Station {" << station.location << "}";
  return os;
}

struct WeatherProtocol
{
};

using WeatherBinaryData = quill::BinaryData<WeatherProtocol>;

template <>
struct fmtquill::formatter<WeatherBinaryData>
{
  constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }

  auto format(::WeatherBinaryData const& bin_data, format_context& ctx) const
  {
    auto out = ctx.out();
    auto size = bin_data.size();
    char const* size_ptr = reinterpret_cast<char const*>(&size);
    out = std::copy(size_ptr, size_ptr + sizeof(size), out);
    char const* data_ptr = reinterpret_cast<char const*>(bin_data.data());
    out = std::copy(data_ptr, data_ptr + size, out);
    return out;
  }
};

template <>
struct quill::Codec<WeatherBinaryData> : quill::BinaryDataDeferredFormatCodec<WeatherBinaryData>
{
};

int main()
{
  // Initialize Quill backend
  quill::BackendOptions backend_options;
  backend_options.check_printable_char = {};
  quill::Backend::start(backend_options);

  quill::PatternFormatterOptions binary_pfo;
  binary_pfo.format_pattern = "%(message)";
  binary_pfo.add_metadata_to_multi_line_logs = false;
  binary_pfo.pattern_suffix = quill::PatternFormatterOptions::NO_SUFFIX;

  // Create a file sink and logger
  auto file_sink = quill::Frontend::create_or_get_sink<quill::FileSink>("weather_data.bin",
                                                                        []()
                                                                        {
                                                                          quill::FileSinkConfig cfg;
                                                                          cfg.set_open_mode("wb");
                                                                          return cfg;
                                                                        }());

  quill::Logger* logger =
    quill::Frontend::create_or_get_logger("weather_monitor", std::move(file_sink), binary_pfo);

  // Buffer for our binary messages
  std::array<uint8_t, 128> buffer{};
  uint32_t message_size{0};

  // Log different types of weather data in rotation
  for (uint32_t i = 0; i < 15; ++i)
  {
    // Simulate encoding into the buffer different weather message types
    if ((i % 3) == 0)
    {
      // encode temperature reading data
      TemperatureReading temp;
      temp.id = 1;
      temp.celsius = 20 + i;
      temp.humidity = 50 + (i * 2);

      std::memcpy(buffer.data(), &temp, sizeof(TemperatureReading));
      message_size = sizeof(TemperatureReading);
    }
    else if ((i % 3) == 1)
    {
      // encode wind data
      WindData wind;
      wind.id = 2;
      wind.timestamp = i * 1000;
      wind.speed_kmh = i * 5;
      wind.storm_warning = (i % 4 == 0); // Storm warning every 4th reading

      std::memcpy(buffer.data(), &wind, sizeof(WindData));
      message_size = sizeof(WindData);
    }
    else if ((i % 3) == 2)
    {
      // encode weather station data
      WeatherStation station;
      station.id = 3;
      auto location = "Station" + std::to_string(i);
      strcpy(&station.location[0], location.c_str());

      std::memcpy(buffer.data(), &station, sizeof(WeatherStation));
      message_size = sizeof(WeatherStation);
    }

    // Pass the buffer for logging
    LOG_INFO(logger, "{}", WeatherBinaryData{buffer.data(), message_size});
  }

  quill::Backend::stop();

  // --- Decoding Phase for demonstration ---
  std::ifstream infile("weather_data.bin", std::ios::binary);
  if (!infile)
  {
    std::cerr << "Failed to open weather_data.bin\n";
    return 1;
  }

  while (true)
  {
    // Read size header
    uint32_t size;
    if (!infile.read(reinterpret_cast<char*>(&size), sizeof(size)))
      break; // EOF

    std::vector<uint8_t> data(size);
    if (!infile.read(reinterpret_cast<char*>(data.data()), size))
      break;

    // Interpret message
    if (size < sizeof(uint32_t))
    {
      std::cout << "Invalid binary data: too small\n";
      continue;
    }

    uint32_t id;
    std::memcpy(&id, data.data(), sizeof(uint32_t));
    std::stringstream oss;

    switch (id)
    {
    case 1:
      if (size >= sizeof(TemperatureReading))
      {
        TemperatureReading temp;
        std::memcpy(&temp, data.data(), sizeof(TemperatureReading));
        oss << temp;
      }
      else
      {
        oss << "Incomplete Temperature message";
      }
      break;
    case 2:
      if (size >= sizeof(WindData))
      {
        WindData wind;
        std::memcpy(&wind, data.data(), sizeof(WindData));
        oss << wind;
      }
      else
      {
        oss << "Incomplete Wind data message";
      }
      break;
    case 3:
      if (size >= sizeof(WeatherStation))
      {
        WeatherStation station;
        std::memcpy(&station, data.data(), sizeof(WeatherStation));
        oss << station;
      }
      else
      {
        oss << "Incomplete Weather Station message";
      }
      break;
    default:
      oss << "Unknown weather message type: " << id;
      break;
    }

    std::cout << oss.str() << "\n";
  }

  return 0;
}

#if defined(_WIN32) && defined(_MSC_VER) && !defined(__GNUC__)
  #pragma warning(pop)
#endif