weaselab/weaseldb
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Decouple parser from CommitRequest

Browse Source
This commit is contained in:
Andrew Noyes
2025-08-17 13:36:53 -04:00
parent db2285dfda
commit fa2a2e4427
10 changed files with 636 additions and 460 deletions
Download Patch File Download Diff File Expand all files Collapse all files

264
src/commit_request.hpp
View File

@@ -1,13 +1,11 @@
#pragma once
#include "arena_allocator.hpp"
#include "json_token_enum.hpp"
#include <optional>
#include <span>
#include <string>
#include <string_view>
#include <vector>
#include <weaseljson/weaseljson.h>
/**
* @brief Represents a precondition for a commit request.
@@ -33,93 +31,13 @@ struct Operation {
};
/**
* @brief Represents a commit request as described in the API specification.
* @brief Format-agnostic commit request data structure.
*
* All string data is stored in the arena allocator to ensure efficient
* memory management and ownership.
* memory management and ownership. This class has no knowledge of any
* specific serialization formats or encoding schemes.
*/
class CommitRequest {
struct PreconditionParseState {
Precondition::Type type;
std::optional<uint64_t> version;
// These are owned by CommitRequest::arena
std::optional<std::string_view> key;
std::optional<std::string_view> begin;
std::optional<std::string_view> end;
};
/**
* @brief Internal state for parsing an operation during JSON processing.
*/
struct OperationParseState {
Operation::Type type;
// These are owned by CommitRequest::arena
std::optional<std::string_view> key;
std::optional<std::string_view> value;
std::optional<std::string_view> begin;
std::optional<std::string_view> end;
};
public:
// Parser state
enum class ParseState {
Root,
PreconditionsArray,
PreconditionObject,
OperationsArray,
OperationObject
};
enum class ParseStatus {
Incomplete, // Still need more data
Complete, // Successfully parsed complete JSON
Error // Parse error occurred
};
struct ParserContext {
using ArenaString = std::basic_string<char, std::char_traits<char>,
ArenaStlAllocator<char>>;
ArenaAllocator arena;
ParseState current_state = ParseState::Root;
JsonTokenType current_key_token;
// Only used if we need to accumulate the current key
ArenaString current_key;
ArenaString current_string;
ArenaString current_number;
bool in_key = false;
const char *parse_error = nullptr;
bool parse_complete = false;
// Current objects being parsed
PreconditionParseState current_precondition{};
OperationParseState current_operation{};
// Parsing state for nested structures
ArenaString precondition_type;
ArenaString operation_type;
// Constructor to initialize arena-allocated containers
explicit ParserContext()
: current_key(ArenaStlAllocator<char>(&arena)),
current_string(ArenaStlAllocator<char>(&arena)),
current_number(ArenaStlAllocator<char>(&arena)),
precondition_type(ArenaStlAllocator<char>(&arena)),
operation_type(ArenaStlAllocator<char>(&arena)) {}
void reset_arena_memory() {
arena.reset();
current_key = ArenaString{ArenaStlAllocator<char>(&arena)};
current_string = ArenaString{ArenaStlAllocator<char>(&arena)};
current_number = ArenaString{ArenaStlAllocator<char>(&arena)};
in_key = false;
current_precondition = {};
current_operation = {};
precondition_type = ArenaString{ArenaStlAllocator<char>(&arena)};
operation_type = ArenaString{ArenaStlAllocator<char>(&arena)};
current_state = ParseState::Root;
}
};
private:
ArenaAllocator arena_;
std::optional<std::string_view> request_id_;
@@ -128,10 +46,6 @@ private:
bool has_read_version_been_set_ = false;
std::vector<Precondition, ArenaStlAllocator<Precondition>> preconditions_;
std::vector<Operation, ArenaStlAllocator<Operation>> operations_;
ParserContext parser_context_;
WeaselJsonParser *json_parser_ =
WeaselJsonParser_create(64, &json_callbacks, this, 0);
static const WeaselJsonCallbacks json_callbacks;
public:
/**
@@ -139,16 +53,7 @@ public:
*/
explicit CommitRequest()
: arena_(), preconditions_(ArenaStlAllocator<Precondition>(&arena_)),
operations_(ArenaStlAllocator<Operation>(&arena_)), parser_context_() {}
/**
* @brief Destructor - cleans up any active parser.
*/
~CommitRequest() {
if (json_parser_) {
WeaselJsonParser_destroy(json_parser_);
}
}
operations_(ArenaStlAllocator<Operation>(&arena_)) {}
// Move constructor
CommitRequest(CommitRequest &&other) noexcept
@@ -156,19 +61,11 @@ public:
leader_id_(other.leader_id_), read_version_(other.read_version_),
has_read_version_been_set_(other.has_read_version_been_set_),
preconditions_(std::move(other.preconditions_)),
operations_(std::move(other.operations_)),
parser_context_(std::move(other.parser_context_)),
json_parser_(other.json_parser_) {
other.json_parser_ = nullptr;
}
operations_(std::move(other.operations_)) {}
// Move assignment operator
CommitRequest &operator=(CommitRequest &&other) noexcept {
if (this != &other) {
if (json_parser_) {
WeaselJsonParser_destroy(json_parser_);
}
arena_ = std::move(other.arena_);
request_id_ = other.request_id_;
leader_id_ = other.leader_id_;
@@ -176,69 +73,14 @@ public:
has_read_version_been_set_ = other.has_read_version_been_set_;
preconditions_ = std::move(other.preconditions_);
operations_ = std::move(other.operations_);
parser_context_ = std::move(other.parser_context_);
json_parser_ = other.json_parser_;
other.json_parser_ = nullptr;
}
return *this;
}
// Copy constructor and assignment are deleted (not safe with parser state)
// Copy constructor and assignment are deleted
CommitRequest(const CommitRequest &) = delete;
CommitRequest &operator=(const CommitRequest &) = delete;
/**
* @brief Parse a JSON string into a CommitRequest object (one-shot parsing).
* @param data Pointer to the JSON data buffer
* @param len Length of the data in bytes
* @return true if parsing succeeded, false otherwise
*/
bool parse_json(char *data, size_t len);
/**
* @brief Initialize streaming JSON parsing.
* @return true if initialization succeeded, false otherwise
*/
bool begin_streaming_parse();
/**
* @brief Parse additional JSON data incrementally.
* @param data Pointer to the data buffer
* @param len Length of the data
* @return ParseStatus indicating current parse state
*/
ParseStatus parse_chunk(char *data, size_t len);
/**
* @brief Finish streaming parse (call when no more data is available).
* @return ParseStatus indicating final parse result
*/
ParseStatus finish_streaming_parse();
/**
* @brief Check if parsing is complete and successful.
* @return true if parsing is complete and successful
*/
bool is_parse_complete() const {
return parser_context_.parse_complete && !parser_context_.parse_error &&
!leader_id_.empty() && has_read_version_been_set_;
}
/**
* @brief Check if there was a parse error.
* @return true if there was a parse error
*/
bool has_parse_error() const {
return parser_context_.parse_error != nullptr;
}
/**
* @brief Get the parse error message if there was an error.
* @return Error message string, or nullptr if no error
*/
const char *get_parse_error() const { return parser_context_.parse_error; }
/**
* @brief Get the request ID if present.
* @return Optional request ID
@@ -259,6 +101,12 @@ public:
*/
uint64_t read_version() const { return read_version_; }
/**
* @brief Check if read version has been explicitly set.
* @return true if read version was set during parsing
*/
bool has_read_version_been_set() const { return has_read_version_been_set_; }
/**
* @brief Get the preconditions.
* @return span of preconditions
@@ -289,6 +137,12 @@ public:
*/
const ArenaAllocator &arena() const { return arena_; }
/**
* @brief Get access to the underlying arena allocator for allocation.
* @return Reference to the arena allocator
*/
ArenaAllocator &arena() { return arena_; }
/**
* @brief Reset the commit request for reuse.
*/
@@ -300,48 +154,64 @@ public:
has_read_version_been_set_ = false;
preconditions_.clear();
operations_.clear();
// Reset parser state
if (json_parser_) {
WeaselJsonParser_reset(json_parser_);
}
parser_context_.reset_arena_memory();
parser_context_.current_state = ParseState::Root;
parser_context_.parse_error = nullptr;
parser_context_.parse_complete = false;
}
// Weaseljson callbacks (public for global callbacks)
static void on_begin_object(void *userdata);
static void on_end_object(void *userdata);
static void on_string_data(void *userdata, const char *buf, int len,
int done);
static void on_key_data(void *userdata, const char *buf, int len, int done);
static void on_begin_array(void *userdata);
static void on_end_array(void *userdata);
static void on_number_data(void *userdata, const char *buf, int len,
int done);
static void on_true_literal(void *userdata);
static void on_false_literal(void *userdata);
static void on_null_literal(void *userdata);
// Builder methods for setting data
// Note: All string_view parameters must point to arena-allocated memory
void set_request_id(std::string_view arena_allocated_request_id) {
request_id_ = arena_allocated_request_id;
}
void set_leader_id(std::string_view arena_allocated_leader_id) {
leader_id_ = arena_allocated_leader_id;
}
void set_read_version(uint64_t read_version) {
read_version_ = read_version;
has_read_version_been_set_ = true;
}
void add_precondition(Precondition::Type type, uint64_t version,
std::string_view arena_allocated_begin,
std::string_view arena_allocated_end = {}) {
preconditions_.push_back(Precondition{type, version, arena_allocated_begin,
arena_allocated_end});
}
void add_operation(Operation::Type type,
std::string_view arena_allocated_param1,
std::string_view arena_allocated_param2 = {}) {
operations_.push_back(
Operation{type, arena_allocated_param1, arena_allocated_param2});
}
private:
/**
* @brief Copy a string into the arena and return a string_view.
* Helper utility for external code that needs to copy data into arena memory.
* @param str The string to copy
* @return String view pointing to arena-allocated memory
*/
std::string_view store_string(std::string_view str);
std::string_view copy_to_arena(std::string_view str) {
if (str.empty()) {
return {};
}
void on_complete();
char *arena_str = arena_.allocate<char>(str.size());
std::memcpy(arena_str, str.data(), str.size());
return std::string_view(arena_str, str.size());
}
/**
* @brief Decode a base64 string and store it in the arena.
* @param base64_str The base64 encoded string
* @return String view of decoded data, or empty view if decoding failed
* @brief Apply any post-processing logic after data has been populated.
* This should be called after all data has been added to the request.
*/
std::string_view decode_base64(std::string_view base64_str);
void handle_completed_string(std::string_view s);
void handle_completed_number(std::string_view s);
};
void finalize() {
// Fill in default read version for preconditions that don't specify one
for (auto &precondition : preconditions_) {
if (precondition.version == 0) {
precondition.version = read_version_;
}
}
}
};

279
src/commit_request.cpp → src/json_commit_request_parser.cpp
View File

@@ -1,4 +1,4 @@
#include "commit_request.hpp"
#include "json_commit_request_parser.hpp"
#include "json_token_enum.hpp"
#include <charconv>
#include <cstring>
@@ -6,41 +6,64 @@
#include <string_view>
// Global callbacks for JSON parsing
const WeaselJsonCallbacks CommitRequest::json_callbacks = {
.on_begin_object = CommitRequest::on_begin_object,
.on_end_object = CommitRequest::on_end_object,
.on_string_data = CommitRequest::on_string_data,
.on_key_data = CommitRequest::on_key_data,
.on_begin_array = CommitRequest::on_begin_array,
.on_end_array = CommitRequest::on_end_array,
.on_number_data = CommitRequest::on_number_data,
.on_true_literal = CommitRequest::on_true_literal,
.on_false_literal = CommitRequest::on_false_literal,
.on_null_literal = CommitRequest::on_null_literal,
const WeaselJsonCallbacks JsonCommitRequestParser::json_callbacks = {
.on_begin_object = JsonCommitRequestParser::on_begin_object,
.on_end_object = JsonCommitRequestParser::on_end_object,
.on_string_data = JsonCommitRequestParser::on_string_data,
.on_key_data = JsonCommitRequestParser::on_key_data,
.on_begin_array = JsonCommitRequestParser::on_begin_array,
.on_end_array = JsonCommitRequestParser::on_end_array,
.on_number_data = JsonCommitRequestParser::on_number_data,
.on_true_literal = JsonCommitRequestParser::on_true_literal,
.on_false_literal = JsonCommitRequestParser::on_false_literal,
.on_null_literal = JsonCommitRequestParser::on_null_literal,
};
std::string_view CommitRequest::store_string(std::string_view str) {
if (str.empty()) {
return {};
JsonCommitRequestParser::JsonCommitRequestParser()
: json_parser_(WeaselJsonParser_create(64, &json_callbacks, this, 0)),
parser_context_(nullptr), current_request_(nullptr) {}
JsonCommitRequestParser::~JsonCommitRequestParser() {
if (json_parser_) {
WeaselJsonParser_destroy(json_parser_);
}
char *arena_str = arena_.allocate<char>(str.size());
std::memcpy(arena_str, str.data(), str.size());
return std::string_view(arena_str, str.size());
}
void CommitRequest::on_complete() {
// Fill in default read version
for (auto &precondition : preconditions_) {
if (precondition.version == 0) {
precondition.version = read_version_;
JsonCommitRequestParser::JsonCommitRequestParser(
JsonCommitRequestParser &&other) noexcept
: json_parser_(other.json_parser_),
parser_context_(std::move(other.parser_context_)),
current_request_(other.current_request_) {
other.json_parser_ = nullptr;
other.current_request_ = nullptr;
}
JsonCommitRequestParser &
JsonCommitRequestParser::operator=(JsonCommitRequestParser &&other) noexcept {
if (this != &other) {
if (json_parser_) {
WeaselJsonParser_destroy(json_parser_);
}
json_parser_ = other.json_parser_;
parser_context_ = std::move(other.parser_context_);
current_request_ = other.current_request_;
other.json_parser_ = nullptr;
other.current_request_ = nullptr;
}
return *this;
}
void JsonCommitRequestParser::on_complete() {
if (current_request_) {
current_request_->finalize();
}
}
std::string_view CommitRequest::decode_base64(std::string_view base64_str) {
if (base64_str.empty()) {
std::string_view
JsonCommitRequestParser::decode_base64(std::string_view base64_str) {
if (base64_str.empty() || !current_request_) {
return {};
}
@@ -52,7 +75,7 @@ std::string_view CommitRequest::decode_base64(std::string_view base64_str) {
return {};
}
char *output = arena_.allocate<char>(max_output_len);
char *output = current_request_->arena().allocate<char>(max_output_len);
if (!output) {
return {};
}
@@ -62,16 +85,16 @@ std::string_view CommitRequest::decode_base64(std::string_view base64_str) {
base64_str.data(), base64_str.size(), output, simdutf::base64_default);
if (result.error != simdutf::error_code::SUCCESS) {
parser_context_.parse_error = "Decoding base64 failed";
parser_context_->parse_error = "Decoding base64 failed";
return {};
}
return std::string_view(output, result.count);
}
void CommitRequest::on_begin_object(void *userdata) {
auto *self = static_cast<CommitRequest *>(userdata);
auto &ctx = self->parser_context_;
void JsonCommitRequestParser::on_begin_object(void *userdata) {
auto *self = static_cast<JsonCommitRequestParser *>(userdata);
auto &ctx = *self->parser_context_;
if (ctx.parse_error)
return;
@@ -96,9 +119,9 @@ void CommitRequest::on_begin_object(void *userdata) {
}
}
void CommitRequest::on_end_object(void *userdata) {
auto *self = static_cast<CommitRequest *>(userdata);
auto &ctx = self->parser_context_;
void JsonCommitRequestParser::on_end_object(void *userdata) {
auto *self = static_cast<JsonCommitRequestParser *>(userdata);
auto &ctx = *self->parser_context_;
if (ctx.parse_error) {
return;
@@ -126,11 +149,10 @@ void CommitRequest::on_end_object(void *userdata) {
ctx.parse_error =
"point_read precondition missing required 'key' field";
} else {
self->preconditions_.push_back(
Precondition{ctx.current_precondition.type,
ctx.current_precondition.version.value_or(0),
ctx.current_precondition.key.value(),
{}});
self->current_request_->add_precondition(
ctx.current_precondition.type,
ctx.current_precondition.version.value_or(0),
ctx.current_precondition.key.value(), {});
}
break;
case Precondition::Type::RangeRead:
@@ -139,11 +161,11 @@ void CommitRequest::on_end_object(void *userdata) {
ctx.parse_error = "range_read precondition missing required 'begin' "
"and/or 'end' fields";
} else {
self->preconditions_.push_back(
Precondition{ctx.current_precondition.type,
ctx.current_precondition.version.value_or(0),
ctx.current_precondition.begin.value(),
ctx.current_precondition.end.value()});
self->current_request_->add_precondition(
ctx.current_precondition.type,
ctx.current_precondition.version.value_or(0),
ctx.current_precondition.begin.value(),
ctx.current_precondition.end.value());
}
break;
}
@@ -156,17 +178,17 @@ void CommitRequest::on_end_object(void *userdata) {
ctx.parse_error =
"write operation missing required 'key' and/or 'value' fields";
} else {
self->operations_.push_back(Operation{
self->current_request_->add_operation(
ctx.current_operation.type, ctx.current_operation.key.value(),
ctx.current_operation.value.value()});
ctx.current_operation.value.value());
}
break;
case Operation::Type::Delete:
if (!ctx.current_operation.key.has_value()) {
ctx.parse_error = "delete operation missing required 'key' field";
} else {
self->operations_.push_back(Operation{
ctx.current_operation.type, ctx.current_operation.key.value(), {}});
self->current_request_->add_operation(
ctx.current_operation.type, ctx.current_operation.key.value(), {});
}
break;
case Operation::Type::RangeDelete:
@@ -175,9 +197,9 @@ void CommitRequest::on_end_object(void *userdata) {
ctx.parse_error = "range_delete operation missing required 'begin' "
"and/or 'end' fields";
} else {
self->operations_.push_back(Operation{
self->current_request_->add_operation(
ctx.current_operation.type, ctx.current_operation.begin.value(),
ctx.current_operation.end.value()});
ctx.current_operation.end.value());
}
break;
}
@@ -187,10 +209,10 @@ void CommitRequest::on_end_object(void *userdata) {
}
}
void CommitRequest::on_string_data(void *userdata, const char *buf, int len,
int done) {
auto *self = static_cast<CommitRequest *>(userdata);
auto &ctx = self->parser_context_;
void JsonCommitRequestParser::on_string_data(void *userdata, const char *buf,
int len, int done) {
auto *self = static_cast<JsonCommitRequestParser *>(userdata);
auto &ctx = *self->parser_context_;
if (ctx.parse_error)
return;
@@ -206,10 +228,10 @@ void CommitRequest::on_string_data(void *userdata, const char *buf, int len,
}
}
void CommitRequest::on_key_data(void *userdata, const char *buf, int len,
int done) {
auto *self = static_cast<CommitRequest *>(userdata);
auto &ctx = self->parser_context_;
void JsonCommitRequestParser::on_key_data(void *userdata, const char *buf,
int len, int done) {
auto *self = static_cast<JsonCommitRequestParser *>(userdata);
auto &ctx = *self->parser_context_;
if (ctx.parse_error)
return;
@@ -224,9 +246,9 @@ void CommitRequest::on_key_data(void *userdata, const char *buf, int len,
}
}
void CommitRequest::on_begin_array(void *userdata) {
auto *self = static_cast<CommitRequest *>(userdata);
auto &ctx = self->parser_context_;
void JsonCommitRequestParser::on_begin_array(void *userdata) {
auto *self = static_cast<JsonCommitRequestParser *>(userdata);
auto &ctx = *self->parser_context_;
if (ctx.parse_error)
return;
@@ -245,9 +267,9 @@ void CommitRequest::on_begin_array(void *userdata) {
}
}
void CommitRequest::on_end_array(void *userdata) {
auto *self = static_cast<CommitRequest *>(userdata);
auto &ctx = self->parser_context_;
void JsonCommitRequestParser::on_end_array(void *userdata) {
auto *self = static_cast<JsonCommitRequestParser *>(userdata);
auto &ctx = *self->parser_context_;
if (ctx.parse_error) {
return;
@@ -260,10 +282,10 @@ void CommitRequest::on_end_array(void *userdata) {
}
}
void CommitRequest::on_number_data(void *userdata, const char *buf, int len,
int done) {
auto *self = static_cast<CommitRequest *>(userdata);
auto &ctx = self->parser_context_;
void JsonCommitRequestParser::on_number_data(void *userdata, const char *buf,
int len, int done) {
auto *self = static_cast<JsonCommitRequestParser *>(userdata);
auto &ctx = *self->parser_context_;
if (ctx.parse_error)
return;
@@ -279,20 +301,20 @@ void CommitRequest::on_number_data(void *userdata, const char *buf, int len,
}
}
void CommitRequest::on_true_literal(void *) {
void JsonCommitRequestParser::on_true_literal(void *) {
// Not used in this API
}
void CommitRequest::on_false_literal(void *) {
void JsonCommitRequestParser::on_false_literal(void *) {
// Not used in this API
}
void CommitRequest::on_null_literal(void *) {
void JsonCommitRequestParser::on_null_literal(void *) {
// Not used in this API
}
void CommitRequest::handle_completed_string(std::string_view s) {
auto &ctx = parser_context_;
void JsonCommitRequestParser::handle_completed_string(std::string_view s) {
auto &ctx = *parser_context_;
ParseState current_state = ctx.current_state;
@@ -300,10 +322,10 @@ void CommitRequest::handle_completed_string(std::string_view s) {
case ParseState::Root: {
switch (ctx.current_key_token) {
case JsonTokenType::RequestId:
request_id_ = store_string(s);
current_request_->set_request_id(current_request_->copy_to_arena(s));
break;
case JsonTokenType::LeaderId:
leader_id_ = store_string(s);
current_request_->set_leader_id(current_request_->copy_to_arena(s));
break;
case JsonTokenType::ReadVersion:
// read_version should be a number, not a string
@@ -389,8 +411,8 @@ void CommitRequest::handle_completed_string(std::string_view s) {
}
}
void CommitRequest::handle_completed_number(std::string_view s) {
auto &ctx = parser_context_;
void JsonCommitRequestParser::handle_completed_number(std::string_view s) {
auto &ctx = *parser_context_;
ParseState current_state = ctx.current_state;
@@ -400,8 +422,7 @@ void CommitRequest::handle_completed_number(std::string_view s) {
uint64_t version;
auto result = std::from_chars(s.data(), s.data() + s.size(), version);
if (result.ec == std::errc{}) {
read_version_ = version;
has_read_version_been_set_ = true;
current_request_->set_read_version(version);
} else {
ctx.parse_error = "Invalid number format for read_version field";
}
@@ -426,33 +447,53 @@ void CommitRequest::handle_completed_number(std::string_view s) {
}
}
bool CommitRequest::parse_json(char *data, size_t len) {
if (!begin_streaming_parse()) {
bool JsonCommitRequestParser::parse(CommitRequest &request, char *data,
size_t len) {
if (!begin_streaming_parse(request)) {
return false;
}
parse_chunk(data, len);
finish_streaming_parse();
return is_parse_complete();
parse_chunk(request, data, len);
finish_streaming_parse(request);
return !has_parse_error() && !request.leader_id().empty() &&
request.has_read_version_been_set();
}
bool CommitRequest::begin_streaming_parse() {
reset();
bool JsonCommitRequestParser::begin_streaming_parse(CommitRequest &request) {
request.reset();
current_request_ = &request;
if (!parser_context_) {
parser_context_ = std::make_unique<ParserContext>(&request.arena());
} else {
parser_context_->reset_arena_memory(&request.arena());
parser_context_->parse_error = nullptr;
parser_context_->parse_complete = false;
}
if (json_parser_) {
WeaselJsonParser_reset(json_parser_);
}
return json_parser_ != nullptr;
}
CommitRequest::ParseStatus CommitRequest::parse_chunk(char *data, size_t len) {
if (!json_parser_) {
JsonCommitRequestParser::ParseStatus
JsonCommitRequestParser::parse_chunk(CommitRequest &request, char *data,
size_t len) {
if (!json_parser_ || !parser_context_) {
return ParseStatus::Error;
}
if (parser_context_.parse_error) {
if (parser_context_->parse_error) {
return ParseStatus::Error;
}
if (parser_context_.parse_complete) {
if (parser_context_->parse_complete) {
return ParseStatus::Complete;
}
current_request_ = &request;
WeaselJsonStatus status = WeaselJsonParser_parse(json_parser_, data, len);
switch (status) {
@@ -464,35 +505,43 @@ CommitRequest::ParseStatus CommitRequest::parse_chunk(char *data, size_t len) {
case WeaselJson_REJECT:
case WeaselJson_OVERFLOW:
default:
parser_context_.parse_error =
parser_context_->parse_error =
"JSON parsing failed - invalid or oversized JSON";
return ParseStatus::Error;
}
}
CommitRequest::ParseStatus CommitRequest::finish_streaming_parse() {
CommitRequest::ParseStatus result;
if (!json_parser_) {
result = ParseStatus::Error;
} else if (parser_context_.parse_error) {
result = ParseStatus::Error;
} else {
// Signal end of input
WeaselJsonStatus status = WeaselJsonParser_parse(json_parser_, nullptr, 0);
if (status == WeaselJson_OK && parser_context_.parse_complete &&
!parser_context_.parse_error) {
result = ParseStatus::Complete;
} else {
parser_context_.parse_error =
"JSON parsing incomplete or failed during finalization";
result = ParseStatus::Error;
}
JsonCommitRequestParser::ParseStatus
JsonCommitRequestParser::finish_streaming_parse(CommitRequest &request) {
if (!json_parser_ || !parser_context_) {
return ParseStatus::Error;
}
// Clear the memory used only during parsing
parser_context_.reset_arena_memory();
return result;
if (parser_context_->parse_error) {
return ParseStatus::Error;
}
current_request_ = &request;
// Signal end of input
WeaselJsonStatus status = WeaselJsonParser_parse(json_parser_, nullptr, 0);
if (status == WeaselJson_OK && parser_context_->parse_complete &&
!parser_context_->parse_error) {
// Clear the memory used only during parsing
parser_context_->reset_arena_memory(&request.arena());
return ParseStatus::Complete;
} else {
parser_context_->parse_error =
"JSON parsing incomplete or failed during finalization";
return ParseStatus::Error;
}
}
bool JsonCommitRequestParser::has_parse_error() const {
return parser_context_ && parser_context_->parse_error != nullptr;
}
const char *JsonCommitRequestParser::get_parse_error() const {
return parser_context_ ? parser_context_->parse_error : nullptr;
}

147
src/json_commit_request_parser.hpp Normal file
View File

@@ -0,0 +1,147 @@
#pragma once
#include "json_token_enum.hpp"
#include "parser_interface.hpp"
#include <memory>
#include <simdutf.h>
#include <weaseljson/weaseljson.h>
/**
* @brief JSON-specific implementation of CommitRequestParser.
*
* This parser uses the weaseljson library to parse JSON-formatted
* commit requests into CommitRequest objects.
*/
class JsonCommitRequestParser : public CommitRequestParser {
public:
// Parser state
enum class ParseState {
Root,
PreconditionsArray,
PreconditionObject,
OperationsArray,
OperationObject
};
private:
struct PreconditionParseState {
Precondition::Type type;
std::optional<uint64_t> version;
// These are owned by CommitRequest::arena
std::optional<std::string_view> key;
std::optional<std::string_view> begin;
std::optional<std::string_view> end;
};
/**
* @brief Internal state for parsing an operation during JSON processing.
*/
struct OperationParseState {
Operation::Type type;
// These are owned by CommitRequest::arena
std::optional<std::string_view> key;
std::optional<std::string_view> value;
std::optional<std::string_view> begin;
std::optional<std::string_view> end;
};
struct ParserContext {
using ArenaString = std::basic_string<char, std::char_traits<char>,
ArenaStlAllocator<char>>;
ParseState current_state = ParseState::Root;
JsonTokenType current_key_token;
// Only used if we need to accumulate the current key
ArenaString current_key;
ArenaString current_string;
ArenaString current_number;
bool in_key = false;
const char *parse_error = nullptr;
bool parse_complete = false;
// Current objects being parsed
PreconditionParseState current_precondition{};
OperationParseState current_operation{};
// Parsing state for nested structures
ArenaString precondition_type;
ArenaString operation_type;
// Constructor to initialize arena-allocated containers
explicit ParserContext(ArenaAllocator *arena)
: current_key(ArenaStlAllocator<char>(arena)),
current_string(ArenaStlAllocator<char>(arena)),
current_number(ArenaStlAllocator<char>(arena)),
precondition_type(ArenaStlAllocator<char>(arena)),
operation_type(ArenaStlAllocator<char>(arena)) {}
void reset_arena_memory(ArenaAllocator *arena) {
current_key = ArenaString{ArenaStlAllocator<char>(arena)};
current_string = ArenaString{ArenaStlAllocator<char>(arena)};
current_number = ArenaString{ArenaStlAllocator<char>(arena)};
in_key = false;
current_precondition = {};
current_operation = {};
precondition_type = ArenaString{ArenaStlAllocator<char>(arena)};
operation_type = ArenaString{ArenaStlAllocator<char>(arena)};
current_state = ParseState::Root;
}
};
WeaselJsonParser *json_parser_ = nullptr;
std::unique_ptr<ParserContext> parser_context_;
CommitRequest *current_request_ = nullptr;
static const WeaselJsonCallbacks json_callbacks;
public:
/**
* @brief Construct a new JsonCommitRequestParser.
*/
JsonCommitRequestParser();
/**
* @brief Destructor - cleans up any active parser.
*/
~JsonCommitRequestParser();
// Non-copyable but movable
JsonCommitRequestParser(const JsonCommitRequestParser &) = delete;
JsonCommitRequestParser &operator=(const JsonCommitRequestParser &) = delete;
JsonCommitRequestParser(JsonCommitRequestParser &&other) noexcept;
JsonCommitRequestParser &operator=(JsonCommitRequestParser &&other) noexcept;
// CommitRequestParser interface implementation
bool parse(CommitRequest &request, char *data, size_t len) override;
bool begin_streaming_parse(CommitRequest &request) override;
ParseStatus parse_chunk(CommitRequest &request, char *data,
size_t len) override;
ParseStatus finish_streaming_parse(CommitRequest &request) override;
bool has_parse_error() const override;
const char *get_parse_error() const override;
// Weaseljson callbacks (public for global callbacks)
static void on_begin_object(void *userdata);
static void on_end_object(void *userdata);
static void on_string_data(void *userdata, const char *buf, int len,
int done);
static void on_key_data(void *userdata, const char *buf, int len, int done);
static void on_begin_array(void *userdata);
static void on_end_array(void *userdata);
static void on_number_data(void *userdata, const char *buf, int len,
int done);
static void on_true_literal(void *userdata);
static void on_false_literal(void *userdata);
static void on_null_literal(void *userdata);
private:
/**
* @brief Decode a base64 string and store it in the arena.
* @param base64_str The base64 encoded string
* @return String view of decoded data, or empty view if decoding failed
*/
std::string_view decode_base64(std::string_view base64_str);
void handle_completed_string(std::string_view s);
void handle_completed_number(std::string_view s);
void on_complete();
};

22
src/main.cpp
View File

@@ -1,5 +1,6 @@
#include "commit_request.hpp"
#include "config.hpp"
#include "json_commit_request_parser.hpp"
#include <iostream>
void print_stats(const CommitRequest &request) {
@@ -63,6 +64,7 @@ int main(int argc, char *argv[]) {
std::cout << "\n--- CommitRequest Demo ---" << std::endl;
CommitRequest request;
JsonCommitRequestParser parser;
const std::string sample_json = R"({
"request_id": "demo-12345",
@@ -85,7 +87,7 @@ int main(int argc, char *argv[]) {
})";
auto copy = sample_json;
if (request.parse_json(copy.data(), copy.size())) {
if (parser.parse(request, copy.data(), copy.size())) {
print_stats(request);
} else {
std::cout << "✗ Failed to parse commit request" << std::endl;
@@ -95,8 +97,9 @@ int main(int argc, char *argv[]) {
std::cout << "\n--- Streaming Parse Demo ---" << std::endl;
CommitRequest streaming_request;
JsonCommitRequestParser streaming_parser;
if (streaming_request.begin_streaming_parse()) {
if (streaming_parser.begin_streaming_parse(streaming_request)) {
std::cout << "✓ Initialized streaming parser" << std::endl;
// Simulate receiving data in small chunks like from a network socket
@@ -106,10 +109,11 @@ int main(int argc, char *argv[]) {
size_t offset = 0;
int chunk_count = 0;
CommitRequest::ParseStatus status = CommitRequest::ParseStatus::Incomplete;
CommitRequestParser::ParseStatus status =
CommitRequestParser::ParseStatus::Incomplete;
while (offset < copy.size() &&
status == CommitRequest::ParseStatus::Incomplete) {
status == CommitRequestParser::ParseStatus::Incomplete) {
size_t len = std::min(chunk_size, copy.size() - offset);
std::string chunk = copy.substr(offset, len);
@@ -118,18 +122,18 @@ int main(int argc, char *argv[]) {
// Need mutable data for weaseljson
std::string mutable_chunk = chunk;
status = streaming_request.parse_chunk(mutable_chunk.data(),
mutable_chunk.size());
status = streaming_parser.parse_chunk(
streaming_request, mutable_chunk.data(), mutable_chunk.size());
offset += len;
}
if (status == CommitRequest::ParseStatus::Incomplete) {
if (status == CommitRequestParser::ParseStatus::Incomplete) {
std::cout << " Finalizing parse..." << std::endl;
status = streaming_request.finish_streaming_parse();
status = streaming_parser.finish_streaming_parse(streaming_request);
}
if (status == CommitRequest::ParseStatus::Complete) {
if (status == CommitRequestParser::ParseStatus::Complete) {
print_stats(streaming_request);
} else {
std::cout << "✗ Streaming parse failed" << std::endl;

66
src/parser_interface.hpp Normal file
View File

@@ -0,0 +1,66 @@
#pragma once
#include "commit_request.hpp"
/**
* @brief Abstract interface for commit request parsers.
*
* This interface defines how parsers should interact with CommitRequest
* objects. Parsers are responsible for reading serialized data in various
* formats and populating CommitRequest objects with arena-allocated memory.
*/
class CommitRequestParser {
public:
enum class ParseStatus {
Incomplete, // Still need more data
Complete, // Successfully parsed complete data
Error // Parse error occurred
};
virtual ~CommitRequestParser() = default;
/**
* @brief Parse data into a CommitRequest object (one-shot parsing).
* @param request The CommitRequest object to populate
* @param data Pointer to the data buffer
* @param len Length of the data in bytes
* @return true if parsing succeeded, false otherwise
*/
virtual bool parse(CommitRequest &request, char *data, size_t len) = 0;
/**
* @brief Initialize streaming parsing.
* @param request The CommitRequest object to populate
* @return true if initialization succeeded, false otherwise
*/
virtual bool begin_streaming_parse(CommitRequest &request) = 0;
/**
* @brief Parse additional data incrementally.
* @param request The CommitRequest object to populate
* @param data Pointer to the data buffer
* @param len Length of the data
* @return ParseStatus indicating current parse state
*/
virtual ParseStatus parse_chunk(CommitRequest &request, char *data,
size_t len) = 0;
/**
* @brief Finish streaming parse (call when no more data is available).
* @param request The CommitRequest object to populate
* @return ParseStatus indicating final parse result
*/
virtual ParseStatus finish_streaming_parse(CommitRequest &request) = 0;
/**
* @brief Check if there was a parse error.
* @return true if there was a parse error
*/
virtual bool has_parse_error() const = 0;
/**
* @brief Get the parse error message if there was an error.
* @return Error message string, or nullptr if no error
*/
virtual const char *get_parse_error() const = 0;
};