weaselab/weaseldb
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Simplify public headers more

Browse Source
This commit is contained in:
Andrew Noyes
2025-08-17 16:38:38 -04:00
parent 8862fdd588
commit 34ebf5725f
6 changed files with 36 additions and 370 deletions
Download Patch File Download Diff File Expand all files Collapse all files

126
src/arena_allocator.cpp
View File

@@ -127,132 +127,6 @@ void *ArenaAllocator::realloc_raw(void *ptr, uint32_t old_size,
return new_ptr; return new_ptr;
} }
std::vector<ArenaAllocator::PointerInfo>
ArenaAllocator::find_intra_arena_pointers() const {
std::vector<PointerInfo> pointers;
if (!current_block_) {
return pointers;
}
// Build list of blocks from current to first
std::vector<Block *> blocks;
Block *block = current_block_;
while (block) {
blocks.push_back(block);
block = block->prev;
}
// Helper function to check if a pointer value points within the used area
// of any block
auto is_intra_arena_pointer = [&blocks,
this](uint64_t pointer_value) -> bool {
for (size_t block_idx = 0; block_idx < blocks.size(); ++block_idx) {
Block *b = blocks[block_idx];
uintptr_t block_start = reinterpret_cast<uintptr_t>(b->data());
// Calculate used bytes in this specific block
size_t block_used = b->offset;
uintptr_t block_used_end = block_start + block_used;
// Check if pointer falls within the used area of this block
if (pointer_value >= block_start && pointer_value < block_used_end) {
return true;
}
}
return false;
};
// Scan each block for pointers
for (size_t block_idx = 0; block_idx < blocks.size(); ++block_idx) {
Block *b = blocks[block_idx];
const char *data = b->data();
// Calculate used bytes in this specific block
size_t block_used;
if (block_idx == 0) {
// Current block - use offset
block_used = current_block_->offset;
} else {
// Previous blocks are fully used
block_used = b->size;
}
// Scan for 64-bit aligned pointers
for (size_t offset = 0; offset + sizeof(uint64_t) <= block_used;
offset += sizeof(uint64_t)) {
uint64_t potential_pointer;
std::memcpy(&potential_pointer, data + offset, sizeof(potential_pointer));
// Check if this value points within the used area of any block
if (is_intra_arena_pointer(potential_pointer)) {
// Find target location within arena
auto target_location = find_address_location(
reinterpret_cast<const void *>(potential_pointer));
pointers.emplace_back(
data + offset, // source address
blocks.size() - block_idx, // source block number (1-based)
offset, // source offset in block
reinterpret_cast<const void *>(potential_pointer), // target address
target_location.found ? target_location.block_number
: 0, // target block number
target_location.found ? target_location.offset_in_block
: 0 // target offset
);
}
}
}
return pointers;
}
ArenaAllocator::AddressLocation
ArenaAllocator::find_address_location(const void *addr) const {
if (!current_block_ || !addr) {
return AddressLocation();
}
uintptr_t target_addr = reinterpret_cast<uintptr_t>(addr);
// Build list of blocks from current to first
std::vector<Block *> blocks;
Block *block = current_block_;
while (block) {
blocks.push_back(block);
block = block->prev;
}
// Check each block to see if the address falls within its used area
for (size_t block_idx = 0; block_idx < blocks.size(); ++block_idx) {
Block *b = blocks[block_idx];
uintptr_t block_start = reinterpret_cast<uintptr_t>(b->data());
// Calculate used bytes in this specific block
size_t block_used;
if (block_idx == 0) {
// Current block - use offset
block_used = current_block_->offset;
} else {
// Previous blocks are fully used
block_used = b->size;
}
uintptr_t block_used_end = block_start + block_used;
// Check if address falls within the used area of this block
if (target_addr >= block_start && target_addr < block_used_end) {
return AddressLocation(
blocks.size() - block_idx, // block number (1-based)
target_addr - block_start // offset within block
);
}
}
return AddressLocation(); // Not found
}
void ArenaAllocator::debug_dump(std::ostream &out, bool show_memory_map, void ArenaAllocator::debug_dump(std::ostream &out, bool show_memory_map,
bool show_content, size_t content_limit) const { bool show_content, size_t content_limit) const {
out << "=== Arena Debug Dump ===" << std::endl; out << "=== Arena Debug Dump ===" << std::endl;

134
src/arena_allocator.hpp
View File

@@ -37,20 +37,10 @@
* *
* ## Usage Examples: * ## Usage Examples:
* ```cpp * ```cpp
* // Basic allocation
* ArenaAllocator arena(1024); * ArenaAllocator arena(1024);
* void* ptr = arena.allocate(100); * void* ptr = arena.allocate_raw(100);
*
* // Construct trivially destructible objects in-place
* int* num = arena.construct<int>(42); * int* num = arena.construct<int>(42);
* MyPOD* obj = arena.construct<MyPOD>(arg1, arg2); // If MyPOD is trivial * arena.reset(); // Reuse arena memory
*
* // Track memory usage
* size_t total = arena.total_allocated();
* size_t used = arena.used_bytes();
*
* // Reset to reuse first block (frees others)
* arena.reset();
* ``` * ```
* *
* ## Memory Management: * ## Memory Management:
@@ -172,18 +162,7 @@ public:
* - Respects alignment requirements with minimal padding * - Respects alignment requirements with minimal padding
* - Automatically creates new blocks when current block is exhausted * - Automatically creates new blocks when current block is exhausted
* *
* ## Example: * @note This method is kept inline for maximum performance (~1ns allocation).
* ```cpp
* void* ptr1 = arena.allocate_raw(100); // Default alignment
* void* ptr2 = arena.allocate_raw(64, 16); // 16-byte aligned
* MyStruct* ptr3 = static_cast<MyStruct*>(
* arena.allocate_raw(sizeof(MyStruct), alignof(MyStruct)));
* ```
*
* ## Performance Note:
* This method is kept inline in the header for maximum performance.
* The allocation path is extremely hot and inlining eliminates function
* call overhead, allowing the ~1ns allocation performance.
*/ */
void *allocate_raw(uint32_t size, void *allocate_raw(uint32_t size,
size_t alignment = alignof(std::max_align_t)) { size_t alignment = alignof(std::max_align_t)) {
@@ -239,13 +218,6 @@ public:
* - If ptr is null, behaves like allocate(new_size, alignment) * - If ptr is null, behaves like allocate(new_size, alignment)
* - If ptr was the last allocation and space exists, extends in place * - If ptr was the last allocation and space exists, extends in place
* *
* ## Example:
* ```cpp
* void* ptr = arena.allocate_raw(100, alignof(int));
* // ... use ptr ...
* ptr = arena.realloc_raw(ptr, 100, 200, alignof(int)); // May extend in
* place or copy
* ```
* *
* ## Safety Notes: * ## Safety Notes:
* - The caller must provide the correct old_size - this is not tracked * - The caller must provide the correct old_size - this is not tracked
@@ -292,13 +264,6 @@ public:
* This prevents subtle bugs since destructors are never called for objects * This prevents subtle bugs since destructors are never called for objects
* constructed in the arena. * constructed in the arena.
* *
* ## Example:
* ```cpp
* int* num = arena.construct<int>(42); // ✓ Trivially
* destructible MyPOD* pod = arena.construct<MyPOD>(arg1, arg2); // ✓ If
* MyPOD is trivial std::string* str = arena.construct<std::string>("hi"); //
* ✗ Compile error!
* ```
* *
* ## Note: * ## Note:
* Objects constructed this way cannot be individually destroyed. * Objects constructed this way cannot be individually destroyed.
@@ -335,18 +300,6 @@ public:
* This ensures consistency with the arena allocator's design where * This ensures consistency with the arena allocator's design where
* destructors are never called. * destructors are never called.
* *
* ## Example:
* ```cpp
* // Allocate space for 100 integers
* int* numbers = arena.allocate<int>(100);
*
* // Allocate space for 50 POD structs
* MyPOD* objects = arena.allocate<MyPOD>(50);
*
* // Initialize some elements (no automatic construction)
* numbers[0] = 42;
* new (&objects[0]) MyPOD(arg1, arg2);
* ```
* *
* ## Note: * ## Note:
* This method only allocates memory - it does not construct objects. * This method only allocates memory - it does not construct objects.
@@ -383,12 +336,6 @@ public:
* - Prevents memory leaks by freeing unused blocks * - Prevents memory leaks by freeing unused blocks
* - Faster than destroying and recreating the allocator * - Faster than destroying and recreating the allocator
* *
* ## Example:
* ```cpp
* arena.allocate(1000); // Creates blocks
* arena.reset(); // Frees extra blocks, keeps first
* arena.allocate(100); // Reuses first block
* ```
*/ */
void reset(); void reset();
@@ -429,6 +376,9 @@ public:
/** /**
* @brief Get the total number of blocks in the allocator. * @brief Get the total number of blocks in the allocator.
*
* @note This function is primarily used for testing and debugging.
* It has O(n) complexity as it traverses the entire block chain.
*/ */
size_t num_blocks() const { size_t num_blocks() const {
size_t result = 0; size_t result = 0;
@@ -438,88 +388,22 @@ public:
return result; return result;
} }
/**
* @brief Debug function to find all intra-arena pointers.
*
* Scans all used memory in the arena for 64-bit aligned values that could be
* pointers to locations within the arena itself. This is useful for
* understanding memory references and potential data structures.
*
* @return Vector of PointerInfo structs containing source and target
* addresses
*/
struct PointerInfo {
const void *source_addr; ///< Address where the pointer was found
size_t source_block_number; ///< Block number containing the source
size_t source_offset; ///< Offset within the source block
const void *target_addr; ///< Address the pointer points to
size_t target_block_number; ///< Block number containing the target
size_t target_offset; ///< Offset within the target block
PointerInfo(const void *src, size_t src_block, size_t src_offset,
const void *target, size_t target_block, size_t target_offset)
: source_addr(src), source_block_number(src_block),
source_offset(src_offset), target_addr(target),
target_block_number(target_block), target_offset(target_offset) {}
};
std::vector<PointerInfo> find_intra_arena_pointers() const;
/**
* @brief Find which block and offset a given address belongs to.
*
* @param addr The address to locate within the arena
* @return PointerInfo with block number and offset, or invalid info if not
* found
*/
struct AddressLocation {
size_t block_number;
size_t offset_in_block;
bool found;
AddressLocation() : block_number(0), offset_in_block(0), found(false) {}
AddressLocation(size_t block, size_t offset)
: block_number(block), offset_in_block(offset), found(true) {}
};
AddressLocation find_address_location(const void *addr) const;
/** /**
* @brief Debug function to visualize the arena's layout and contents. * @brief Debug function to visualize the arena's layout and contents.
* *
* @note This function is intended for testing, debugging, and development
* only. It should not be used in production code due to performance overhead.
*
* Prints a detailed breakdown of all blocks, memory usage, and allocation * Prints a detailed breakdown of all blocks, memory usage, and allocation
* patterns. This is useful for understanding memory fragmentation and * patterns. This is useful for understanding memory fragmentation and
* allocation behavior during development and debugging. * allocation behavior during development and debugging.
* *
* Output includes:
* - Overall arena statistics (total allocated, used, blocks)
* - Per-block breakdown with sizes and usage
* - Memory utilization percentages
* - Block chain visualization
* - Optional memory content visualization
*
* @param out Output stream to write debug information to (default: std::cout) * @param out Output stream to write debug information to (default: std::cout)
* @param show_memory_map If true, shows a visual memory map of used/free * @param show_memory_map If true, shows a visual memory map of used/free
* space * space
* @param show_content If true, shows actual memory contents in hex and ASCII * @param show_content If true, shows actual memory contents in hex and ASCII
* @param content_limit Maximum bytes of content to show per block (default: * @param content_limit Maximum bytes of content to show per block (default:
* 256) * 256)
*
* ## Example Output:
* ```
* === Arena Debug Dump ===
* Total allocated: 3072 bytes across 2 blocks
* Currently used: 1500 bytes (48.8% utilization)
* Available in current: 572 bytes
*
* Block Chain (newest to oldest):
* Block #2: 2048 bytes [used: 572/2048 = 27.9%] <- current
* Block #1: 1024 bytes [used: 1024/1024 = 100.0%]
*
* Memory Contents:
* Block #2 (first 256 bytes):
* 0x0000: 48656c6c 6f20576f 726c6400 54657374 |Hello World.Test|
* ```
*/ */
void debug_dump(std::ostream &out = std::cout, bool show_memory_map = false, void debug_dump(std::ostream &out = std::cout, bool show_memory_map = false,
bool show_content = false, size_t content_limit = 256) const; bool show_content = false, size_t content_limit = 256) const;

10
src/commit_request.hpp
View File

@@ -147,12 +147,22 @@ public:
/** /**
* @brief Get access to the underlying arena allocator for debugging. * @brief Get access to the underlying arena allocator for debugging.
*
* @note This function is primarily used for testing and debugging.
* Production code should prefer the specific accessor methods like
* total_allocated() and used_bytes() instead of direct arena access.
*
* @return Reference to the arena allocator * @return Reference to the arena allocator
*/ */
const ArenaAllocator &arena() const { return arena_; } const ArenaAllocator &arena() const { return arena_; }
/** /**
* @brief Get access to the underlying arena allocator for allocation. * @brief Get access to the underlying arena allocator for allocation.
*
* @note This function exposes the internal arena for direct manipulation.
* It should be used carefully and primarily for internal parser
* implementation or testing purposes.
*
* @return Reference to the arena allocator * @return Reference to the arena allocator
*/ */
ArenaAllocator &arena() { return arena_; } ArenaAllocator &arena() { return arena_; }

16
src/config.hpp
View File

@@ -15,8 +15,7 @@ struct ServerConfig {
/// TCP port number for the server to listen on /// TCP port number for the server to listen on
int port = 8080; int port = 8080;
/// Maximum size in bytes for incoming HTTP requests (default: 1MB) /// Maximum size in bytes for incoming HTTP requests (default: 1MB)
size_t max_request_size_bytes = size_t max_request_size_bytes = 1024 * 1024;
1024 * 1024; // 1MB default for 413 Content Too Large
}; };
/** /**
@@ -24,13 +23,11 @@ struct ServerConfig {
*/ */
struct CommitConfig { struct CommitConfig {
/// Minimum required length for request_id to ensure sufficient entropy /// Minimum required length for request_id to ensure sufficient entropy
size_t min_request_id_length = 20; // Minimum length for request_id entropy size_t min_request_id_length = 20;
/// How long to retain request IDs for duplicate detection /// How long to retain request IDs for duplicate detection
std::chrono::hours request_id_retention_hours{ std::chrono::hours request_id_retention_hours{24};
24}; // How long to keep request IDs
/// Minimum number of commit versions to retain request IDs for /// Minimum number of commit versions to retain request IDs for
size_t request_id_retention_versions = size_t request_id_retention_versions = 100000000;
100000000; // Min versions to retain request IDs
}; };
/** /**
@@ -38,10 +35,9 @@ struct CommitConfig {
*/ */
struct SubscriptionConfig { struct SubscriptionConfig {
/// Maximum buffer size for unconsumed subscription data before backpressure /// Maximum buffer size for unconsumed subscription data before backpressure
size_t max_buffer_size_bytes = size_t max_buffer_size_bytes = 10 * 1024 * 1024;
10 * 1024 * 1024; // 10MB buffer for unconsumed data
/// Interval between keepalive comments in subscription streams /// Interval between keepalive comments in subscription streams
std::chrono::seconds keepalive_interval{30}; // Keepalive comment frequency std::chrono::seconds keepalive_interval{30};
}; };
/** /**

5
src/json_commit_request_parser.hpp
View File

@@ -120,6 +120,11 @@ public:
/** /**
* @brief Check if read version has been explicitly set during parsing. * @brief Check if read version has been explicitly set during parsing.
*
* @note This function is primarily used for testing to verify parser
* behavior. It exposes internal parser state that is not typically needed in
* production code.
*
* @return true if read version was set during parsing * @return true if read version was set during parsing
*/ */
bool has_read_version_been_set() const; bool has_read_version_been_set() const;

115
tools/debug_arena.cpp
View File

@@ -84,117 +84,14 @@ struct ArenaDebugger {
private: private:
void scan_arena_pointers() { void scan_arena_pointers() {
std::cout << "Scanning all used arena memory for 64-bit aligned pointers..." std::cout << "Arena memory analysis:" << std::endl;
<< std::endl;
// Use the arena's comprehensive pointer scanning method
auto pointers = arena.find_intra_arena_pointers();
std::cout << "Arena memory scan complete:" << std::endl;
std::cout << "- Total scanned: " << arena.used_bytes() << " bytes across " std::cout << "- Total scanned: " << arena.used_bytes() << " bytes across "
<< arena.num_blocks() << " blocks" << std::endl; << arena.num_blocks() << " blocks" << std::endl;
std::cout << "- Intra-arena pointers found: " << pointers.size() std::cout << "- Memory utilization: "
<< std::endl; << (arena.total_allocated() > 0
? (100.0 * arena.used_bytes() / arena.total_allocated())
if (pointers.empty()) { : 0.0)
std::cout << "No intra-arena pointers detected." << std::endl; << "%" << std::endl;
return;
}
std::cout << std::endl;
std::cout << "Detected pointers:" << std::endl;
for (size_t i = 0; i < pointers.size(); ++i) {
const auto &ptr_info = pointers[i];
std::cout << "Pointer #" << (i + 1) << ":" << std::endl;
std::cout << " Source: " << ptr_info.source_addr << " (Block #"
<< ptr_info.source_block_number << ", offset +0x" << std::hex
<< ptr_info.source_offset << std::dec << ")" << std::endl;
std::cout << " Target: " << ptr_info.target_addr << " (Block #"
<< ptr_info.target_block_number << ", offset +0x" << std::hex
<< ptr_info.target_offset << std::dec << ")" << std::endl;
// Try to identify what this pointer might be pointing to
identify_pointer_target(ptr_info.target_addr);
std::cout << std::endl;
}
}
void identify_pointer_target(const void *target_addr) {
// Check if this target address matches any of our known string data
std::cout << " Points to: ";
bool found_match = false;
// Check request_id
if (commit_request.request_id().has_value()) {
const auto &req_id = *commit_request.request_id();
if (target_addr >= req_id.data() &&
target_addr < req_id.data() + req_id.size()) {
std::cout << "request_id string";
found_match = true;
}
}
// Check leader_id
if (!found_match) {
const auto &leader_id = commit_request.leader_id();
if (target_addr >= leader_id.data() &&
target_addr < leader_id.data() + leader_id.size()) {
std::cout << "leader_id string";
found_match = true;
}
}
// Check preconditions
if (!found_match) {
for (size_t i = 0; i < commit_request.preconditions().size(); ++i) {
const auto &precond = commit_request.preconditions()[i];
if (!precond.begin.empty() && target_addr >= precond.begin.data() &&
target_addr < precond.begin.data() + precond.begin.size()) {
std::cout << "precondition[" << i << "].begin string";
found_match = true;
break;
}
if (!precond.end.empty() && target_addr >= precond.end.data() &&
target_addr < precond.end.data() + precond.end.size()) {
std::cout << "precondition[" << i << "].end string";
found_match = true;
break;
}
}
}
// Check operations
if (!found_match) {
for (size_t i = 0; i < commit_request.operations().size(); ++i) {
const auto &op = commit_request.operations()[i];
if (!op.param1.empty() && target_addr >= op.param1.data() &&
target_addr < op.param1.data() + op.param1.size()) {
std::cout << "operation[" << i << "].param1 string";
found_match = true;
break;
}
if (!op.param2.empty() && target_addr >= op.param2.data() &&
target_addr < op.param2.data() + op.param2.size()) {
std::cout << "operation[" << i << "].param2 string";
found_match = true;
break;
}
}
}
if (!found_match) {
std::cout << "unknown arena data";
}
std::cout << std::endl;
} }
std::string_view find_string_view_for_data(const char *data) { std::string_view find_string_view_for_data(const char *data) {