weaselab/weaseldb
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove release_back_to_server

Browse Source
This commit is contained in:
Andrew Noyes
2025-09-12 19:43:39 -04:00
parent e887906da8
commit e96a493835
10 changed files with 295 additions and 342 deletions
Download Patch File Download Diff File Expand all files Collapse all files

171
src/connection.cpp
View File

@@ -4,9 +4,10 @@
#include <climits>
#include <cstdio>
#include <cstdlib>
#include <sys/epoll.h>
#include "metric.hpp"
#include "server.hpp" // Need this for release_back_to_server implementation
#include "server.hpp" // Need this for server reference
namespace {
// Thread-local metric instances
@@ -35,11 +36,13 @@ thread_local auto write_eagain_failures =
// Static thread-local storage for iovec buffer
static thread_local std::vector<struct iovec> g_iovec_buffer{IOV_MAX};
// Thread-local storage for arenas to be freed after unlocking
static thread_local std::vector<Arena> g_arenas_to_free;
Connection::Connection(struct sockaddr_storage addr, int fd, int64_t id,
size_t epoll_index, ConnectionHandler *handler,
WeakRef<Server> server)
: fd_(fd), id_(id), epoll_index_(epoll_index), addr_(addr), arena_(),
: fd_(fd), id_(id), epoll_index_(epoll_index), addr_(addr),
handler_(handler), server_(std::move(server)) {
auto server_ref = server_.lock();
// This should only be called from a member of Server itself, so I should
@@ -75,15 +78,47 @@ Connection::~Connection() {
// EINTR ignored - fd is guaranteed closed on Linux
}
void Connection::append_message(std::string_view s, bool copy_to_arena) {
if (copy_to_arena) {
char *arena_str = arena_.allocate<char>(s.size());
std::memcpy(arena_str, s.data(), s.size());
messages_.emplace_back(arena_str, s.size());
} else {
messages_.push_back(s);
void Connection::append_message(std::span<std::string_view> data_parts,
Arena arena, bool close_after_send) {
// Calculate total bytes for this message. Don't need to hold the lock yet.
size_t total_bytes = 0;
for (const auto &part : data_parts) {
total_bytes += part.size();
}
std::unique_lock<std::mutex> lock(mutex_);
if (is_closed_) {
return; // Connection is closed, ignore message
}
// Check if queue was empty to determine if we need to enable EPOLLOUT
bool was_empty = message_queue_.empty();
// Add message to queue
message_queue_.emplace_back(
Message{std::move(arena), data_parts, close_after_send});
outgoing_bytes_queued_ += total_bytes;
// If this message has close_after_send flag, set connection flag
if (close_after_send) {
close_after_send_ = true;
}
lock.unlock();
// If queue was empty, we need to add EPOLLOUT interest. We don't need to hold
// the lock
if (was_empty) {
auto server = server_.lock();
if (server) {
// Add EPOLLOUT interest - pipeline thread manages epoll
struct epoll_event event;
event.data.fd = fd_;
event.events = EPOLLIN | EPOLLOUT;
epoll_ctl(server->epoll_fds_[epoll_index_], EPOLL_CTL_MOD, fd_, &event);
}
}
outgoing_bytes_queued_ += s.size();
}
int Connection::readBytes(char *buf, size_t buffer_size) {
@@ -115,27 +150,47 @@ int Connection::readBytes(char *buf, size_t buffer_size) {
bool Connection::writeBytes() {
ssize_t total_bytes_written = 0;
while (!messages_.empty()) {
// Build iovec array up to IOV_MAX limit using thread-local vector
assert(g_iovec_buffer.size() == IOV_MAX);
struct iovec *iov = g_iovec_buffer.data();
while (true) {
// Build iovec array while holding mutex using thread-local buffer
int iov_count = 0;
{
std::lock_guard lock(mutex_);
for (auto it = messages_.begin();
it != messages_.end() && iov_count < IOV_MAX; ++it) {
const auto &msg = *it;
iov[iov_count] = {
const_cast<void *>(static_cast<const void *>(msg.data())),
msg.size()};
iov_count++;
}
if (is_closed_ || message_queue_.empty()) {
break;
}
assert(iov_count > 0);
// Build iovec array up to IOV_MAX limit using thread-local vector
assert(g_iovec_buffer.size() == IOV_MAX);
struct iovec *iov = g_iovec_buffer.data();
for (auto &message : message_queue_) {
if (iov_count >= IOV_MAX)
break;
for (const auto &part : message.data_parts) {
if (iov_count >= IOV_MAX)
break;
if (part.empty())
continue;
iov[iov_count] = {
const_cast<void *>(static_cast<const void *>(part.data())),
part.size()};
iov_count++;
}
}
if (iov_count == 0)
break;
} // Release mutex during I/O
// Perform I/O without holding mutex
ssize_t w;
for (;;) {
struct msghdr msg = {};
msg.msg_iov = iov;
msg.msg_iov = g_iovec_buffer.data();
msg.msg_iovlen = iov_count;
w = sendmsg(fd_, &msg, MSG_NOSIGNAL);
@@ -146,7 +201,6 @@ bool Connection::writeBytes() {
if (errno == EAGAIN) {
// Increment EAGAIN failure metric
write_eagain_failures.inc();
// Increment bytes written metric before returning
if (total_bytes_written > 0) {
bytes_written.inc(total_bytes_written);
}
@@ -161,30 +215,67 @@ bool Connection::writeBytes() {
assert(w > 0);
total_bytes_written += w;
// Handle partial writes by updating string_view data/size
size_t bytes_written = static_cast<size_t>(w);
outgoing_bytes_queued_ -= bytes_written;
while (bytes_written > 0 && !messages_.empty()) {
auto &front = messages_.front();
// Handle partial writes by updating message data_parts
{
std::lock_guard lock(mutex_);
outgoing_bytes_queued_ -= w;
size_t bytes_remaining = static_cast<size_t>(w);
if (bytes_written >= front.size()) {
// This message is completely written
bytes_written -= front.size();
messages_.pop_front();
} else {
// Partial write of this message - update string_view
front = std::string_view(front.data() + bytes_written,
front.size() - bytes_written);
bytes_written = 0;
while (bytes_remaining > 0 && !message_queue_.empty()) {
auto &front_message = message_queue_.front();
bool message_complete = true;
for (auto &part : front_message.data_parts) {
if (part.empty())
continue;
if (bytes_remaining >= part.size()) {
// This part is completely written
bytes_remaining -= part.size();
part = std::string_view(); // Mark as consumed
} else {
// Partial write of this part
part = std::string_view(part.data() + bytes_remaining,
part.size() - bytes_remaining);
bytes_remaining = 0;
message_complete = false;
break;
}
}
if (message_complete) {
// Move arena to thread-local vector for deferred cleanup
g_arenas_to_free.emplace_back(std::move(front_message.arena));
message_queue_.pop_front();
} else {
break;
}
}
}
}
// Check if queue is empty and remove EPOLLOUT interest
{
std::lock_guard lock(mutex_);
if (message_queue_.empty()) {
auto server = server_.lock();
if (server) {
struct epoll_event event;
event.data.fd = fd_;
event.events = EPOLLIN; // Remove EPOLLOUT
epoll_ctl(server->epoll_fds_[epoll_index_], EPOLL_CTL_MOD, fd_, &event);
}
}
}
assert(messages_.empty());
// Increment bytes written metric
if (total_bytes_written > 0) {
bytes_written.inc(total_bytes_written);
}
// Clean up arenas after all mutex operations are complete
// This avoids holding the connection mutex while free() potentially contends
g_arenas_to_free.clear();
return false;
}