Enable interleaved range writes
This commit is contained in:
237
ConflictSet.cpp
237
ConflictSet.cpp
@@ -3113,13 +3113,16 @@ struct AddedWriteRange {
|
|||||||
Node *endNode;
|
Node *endNode;
|
||||||
};
|
};
|
||||||
|
|
||||||
AddedWriteRange
|
AddedWriteRange addWriteRange(Node *beginRoot, std::span<const uint8_t> begin,
|
||||||
addWriteRange(TaggedNodePointer &beginRoot, std::span<const uint8_t> begin,
|
Node *endRoot, std::span<const uint8_t> end,
|
||||||
TaggedNodePointer &endRoot, std::span<const uint8_t> end,
|
InternalVersionT writeVersion,
|
||||||
InternalVersionT writeVersion, WriteContext *writeContext) {
|
WriteContext *writeContext,
|
||||||
|
ConflictSet::Impl *impl) {
|
||||||
|
|
||||||
++writeContext->accum.range_writes;
|
++writeContext->accum.range_writes;
|
||||||
|
|
||||||
Node *beginNode = *insert(&beginRoot, begin, writeVersion, writeContext);
|
Node *beginNode =
|
||||||
|
*insert(&getInTree(beginRoot, impl), begin, writeVersion, writeContext);
|
||||||
addKey(beginNode);
|
addKey(beginNode);
|
||||||
if (!beginNode->entryPresent) {
|
if (!beginNode->entryPresent) {
|
||||||
++writeContext->accum.entries_inserted;
|
++writeContext->accum.entries_inserted;
|
||||||
@@ -3131,7 +3134,12 @@ addWriteRange(TaggedNodePointer &beginRoot, std::span<const uint8_t> begin,
|
|||||||
}
|
}
|
||||||
beginNode->entry.pointVersion = writeVersion;
|
beginNode->entry.pointVersion = writeVersion;
|
||||||
|
|
||||||
Node *endNode = *insert(&endRoot, end, writeVersion, writeContext);
|
while (endRoot->releaseDeferred) {
|
||||||
|
endRoot = endRoot->forwardTo;
|
||||||
|
}
|
||||||
|
Node *endNode =
|
||||||
|
*insert(&getInTree(endRoot, impl), end, writeVersion, writeContext);
|
||||||
|
|
||||||
addKey(endNode);
|
addKey(endNode);
|
||||||
if (!endNode->entryPresent) {
|
if (!endNode->entryPresent) {
|
||||||
++writeContext->accum.entries_inserted;
|
++writeContext->accum.entries_inserted;
|
||||||
@@ -3182,7 +3190,7 @@ void addWriteRange(TaggedNodePointer &root, std::span<const uint8_t> begin,
|
|||||||
|
|
||||||
auto [beginNode, endNode] = addWriteRange(
|
auto [beginNode, endNode] = addWriteRange(
|
||||||
*useAsRoot, begin.subspan(lcp, begin.size() - lcp), *useAsRoot,
|
*useAsRoot, begin.subspan(lcp, begin.size() - lcp), *useAsRoot,
|
||||||
end.subspan(lcp, end.size() - lcp), writeVersion, writeContext);
|
end.subspan(lcp, end.size() - lcp), writeVersion, writeContext, impl);
|
||||||
|
|
||||||
eraseInRange(beginNode, endNode, writeContext, impl);
|
eraseInRange(beginNode, endNode, writeContext, impl);
|
||||||
}
|
}
|
||||||
@@ -3350,10 +3358,6 @@ static Continuation iterTable[] = {iter<Node0>, iter<Node3>, iter<Node16>,
|
|||||||
|
|
||||||
void begin(Job *job, Context *context) {
|
void begin(Job *job, Context *context) {
|
||||||
++context->readContext.point_read_accum;
|
++context->readContext.point_read_accum;
|
||||||
#if DEBUG_VERBOSE && !defined(NDEBUG)
|
|
||||||
fprintf(stderr, "Check point read: %s\n", printable(key).c_str());
|
|
||||||
#endif
|
|
||||||
|
|
||||||
if (job->begin.size() == 0) [[unlikely]] {
|
if (job->begin.size() == 0) [[unlikely]] {
|
||||||
// We don't erase the root
|
// We don't erase the root
|
||||||
assert(job->n->entryPresent);
|
assert(job->n->entryPresent);
|
||||||
@@ -3479,9 +3483,6 @@ static Continuation iterTable[] = {iter<Node0>, iter<Node3>, iter<Node16>,
|
|||||||
|
|
||||||
void begin(Job *job, Context *context) {
|
void begin(Job *job, Context *context) {
|
||||||
++context->readContext.prefix_read_accum;
|
++context->readContext.prefix_read_accum;
|
||||||
#if DEBUG_VERBOSE && !defined(NDEBUG)
|
|
||||||
fprintf(stderr, "Check prefix read: %s\n", printable(key).c_str());
|
|
||||||
#endif
|
|
||||||
|
|
||||||
// There's no way to encode a prefix read of ""
|
// There's no way to encode a prefix read of ""
|
||||||
assert(job->begin.size() > 0);
|
assert(job->begin.size() > 0);
|
||||||
@@ -4098,6 +4099,10 @@ struct Job {
|
|||||||
std::span<const uint8_t> remaining;
|
std::span<const uint8_t> remaining;
|
||||||
Node *n;
|
Node *n;
|
||||||
int index;
|
int index;
|
||||||
|
std::span<const uint8_t> begin; // Range write only
|
||||||
|
std::span<const uint8_t> end; // Range write only
|
||||||
|
Node *endNode; // Range write only
|
||||||
|
int commonPrefixLen; // Range write only
|
||||||
|
|
||||||
// State for context switching machinery - not application specific
|
// State for context switching machinery - not application specific
|
||||||
Continuation continuation;
|
Continuation continuation;
|
||||||
@@ -4112,6 +4117,9 @@ struct Job {
|
|||||||
struct Result {
|
struct Result {
|
||||||
Node *insertionPoint;
|
Node *insertionPoint;
|
||||||
std::span<const uint8_t> remaining;
|
std::span<const uint8_t> remaining;
|
||||||
|
|
||||||
|
Node *endInsertionPoint; // Range write only
|
||||||
|
std::span<const uint8_t> endRemaining; // Range write only
|
||||||
};
|
};
|
||||||
|
|
||||||
// State relevant to every insertion
|
// State relevant to every insertion
|
||||||
@@ -4146,43 +4154,172 @@ PRESERVE_NONE void complete(Job *job, Context *context) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
template <class NodeT> PRESERVE_NONE void iter(Job *, Context *);
|
template <class NodeT> PRESERVE_NONE void pointIter(Job *, Context *);
|
||||||
|
|
||||||
static Continuation iterTable[] = {iter<Node0>, iter<Node3>, iter<Node16>,
|
static Continuation pointIterTable[] = {pointIter<Node0>, pointIter<Node3>,
|
||||||
iter<Node48>, iter<Node256>};
|
pointIter<Node16>, pointIter<Node48>,
|
||||||
|
pointIter<Node256>};
|
||||||
|
|
||||||
template <class NodeT> void iter(Job *job, Context *context) {
|
template <class NodeT> void pointIter(Job *job, Context *context) {
|
||||||
assert(NodeT::kType == job->n->getType());
|
assert(NodeT::kType == job->n->getType());
|
||||||
NodeT *n = static_cast<NodeT *>(job->n);
|
NodeT *n = static_cast<NodeT *>(job->n);
|
||||||
|
|
||||||
TaggedNodePointer *child =
|
TaggedNodePointer *child =
|
||||||
getChildUpdatingMaxVersion(n, job->remaining, context->writeVersion);
|
getChildUpdatingMaxVersion(n, job->remaining, context->writeVersion);
|
||||||
if (child == nullptr) [[unlikely]] {
|
if (child == nullptr) [[unlikely]] {
|
||||||
context->results[job->index] = {job->n, job->remaining};
|
context->results[job->index] = {job->n, job->remaining, nullptr, {}};
|
||||||
MUSTTAIL return complete(job, context);
|
MUSTTAIL return complete(job, context);
|
||||||
}
|
}
|
||||||
job->n = *child;
|
job->n = *child;
|
||||||
if (job->remaining.size() == 0) [[unlikely]] {
|
if (job->remaining.size() == 0) [[unlikely]] {
|
||||||
context->results[job->index] = {job->n, job->remaining};
|
context->results[job->index] = {job->n, job->remaining, nullptr, {}};
|
||||||
MUSTTAIL return complete(job, context);
|
MUSTTAIL return complete(job, context);
|
||||||
}
|
}
|
||||||
++context->iterations;
|
++context->iterations;
|
||||||
job->continuation = iterTable[child->getType()];
|
job->continuation = pointIterTable[child->getType()];
|
||||||
__builtin_prefetch(job->n);
|
__builtin_prefetch(job->n);
|
||||||
MUSTTAIL return keepGoing(job, context);
|
MUSTTAIL return keepGoing(job, context);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
template <class NodeT> PRESERVE_NONE void commonPrefixIter(Job *, Context *);
|
||||||
|
template <class NodeT> PRESERVE_NONE void beginIter(Job *, Context *);
|
||||||
|
template <class NodeT> PRESERVE_NONE void endIter(Job *, Context *);
|
||||||
|
|
||||||
|
static Continuation commonPrefixIterTable[] = {
|
||||||
|
commonPrefixIter<Node0>, commonPrefixIter<Node3>, commonPrefixIter<Node16>,
|
||||||
|
commonPrefixIter<Node48>, commonPrefixIter<Node256>};
|
||||||
|
|
||||||
|
static Continuation beginIterTable[] = {beginIter<Node0>, beginIter<Node3>,
|
||||||
|
beginIter<Node16>, beginIter<Node48>,
|
||||||
|
beginIter<Node256>};
|
||||||
|
|
||||||
|
static Continuation endIterTable[] = {endIter<Node0>, endIter<Node3>,
|
||||||
|
endIter<Node16>, endIter<Node48>,
|
||||||
|
endIter<Node256>};
|
||||||
|
|
||||||
|
template <class NodeT> void commonPrefixIter(Job *job, Context *context) {
|
||||||
|
assert(NodeT::kType == job->n->getType());
|
||||||
|
NodeT *n = static_cast<NodeT *>(job->n);
|
||||||
|
|
||||||
|
TaggedNodePointer *child =
|
||||||
|
getChildUpdatingMaxVersion(n, job->remaining, context->writeVersion);
|
||||||
|
if (child == nullptr) [[unlikely]] {
|
||||||
|
int prefixLen = job->commonPrefixLen - job->remaining.size();
|
||||||
|
assert(prefixLen >= 0);
|
||||||
|
assert(job->n != nullptr);
|
||||||
|
context->results[job->index] = {
|
||||||
|
job->n,
|
||||||
|
job->begin.subspan(prefixLen, job->begin.size() - prefixLen),
|
||||||
|
job->n,
|
||||||
|
job->end.subspan(prefixLen, job->end.size() - prefixLen),
|
||||||
|
};
|
||||||
|
MUSTTAIL return complete(job, context);
|
||||||
|
}
|
||||||
|
job->n = *child;
|
||||||
|
++context->iterations;
|
||||||
|
if (job->remaining.size() == 0) [[unlikely]] {
|
||||||
|
job->endNode = job->n;
|
||||||
|
job->begin = job->begin.subspan(job->commonPrefixLen,
|
||||||
|
job->begin.size() - job->commonPrefixLen);
|
||||||
|
job->end = job->end.subspan(job->commonPrefixLen,
|
||||||
|
job->end.size() - job->commonPrefixLen);
|
||||||
|
if (job->begin.size() == 0) [[unlikely]] {
|
||||||
|
job->continuation = endIterTable[child->getType()];
|
||||||
|
} else {
|
||||||
|
job->continuation = beginIterTable[child->getType()];
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
job->continuation = commonPrefixIterTable[child->getType()];
|
||||||
|
}
|
||||||
|
__builtin_prefetch(job->n);
|
||||||
|
MUSTTAIL return keepGoing(job, context);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <class NodeT> void beginIter(Job *job, Context *context) {
|
||||||
|
assert(NodeT::kType == job->n->getType());
|
||||||
|
NodeT *n = static_cast<NodeT *>(job->n);
|
||||||
|
|
||||||
|
TaggedNodePointer *child =
|
||||||
|
getChildUpdatingMaxVersion(n, job->begin, context->writeVersion);
|
||||||
|
if (child == nullptr) [[unlikely]] {
|
||||||
|
MUSTTAIL return endIterTable[job->endNode->getType()](job, context);
|
||||||
|
}
|
||||||
|
job->n = *child;
|
||||||
|
if (job->begin.size() == 0) [[unlikely]] {
|
||||||
|
MUSTTAIL return endIterTable[job->endNode->getType()](job, context);
|
||||||
|
}
|
||||||
|
++context->iterations;
|
||||||
|
job->continuation = beginIterTable[child->getType()];
|
||||||
|
__builtin_prefetch(job->n);
|
||||||
|
MUSTTAIL return keepGoing(job, context);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <class NodeT> void endIter(Job *job, Context *context) {
|
||||||
|
assert(NodeT::kType == job->endNode->getType());
|
||||||
|
NodeT *endNode = static_cast<NodeT *>(job->endNode);
|
||||||
|
|
||||||
|
TaggedNodePointer *child =
|
||||||
|
getChildUpdatingMaxVersion(endNode, job->end, context->writeVersion);
|
||||||
|
if (child == nullptr) [[unlikely]] {
|
||||||
|
context->results[job->index] = {job->n, job->begin, job->endNode, job->end};
|
||||||
|
assert(job->endNode != nullptr);
|
||||||
|
MUSTTAIL return complete(job, context);
|
||||||
|
}
|
||||||
|
job->endNode = *child;
|
||||||
|
if (job->remaining.size() == 0) [[unlikely]] {
|
||||||
|
context->results[job->index] = {job->n, job->begin, job->endNode, job->end};
|
||||||
|
assert(job->endNode != nullptr);
|
||||||
|
MUSTTAIL return complete(job, context);
|
||||||
|
}
|
||||||
|
++context->iterations;
|
||||||
|
job->continuation = endIterTable[child->getType()];
|
||||||
|
__builtin_prefetch(job->endNode);
|
||||||
|
MUSTTAIL return keepGoing(job, context);
|
||||||
|
}
|
||||||
|
|
||||||
void Job::init(Context *context, int index) {
|
void Job::init(Context *context, int index) {
|
||||||
this->index = index;
|
this->index = index;
|
||||||
remaining = std::span<const uint8_t>(context->writes[index].begin.p,
|
|
||||||
context->writes[index].begin.len);
|
|
||||||
n = context->root;
|
n = context->root;
|
||||||
|
|
||||||
if (remaining.size() == 0) [[unlikely]] {
|
if (context->writes[index].end.len == 0) {
|
||||||
context->results[index] = {n, remaining};
|
goto pointWrite;
|
||||||
continuation = interleaved_insert::complete;
|
}
|
||||||
|
|
||||||
|
begin = std::span<const uint8_t>(context->writes[index].begin.p,
|
||||||
|
context->writes[index].begin.len);
|
||||||
|
end = std::span<const uint8_t>(context->writes[index].end.p,
|
||||||
|
context->writes[index].end.len);
|
||||||
|
|
||||||
|
commonPrefixLen = longestCommonPrefix(begin.data(), end.data(),
|
||||||
|
std::min(begin.size(), end.size()));
|
||||||
|
if (commonPrefixLen == int(begin.size()) && end.size() == begin.size() + 1 &&
|
||||||
|
end.back() == 0) {
|
||||||
|
goto pointWrite;
|
||||||
|
}
|
||||||
|
|
||||||
|
remaining =
|
||||||
|
std::span<const uint8_t>(context->writes[index].begin.p, commonPrefixLen);
|
||||||
|
|
||||||
|
if (commonPrefixLen > 0) {
|
||||||
|
// common prefix iter will set endNode
|
||||||
|
continuation = commonPrefixIterTable[n->getType()];
|
||||||
|
} else if (begin.size() > 0) {
|
||||||
|
endNode = n;
|
||||||
|
continuation = beginIterTable[n->getType()];
|
||||||
} else {
|
} else {
|
||||||
continuation = iterTable[n->getType()];
|
endNode = n;
|
||||||
|
continuation = endIterTable[n->getType()];
|
||||||
|
}
|
||||||
|
|
||||||
|
return;
|
||||||
|
pointWrite:
|
||||||
|
remaining = std::span<const uint8_t>(context->writes[index].begin.p,
|
||||||
|
context->writes[index].begin.len);
|
||||||
|
if (remaining.size() == 0) [[unlikely]] {
|
||||||
|
context->results[index] = {n, remaining, nullptr, {}};
|
||||||
|
continuation = complete;
|
||||||
|
} else {
|
||||||
|
continuation = pointIterTable[n->getType()];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -4763,12 +4900,51 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
|
|||||||
// phase 2.
|
// phase 2.
|
||||||
|
|
||||||
for (int i = 0; i < count; ++i) {
|
for (int i = 0; i < count; ++i) {
|
||||||
|
|
||||||
|
#if DEBUG_VERBOSE && !defined(NDEBUG)
|
||||||
|
fprintf(stderr, "search path: %s, begin: %s\n",
|
||||||
|
getSearchPathPrintable(context.results[i].insertionPoint).c_str(),
|
||||||
|
printable(writes[i].begin).c_str());
|
||||||
|
fprintf(
|
||||||
|
stderr, "search path: %s, end: %s\n",
|
||||||
|
getSearchPathPrintable(context.results[i].endInsertionPoint).c_str(),
|
||||||
|
printable(writes[i].end).c_str());
|
||||||
|
#endif
|
||||||
|
|
||||||
while (context.results[i].insertionPoint->releaseDeferred) {
|
while (context.results[i].insertionPoint->releaseDeferred) {
|
||||||
context.results[i].insertionPoint =
|
context.results[i].insertionPoint =
|
||||||
context.results[i].insertionPoint->forwardTo;
|
context.results[i].insertionPoint->forwardTo;
|
||||||
}
|
}
|
||||||
addPointWrite(getInTree(context.results[i].insertionPoint, this),
|
if (context.results[i].endInsertionPoint == nullptr) {
|
||||||
context.results[i].remaining, writeVersion, &writeContext);
|
addPointWrite(getInTree(context.results[i].insertionPoint, this),
|
||||||
|
context.results[i].remaining, writeVersion,
|
||||||
|
&writeContext);
|
||||||
|
} else {
|
||||||
|
auto [beginNode, endNode] = addWriteRange(
|
||||||
|
context.results[i].insertionPoint, context.results[i].remaining,
|
||||||
|
context.results[i].endInsertionPoint,
|
||||||
|
context.results[i].endRemaining, writeVersion, &writeContext, this);
|
||||||
|
context.results[i].insertionPoint = beginNode;
|
||||||
|
context.results[i].endInsertionPoint = endNode;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Phase 3: Erase nodes within written ranges. Going left to right ensures
|
||||||
|
// that nothing later is on the search path of anything earlier, so we don't
|
||||||
|
// encounter invalidated nodes.
|
||||||
|
for (int i = 0; i < count; ++i) {
|
||||||
|
if (context.results[i].endInsertionPoint != nullptr) {
|
||||||
|
while (context.results[i].insertionPoint->releaseDeferred) {
|
||||||
|
context.results[i].insertionPoint =
|
||||||
|
context.results[i].insertionPoint->forwardTo;
|
||||||
|
}
|
||||||
|
while (context.results[i].endInsertionPoint->releaseDeferred) {
|
||||||
|
context.results[i].endInsertionPoint =
|
||||||
|
context.results[i].endInsertionPoint->forwardTo;
|
||||||
|
}
|
||||||
|
eraseInRange(context.results[i].insertionPoint,
|
||||||
|
context.results[i].endInsertionPoint, &writeContext, this);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (count > int(sizeof(stackResults) / sizeof(stackResults[0])))
|
if (count > int(sizeof(stackResults) / sizeof(stackResults[0])))
|
||||||
@@ -4792,8 +4968,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
|
|||||||
#endif
|
#endif
|
||||||
|
|
||||||
#if __has_attribute(preserve_none) && __has_attribute(musttail)
|
#if __has_attribute(preserve_none) && __has_attribute(musttail)
|
||||||
// TODO make this work for sorted range writes
|
constexpr bool kEnableInterleaved = true;
|
||||||
constexpr bool kEnableInterleaved = false;
|
|
||||||
#else
|
#else
|
||||||
constexpr bool kEnableInterleaved = false;
|
constexpr bool kEnableInterleaved = false;
|
||||||
#endif
|
#endif
|
||||||
|
Reference in New Issue
Block a user