Use interleaved even if preserve_none is missing

But still run it to account for the version potentially increasing

ConflictSet.cpp

@@ -1213,135 +1213,6 @@ TaggedNodePointer getChild(Node *self, uint8_t index) {
   }
 }
 
-TaggedNodePointer *getChildUpdatingMaxVersion(Node0 *, TrivialSpan &,
-                                              InternalVersionT) {
-  return nullptr;
-}
-TaggedNodePointer *getChildUpdatingMaxVersion(Node3 *self,
-                                              TrivialSpan &remaining,
-                                              InternalVersionT maxVersion) {
-  assert(remaining.size() > 0);
-  int index = remaining.front();
-  auto key = remaining.subspan(1, remaining.size() - 1);
-  int i = getNodeIndex(self, index);
-  if (i < 0) {
-    return nullptr;
-  }
-  Node *c = self->children[i];
-  if (c->partialKeyLen > 0) {
-    int commonLen = std::min<int>(c->partialKeyLen, key.size());
-    int partialKeyIndex =
-        longestCommonPrefix(c->partialKey(), key.data(), commonLen);
-    if (partialKeyIndex < c->partialKeyLen) {
-      return nullptr;
-    }
-  }
-  remaining = key.subspan(c->partialKeyLen, key.size() - c->partialKeyLen);
-  self->childMaxVersion[i] = maxVersion;
-  return &self->children[i];
-}
-TaggedNodePointer *getChildUpdatingMaxVersion(Node16 *self,
-                                              TrivialSpan &remaining,
-                                              InternalVersionT maxVersion) {
-  assert(remaining.size() > 0);
-  int index = remaining.front();
-  auto key = remaining.subspan(1, remaining.size() - 1);
-  int i = getNodeIndex(self, index);
-  if (i < 0) {
-    return nullptr;
-  }
-  Node *c = self->children[i];
-  if (c->partialKeyLen > 0) {
-    int commonLen = std::min<int>(c->partialKeyLen, key.size());
-    int partialKeyIndex =
-        longestCommonPrefix(c->partialKey(), key.data(), commonLen);
-    if (partialKeyIndex < c->partialKeyLen) {
-      return nullptr;
-    }
-  }
-  remaining = key.subspan(c->partialKeyLen, key.size() - c->partialKeyLen);
-  self->childMaxVersion[i] = maxVersion;
-  return &self->children[i];
-}
-TaggedNodePointer *getChildUpdatingMaxVersion(Node48 *self,
-                                              TrivialSpan &remaining,
-                                              InternalVersionT maxVersion) {
-  assert(remaining.size() > 0);
-  int index = remaining.front();
-  auto key = remaining.subspan(1, remaining.size() - 1);
-  int i = self->index[index];
-  if (i < 0) {
-    return nullptr;
-  }
-  Node *c = self->children[i];
-  if (c->partialKeyLen > 0) {
-    int commonLen = std::min<int>(c->partialKeyLen, key.size());
-    int partialKeyIndex =
-        longestCommonPrefix(c->partialKey(), key.data(), commonLen);
-    if (partialKeyIndex < c->partialKeyLen) {
-      return nullptr;
-    }
-  }
-  remaining = key.subspan(c->partialKeyLen, key.size() - c->partialKeyLen);
-  self->childMaxVersion[i] = maxVersion;
-  self->maxOfMax[i >> Node48::kMaxOfMaxShift] =
-      std::max(self->maxOfMax[i >> Node48::kMaxOfMaxShift], maxVersion);
-  return &self->children[i];
-}
-TaggedNodePointer *getChildUpdatingMaxVersion(Node256 *self,
-                                              TrivialSpan &remaining,
-                                              InternalVersionT maxVersion) {
-  assert(remaining.size() > 0);
-  int index = remaining.front();
-  auto key = remaining.subspan(1, remaining.size() - 1);
-  auto &n = self->children[index];
-  if (n == nullptr) {
-    return nullptr;
-  }
-  Node *c = n;
-  if (c->partialKeyLen > 0) {
-    int commonLen = std::min<int>(c->partialKeyLen, key.size());
-    int partialKeyIndex =
-        longestCommonPrefix(c->partialKey(), key.data(), commonLen);
-    if (partialKeyIndex < c->partialKeyLen) {
-      return nullptr;
-    }
-  }
-  remaining = key.subspan(c->partialKeyLen, key.size() - c->partialKeyLen);
-  self->childMaxVersion[index] = maxVersion;
-  self->maxOfMax[index >> Node256::kMaxOfMaxShift] =
-      std::max(self->maxOfMax[index >> Node256::kMaxOfMaxShift], maxVersion);
-  return &n;
-}
-
-// Precondition: remaining.size() > 0
-// If a child of self lies along the search path of remaining, return a pointer
-// to that child, update max version, and consume the matching prefix bytes from
-// remaining. Otherwise return nullptr without changing the tree at all.
-TaggedNodePointer *getChildUpdatingMaxVersion(Node *self,
-                                              TrivialSpan &remaining,
-                                              InternalVersionT maxVersion) {
-  switch (self->getType()) {
-  case Type_Node0:
-    return getChildUpdatingMaxVersion(static_cast<Node0 *>(self), remaining,
-                                      maxVersion);
-  case Type_Node3:
-    return getChildUpdatingMaxVersion(static_cast<Node3 *>(self), remaining,
-                                      maxVersion);
-  case Type_Node16:
-    return getChildUpdatingMaxVersion(static_cast<Node16 *>(self), remaining,
-                                      maxVersion);
-  case Type_Node48:
-    return getChildUpdatingMaxVersion(static_cast<Node48 *>(self), remaining,
-                                      maxVersion);
-  case Type_Node256:
-    return getChildUpdatingMaxVersion(static_cast<Node256 *>(self), remaining,
-                                      maxVersion);
-  default:                   // GCOVR_EXCL_LINE
-    __builtin_unreachable(); // GCOVR_EXCL_LINE
-  }
-}
-
 struct ChildAndMaxVersion {
   TaggedNodePointer child;
   InternalVersionT maxVersion;
@@ -4283,42 +4154,84 @@ void pointIter(Job *job, Context *context) {
   MUSTTAIL return keepGoing(job, context);
 }
 
-template <class NodeT> PRESERVE_NONE void commonPrefixIter(Job *, Context *);
-template <class NodeT> PRESERVE_NONE void beginIter(Job *, Context *);
-template <class NodeT> PRESERVE_NONE void endIter(Job *, Context *);
+template <class NodeTFrom, class NodeTTo>
+PRESERVE_NONE void prefixIter(Job *, Context *);
+template <class NodeTFrom, class NodeTTo>
+PRESERVE_NONE void beginIter(Job *, Context *);
+template <class NodeTFrom, class NodeTTo>
+PRESERVE_NONE void endIter(Job *, Context *);
 
-static Continuation commonPrefixIterTable[] = {
-    commonPrefixIter<Node0>, commonPrefixIter<Node3>, commonPrefixIter<Node16>,
-    commonPrefixIter<Node48>, commonPrefixIter<Node256>};
+template <class NodeTFrom> struct PrefixIterTable {
+  static constexpr Continuation table[] = {
+      prefixIter<NodeTFrom, Node0>, prefixIter<NodeTFrom, Node3>,
+      prefixIter<NodeTFrom, Node16>, prefixIter<NodeTFrom, Node48>,
+      prefixIter<NodeTFrom, Node256>};
+};
 
-static Continuation beginIterTable[] = {beginIter<Node0>, beginIter<Node3>,
-                                        beginIter<Node16>, beginIter<Node48>,
-                                        beginIter<Node256>};
+static constexpr Continuation const *prefixIterTable[] = {
+    PrefixIterTable<Node0>::table,   PrefixIterTable<Node3>::table,
+    PrefixIterTable<Node16>::table,  PrefixIterTable<Node48>::table,
+    PrefixIterTable<Node256>::table,
+};
 
-static Continuation endIterTable[] = {endIter<Node0>, endIter<Node3>,
-                                      endIter<Node16>, endIter<Node48>,
-                                      endIter<Node256>};
+template <class NodeTFrom> struct BeginIterTable {
+  static constexpr Continuation table[] = {
+      beginIter<NodeTFrom, Node0>, beginIter<NodeTFrom, Node3>,
+      beginIter<NodeTFrom, Node16>, beginIter<NodeTFrom, Node48>,
+      beginIter<NodeTFrom, Node256>};
+};
 
-template <class NodeT> void commonPrefixIter(Job *job, Context *context) {
-  assert(NodeT::kType == job->n->getType());
-  NodeT *n = static_cast<NodeT *>(job->n);
+static constexpr Continuation const *beginIterTable[] = {
+    BeginIterTable<Node0>::table,   BeginIterTable<Node3>::table,
+    BeginIterTable<Node16>::table,  BeginIterTable<Node48>::table,
+    BeginIterTable<Node256>::table,
+};
 
-  TaggedNodePointer *child =
-      getChildUpdatingMaxVersion(n, job->remaining, context->writeVersion);
-  if (child == nullptr) [[unlikely]] {
-    int prefixLen = job->commonPrefixLen - job->remaining.size();
-    assert(prefixLen >= 0);
-    assert(job->n != nullptr);
-    *job->result = {
-        job->n,
-        job->begin.subspan(prefixLen, job->begin.size() - prefixLen),
-        job->n,
-        job->end.subspan(prefixLen, job->end.size() - prefixLen),
-    };
-    MUSTTAIL return complete(job, context);
+template <class NodeTFrom> struct EndIterTable {
+  static constexpr Continuation table[] = {
+      endIter<NodeTFrom, Node0>, endIter<NodeTFrom, Node3>,
+      endIter<NodeTFrom, Node16>, endIter<NodeTFrom, Node48>,
+      endIter<NodeTFrom, Node256>};
+};
+
+static constexpr Continuation const *endIterTable[] = {
+    EndIterTable<Node0>::table,   EndIterTable<Node3>::table,
+    EndIterTable<Node16>::table,  EndIterTable<Node48>::table,
+    EndIterTable<Node256>::table,
+};
+
+template <class NodeTFrom, class NodeTTo>
+void prefixIter(Job *job, Context *context) {
+  assert(NodeTFrom::kType == job->n->getType());
+  NodeTFrom *n = static_cast<NodeTFrom *>(job->n);
+  assert(NodeTTo::kType == job->child->getType());
+  NodeTTo *child = static_cast<NodeTTo *>(job->child);
+
+  auto key = job->remaining.subspan(1, job->remaining.size() - 1);
+  if (child->partialKeyLen > 0) {
+    int commonLen = std::min<int>(child->partialKeyLen, key.size());
+    int partialKeyIndex =
+        longestCommonPrefix(child->partialKey(), key.data(), commonLen);
+    if (partialKeyIndex < child->partialKeyLen) {
+      goto noNodeOnSearchPath;
+    }
   }
-  job->n = *child;
-  ++context->iterations;
+
+  // child is on the search path. Commit to advancing and updating max version
+  job->n = child;
+  job->remaining =
+      key.subspan(child->partialKeyLen, key.size() - child->partialKeyLen);
+  if constexpr (std::is_same_v<NodeTFrom, Node3> ||
+                std::is_same_v<NodeTFrom, Node16>) {
+    n->childMaxVersion[job->childIndex] = context->writeVersion;
+  } else if constexpr (std::is_same_v<NodeTFrom, Node48> ||
+                       std::is_same_v<NodeTFrom, Node256>) {
+    n->maxOfMax[job->childIndex >> NodeTFrom::kMaxOfMaxShift] =
+        std::max(n->maxOfMax[job->childIndex >> NodeTFrom::kMaxOfMaxShift],
+                 context->writeVersion);
+    n->childMaxVersion[job->childIndex] = context->writeVersion;
+  }
+
   if (job->remaining.size() == 0) [[unlikely]] {
     job->endNode = job->n;
     job->begin = job->begin.subspan(job->commonPrefixLen,
@@ -4326,56 +4239,163 @@ template <class NodeT> void commonPrefixIter(Job *job, Context *context) {
     job->end = job->end.subspan(job->commonPrefixLen,
                                 job->end.size() - job->commonPrefixLen);
     if (job->begin.size() == 0) [[unlikely]] {
-      job->continuation = endIterTable[child->getType()];
+      goto gotoEndIter;
+    } else if (!job->getChildAndIndex(child, job->begin.front())) [[unlikely]] {
+      goto gotoEndIter;
     } else {
-      job->continuation = beginIterTable[child->getType()];
+      job->continuation = BeginIterTable<NodeTTo>::table[job->child.getType()];
+      __builtin_prefetch(job->child);
+      MUSTTAIL return keepGoing(job, context);
     }
+  }
+
+  if (!job->getChildAndIndex(child, job->remaining.front())) [[unlikely]] {
+    goto noNodeOnSearchPath;
+  }
+
+  ++context->iterations;
+  job->continuation = PrefixIterTable<NodeTTo>::table[job->child.getType()];
+  __builtin_prefetch(job->child);
+  MUSTTAIL return keepGoing(job, context);
+
+noNodeOnSearchPath: {
+  int prefixLen = job->commonPrefixLen - job->remaining.size();
+  assert(prefixLen >= 0);
+  assert(job->n != nullptr);
+  *job->result = {
+      job->n,
+      job->begin.subspan(prefixLen, job->begin.size() - prefixLen),
+      job->n,
+      job->end.subspan(prefixLen, job->end.size() - prefixLen),
+  };
+  MUSTTAIL return complete(job, context);
+}
+
+gotoEndIter:
+  if (!job->getChildAndIndex(child, job->end.front())) [[unlikely]] {
+    *job->result = {
+        job->n,
+        job->begin,
+        job->n,
+        job->end,
+    };
+    MUSTTAIL return complete(job, context);
   } else {
-    job->continuation = commonPrefixIterTable[child->getType()];
+    job->continuation = EndIterTable<NodeTTo>::table[job->child.getType()];
+    __builtin_prefetch(job->child);
+    MUSTTAIL return keepGoing(job, context);
   }
-  __builtin_prefetch(job->n);
-  MUSTTAIL return keepGoing(job, context);
 }
 
-template <class NodeT> void beginIter(Job *job, Context *context) {
-  assert(NodeT::kType == job->n->getType());
-  NodeT *n = static_cast<NodeT *>(job->n);
+template <class NodeTFrom, class NodeTTo>
+void beginIter(Job *job, Context *context) {
+  assert(NodeTFrom::kType == job->n->getType());
+  NodeTFrom *n = static_cast<NodeTFrom *>(job->n);
+  assert(NodeTTo::kType == job->child->getType());
+  NodeTTo *child = static_cast<NodeTTo *>(job->child);
 
-  TaggedNodePointer *child =
-      getChildUpdatingMaxVersion(n, job->begin, context->writeVersion);
-  if (child == nullptr) [[unlikely]] {
-    MUSTTAIL return endIterTable[job->endNode->getType()](job, context);
+  auto key = job->begin.subspan(1, job->begin.size() - 1);
+  if (child->partialKeyLen > 0) {
+    int commonLen = std::min<int>(child->partialKeyLen, key.size());
+    int partialKeyIndex =
+        longestCommonPrefix(child->partialKey(), key.data(), commonLen);
+    if (partialKeyIndex < child->partialKeyLen) {
+      goto gotoEndIter;
+    }
   }
-  job->n = *child;
+
+  // child is on the search path. Commit to advancing and updating max version
+  job->n = child;
+  job->begin =
+      key.subspan(child->partialKeyLen, key.size() - child->partialKeyLen);
+  if constexpr (std::is_same_v<NodeTFrom, Node3> ||
+                std::is_same_v<NodeTFrom, Node16>) {
+    n->childMaxVersion[job->childIndex] = context->writeVersion;
+  } else if constexpr (std::is_same_v<NodeTFrom, Node48> ||
+                       std::is_same_v<NodeTFrom, Node256>) {
+    n->maxOfMax[job->childIndex >> NodeTFrom::kMaxOfMaxShift] =
+        std::max(n->maxOfMax[job->childIndex >> NodeTFrom::kMaxOfMaxShift],
+                 context->writeVersion);
+    n->childMaxVersion[job->childIndex] = context->writeVersion;
+  }
+
   if (job->begin.size() == 0) [[unlikely]] {
-    MUSTTAIL return endIterTable[job->endNode->getType()](job, context);
+    goto gotoEndIter;
   }
+
+  if (!job->getChildAndIndex(child, job->begin.front())) [[unlikely]] {
+    goto gotoEndIter;
+  }
+
   ++context->iterations;
-  job->continuation = beginIterTable[child->getType()];
-  __builtin_prefetch(job->n);
+  job->continuation = BeginIterTable<NodeTTo>::table[job->child.getType()];
+  __builtin_prefetch(job->child);
   MUSTTAIL return keepGoing(job, context);
+
+gotoEndIter:
+  if (!job->getChildAndIndex(job->endNode, job->end.front())) [[unlikely]] {
+    *job->result = {
+        job->n,
+        job->begin,
+        job->endNode,
+        job->end,
+    };
+    MUSTTAIL return complete(job, context);
+  } else {
+    MUSTTAIL return endIterTable[job->endNode->getType()][job->child.getType()](
+        job, context);
+  }
 }
 
-template <class NodeT> void endIter(Job *job, Context *context) {
-  assert(NodeT::kType == job->endNode->getType());
-  NodeT *endNode = static_cast<NodeT *>(job->endNode);
+template <class NodeTFrom, class NodeTTo>
+void endIter(Job *job, Context *context) {
+  assert(NodeTFrom::kType == job->endNode->getType());
+  NodeTFrom *endNode = static_cast<NodeTFrom *>(job->endNode);
+  assert(NodeTTo::kType == job->child->getType());
+  NodeTTo *child = static_cast<NodeTTo *>(job->child);
 
-  TaggedNodePointer *child =
-      getChildUpdatingMaxVersion(endNode, job->end, context->writeVersion);
-  if (child == nullptr) [[unlikely]] {
+  auto key = job->end.subspan(1, job->end.size() - 1);
+  if (child->partialKeyLen > 0) {
+    int commonLen = std::min<int>(child->partialKeyLen, key.size());
+    int partialKeyIndex =
+        longestCommonPrefix(child->partialKey(), key.data(), commonLen);
+    if (partialKeyIndex < child->partialKeyLen) {
+      *job->result = {job->n, job->begin, job->endNode, job->end};
+      assert(job->endNode != nullptr);
+      MUSTTAIL return complete(job, context);
+    }
+  }
+
+  // child is on the search path. Commit to advancing and updating max version
+  job->endNode = child;
+  job->end =
+      key.subspan(child->partialKeyLen, key.size() - child->partialKeyLen);
+  if constexpr (std::is_same_v<NodeTFrom, Node3> ||
+                std::is_same_v<NodeTFrom, Node16>) {
+    endNode->childMaxVersion[job->childIndex] = context->writeVersion;
+  } else if constexpr (std::is_same_v<NodeTFrom, Node48> ||
+                       std::is_same_v<NodeTFrom, Node256>) {
+    endNode->maxOfMax[job->childIndex >> NodeTFrom::kMaxOfMaxShift] = std::max(
+        endNode->maxOfMax[job->childIndex >> NodeTFrom::kMaxOfMaxShift],
+        context->writeVersion);
+    endNode->childMaxVersion[job->childIndex] = context->writeVersion;
+  }
+
+  if (job->end.size() == 0) [[unlikely]] {
     *job->result = {job->n, job->begin, job->endNode, job->end};
     assert(job->endNode != nullptr);
     MUSTTAIL return complete(job, context);
   }
-  job->endNode = *child;
-  if (job->remaining.size() == 0) [[unlikely]] {
+
+  if (!job->getChildAndIndex(child, job->end.front())) [[unlikely]] {
     *job->result = {job->n, job->begin, job->endNode, job->end};
     assert(job->endNode != nullptr);
     MUSTTAIL return complete(job, context);
   }
+
   ++context->iterations;
-  job->continuation = endIterTable[child->getType()];
-  __builtin_prefetch(job->endNode);
+  job->continuation = EndIterTable<NodeTTo>::table[job->child.getType()];
+  __builtin_prefetch(job->child);
   MUSTTAIL return keepGoing(job, context);
 }
 
@@ -4403,13 +4423,34 @@ void Job::init(Context *context, int index) {
 
   if (commonPrefixLen > 0) {
     // common prefix iter will set endNode
-    continuation = commonPrefixIterTable[n->getType()];
-  } else if (begin.size() > 0) {
+    if (!getChildAndIndex(n, remaining.front())) [[unlikely]] {
+      *result = {
+          n,
+          begin,
+          n,
+          end,
+      };
+      continuation = complete;
+    } else {
+      continuation = prefixIterTable[n->getType()][child.getType()];
+    }
+  } else if (begin.size() > 0 && getChildAndIndex(n, begin.front())) {
     endNode = n;
-    continuation = beginIterTable[n->getType()];
+    continuation = beginIterTable[n->getType()][child.getType()];
   } else {
+    assert(end.size() > 0);
     endNode = n;
-    continuation = endIterTable[n->getType()];
+    if (!getChildAndIndex(n, end.front())) [[unlikely]] {
+      *result = {
+          n,
+          begin,
+          n,
+          end,
+      };
+      continuation = complete;
+    } else {
+      continuation = endIterTable[n->getType()][child.getType()];
+    }
   }
 
   return;
@@ -4417,11 +4458,11 @@ pointWrite:
   remaining = TrivialSpan(context->writes[index].begin.p,
                           context->writes[index].begin.len);
   if (remaining.size() == 0) [[unlikely]] {
-    context->results[index] = {n, remaining};
+    *result = {n, remaining};
     continuation = complete;
   } else {
     if (!getChildAndIndex(n, remaining.front())) [[unlikely]] {
-      context->results[index] = {n, remaining};
+      *result = {n, remaining};
       continuation = complete;
     } else {
       continuation = pointIterTable[n->getType()][child.getType()];
@@ -4876,7 +4917,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
     check::Context context;
     context.readContext.impl = this;
 
-#if __has_attribute(preserve_none) && __has_attribute(musttail)
+#if __has_attribute(musttail)
     if (count == 1) {
       useSequential(reads, result, count, context);
     } else {
@@ -5093,7 +5134,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
     assert(allPointWrites || sorted);
 #endif
 
-#if __has_attribute(preserve_none) && __has_attribute(musttail)
+#if __has_attribute(musttail)
     constexpr bool kEnableInterleaved = true;
 #else
     constexpr bool kEnableInterleaved = false;
@@ -5128,9 +5169,6 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
     assert(writeContext.accum.entries_erased == 0);
     assert(writeContext.accum.entries_inserted == 0);
 
-    // Queue up some gc work just for (potentially) increasing the version
-    ++keyUpdates;
-
     if (oldestExtantVersion < writeVersion - kMaxCorrectVersionWindow)
         [[unlikely]] {
       if (writeVersion > newestVersionFullPrecision + kNominalVersionWindow) {
@@ -5187,10 +5225,11 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
 
     writeContext.releaseDeferred();
 
-    // Run gc at least 200% the rate we're inserting entries
+    // Run gc at least 200% the rate we're inserting entries, and at least run
+    // some gc just for potentially increasing the version.
     keyUpdates += std::max<int64_t>(writeContext.accum.entries_inserted -
                                         writeContext.accum.entries_erased,
-                                    0) *
+                                    1) *
                   2;
 
     point_writes_total.add(writeContext.accum.point_writes);

aarch64-symbol-imports.txt

@@ -5,6 +5,7 @@ __stack_chk_guard@GLIBC_2.17
 abort@GLIBC_2.17
 free@GLIBC_2.17
 malloc@GLIBC_2.17
+memcmp@GLIBC_2.17
 memcpy@GLIBC_2.17
 memmove@GLIBC_2.17
 memset@GLIBC_2.17