Convert checkPointRead to IteratorBase

ConflictSet.cpp

@@ -216,6 +216,9 @@ enum Type : int8_t {
 
 template <class T> struct NodeAllocator;
 
+// Make sure tag fits in low 4 bits
+static_assert(alignof(std::max_align_t) >= 16);
+
 struct TaggedNodePointer {
   TaggedNodePointer() = default;
   operator struct Node *() { return (struct Node *)withoutType(); }
@@ -271,11 +274,11 @@ struct TaggedNodePointer {
 
 private:
   TaggedNodePointer(struct Node *p, Type t) : p((uintptr_t)p) {
-    assert((this->p & 7) == 0);
+    assert((this->p & 15) == 0);
     this->p |= t;
     assume(p != 0);
   }
-  uintptr_t withoutType() const { return p & ~uintptr_t(7); }
+  uintptr_t withoutType() const { return p & ~uintptr_t(15); }
   uintptr_t p;
 };
 
@@ -317,7 +320,7 @@ TaggedNodePointer::TaggedNodePointer(Node *n)
 Type TaggedNodePointer::getType() {
   assert(p != 0);
   prefetch();
-  return Type(p & uintptr_t(7));
+  return Type(p & uintptr_t(15));
 }
 
 // copyChildrenAndKeyFrom is responsible for copying all
@@ -1882,6 +1885,163 @@ void downsize(Node *self, WriteContext *writeContext) {
   }
 }
 
+template <class T> struct Iterator;
+
+// Higher-level handle to a position in the tree
+struct IteratorBase {
+
+  explicit IteratorBase(Node *node) : node(node) {}
+
+  bool entryPresent() { return node->entryPresent; }
+  Entry getEntry() {
+    assert(node->entryPresent);
+    return node->entry;
+  }
+  void setEntry(Entry e) {
+    node->entryPresent = true;
+    node->entry = e;
+  }
+
+  bool operator==(const IteratorBase &other) const {
+    assert(!node->releaseDeferred);
+    assert(!other.node->releaseDeferred);
+    return node == other.node;
+  }
+
+  bool valid() { return node != nullptr; }
+
+  void followForwardingPointers() {
+    while (node->releaseDeferred) [[unlikely]] {
+      node = node->forwardTo;
+    }
+  }
+
+  Type getType() { return node->getType(); }
+
+  template <class T> Iterator<T> as() {
+    assert(getType() == T::kType);
+    return static_cast<Iterator<T>>(*this);
+  }
+
+  IteratorBase nextLogical();
+  IteratorBase nextPhysical();
+  InternalVersionT getMaxVersion();
+  InternalVersionT exchangeMaxVersion(InternalVersionT);
+  InternalVersionT setMaxVersion();
+  TrivialSpan partialKey();
+  IteratorBase getFirstChild();
+  bool checkRangeVersionOfFirstGeq(InternalVersionT readVersion);
+  struct ChildAndMaxVersion;
+  ChildAndMaxVersion getChildAndMaxVersion(int index);
+  IteratorBase getChildGeq(int index);
+  IteratorBase nextSibling();
+
+protected:
+  Node *node;
+  // index into children array of particular leaf type
+  int index;
+};
+
+struct IteratorBase::ChildAndMaxVersion {
+  IteratorBase child;
+  InternalVersionT maxVersion;
+};
+
+template <class T> struct Iterator : IteratorBase {
+  IteratorBase getFirstChild() {
+    return IteratorBase(::getFirstChild(static_cast<T *>(node)));
+  }
+  ChildAndMaxVersion getChildAndMaxVersion(int index) {
+    auto x = ::getChildAndMaxVersion(static_cast<T *>(node), index);
+    return {IteratorBase{x.child}, x.maxVersion};
+  }
+  IteratorBase getChildGeq(int index) {
+    return IteratorBase{::getChildGeq(static_cast<T *>(node), index)};
+  }
+  TrivialSpan partialKey() {
+    return {static_cast<T *>(node)->partialKey(), node->partialKeyLen};
+  }
+};
+
+TrivialSpan IteratorBase::partialKey() {
+  switch (node->getType()) {
+  case Type_Node0:
+    return as<Node0>().partialKey();
+  case Type_Node3:
+    return as<Node3>().partialKey();
+  case Type_Node16:
+    return as<Node16>().partialKey();
+  case Type_Node48:
+    return as<Node48>().partialKey();
+  case Type_Node256:
+    return as<Node256>().partialKey();
+  default:                   // GCOVR_EXCL_LINE
+    __builtin_unreachable(); // GCOVR_EXCL_LINE
+  }
+}
+
+IteratorBase IteratorBase::getFirstChild() {
+  switch (node->getType()) {
+  case Type_Node0:
+    return as<Node0>().getFirstChild();
+  case Type_Node3:
+    return as<Node3>().getFirstChild();
+  case Type_Node16:
+    return as<Node16>().getFirstChild();
+  case Type_Node48:
+    return as<Node48>().getFirstChild();
+  case Type_Node256:
+    return as<Node256>().getFirstChild();
+  default:                   // GCOVR_EXCL_LINE
+    __builtin_unreachable(); // GCOVR_EXCL_LINE
+  }
+}
+
+IteratorBase::ChildAndMaxVersion
+IteratorBase::getChildAndMaxVersion(int index) {
+  switch (node->getType()) {
+  case Type_Node0:
+    return as<Node0>().getChildAndMaxVersion(index);
+  case Type_Node3:
+    return as<Node3>().getChildAndMaxVersion(index);
+  case Type_Node16:
+    return as<Node16>().getChildAndMaxVersion(index);
+  case Type_Node48:
+    return as<Node48>().getChildAndMaxVersion(index);
+  case Type_Node256:
+    return as<Node256>().getChildAndMaxVersion(index);
+  default:                   // GCOVR_EXCL_LINE
+    __builtin_unreachable(); // GCOVR_EXCL_LINE
+  }
+}
+
+IteratorBase IteratorBase::getChildGeq(int index) {
+  switch (node->getType()) {
+  case Type_Node0:
+    return as<Node0>().getChildGeq(index);
+  case Type_Node3:
+    return as<Node3>().getChildGeq(index);
+  case Type_Node16:
+    return as<Node16>().getChildGeq(index);
+  case Type_Node48:
+    return as<Node48>().getChildGeq(index);
+  case Type_Node256:
+    return as<Node256>().getChildGeq(index);
+  default:                   // GCOVR_EXCL_LINE
+    __builtin_unreachable(); // GCOVR_EXCL_LINE
+  }
+}
+
+bool IteratorBase::checkRangeVersionOfFirstGeq(InternalVersionT readVersion) {
+  return ::checkRangeVersionOfFirstGeq(node, readVersion);
+}
+
+TaggedNodePointer nextSibling(Node *node);
+
+IteratorBase IteratorBase::nextSibling() {
+  return IteratorBase{::nextSibling(node)};
+}
+
 // Precondition: self is not the root. May invalidate nodes along the search
 // path to self. May invalidate children of self->parent. Returns a pointer to
 // the node after self. Precondition: `self->entryPresent`
@@ -4381,7 +4541,7 @@ namespace {
 // Logically this is the same as performing firstGeq and then checking against
 // point or range version according to cmp, but this version short circuits as
 // soon as it can prove that there's no conflict.
-bool checkPointRead(Node *n, const TrivialSpan key,
+bool checkPointRead(IteratorBase n, const TrivialSpan key,
                     InternalVersionT readVersion, ReadContext *readContext) {
   ++readContext->point_read_accum;
 #if DEBUG_VERBOSE && !defined(NDEBUG)
@@ -4390,54 +4550,54 @@ bool checkPointRead(Node *n, const TrivialSpan key,
   auto remaining = key;
   for (;; ++readContext->point_read_iterations_accum) {
     if (remaining.size() == 0) {
-      if (n->entryPresent) {
-        return n->entry.pointVersion <= readVersion;
+      if (n.entryPresent()) {
+        return n.getEntry().pointVersion <= readVersion;
       }
-      n = getFirstChild(n);
-      return checkRangeVersionOfFirstGeq(n, readVersion);
+      return n.getFirstChild().checkRangeVersionOfFirstGeq(readVersion);
     }
 
-    auto [c, maxV] = getChildAndMaxVersion(n, remaining[0]);
-    Node *child = c;
-    if (child == nullptr) {
-      auto c = getChildGeq(n, remaining[0]);
-      if (c != nullptr) {
-        n = c;
-        return checkRangeVersionOfFirstGeq(n, readVersion);
+    auto [c, maxV] = n.getChildAndMaxVersion(remaining[0]);
+    IteratorBase child = c;
+    if (!child.valid()) {
+      auto c = n.getChildGeq(remaining[0]);
+      if (c.valid()) {
+        return c.checkRangeVersionOfFirstGeq(readVersion);
       } else {
-        n = nextSibling(n);
-        if (n == nullptr) {
+        n = n.nextSibling();
+        if (!n.valid()) {
           return true;
         }
-        return checkRangeVersionOfFirstGeq(n, readVersion);
+        return n.checkRangeVersionOfFirstGeq(readVersion);
       }
     }
 
     n = child;
     remaining = remaining.subspan(1, remaining.size() - 1);
 
-    if (n->partialKeyLen > 0) {
-      int commonLen = std::min<int>(n->partialKeyLen, remaining.size());
-      int i = longestCommonPrefix(n->partialKey(), remaining.data(), commonLen);
+    auto partialKey = n.partialKey();
+    if (partialKey.size() > 0) {
+      int commonLen = std::min<int>(partialKey.size(), remaining.size());
+      int i =
+          longestCommonPrefix(partialKey.data(), remaining.data(), commonLen);
       if (i < commonLen) {
-        auto c = n->partialKey()[i] <=> remaining[i];
+        auto c = partialKey[i] <=> remaining[i];
         if (c > 0) {
-          return checkRangeVersionOfFirstGeq(n, readVersion);
+          return n.checkRangeVersionOfFirstGeq(readVersion);
         } else {
-          n = nextSibling(n);
-          if (n == nullptr) {
+          n = n.nextSibling();
+          if (!n.valid()) {
             return true;
           }
-          return checkRangeVersionOfFirstGeq(n, readVersion);
+          return n.checkRangeVersionOfFirstGeq(readVersion);
         }
       }
-      if (commonLen == n->partialKeyLen) {
+      if (commonLen == partialKey.size()) {
         // partial key matches
         remaining = remaining.subspan(commonLen, remaining.size() - commonLen);
-      } else if (n->partialKeyLen > remaining.size()) {
+      } else if (partialKey.size() > remaining.size()) {
         // n is the first physical node greater than remaining, and there's no
         // eq node
-        return checkRangeVersionOfFirstGeq(n, readVersion);
+        return n.checkRangeVersionOfFirstGeq(readVersion);
       }
     }
 
@@ -4697,7 +4857,7 @@ bool checkRangeRead(Node *n, TrivialSpan begin, TrivialSpan end,
                                 std::min(begin.size(), end.size()));
   if (lcp == begin.size() && end.size() == begin.size() + 1 &&
       end.back() == 0) {
-    return checkPointRead(n, begin, readVersion, readContext);
+    return checkPointRead(IteratorBase{n}, begin, readVersion, readContext);
   }
   if (lcp == begin.size() - 1 && end.size() == begin.size() &&
       begin.back() + 1 == end.back()) {
@@ -4773,7 +4933,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
       } else {
         bool ok;
         if (reads[i].end.len == 0) {
-          ok = checkPointRead(rootParent->children[0],
+          ok = checkPointRead(IteratorBase{rootParent->children[0]},
                               TrivialSpan(reads[i].begin.p, reads[i].begin.len),
                               InternalVersionT(reads[i].readVersion),
                               &context.readContext);
@@ -4984,15 +5144,16 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
     // encounter invalidated nodes.
     for (auto *iter = firstRangeWrite; iter != nullptr;
          iter = iter->nextRangeWrite) {
-      assert(iter->endInsertionPoint != nullptr);
-      while (iter->insertionPoint->releaseDeferred) {
-        iter->insertionPoint = iter->insertionPoint->forwardTo;
+      if (iter->endInsertionPoint != nullptr) {
+        while (iter->insertionPoint->releaseDeferred) {
+          iter->insertionPoint = iter->insertionPoint->forwardTo;
+        }
+        while (iter->endInsertionPoint->releaseDeferred) {
+          iter->endInsertionPoint = iter->endInsertionPoint->forwardTo;
+        }
+        eraseInRange(iter->insertionPoint, iter->endInsertionPoint,
+                     &writeContext);
       }
-      while (iter->endInsertionPoint->releaseDeferred) {
-        iter->endInsertionPoint = iter->endInsertionPoint->forwardTo;
-      }
-      eraseInRange(iter->insertionPoint, iter->endInsertionPoint,
-                   &writeContext);
     }
 
     if (count > kStackResultMax) [[unlikely]] {