Add to corpus

ConflictSet.cpp

@@ -2147,7 +2147,14 @@ bool scan16(const InternalVersionT *vs, int begin, int end,
 // path of n + [child], where child in (begin, end) is <= readVersion. Does not
 // account for the range version of firstGt(searchpath(n) + [end - 1])
 template <bool kAVX512>
-bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
+bool checkMaxBetweenExclusiveImpl(Node0 *, int, int, InternalVersionT,
+                                  ReadContext *tls) {
+  ++tls->range_read_node_scan_accum;
+  return true;
+}
+
+template <bool kAVX512>
+bool checkMaxBetweenExclusiveImpl(Node3 *n, int begin, int end,
                                   InternalVersionT readVersion,
                                   ReadContext *tls) {
   ++tls->range_read_node_scan_accum;
@@ -2156,22 +2163,15 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
   assume(-1 <= end);
   assume(end <= 256);
   assume(begin < end);
-
   assert(!(begin == -1 && end == 256));
 
-  switch (n->getType()) {
-  case Type_Node0:
-    return true;
-  case Type_Node3: {
   auto *self = static_cast<Node3 *>(n);
 
   ++begin;
 
   const unsigned shiftUpperBound = end - begin;
   const unsigned shiftAmount = begin;
-    auto inBounds = [&](unsigned c) {
+  auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; };
-      return c - shiftAmount < shiftUpperBound;
-    };
 
   uint32_t mask = 0;
   for (int i = 0; i < Node3::kMaxNodes; ++i) {
@@ -2190,8 +2190,20 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
   }
 
   return !(compared & mask) && firstRangeOk;
-  }
+}
-  case Type_Node16: {
+
+template <bool kAVX512>
+bool checkMaxBetweenExclusiveImpl(Node16 *n, int begin, int end,
+                                  InternalVersionT readVersion,
+                                  ReadContext *tls) {
+  ++tls->range_read_node_scan_accum;
+  assume(-1 <= begin);
+  assume(begin <= 256);
+  assume(-1 <= end);
+  assume(end <= 256);
+  assume(begin < end);
+  assert(!(begin == -1 && end == 256));
+
   auto *self = static_cast<Node16 *>(n);
 
   ++begin;
@@ -2231,8 +2243,8 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
 
   uint16x4_t conflicting[4];
   for (int i = 0; i < 4; ++i) {
-      conflicting[i] = vmovn_u32(
+    conflicting[i] =
-          vcgtq_s32(vreinterpretq_s32_u32(vsubq_u32(w4[i], rvVec)), z));
+        vmovn_u32(vcgtq_s32(vreinterpretq_s32_u32(vsubq_u32(w4[i], rvVec)), z));
   }
   auto combined =
       vcombine_u8(vmovn_u16(vcombine_u16(conflicting[0], conflicting[1])),
@@ -2249,8 +2261,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
   memcpy(&indices, self->index, 16);
   indices = _mm_sub_epi8(indices, _mm_set1_epi8(begin));
   uint32_t mask =
-        0xffff &
+      0xffff & ~_mm_movemask_epi8(_mm_cmpeq_epi8(
-        ~_mm_movemask_epi8(_mm_cmpeq_epi8(
                    indices, _mm_max_epu8(indices, _mm_set1_epi8(end - begin))));
   mask &= (1 << self->numChildren) - 1;
   if (!mask) {
@@ -2272,9 +2283,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
 
   const unsigned shiftUpperBound = end - begin;
   const unsigned shiftAmount = begin;
-    auto inBounds = [&](unsigned c) {
+  auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; };
-      return c - shiftAmount < shiftUpperBound;
-    };
 
   uint32_t mask = 0;
   for (int i = 0; i < 16; ++i) {
@@ -2294,8 +2303,20 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
   return !(compared & mask) && firstRangeOk;
 
 #endif
-  }
+}
-  case Type_Node48: {
+
+template <bool kAVX512>
+bool checkMaxBetweenExclusiveImpl(Node48 *n, int begin, int end,
+                                  InternalVersionT readVersion,
+                                  ReadContext *tls) {
+  ++tls->range_read_node_scan_accum;
+  assume(-1 <= begin);
+  assume(begin <= 256);
+  assume(-1 <= end);
+  assume(end <= 256);
+  assume(begin < end);
+  assert(!(begin == -1 && end == 256));
+
   auto *self = static_cast<Node48 *>(n);
 
   {
@@ -2327,8 +2348,20 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
     }
   }
   return true;
-  }
+}
-  case Type_Node256: {
+
+template <bool kAVX512>
+bool checkMaxBetweenExclusiveImpl(Node256 *n, int begin, int end,
+                                  InternalVersionT readVersion,
+                                  ReadContext *tls) {
+  ++tls->range_read_node_scan_accum;
+  assume(-1 <= begin);
+  assume(begin <= 256);
+  assume(-1 <= end);
+  assume(end <= 256);
+  assume(begin < end);
+  assert(!(begin == -1 && end == 256));
+
   static_assert(Node256::kMaxOfMaxTotalPages == 16);
   auto *self = static_cast<Node256 *>(n);
 
@@ -2380,8 +2413,32 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
     }
   }
   // Check inner pages
-    return scan16<kAVX512>(self->maxOfMax, firstPage + 1, lastPage,
+  return scan16<kAVX512>(self->maxOfMax, firstPage + 1, lastPage, readVersion);
-                           readVersion);
+}
+
+template <bool kAVX512>
+bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
+                                  InternalVersionT readVersion,
+                                  ReadContext *tls) {
+  switch (n->getType()) {
+  case Type_Node0:
+    return checkMaxBetweenExclusiveImpl<kAVX512>(static_cast<Node0 *>(n), begin,
+                                                 end, readVersion, tls);
+  case Type_Node3: {
+    return checkMaxBetweenExclusiveImpl<kAVX512>(static_cast<Node3 *>(n), begin,
+                                                 end, readVersion, tls);
+  }
+  case Type_Node16: {
+    return checkMaxBetweenExclusiveImpl<kAVX512>(static_cast<Node16 *>(n),
+                                                 begin, end, readVersion, tls);
+  }
+  case Type_Node48: {
+    return checkMaxBetweenExclusiveImpl<kAVX512>(static_cast<Node48 *>(n),
+                                                 begin, end, readVersion, tls);
+  }
+  case Type_Node256: {
+    return checkMaxBetweenExclusiveImpl<kAVX512>(static_cast<Node256 *>(n),
+                                                 begin, end, readVersion, tls);
   }
   default:                   // GCOVR_EXCL_LINE
     __builtin_unreachable(); // GCOVR_EXCL_LINE
@@ -2402,6 +2459,58 @@ bool checkMaxBetweenExclusive(Node *n, int begin, int end,
   return checkMaxBetweenExclusiveImpl<false>(n, begin, end, readVersion, tls);
 }
 
+bool checkMaxBetweenExclusive(Node0 *n, int begin, int end,
+                              InternalVersionT readVersion, ReadContext *tls) {
+  return checkMaxBetweenExclusiveImpl<false>(n, begin, end, readVersion, tls);
+}
+
+bool checkMaxBetweenExclusive(Node3 *n, int begin, int end,
+                              InternalVersionT readVersion, ReadContext *tls) {
+  return checkMaxBetweenExclusiveImpl<false>(n, begin, end, readVersion, tls);
+}
+
+#if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__)
+__attribute__((target("avx512f"))) bool
+checkMaxBetweenExclusive(Node16 *n, int begin, int end,
+                         InternalVersionT readVersion, ReadContext *tls) {
+  return checkMaxBetweenExclusiveImpl<true>(n, begin, end, readVersion, tls);
+}
+__attribute__((target("default")))
+#endif
+
+bool checkMaxBetweenExclusive(Node16 *n, int begin, int end,
+                         InternalVersionT readVersion, ReadContext *tls) {
+  return checkMaxBetweenExclusiveImpl<false>(n, begin, end, readVersion, tls);
+}
+
+#if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__)
+__attribute__((target("avx512f"))) bool
+checkMaxBetweenExclusive(Node48 *n, int begin, int end,
+                         InternalVersionT readVersion, ReadContext *tls) {
+  return checkMaxBetweenExclusiveImpl<true>(n, begin, end, readVersion, tls);
+}
+__attribute__((target("default")))
+#endif
+
+bool checkMaxBetweenExclusive(Node48 *n, int begin, int end,
+                         InternalVersionT readVersion, ReadContext *tls) {
+  return checkMaxBetweenExclusiveImpl<false>(n, begin, end, readVersion, tls);
+}
+
+#if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__)
+__attribute__((target("avx512f"))) bool
+checkMaxBetweenExclusive(Node256 *n, int begin, int end,
+                         InternalVersionT readVersion, ReadContext *tls) {
+  return checkMaxBetweenExclusiveImpl<true>(n, begin, end, readVersion, tls);
+}
+__attribute__((target("default")))
+#endif
+
+bool checkMaxBetweenExclusive(Node256 *n, int begin, int end,
+                         InternalVersionT readVersion, ReadContext *tls) {
+  return checkMaxBetweenExclusiveImpl<false>(n, begin, end, readVersion, tls);
+}
+
 Vector<uint8_t> getSearchPath(Arena &arena, Node *n) {
   assert(n != nullptr);
   auto result = vector<uint8_t>(arena);
@@ -2424,25 +2533,27 @@ Vector<uint8_t> getSearchPath(Arena &arena, Node *n) {
 //
 // Precondition: transitively, no child of n has a search path that's a longer
 // prefix of key than n
-bool checkRangeStartsWith(Node *n, std::span<const uint8_t> key, int begin,
+template <class NodeT>
-                          int end, InternalVersionT readVersion,
+bool checkRangeStartsWith(NodeT *nTyped, std::span<const uint8_t> key,
+                          int begin, int end, InternalVersionT readVersion,
                           ReadContext *tls) {
+  Node *n;
 #if DEBUG_VERBOSE && !defined(NDEBUG)
   fprintf(stderr, "%s(%02x,%02x)*\n", printable(key).c_str(), begin, end);
 #endif
   auto remaining = key;
   if (remaining.size() == 0) {
-    return checkMaxBetweenExclusive(n, begin, end, readVersion, tls);
+    return checkMaxBetweenExclusive(nTyped, begin, end, readVersion, tls);
   }
 
-  Node *child = getChild(n, remaining[0]);
+  Node *child = getChild(nTyped, remaining[0]);
   if (child == nullptr) {
-    auto c = getChildGeq(n, remaining[0]);
+    auto c = getChildGeq(nTyped, remaining[0]);
     if (c != nullptr) {
       n = c;
       goto downLeftSpine;
     } else {
-      n = nextSibling(n);
+      n = nextSibling(nTyped);
       if (n == nullptr) {
         return true;
       }
@@ -2501,6 +2612,15 @@ scan16<true>(const InternalVersionT *vs, int begin, int end,
 template __attribute__((target("avx512f"))) bool
 checkMaxBetweenExclusiveImpl<true>(Node *n, int begin, int end,
                                    InternalVersionT readVersion, ReadContext *);
+template __attribute__((target("avx512f"))) bool
+checkMaxBetweenExclusiveImpl<true>(Node16 *n, int begin, int end,
+                                   InternalVersionT readVersion, ReadContext *);
+template __attribute__((target("avx512f"))) bool
+checkMaxBetweenExclusiveImpl<true>(Node48 *n, int begin, int end,
+                                   InternalVersionT readVersion, ReadContext *);
+template __attribute__((target("avx512f"))) bool
+checkMaxBetweenExclusiveImpl<true>(Node256 *n, int begin, int end,
+                                   InternalVersionT readVersion, ReadContext *);
 #endif
 
 // Returns a pointer the pointer to the newly inserted node in the tree.  Caller
@@ -2858,7 +2978,7 @@ struct CheckContext {
   const ConflictSet::ReadRange *queries;
   ConflictSet::Result *results;
   int64_t started;
-  ReadContext *tls;
+  ReadContext tls;
 #if !__has_attribute(musttail)
   CheckJob *job;
   bool done;
@@ -2914,7 +3034,7 @@ static Continuation iterTable[] = {iter<Node0>, iter<Node3>, iter<Node16>,
                                    iter<Node48>, iter<Node256>};
 
 void begin(CheckJob *job, CheckContext *context) {
-  ++context->tls->point_read_accum;
+  ++context->tls.point_read_accum;
 #if DEBUG_VERBOSE && !defined(NDEBUG)
   fprintf(stderr, "Check point read: %s\n", printable(key).c_str());
 #endif
@@ -2984,11 +3104,11 @@ template <class NodeT> void iter(CheckJob *job, CheckContext *context) {
     }
   }
 
-  ++context->tls->point_read_iterations_accum;
+  ++context->tls.point_read_iterations_accum;
 
   if (job->maxV <= job->readVersion) {
     job->setResult(true);
-    ++context->tls->point_read_short_circuit_accum;
+    ++context->tls.point_read_short_circuit_accum;
     MUSTTAIL return complete(job, context);
   }
 
@@ -3042,7 +3162,7 @@ static Continuation iterTable[] = {iter<Node0>, iter<Node3>, iter<Node16>,
                                    iter<Node48>, iter<Node256>};
 
 void begin(CheckJob *job, CheckContext *context) {
-  ++context->tls->prefix_read_accum;
+  ++context->tls.prefix_read_accum;
 #if DEBUG_VERBOSE && !defined(NDEBUG)
   fprintf(stderr, "Check prefix read: %s\n", printable(key).c_str());
 #endif
@@ -3113,11 +3233,11 @@ template <class NodeT> void iter(CheckJob *job, CheckContext *context) {
     }
   }
 
-  ++context->tls->prefix_read_iterations_accum;
+  ++context->tls.prefix_read_iterations_accum;
 
   if (job->maxV <= job->readVersion) {
     job->setResult(true);
-    ++context->tls->prefix_read_short_circuit_accum;
+    ++context->tls.prefix_read_short_circuit_accum;
     MUSTTAIL return complete(job, context);
   }
 
@@ -3161,6 +3281,7 @@ PRESERVE_NONE void begin(CheckJob *, CheckContext *);
 template <class NodeT>
 PRESERVE_NONE void common_prefix_iter(CheckJob *, CheckContext *);
 
+template <class NodeT>
 PRESERVE_NONE void done_common_prefix_iter(CheckJob *, CheckContext *);
 
 static Continuation commonPrefixIterTable[] = {
@@ -3168,6 +3289,11 @@ static Continuation commonPrefixIterTable[] = {
     common_prefix_iter<Node16>, common_prefix_iter<Node48>,
     common_prefix_iter<Node256>};
 
+static Continuation doneCommonPrefixIterTable[] = {
+    done_common_prefix_iter<Node0>, done_common_prefix_iter<Node3>,
+    done_common_prefix_iter<Node16>, done_common_prefix_iter<Node48>,
+    done_common_prefix_iter<Node256>};
+
 template <class NodeT>
 PRESERVE_NONE void left_side_iter(CheckJob *, CheckContext *);
 
@@ -3203,18 +3329,20 @@ PRESERVE_NONE void begin(CheckJob *job, CheckContext *context) {
     MUSTTAIL return job->continuation(job, context);
   }
 
-  ++context->tls->range_read_accum;
+  ++context->tls.range_read_accum;
   job->remaining = job->begin.subspan(0, job->lcp);
 
   if (job->remaining.size() == 0) {
-    MUSTTAIL return done_common_prefix_iter(job, context);
+    job->continuation = doneCommonPrefixIterTable[job->n->getType()];
+    MUSTTAIL return job->continuation(job, context);
   }
 
   auto [c, maxV] = getChildAndMaxVersion(job->n, job->remaining[0]);
   job->maxV = maxV;
   job->child = c;
   if (job->child == nullptr) {
-    MUSTTAIL return done_common_prefix_iter(job, context);
+    job->continuation = doneCommonPrefixIterTable[job->n->getType()];
+    MUSTTAIL return job->continuation(job, context);
   }
 
   job->continuation = commonPrefixIterTable[c.getType()];
@@ -3234,7 +3362,8 @@ void common_prefix_iter(CheckJob *job, CheckContext *context) {
     int i =
         longestCommonPrefix(child->partialKey(), job->remaining.data() + 1, cl);
     if (i != child->partialKeyLen) {
-      MUSTTAIL return done_common_prefix_iter(job, context);
+      job->continuation = doneCommonPrefixIterTable[job->n->getType()];
+      MUSTTAIL return job->continuation(job, context);
     }
   }
   job->n = child;
@@ -3242,23 +3371,25 @@ void common_prefix_iter(CheckJob *job, CheckContext *context) {
                                           job->remaining.size() -
                                               (1 + child->partialKeyLen));
 
-  ++context->tls->range_read_iterations_accum;
+  ++context->tls.range_read_iterations_accum;
 
   if (job->maxV <= job->readVersion) {
     job->setResult(true);
-    ++context->tls->range_read_short_circuit_accum;
+    ++context->tls.range_read_short_circuit_accum;
     MUSTTAIL return complete(job, context);
   }
 
   if (job->remaining.size() == 0) {
-    MUSTTAIL return done_common_prefix_iter(job, context);
+    job->continuation = done_common_prefix_iter<NodeT>;
+    MUSTTAIL return job->continuation(job, context);
   }
 
   auto [c, maxV] = getChildAndMaxVersion(child, job->remaining[0]);
   job->maxV = maxV;
   job->child = c;
   if (job->child == nullptr) {
-    MUSTTAIL return done_common_prefix_iter(job, context);
+    job->continuation = done_common_prefix_iter<NodeT>;
+    MUSTTAIL return job->continuation(job, context);
   }
 
   job->continuation = commonPrefixIterTable[c.getType()];
@@ -3266,12 +3397,15 @@ void common_prefix_iter(CheckJob *job, CheckContext *context) {
   MUSTTAIL return keepGoing(job, context);
 }
 
+template <class NodeT>
 PRESERVE_NONE void done_common_prefix_iter(CheckJob *job,
                                            CheckContext *context) {
+  assert(NodeT::kType == job->n->getType());
+  NodeT *n = static_cast<NodeT *>(job->n);
 
   {
     Arena arena;
-    assert(getSearchPath(arena, job->n) <=>
+    assert(getSearchPath(arena, n) <=>
                job->begin.subspan(0, job->lcp - job->remaining.size()) ==
            0);
   }
@@ -3282,19 +3416,18 @@ PRESERVE_NONE void done_common_prefix_iter(CheckJob *job,
   job->begin = job->begin.subspan(consumed, int(job->begin.size()) - consumed);
   job->end = job->end.subspan(consumed, int(job->end.size()) - consumed);
   job->lcp -= consumed;
-  job->commonPrefixNode = job->n;
+  job->commonPrefixNode = n;
 
   if (job->lcp == int(job->begin.size())) {
     job->remaining = job->end;
     if (job->lcp == 0) {
-      if (job->n->entryPresent &&
+      if (n->entryPresent && n->entry.pointVersion > job->readVersion) {
-          job->n->entry.pointVersion > job->readVersion) {
         job->setResult(false);
         MUSTTAIL return complete(job, context);
       }
 
-      if (!checkMaxBetweenExclusive(job->n, -1, job->remaining[0],
+      if (!checkMaxBetweenExclusive(n, -1, job->remaining[0], job->readVersion,
-                                    job->readVersion, context->tls)) {
+                                    &context->tls)) {
         job->setResult(false);
         MUSTTAIL return complete(job, context);
       }
@@ -3303,10 +3436,10 @@ PRESERVE_NONE void done_common_prefix_iter(CheckJob *job,
     // This is a hack
     --job->lcp;
 
-    auto c = getChild(job->n, job->remaining[0]);
+    auto c = getChild(n, job->remaining[0]);
     Node *child = c;
     if (child == nullptr) {
-      auto c = getChildGeq(job->n, job->remaining[0]);
+      auto c = getChildGeq(n, job->remaining[0]);
       if (c != nullptr) {
         job->n = c;
         job->continuation = down_left_spine;
@@ -3331,20 +3464,20 @@ PRESERVE_NONE void done_common_prefix_iter(CheckJob *job,
   // If this were not true we would have returned above
   assert(job->begin.size() > 0);
 
-  if (!checkRangeStartsWith(job->n, job->begin.subspan(0, job->lcp),
+  if (!checkRangeStartsWith(n, job->begin.subspan(0, job->lcp),
                             job->begin[job->lcp], job->end[job->lcp],
-                            job->readVersion, context->tls)) {
+                            job->readVersion, &context->tls)) {
     job->setResult(false);
     MUSTTAIL return complete(job, context);
   }
 
   job->remaining = job->begin;
 
-  auto [c, maxV] = getChildAndMaxVersion(job->n, job->remaining[0]);
+  auto [c, maxV] = getChildAndMaxVersion(n, job->remaining[0]);
   job->maxV = maxV;
   Node *child = c;
   if (child == nullptr) {
-    auto c = getChildGeq(job->n, job->remaining[0]);
+    auto c = getChildGeq(n, job->remaining[0]);
     if (c != nullptr) {
       job->n = c;
       job->continuation = left_side_down_left_spine;
@@ -3428,7 +3561,7 @@ PRESERVE_NONE void left_side_iter(CheckJob *job, CheckContext *context) {
     }
   }
 
-  ++context->tls->range_read_iterations_accum;
+  ++context->tls.range_read_iterations_accum;
 
   if (job->maxV <= job->readVersion) {
     job->continuation = done_left_side_iter;
@@ -3442,7 +3575,7 @@ PRESERVE_NONE void left_side_iter(CheckJob *job, CheckContext *context) {
   }
 
   if (!checkMaxBetweenExclusive(n, job->remaining[0], 256, job->readVersion,
-                                context->tls)) {
+                                &context->tls)) {
     job->setResult(false);
     MUSTTAIL return complete(job, context);
   }
@@ -3566,7 +3699,7 @@ PRESERVE_NONE void right_side_iter(CheckJob *job, CheckContext *context) {
     }
   }
 
-  ++context->tls->range_read_iterations_accum;
+  ++context->tls.range_read_iterations_accum;
 
   if (job->remaining.size() == 0) {
     job->continuation = down_left_spine;
@@ -3580,7 +3713,7 @@ PRESERVE_NONE void right_side_iter(CheckJob *job, CheckContext *context) {
   }
 
   if (!checkMaxBetweenExclusive(n, -1, job->remaining[0], job->readVersion,
-                                context->tls)) {
+                                &context->tls)) {
     job->setResult(false);
     MUSTTAIL return complete(job, context);
   }
@@ -3646,18 +3779,16 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
       return;
     }
 
-    ReadContext tls;
-    tls.impl = this;
     int64_t check_byte_accum = 0;
     constexpr int kConcurrent = 16;
     CheckJob inProgress[kConcurrent];
     CheckContext context;
+    context.tls.impl = this;
     context.count = count;
     context.oldestVersionFullPrecision = oldestVersionFullPrecision;
     context.root = root;
     context.queries = reads;
     context.results = result;
-    context.tls = &tls;
     int64_t started = std::min(kConcurrent, count);
     context.started = started;
     for (int i = 0; i < started; i++) {
@@ -3690,24 +3821,27 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
       assert(reads[i].readVersion <= newestVersionFullPrecision);
       const auto &r = reads[i];
       check_byte_accum += r.begin.len + r.end.len;
-      tls.commits_accum += result[i] == Commit;
+      context.tls.commits_accum += result[i] == Commit;
-      tls.conflicts_accum += result[i] == Conflict;
+      context.tls.conflicts_accum += result[i] == Conflict;
-      tls.too_olds_accum += result[i] == TooOld;
+      context.tls.too_olds_accum += result[i] == TooOld;
     }
 
-    point_read_total.add(tls.point_read_accum);
+    point_read_total.add(context.tls.point_read_accum);
-    prefix_read_total.add(tls.prefix_read_accum);
+    prefix_read_total.add(context.tls.prefix_read_accum);
-    range_read_total.add(tls.range_read_accum);
+    range_read_total.add(context.tls.range_read_accum);
-    range_read_node_scan_total.add(tls.range_read_node_scan_accum);
+    range_read_node_scan_total.add(context.tls.range_read_node_scan_accum);
-    point_read_short_circuit_total.add(tls.point_read_short_circuit_accum);
+    point_read_short_circuit_total.add(
-    prefix_read_short_circuit_total.add(tls.prefix_read_short_circuit_accum);
+        context.tls.point_read_short_circuit_accum);
-    range_read_short_circuit_total.add(tls.range_read_short_circuit_accum);
+    prefix_read_short_circuit_total.add(
-    point_read_iterations_total.add(tls.point_read_iterations_accum);
+        context.tls.prefix_read_short_circuit_accum);
-    prefix_read_iterations_total.add(tls.prefix_read_iterations_accum);
+    range_read_short_circuit_total.add(
-    range_read_iterations_total.add(tls.range_read_iterations_accum);
+        context.tls.range_read_short_circuit_accum);
-    commits_total.add(tls.commits_accum);
+    point_read_iterations_total.add(context.tls.point_read_iterations_accum);
-    conflicts_total.add(tls.conflicts_accum);
+    prefix_read_iterations_total.add(context.tls.prefix_read_iterations_accum);
-    too_olds_total.add(tls.too_olds_accum);
+    range_read_iterations_total.add(context.tls.range_read_iterations_accum);
+    commits_total.add(context.tls.commits_accum);
+    conflicts_total.add(context.tls.conflicts_accum);
+    too_olds_total.add(context.tls.too_olds_accum);
     check_bytes_total.add(check_byte_accum);
   }