include/weaseljson.h

@@ -50,14 +50,10 @@ enum WeaselJsonFlags {
 };
 
 /** Create a parser. Increasing stack size increases memory usage but also
- * increases the depth of nested json accepted. `stackSize` is an exact budget:
+ * increases the depth of nested json accepted. Whether a document is accepted,
- * whether a document is accepted, rejected or reported as too deeply nested
+ * rejected or reported as too deeply nested depends only on its bytes and on
- * depends only on its bytes and on `stackSize`, not on how the bytes are split
+ * `stackSize`, not on how the bytes are split across WeaselJsonParser_parse
- * across WeaselJsonParser_parse calls. To get that guarantee, the parser
+ * calls. `callbacks` and `userdata` must outlive the returned parser. Returns
- * allocates 6 slots beyond `stackSize` for tokenizer progress: literals
- * ("true", "false", "null") and \uXXXX escapes that straddle the end of a chunk
- * record their unmatched characters there instead of spending the nesting
- * budget. `callbacks` and `userdata` must outlive the returned parser. Returns
  * null if there's insufficient available memory, or if `stackSize` is negative
  * or too small to hold a minimal document. */
 WeaselJsonParser *WeaselJsonParser_create(int stackSize,

src/parser3.h

@@ -73,24 +73,27 @@ enum Symbol : uint8_t {
   N_SYMBOL_COUNT, // Must be last
 };
 
-// Symbols that are only on the stack because a literal (`true`/`false`/`null`)
+// Some symbols are on the stack only because a literal (`true`/`false`/`null`)
 // or a `\uXXXX` escape straddles the end of a chunk. The whole-buffer path
 // consumes those characters inline and pushes nothing, so these symbols must
 // not spend the caller's nesting budget: otherwise the stackSize a document
-// needs would depend on where the parse() boundaries happen to fall (see issue
+// needs would depend on where the parse() boundaries happen to fall (issue
-// #71). WeaselJsonParser_create() therefore allocates this many extra slots on
+// #71). WeaselJsonParser_create() allocates this many slots beyond `stackSize`
-// top of `stackSize`; see Parser3::pushTransient().
+// to hold them, and Parser3::pushTransient() pushes them there.
 //
-// These are all the pushes of such symbols, with their sizes:
+// All the pushes of such symbols, with their sizes:
 //   n_value()                    3, 4, 3  (true, false, null)
 //   n_string2()                  1        (N_STRING_FOLLOWING_ESCAPE)
 //   n_string_following_escape()  4        (the four hex digits)
 //   t_hex2()                     6        (utf-16 surrogate pair)
-// No transient symbol is ever pushed while another one is on the stack (each of
+//
-// these continuations pops its own symbol before pushing, and the literal ones
+// Upper bound on how many of them are on the stack at once: 6. These are
-// run with N_VALUE already popped), so the largest single push - the six
+// pushdown symbols, so the ones that are on the stack are consumed before
-// symbols that start a surrogate pair - bounds the number of transient symbols
+// anything else is pushed; and none of those continuations pushes while
-// on the stack.
+// another's symbols are still pending, because each pops the symbol it took
+// before pushing (the literal ones pop N_VALUE before n_value() pushes, and the
+// rest pop themselves). The number of pending symbols is therefore bounded by
+// the largest single push, the six symbols that start a surrogate pair.
 inline constexpr int kTransientSymbols = 6;
 
 struct Parser3 {
@@ -164,10 +167,10 @@ struct Parser3 {
   }
 
   // Push tokenizer progress symbols for a literal or escape that straddles a
-  // chunk boundary. These go in the reserve past `stackEnd`, so they don't
+  // chunk boundary. Unlike push(), these may grow past `stackEnd` into the
-  // spend the caller's nesting budget and can't make the parser overflow on
+  // reserve, so they don't spend the caller's nesting budget and can never make
-  // input it would have accepted in one piece. See kTransientSymbols for why
+  // the parser overflow on input it would have accepted in one piece. See
-  // the reserve is big enough to always succeed.
+  // kTransientSymbols for why the reserve is big enough to always succeed.
   [[nodiscard]] WeaselJsonStatus
   pushTransient(std::initializer_list<Symbol> symbols) {
     assert(ptrdiff_t(symbols.size()) <= kTransientSymbols);

src/test.cpp

@@ -154,19 +154,30 @@ void testStreaming(std::string const &json) {
   CHECK(streaming.result == batch.result);
 }
 
-// Feed `doc` to a fresh parser with the given stackSize, in chunks of `stride`
+// Feed `doc` to a fresh parser with the given stackSize, cutting it into chunks
-// bytes (`stride` <= 0 means all at once), and return the final status.
+// at the offsets in `cuts` (ascending, each > 0 and < doc.size(); an empty
+// `cuts` feeds the whole document at once), and return the final status.
 WeaselJsonStatus parseChunked(std::string const &doc, int stackSize,
-                              int stride) {
+                              std::vector<size_t> const &cuts) {
   auto c = noopCallbacks();
   std::string copy = doc;
   auto *parser = WeaselJsonParser_create(stackSize, &c, nullptr, 0);
   REQUIRE(parser != nullptr);
-  int step = stride > 0 ? stride : std::max(1, (int)copy.size());
   WeaselJsonStatus status = WeaselJson_AGAIN;
-  for (size_t i = 0; i < copy.size() && status == WeaselJson_AGAIN; i += step) {
+  size_t pos = 0;
-    status = WeaselJsonParser_parse(parser, copy.data() + i,
+  for (size_t cut : cuts) {
-                                    std::min<int>(step, copy.size() - i));
+    if (status != WeaselJson_AGAIN) {
+      break;
+    }
+    if (cut > pos && cut < copy.size()) {
+      status =
+          WeaselJsonParser_parse(parser, copy.data() + pos, int(cut - pos));
+      pos = cut;
+    }
+  }
+  if (status == WeaselJson_AGAIN && pos < copy.size()) {
+    status = WeaselJsonParser_parse(parser, copy.data() + pos,
+                                    int(copy.size() - pos));
   }
   if (status == WeaselJson_AGAIN) {
     status = WeaselJsonParser_parse(parser, nullptr, 0);
@@ -175,6 +186,15 @@ WeaselJsonStatus parseChunked(std::string const &doc, int stackSize,
   return status;
 }
 
+// The offsets to cut `doc` into chunks of `stride` bytes.
+std::vector<size_t> stridedCuts(std::string const &doc, size_t stride) {
+  std::vector<size_t> cuts;
+  for (size_t cut = stride; cut < doc.size(); cut += stride) {
+    cuts.push_back(cut);
+  }
+  return cuts;
+}
+
 } // namespace
 
 TEST_CASE("parser3") {
@@ -314,10 +334,17 @@ TEST_CASE("acceptance is independent of the chunking") {
     // sweeps from "everything overflows" to "everything fits".
     for (int stackSize = 3; stackSize <= 16; ++stackSize) {
       CAPTURE(stackSize);
-      auto whole = parseChunked(doc, stackSize, 0);
+      auto whole = parseChunked(doc, stackSize, {});
-      for (int stride = 1; stride <= 12; ++stride) {
+      // Every way to split the document in two, so that every literal and
+      // escape in `docs` straddles a chunk boundary at some point.
+      for (size_t cut = 1; cut < doc.size(); ++cut) {
+        CAPTURE(cut);
+        REQUIRE(parseChunked(doc, stackSize, {cut}) == whole);
+      }
+      for (size_t stride = 1; stride <= 12; ++stride) {
         CAPTURE(stride);
-        REQUIRE(parseChunked(doc, stackSize, stride) == whole);
+        REQUIRE(parseChunked(doc, stackSize, stridedCuts(doc, stride)) ==
+                whole);
       }
     }
   }