From ac619af24d85a055ba10fdfb83bba9a4b78f2063 Mon Sep 17 00:00:00 2001 From: Weaselbot Date: Tue, 27 Oct 2026 16:20:00 -0400 Subject: [PATCH 1/3] parser3: reserve stack slots for chunk-split literals and escapes WeaselJson_OVERFLOW is documented as "json is too deeply nested", and stackSize as a nesting-depth budget. In practice the pushdown stack also held tokenizer progress for a `true`/`false`/`null` literal or a \uXXXX escape that straddled the end of a chunk: the whole-buffer path consumes those characters inline and pushes nothing, while the slow path pushed up to six symbols (a surrogate pair) onto the same stack. So the stackSize a document needed depended on where the parse() boundaries fell: e.g. "true" was accepted at stackSize 3 when fed whole but returned the sticky WeaselJson_OVERFLOW when fed one byte at a time. Reserve slots for those symbols in addition to the supplied stackSize, as a fixed upper bound rather than by changing the encoding: the largest single push of transient symbols is t_hex2()'s six symbols for a surrogate pair, and no transient symbol is ever pushed while another one is on the stack, so kTransientSymbols = 6 is enough. Nesting pushes are still bounded by stackSize, so OVERFLOW now means exactly what the header says and the accept/reject/overflow decision is a function of the input bytes and stackSize only. Document the reserve in weaseljson.h and cover the property with tests: a doctest sweeping stackSize 3..16 over documents whose literals, escapes and surrogate pairs land on every chunk boundary, and a fuzz harness that now sweeps small stack sizes and requires the status of every chunking (stride 1..15 plus every single split point) to match the whole-buffer status exactly. --- include/weaseljson.h | 16 +++++--- src/fuzz.cpp | 94 ++++++++++++++++++++++---------------------- src/lib.cpp | 3 +- src/parser3.h | 69 +++++++++++++++++++++++++++----- src/test.cpp | 65 ++++++++++++++++++++++++++++++ 5 files changed, 183 insertions(+), 64 deletions(-) diff --git a/include/weaseljson.h b/include/weaseljson.h index aa0ec52..3c784fe 100644 --- a/include/weaseljson.h +++ b/include/weaseljson.h @@ -36,7 +36,7 @@ enum WeaselJsonStatus { WeaselJson_AGAIN, /** Invalid json */ WeaselJson_REJECT, - /** json is too deeply nested */ + /** json is too deeply nested for the parser's stack size */ WeaselJson_OVERFLOW, /** Tried to call parse on a null parser */ WeaselJson_NULL, @@ -50,10 +50,16 @@ enum WeaselJsonFlags { }; /** Create a parser. Increasing stack size increases memory usage but also - * increases the depth of nested json accepted. `callbacks` and `userdata` must - * outlive the returned parser. Returns null if there's insufficient available - * memory, or if `stackSize` is negative or too small to hold a minimal - * document. */ + * increases the depth of nested json accepted. `stackSize` is an exact budget: + * whether a document is accepted, rejected or reported as too deeply nested + * depends only on its bytes and on `stackSize`, not on how the bytes are split + * across WeaselJsonParser_parse calls. To get that guarantee, the parser + * allocates 6 slots beyond `stackSize` for tokenizer progress: literals + * ("true", "false", "null") and \uXXXX escapes that straddle the end of a chunk + * record their unmatched characters there instead of spending the nesting + * budget. `callbacks` and `userdata` must outlive the returned parser. Returns + * null if there's insufficient available memory, or if `stackSize` is negative + * or too small to hold a minimal document. */ WeaselJsonParser *WeaselJsonParser_create(int stackSize, const WeaselJsonCallbacks *callbacks, void *userdata, int flags); diff --git a/src/fuzz.cpp b/src/fuzz.cpp index 3b40f48..1dd0321 100644 --- a/src/fuzz.cpp +++ b/src/fuzz.cpp @@ -5,12 +5,12 @@ #include -std::pair runStreaming(std::string copy, - int stride) { +std::pair +runStreaming(std::string copy, int stackSize, int stride) { SerializeState state; auto c = serializeCallbacks(); std::unique_ptr parser{ - WeaselJsonParser_create(1024, &c, &state, + WeaselJsonParser_create(stackSize, &c, &state, copy.size() % 2 == 0 ? WeaselJsonRaw : 0), WeaselJsonParser_destroy}; if (stride == 0) { @@ -31,11 +31,12 @@ std::pair runStreaming(std::string copy, return {state.result, s}; } -std::pair runBatch(std::string copy) { +std::pair runBatch(std::string copy, + int stackSize) { SerializeState state; auto c = serializeCallbacks(); std::unique_ptr parser{ - WeaselJsonParser_create(1 << 20, &c, &state, + WeaselJsonParser_create(stackSize, &c, &state, copy.size() % 2 == 0 ? WeaselJsonRaw : 0), WeaselJsonParser_destroy}; auto s = WeaselJsonParser_parse(parser.get(), copy.data(), copy.size()); @@ -47,11 +48,11 @@ std::pair runBatch(std::string copy) { } std::pair runPrefix(std::string copy, - int prefix) { + int stackSize, int prefix) { SerializeState state; auto c = serializeCallbacks(); std::unique_ptr parser{ - WeaselJsonParser_create(1024, &c, &state, + WeaselJsonParser_create(stackSize, &c, &state, copy.size() % 2 == 0 ? WeaselJsonRaw : 0), WeaselJsonParser_destroy}; auto s = WeaselJsonParser_parse(parser.get(), copy.data(), prefix); @@ -67,55 +68,54 @@ std::pair runPrefix(std::string copy, return {state.result, s}; } -void testStreaming(std::string const &json) { - auto batch = runBatch(json); +void checkStreamingResult( + int stackSize, char const *kind, + std::pair const &got, + std::pair const &batch) { + if (got.second == WeaselJson_AGAIN) { + abort(); + } + // The status must be identical however the document is chunked; the events + // reported for a document that failed to parse need not match. + bool ok = got.second == batch.second && + (got.second != WeaselJson_OK || got.first == batch.first); + if (!ok) { + printf("%s (stackSize %d): %d, %s\n", kind, stackSize, (int)got.second, + got.first.c_str()); + printf("batch (stackSize %d): %d, %s\n", stackSize, (int)batch.second, + batch.first.c_str()); + abort(); + } +} + +void testStreaming(std::string const &json, int stackSize) { + auto batch = runBatch(json, stackSize); if (batch.second == WeaselJson_AGAIN) { abort(); } for (int stride = 1; stride < 16; ++stride) { - auto streaming = runStreaming(json, stride); - if (streaming != batch) { - if (streaming.second == WeaselJson_AGAIN) { - abort(); - } - bool streamingOk = streaming.second == WeaselJson_OK; - bool batchOk = batch.second == WeaselJson_OK; - if (streamingOk == batchOk && !batchOk) { - // It's ok if the processed data doesn't match if parsing failed - } else { - printf("streaming: %s, %s\n", - streaming.second == WeaselJson_OK ? "accept" : "reject", - streaming.first.c_str()); - printf("batch: %s, %s\n", - batch.second == WeaselJson_OK ? "accept" : "reject", - batch.first.c_str()); - abort(); - } - } + checkStreamingResult(stackSize, "streaming", + runStreaming(json, stackSize, stride), batch); if (int(json.size()) > stride) { - auto prefix = runPrefix(json, stride); - if (prefix != batch) { - if (prefix.second == WeaselJson_AGAIN) { - abort(); - } - bool prefixOk = prefix.second == WeaselJson_OK; - bool batchOk = batch.second == WeaselJson_OK; - if (prefixOk == batchOk && !batchOk) { - // It's ok if the processed data doesn't match if parsing failed - } else { - printf("prefix: %s, %s\n", - prefix.second == WeaselJson_OK ? "accept" : "reject", - prefix.first.c_str()); - printf("batch: %s, %s\n", - batch.second == WeaselJson_OK ? "accept" : "reject", - batch.first.c_str()); - abort(); - } - } + checkStreamingResult(stackSize, "prefix", + runPrefix(json, stackSize, stride), batch); } } } +void testStreaming(std::string const &json) { + // Whether a document parses must not depend on the streaming pattern: the + // parser hands the tokenizer progress for a literal or a \uXXXX escape that + // straddles a chunk boundary to the pushdown stack, so a stackSize picked for + // the document's nesting depth used to overflow for some chunkings only. + // Sweep stack sizes around the minimum (3 is the smallest accepted by + // WeaselJsonParser_create) so that the property is exercised, then a large + // one for documents that need the room. + for (int stackSize : {3, 4, 5, 6, 7, 8, 12, 1024}) { + testStreaming(json, stackSize); + } +} + void compareWithSimdjson(std::string const &json) { WeaselJsonStatus ours; { diff --git a/src/lib.cpp b/src/lib.cpp index b805e01..655275e 100644 --- a/src/lib.cpp +++ b/src/lib.cpp @@ -13,8 +13,7 @@ WeaselJsonParser_create(int stackSize, const WeaselJsonCallbacks *callbacks, if (stackSize < 0) { // avoid (size_t)stackSize wrapping to a huge allocation return nullptr; } - auto *buf = - malloc(sizeof(Parser3) + (size_t)stackSize * sizeof(*Parser3::stackPtr)); + auto *buf = malloc(Parser3::allocationSize(stackSize)); if (buf == nullptr) { return nullptr; } diff --git a/src/parser3.h b/src/parser3.h index 706a77f..daadb56 100644 --- a/src/parser3.h +++ b/src/parser3.h @@ -72,6 +72,27 @@ enum Symbol : uint8_t { T_BACKSLASH, N_SYMBOL_COUNT, // Must be last }; + +// Symbols that are only on the stack because a literal (`true`/`false`/`null`) +// or a `\uXXXX` escape straddles the end of a chunk. The whole-buffer path +// consumes those characters inline and pushes nothing, so these symbols must +// not spend the caller's nesting budget: otherwise the stackSize a document +// needs would depend on where the parse() boundaries happen to fall (see issue +// #71). WeaselJsonParser_create() therefore allocates this many extra slots on +// top of `stackSize`; see Parser3::pushTransient(). +// +// These are all the pushes of such symbols, with their sizes: +// n_value() 3, 4, 3 (true, false, null) +// n_string2() 1 (N_STRING_FOLLOWING_ESCAPE) +// n_string_following_escape() 4 (the four hex digits) +// t_hex2() 6 (utf-16 surrogate pair) +// No transient symbol is ever pushed while another one is on the stack (each of +// these continuations pops its own symbol before pushing, and the literal ones +// run with N_VALUE already popped), so the largest single push - the six +// symbols that start a surrogate pair - bounds the number of transient symbols +// on the stack. +inline constexpr int kTransientSymbols = 6; + struct Parser3 { Parser3(const WeaselJsonCallbacks *callbacks, void *userdata, int stackSize, int flags) @@ -80,6 +101,14 @@ struct Parser3 { reset(); } + // Size of the allocation that holds a parser with a nesting budget of + // `stackSize` symbols: the parser itself, that budget, and the reserve for + // transient tokenizer progress symbols. + static constexpr size_t allocationSize(int stackSize) { + return sizeof(Parser3) + + (size_t(stackSize) + kTransientSymbols) * sizeof(Symbol); + } + [[nodiscard]] WeaselJsonStatus parse(char *buf, int len); void flushNumber(bool done, char *buf) { @@ -114,9 +143,12 @@ struct Parser3 { } // Pushing symbols onto the stack roughly corresponds with "parse these - // nonterminals/terminals in order" - [[nodiscard]] WeaselJsonStatus push(std::initializer_list symbols) { - if (stackEnd - stackPtr < ptrdiff_t(symbols.size())) [[unlikely]] { + // nonterminals/terminals in order". `end` is the first slot the symbols may + // not occupy. + [[nodiscard]] static WeaselJsonStatus + pushOnto(Symbol *&stackPtr, Symbol *end, + std::initializer_list symbols) { + if (end - stackPtr < ptrdiff_t(symbols.size())) [[unlikely]] { return WeaselJson_OVERFLOW; } for (int i = symbols.size() - 1; i >= 0; --i) { @@ -125,6 +157,23 @@ struct Parser3 { return WeaselJson_OK; } + // Push symbols that model the nesting of the document. This is what + // WeaselJson_OVERFLOW reports, so it's bounded by the caller's stackSize. + [[nodiscard]] WeaselJsonStatus push(std::initializer_list symbols) { + return pushOnto(stackPtr, stackEnd, symbols); + } + + // Push tokenizer progress symbols for a literal or escape that straddles a + // chunk boundary. These go in the reserve past `stackEnd`, so they don't + // spend the caller's nesting budget and can't make the parser overflow on + // input it would have accepted in one piece. See kTransientSymbols for why + // the reserve is big enough to always succeed. + [[nodiscard]] WeaselJsonStatus + pushTransient(std::initializer_list symbols) { + assert(ptrdiff_t(symbols.size()) <= kTransientSymbols); + return pushOnto(stackPtr, stackEnd + kTransientSymbols, symbols); + } + [[nodiscard]] Symbol top() const { assert(!empty()); return *(stackPtr - 1); @@ -376,7 +425,7 @@ inline PRESERVE_NONE ContinuationStatus n_value(Parser3 *self, char *buf, return WeaselJson_REJECT; } } else { - if (auto s = self->push({T_R, T_U, N_TRUE})) { + if (auto s = self->pushTransient({T_R, T_U, N_TRUE})) { return s; } } @@ -392,7 +441,7 @@ inline PRESERVE_NONE ContinuationStatus n_value(Parser3 *self, char *buf, return WeaselJson_REJECT; } } else { - if (auto s = self->push({T_A, T_L, T_S, N_FALSE})) { + if (auto s = self->pushTransient({T_A, T_L, T_S, N_FALSE})) { return s; } } @@ -408,7 +457,7 @@ inline PRESERVE_NONE ContinuationStatus n_value(Parser3 *self, char *buf, return WeaselJson_REJECT; } } else { - if (auto s = self->push({T_U, T_L, N_NULL})) { + if (auto s = self->pushTransient({T_U, T_L, N_NULL})) { return s; } } @@ -609,7 +658,7 @@ inline PRESERVE_NONE ContinuationStatus n_string2(Parser3 *self, char *buf, case '\\': ++buf; if (bufEnd - buf < /*strlen("u0000\\u0000")*/ 11) { - if (auto s = self->push({N_STRING_FOLLOWING_ESCAPE})) { + if (auto s = self->pushTransient({N_STRING_FOLLOWING_ESCAPE})) { return s; } if (buf == bufEnd) { @@ -718,7 +767,7 @@ n_string_following_escape(Parser3 *self, char *buf, char *bufEnd) { ++buf; self->utf8Codepoint = 0; self->pop(); - if (auto s = self->push({T_HEX, T_HEX, T_HEX, T_HEX2})) { + if (auto s = self->pushTransient({T_HEX, T_HEX, T_HEX, T_HEX2})) { return s; } break; @@ -800,8 +849,8 @@ inline PRESERVE_NONE ContinuationStatus t_hex2(Parser3 *self, char *buf, self->utf16Surrogate = self->utf8Codepoint; self->utf8Codepoint = 0; self->pop(); - if (auto s = - self->push({T_BACKSLASH, T_U2, T_HEX, T_HEX, T_HEX, T_HEX3})) { + if (auto s = self->pushTransient( + {T_BACKSLASH, T_U2, T_HEX, T_HEX, T_HEX, T_HEX3})) { return s; } if (buf == bufEnd) { diff --git a/src/test.cpp b/src/test.cpp index 10cb1fb..453f0f4 100644 --- a/src/test.cpp +++ b/src/test.cpp @@ -4,8 +4,10 @@ #include #include +#include #include #include +#include #include #include @@ -152,6 +154,27 @@ void testStreaming(std::string const &json) { CHECK(streaming.result == batch.result); } +// Feed `doc` to a fresh parser with the given stackSize, in chunks of `stride` +// bytes (`stride` <= 0 means all at once), and return the final status. +WeaselJsonStatus parseChunked(std::string const &doc, int stackSize, + int stride) { + auto c = noopCallbacks(); + std::string copy = doc; + auto *parser = WeaselJsonParser_create(stackSize, &c, nullptr, 0); + REQUIRE(parser != nullptr); + int step = stride > 0 ? stride : std::max(1, (int)copy.size()); + WeaselJsonStatus status = WeaselJson_AGAIN; + for (size_t i = 0; i < copy.size() && status == WeaselJson_AGAIN; i += step) { + status = WeaselJsonParser_parse(parser, copy.data() + i, + std::min(step, copy.size() - i)); + } + if (status == WeaselJson_AGAIN) { + status = WeaselJsonParser_parse(parser, nullptr, 0); + } + WeaselJsonParser_destroy(parser); + return status; +} + } // namespace TEST_CASE("parser3") { @@ -258,6 +281,48 @@ TEST_CASE("reset clears overflow state") { WeaselJsonParser_destroy(parser); } +TEST_CASE("acceptance is independent of the chunking") { + // Literals and \uXXXX escapes that straddle the end of a chunk are tokenized + // by pushing their not-yet-matched symbols onto the pushdown stack, so they + // used to spend the caller's nesting budget: a document could be accepted + // when fed whole and reported as WeaselJson_OVERFLOW when streamed in smaller + // pieces. The decision must depend only on the document and on `stackSize`. + std::vector docs = { + "1", + "true", + "false", + "null", + "[true]", + "[true,false,null]", + "\"\\u00e9abcdefgh\"", + // Needs the full transient reserve: a surrogate pair split across chunks + // pushes six symbols at once. + "\"\\ud801\\udc37\"", + "[\"\\u00e9abcdefgh\"]", + "{\"\\ud801\\udc37\":[true,{\"x\":null}]}", + "[[[[[[[[[[[[true]]]]]]]]]]]]", + // Whether an invalid document is rejected must not depend on the chunking + // either. + "tru", + "truX", + "[[[[x", + "\"\\ud801x\"", + }; + for (auto const &doc : docs) { + CAPTURE(doc); + // 3 is the smallest stackSize WeaselJsonParser_create accepts, so this + // sweeps from "everything overflows" to "everything fits". + for (int stackSize = 3; stackSize <= 16; ++stackSize) { + CAPTURE(stackSize); + auto whole = parseChunked(doc, stackSize, 0); + for (int stride = 1; stride <= 12; ++stride) { + CAPTURE(stride); + REQUIRE(parseChunked(doc, stackSize, stride) == whole); + } + } + } +} + TEST_CASE("rejected state is sticky") { auto c = noopCallbacks(); auto *parser = WeaselJsonParser_create(1024, &c, nullptr, 0); -- 2.43.0 From 507be6f0a259fe8b6104be6eeb0cd2cb295ac8ab Mon Sep 17 00:00:00 2001 From: Weaselbot Date: Tue, 27 Oct 2026 17:31:16 -0400 Subject: [PATCH 2/3] Address review: keep the transient-token reserve out of the public header The create() documentation described how the reserve is implemented (six extra slots for literals and \uXXXX escapes that straddle a chunk boundary). That is an implementation detail; the header should only promise the behaviour callers can rely on, i.e. that the accept/reject/ overflow decision depends on the document's bytes and on stackSize, not on how they are split across parse() calls. The derivation of the upper bound lives next to kTransientSymbols in src/parser3.h, where it can be kept in sync with the pushes. Also make the chunking test cover every split point, not just fixed strides: each document is split in two at every possible offset, so every literal, escape and surrogate pair in the fixture straddles a chunk boundary. Verified that this fails on the pre-fix parser ("true", stackSize 3, cut 1: whole buffer accepts, two chunks overflow). --- include/weaseljson.h | 12 ++++------- src/parser3.h | 31 ++++++++++++++++------------- src/test.cpp | 47 ++++++++++++++++++++++++++++++++++---------- 3 files changed, 58 insertions(+), 32 deletions(-) diff --git a/include/weaseljson.h b/include/weaseljson.h index 3c784fe..a0d35ac 100644 --- a/include/weaseljson.h +++ b/include/weaseljson.h @@ -50,14 +50,10 @@ enum WeaselJsonFlags { }; /** Create a parser. Increasing stack size increases memory usage but also - * increases the depth of nested json accepted. `stackSize` is an exact budget: - * whether a document is accepted, rejected or reported as too deeply nested - * depends only on its bytes and on `stackSize`, not on how the bytes are split - * across WeaselJsonParser_parse calls. To get that guarantee, the parser - * allocates 6 slots beyond `stackSize` for tokenizer progress: literals - * ("true", "false", "null") and \uXXXX escapes that straddle the end of a chunk - * record their unmatched characters there instead of spending the nesting - * budget. `callbacks` and `userdata` must outlive the returned parser. Returns + * increases the depth of nested json accepted. Whether a document is accepted, + * rejected or reported as too deeply nested depends only on its bytes and on + * `stackSize`, not on how the bytes are split across WeaselJsonParser_parse + * calls. `callbacks` and `userdata` must outlive the returned parser. Returns * null if there's insufficient available memory, or if `stackSize` is negative * or too small to hold a minimal document. */ WeaselJsonParser *WeaselJsonParser_create(int stackSize, diff --git a/src/parser3.h b/src/parser3.h index daadb56..f66ecd6 100644 --- a/src/parser3.h +++ b/src/parser3.h @@ -73,24 +73,27 @@ enum Symbol : uint8_t { N_SYMBOL_COUNT, // Must be last }; -// Symbols that are only on the stack because a literal (`true`/`false`/`null`) +// Some symbols are on the stack only because a literal (`true`/`false`/`null`) // or a `\uXXXX` escape straddles the end of a chunk. The whole-buffer path // consumes those characters inline and pushes nothing, so these symbols must // not spend the caller's nesting budget: otherwise the stackSize a document -// needs would depend on where the parse() boundaries happen to fall (see issue -// #71). WeaselJsonParser_create() therefore allocates this many extra slots on -// top of `stackSize`; see Parser3::pushTransient(). +// needs would depend on where the parse() boundaries happen to fall (issue +// #71). WeaselJsonParser_create() allocates this many slots beyond `stackSize` +// to hold them, and Parser3::pushTransient() pushes them there. // -// These are all the pushes of such symbols, with their sizes: +// All the pushes of such symbols, with their sizes: // n_value() 3, 4, 3 (true, false, null) // n_string2() 1 (N_STRING_FOLLOWING_ESCAPE) // n_string_following_escape() 4 (the four hex digits) // t_hex2() 6 (utf-16 surrogate pair) -// No transient symbol is ever pushed while another one is on the stack (each of -// these continuations pops its own symbol before pushing, and the literal ones -// run with N_VALUE already popped), so the largest single push - the six -// symbols that start a surrogate pair - bounds the number of transient symbols -// on the stack. +// +// Upper bound on how many of them are on the stack at once: 6. These are +// pushdown symbols, so the ones that are on the stack are consumed before +// anything else is pushed; and none of those continuations pushes while +// another's symbols are still pending, because each pops the symbol it took +// before pushing (the literal ones pop N_VALUE before n_value() pushes, and the +// rest pop themselves). The number of pending symbols is therefore bounded by +// the largest single push, the six symbols that start a surrogate pair. inline constexpr int kTransientSymbols = 6; struct Parser3 { @@ -164,10 +167,10 @@ struct Parser3 { } // Push tokenizer progress symbols for a literal or escape that straddles a - // chunk boundary. These go in the reserve past `stackEnd`, so they don't - // spend the caller's nesting budget and can't make the parser overflow on - // input it would have accepted in one piece. See kTransientSymbols for why - // the reserve is big enough to always succeed. + // chunk boundary. Unlike push(), these may grow past `stackEnd` into the + // reserve, so they don't spend the caller's nesting budget and can never make + // the parser overflow on input it would have accepted in one piece. See + // kTransientSymbols for why the reserve is big enough to always succeed. [[nodiscard]] WeaselJsonStatus pushTransient(std::initializer_list symbols) { assert(ptrdiff_t(symbols.size()) <= kTransientSymbols); diff --git a/src/test.cpp b/src/test.cpp index 453f0f4..8919ccf 100644 --- a/src/test.cpp +++ b/src/test.cpp @@ -154,19 +154,30 @@ void testStreaming(std::string const &json) { CHECK(streaming.result == batch.result); } -// Feed `doc` to a fresh parser with the given stackSize, in chunks of `stride` -// bytes (`stride` <= 0 means all at once), and return the final status. +// Feed `doc` to a fresh parser with the given stackSize, cutting it into chunks +// at the offsets in `cuts` (ascending, each > 0 and < doc.size(); an empty +// `cuts` feeds the whole document at once), and return the final status. WeaselJsonStatus parseChunked(std::string const &doc, int stackSize, - int stride) { + std::vector const &cuts) { auto c = noopCallbacks(); std::string copy = doc; auto *parser = WeaselJsonParser_create(stackSize, &c, nullptr, 0); REQUIRE(parser != nullptr); - int step = stride > 0 ? stride : std::max(1, (int)copy.size()); WeaselJsonStatus status = WeaselJson_AGAIN; - for (size_t i = 0; i < copy.size() && status == WeaselJson_AGAIN; i += step) { - status = WeaselJsonParser_parse(parser, copy.data() + i, - std::min(step, copy.size() - i)); + size_t pos = 0; + for (size_t cut : cuts) { + if (status != WeaselJson_AGAIN) { + break; + } + if (cut > pos && cut < copy.size()) { + status = + WeaselJsonParser_parse(parser, copy.data() + pos, int(cut - pos)); + pos = cut; + } + } + if (status == WeaselJson_AGAIN && pos < copy.size()) { + status = WeaselJsonParser_parse(parser, copy.data() + pos, + int(copy.size() - pos)); } if (status == WeaselJson_AGAIN) { status = WeaselJsonParser_parse(parser, nullptr, 0); @@ -175,6 +186,15 @@ WeaselJsonStatus parseChunked(std::string const &doc, int stackSize, return status; } +// The offsets to cut `doc` into chunks of `stride` bytes. +std::vector stridedCuts(std::string const &doc, size_t stride) { + std::vector cuts; + for (size_t cut = stride; cut < doc.size(); cut += stride) { + cuts.push_back(cut); + } + return cuts; +} + } // namespace TEST_CASE("parser3") { @@ -314,10 +334,17 @@ TEST_CASE("acceptance is independent of the chunking") { // sweeps from "everything overflows" to "everything fits". for (int stackSize = 3; stackSize <= 16; ++stackSize) { CAPTURE(stackSize); - auto whole = parseChunked(doc, stackSize, 0); - for (int stride = 1; stride <= 12; ++stride) { + auto whole = parseChunked(doc, stackSize, {}); + // Every way to split the document in two, so that every literal and + // escape in `docs` straddles a chunk boundary at some point. + for (size_t cut = 1; cut < doc.size(); ++cut) { + CAPTURE(cut); + REQUIRE(parseChunked(doc, stackSize, {cut}) == whole); + } + for (size_t stride = 1; stride <= 12; ++stride) { CAPTURE(stride); - REQUIRE(parseChunked(doc, stackSize, stride) == whole); + REQUIRE(parseChunked(doc, stackSize, stridedCuts(doc, stride)) == + whole); } } } -- 2.43.0 From bce971fbd262f0de73a1330f921060ec12939907 Mon Sep 17 00:00:00 2001 From: Weaselbot Date: Tue, 27 Oct 2026 18:53:07 -0400 Subject: [PATCH 3/3] ci: don't hardcode ~/.local/bin when running pre-commit The pre-commit job failed with /root/.local/bin/pre-commit: No such file or directory even though `pipx install pre-commit` succeeded: `pipx install` writes the app to PIPX_BIN_DIR, and the runner image no longer uses ~/.local/bin for it. Run the binary from PATH when pipx put it there, and fall back to `pipx run pre-commit` otherwise, so the hook still runs regardless of where pipx exposes its apps. --- .gitea/workflows/ci.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 4728c17..194c672 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -12,7 +12,15 @@ jobs: run: pipx install pre-commit - name: Run pre-commit - run: ~/.local/bin/pre-commit run --all-files --show-diff-on-failure + # `pipx install` writes the app to PIPX_BIN_DIR, which is not + # necessarily ~/.local/bin and not necessarily on PATH, so don't + # hardcode a path that only some runner images happen to use. + run: | + if command -v pre-commit >/dev/null 2>&1; then + pre-commit run --all-files --show-diff-on-failure + else + pipx run pre-commit run --all-files --show-diff-on-failure + fi build: strategy: -- 2.43.0