ci: download mc from weaselab mirror

Reviewed-on: weaselab/weaseljson#72

.gitea/workflows/ci.yml

@@ -12,7 +12,15 @@ jobs:
         run: pipx install pre-commit
 
       - name: Run pre-commit
-        run: ~/.local/bin/pre-commit run --all-files --show-diff-on-failure
+        # `pipx install` writes the app to PIPX_BIN_DIR, which is not
+        # necessarily ~/.local/bin and not necessarily on PATH, so don't
+        # hardcode a path that only some runner images happen to use.
+        run: |
+          if command -v pre-commit >/dev/null 2>&1; then
+            pre-commit run --all-files --show-diff-on-failure
+          else
+            pipx run pre-commit run --all-files --show-diff-on-failure
+          fi
 
   build:
     strategy:
@@ -76,7 +84,7 @@ jobs:
             echo "MinIO credentials not configured; skipping upload"
             exit 0
           fi
-          curl -Ls "https://dl.min.io/client/mc/release/linux-$(dpkg --print-architecture)/mc" \
+          curl -fLs "https://minio.weaselab.dev/public/$(uname -m)/mc.RELEASE.2025-08-13T08-35-41Z" \
             -o /tmp/mc && chmod +x /tmp/mc
           dest="minio/jenkins/weaseljson/${{ gitea.run_number }}/${{ matrix.name }}/"
           /tmp/mc cp build/*.deb "$dest"

contrib/schemagen/README.md

@@ -60,8 +60,10 @@ into the result, so it is non-movable.
 ## Not supported (rejected at generation time, no fallback)
 
 `oneOf` / `anyOf` / `allOf` / `not` / `if`-`then`-`else`,
-`patternProperties`, `additionalProperties` with a schema (typed map),
-`additionalProperties: true`, `prefixItems` (tuples), `const`,
+`patternProperties`, `additionalProperties` absent or set to `true`,
+`additionalProperties` with a schema (typed map), `prefixItems` (tuples),
+`const`, `required` entries with no matching `properties` key (the schema
+would be unsatisfiable),
 `dependentSchemas`/`dependentRequired`, union `type` lists other than
 `["T", "null"]`, non-string enums, and remote (`$ref` to other documents).
 

contrib/schemagen/example.schema.json

@@ -56,6 +56,7 @@
       "type": "array",
       "items": {
         "type": "object",
+        "additionalProperties": false,
         "required": [
           "name"
         ],

contrib/schemagen/test_schemagen.py

@@ -32,6 +32,7 @@ class SchemagenEnumTest(unittest.TestCase):
     def test_colliding_enum_values_deduplicate(self):
         schema = {
             "type": "object",
+            "additionalProperties": False,
             "properties": {"role": {"enum": ["foo-bar", "foo_bar"]}},
         }
         rc, stdout, stderr = self.run_schemagen(schema)
@@ -45,6 +46,7 @@ class SchemagenEnumTest(unittest.TestCase):
     def test_distinct_enum_values_generate(self):
         schema = {
             "type": "object",
+            "additionalProperties": False,
             "properties": {"role": {"enum": ["admin", "user", "guest"]}},
         }
         rc, stdout, stderr = self.run_schemagen(schema)
@@ -80,7 +82,7 @@ class SchemagenKeywordTest(unittest.TestCase):
             "module",
             "import",
         ]
-        schema = {"type": "object", "properties": {}}
+        schema = {"type": "object", "additionalProperties": False, "properties": {}}
         for kw in keywords:
             schema["properties"][kw] = {"type": "string"}
         rc, stdout, stderr = self.run_schemagen(schema)
@@ -164,11 +166,18 @@ class SchemagenCollisionTest(unittest.TestCase):
     def test_kind_enum_does_not_duplicate_arr0(self):
         schema = {
             "type": "object",
+            "additionalProperties": False,
             "properties": {
                 "arr": {"type": "array", "items": {"type": "string"}},
                 "obj": {"$ref": "#/$defs/Arr0"},
             },
-            "$defs": {"Arr0": {"type": "object", "properties": {}}},
+            "$defs": {
+                "Arr0": {
+                    "type": "object",
+                    "additionalProperties": False,
+                    "properties": {},
+                }
+            },
         }
         out = self.generate_and_compile(schema)
         self.assertIn("struct Arr0", out)
@@ -183,7 +192,13 @@ class SchemagenCollisionTest(unittest.TestCase):
         schema = {
             "type": "array",
             "items": {"$ref": "#/$defs/Skip"},
-            "$defs": {"Skip": {"type": "object", "properties": {}}},
+            "$defs": {
+                "Skip": {
+                    "type": "object",
+                    "additionalProperties": False,
+                    "properties": {},
+                }
+            },
         }
         out = self.generate_and_compile(schema)
         self.assertIn("struct Skip", out)
@@ -199,6 +214,7 @@ class SchemagenCollisionTest(unittest.TestCase):
         """Regression test for issue #17: enum and array $defs must be reused."""
         schema = {
             "type": "object",
+            "additionalProperties": False,
             "properties": {
                 "role1": {"$ref": "#/$defs/Role"},
                 "role2": {"$ref": "#/$defs/Role"},
@@ -252,17 +268,14 @@ class SchemagenAdditionalPropertiesTest(unittest.TestCase):
         self.assertNotEqual(rc, 0)
         self.assertIn("additionalProperties: true is not supported", stderr)
 
-    def test_additional_properties_absent_defaults_to_strict(self):
+    def test_additional_properties_absent_rejected(self):
         schema = {
             "type": "object",
             "properties": {"name": {"type": "string"}},
         }
         rc, stdout, stderr = self.run_schemagen(schema)
-        self.assertEqual(rc, 0, msg=stderr)
-        # The generated parser should reject unknown keys. Verify the key-matching
-        # helper returns -1 for an unknown key and cbKeyData rejects it.
-        self.assertIn("int matchKey(Kind k, std::string_view key) const {", stdout)
-        self.assertNotIn("bool isStrict(Kind k) const", stdout)
+        self.assertNotEqual(rc, 0)
+        self.assertIn("additionalProperties is required for object schemas", stderr)
 
     def test_additional_properties_false_accepted(self):
         schema = {
@@ -274,6 +287,94 @@ class SchemagenAdditionalPropertiesTest(unittest.TestCase):
         self.assertEqual(rc, 0, msg=stderr)
 
 
+class SchemagenRequiredPropertyTest(unittest.TestCase):
+    """Regression tests for issue #63: `required` entries with no matching
+    `properties` key must be rejected at generation time. With the mandatory
+    `additionalProperties: false` the schema is unsatisfiable, and the
+    generated parser has no seen bit for such keys, so it would otherwise
+    silently accept documents missing the required key."""
+
+    def run_schemagen(self, schema, args=None):
+        """Run schemagen on a schema dict. Returns (returncode, stdout, stderr)."""
+        with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as fp:
+            json.dump(schema, fp)
+            schema_path = fp.name
+        try:
+            cmd = [sys.executable, SCRIPT, schema_path]
+            if args:
+                cmd.extend(args)
+            result = subprocess.run(cmd, capture_output=True, text=True, check=False)
+            return result.returncode, result.stdout, result.stderr
+        finally:
+            os.unlink(schema_path)
+
+    def test_required_key_without_properties_entry_rejected(self):
+        schema = {
+            "type": "object",
+            "additionalProperties": False,
+            "properties": {"a": {"type": "integer"}},
+            "required": ["b"],
+        }
+        rc, stdout, stderr = self.run_schemagen(schema)
+        self.assertNotEqual(rc, 0)
+        self.assertIn("required key 'b' has no matching properties entry", stderr)
+
+    def test_multiple_dangling_required_keys_reported(self):
+        schema = {
+            "type": "object",
+            "additionalProperties": False,
+            "properties": {"a": {"type": "integer"}},
+            "required": ["a", "x", "y"],
+        }
+        rc, stdout, stderr = self.run_schemagen(schema)
+        self.assertNotEqual(rc, 0)
+        self.assertIn(
+            "required keys 'x', 'y' have no matching properties entries",
+            stderr,
+        )
+
+    def test_required_without_properties_rejected(self):
+        schema = {
+            "type": "object",
+            "additionalProperties": False,
+            "required": ["any"],
+        }
+        rc, stdout, stderr = self.run_schemagen(schema)
+        self.assertNotEqual(rc, 0)
+        self.assertIn("required key 'any' has no matching properties entry", stderr)
+
+    def test_nested_required_key_without_properties_entry_rejected(self):
+        schema = {
+            "type": "object",
+            "additionalProperties": False,
+            "properties": {"child": {"$ref": "#/$defs/Child"}},
+            "$defs": {
+                "Child": {
+                    "type": "object",
+                    "additionalProperties": False,
+                    "properties": {"x": {"type": "string"}},
+                    "required": ["misspelled"],
+                }
+            },
+        }
+        rc, stdout, stderr = self.run_schemagen(schema)
+        self.assertNotEqual(rc, 0)
+        self.assertIn(
+            "required key 'misspelled' has no matching properties entry",
+            stderr,
+        )
+
+    def test_required_key_with_properties_entry_accepted(self):
+        schema = {
+            "type": "object",
+            "additionalProperties": False,
+            "properties": {"a": {"type": "integer"}, "b": {"type": "integer"}},
+            "required": ["b"],
+        }
+        rc, stdout, stderr = self.run_schemagen(schema)
+        self.assertEqual(rc, 0, msg=stderr)
+
+
 class SchemagenCyclicArrayTest(unittest.TestCase):
     """Regression tests for issue #33: cyclic array $ref targets."""
 
@@ -310,6 +411,7 @@ class SchemagenCyclicArrayTest(unittest.TestCase):
     def test_object_field_to_self_referential_array_rejected(self):
         schema = {
             "type": "object",
+            "additionalProperties": False,
             "properties": {"items": {"$ref": "#/$defs/Items"}},
             "$defs": {
                 "Items": {
@@ -326,6 +428,7 @@ class SchemagenCyclicArrayTest(unittest.TestCase):
     def test_chain_of_array_refs_rejected(self):
         schema = {
             "type": "object",
+            "additionalProperties": False,
             "properties": {"x": {"$ref": "#/$defs/A"}},
             "$defs": {
                 "A": {"type": "array", "items": {"$ref": "#/$defs/B"}},
@@ -339,6 +442,7 @@ class SchemagenCyclicArrayTest(unittest.TestCase):
     def test_non_recursive_array_refs_still_allowed(self):
         schema = {
             "type": "object",
+            "additionalProperties": False,
             "properties": {"roles": {"$ref": "#/$defs/Roles"}},
             "$defs": {
                 "Role": {"enum": ["admin", "user"]},
@@ -439,10 +543,12 @@ class SchemagenCyclicNullableObjectTest(unittest.TestCase):
             self.skipTest("C++ compiler not available")
         schema = {
             "type": "object",
+            "additionalProperties": False,
             "properties": {"self": {"$ref": "#/$defs/Self"}},
             "$defs": {
                 "Self": {
                     "type": ["object", "null"],
+                    "additionalProperties": False,
                     "properties": {"self": {"$ref": "#/$defs/Self"}},
                 }
             },
@@ -592,6 +698,13 @@ class SchemagenIntegerBoundaryTest(unittest.TestCase):
             ("123.0", "WeaselJson_OK", 123),
             ("9e18", "WeaselJson_OK", 9000000000000000000),
             ("10e18", "WeaselJson_REJECT", 0),
+            # Issue #56: zero written with a negative exponent must be accepted.
+            ("0e-1", "WeaselJson_OK", 0),
+            ("0e-2", "WeaselJson_OK", 0),
+            ("0.0e-2", "WeaselJson_OK", 0),
+            ("-0e-2", "WeaselJson_OK", 0),
+            ("0e-20", "WeaselJson_OK", 0),
+            ("0.000e-5", "WeaselJson_OK", 0),
         ]
         cases_src = self._build_cases_array("root", cases)
         harness = textwrap.dedent(
@@ -634,6 +747,9 @@ class SchemagenIntegerBoundaryTest(unittest.TestCase):
             ('{"age":2.0}', "WeaselJson_OK", 2),
             ('{"age":0.0001e4}', "WeaselJson_OK", 1),
             ('{"age":0.001}', "WeaselJson_REJECT", 0),
+            # Issue #56: zero written with a negative exponent must be accepted.
+            ('{"age":0e-2}', "WeaselJson_OK", 0),
+            ('{"age":-0e-20}', "WeaselJson_OK", 0),
             (
                 '{"age":-9223372036854775808.0}',
                 "WeaselJson_OK",
@@ -670,6 +786,7 @@ class SchemagenIntegerBoundaryTest(unittest.TestCase):
         )
         schema = {
             "type": "object",
+            "additionalProperties": False,
             "properties": {"age": {"type": "integer"}},
             "required": ["age"],
         }
@@ -785,14 +902,22 @@ class SchemagenStringEscapeTest(unittest.TestCase):
 
     def test_newline_in_property_key(self):
         """A JSON key containing a newline must become a valid C++ literal."""
-        schema = {"type": "object", "properties": {"a\nb": {"type": "string"}}}
+        schema = {
+            "type": "object",
+            "additionalProperties": False,
+            "properties": {"a\nb": {"type": "string"}},
+        }
         out = self.generate_and_compile(schema)
         self.assertIn('// "a\\nb"', out)
         self.assertIn('if (key == "a\\nb") return 0;', out)
 
     def test_newline_in_enum_value(self):
         """An enum value containing a newline must become a valid C++ literal."""
-        schema = {"type": "object", "properties": {"x": {"enum": ["a\nb"]}}}
+        schema = {
+            "type": "object",
+            "additionalProperties": False,
+            "properties": {"x": {"enum": ["a\nb"]}},
+        }
         out = self.generate_and_compile(schema)
         self.assertIn('static constexpr const char *X_names[] = { "a\\nb" };', out)
 
@@ -800,6 +925,7 @@ class SchemagenStringEscapeTest(unittest.TestCase):
         """Mixed control characters in an enum value must be escaped."""
         schema = {
             "type": "object",
+            "additionalProperties": False,
             "properties": {
                 "x": {"enum": ["x\ny\rz\tw\vq\x00\x01"]},
             },
@@ -812,7 +938,11 @@ class SchemagenStringEscapeTest(unittest.TestCase):
 
     def test_backslash_and_quote_still_escaped(self):
         """Existing escaping for backslash and double quote must remain correct."""
-        schema = {"type": "object", "properties": {'a"b\\c': {"type": "string"}}}
+        schema = {
+            "type": "object",
+            "additionalProperties": False,
+            "properties": {'a"b\\c': {"type": "string"}},
+        }
         out = self.generate_and_compile(schema)
         self.assertIn('// "a\\"b\\\\c"', out)
         self.assertIn('if (key == "a\\"b\\\\c") return 0;', out)

contrib/schemagen/weaseljson_schemagen.py

@@ -417,7 +417,13 @@ class Builder:
         tobj = TObj(name)
         if defname is not None:
             self._building[defname] = (tobj, nullable)
-        ap = node.get("additionalProperties", False)
+        ap = node.get("additionalProperties", None)
+        if ap is None:
+            raise GenError(
+                "additionalProperties is required for object schemas; set it "
+                "explicitly to false to reject unknown keys (absent "
+                "additionalProperties is not supported)"
+            )
         if ap is True:
             raise GenError("additionalProperties: true is not supported")
         if isinstance(ap, dict):
@@ -426,6 +432,22 @@ class Builder:
             )
         required = set(node.get("required", []))
         props = node.get("properties", {})
+        # A `required` entry with no matching `properties` key makes the schema
+        # unsatisfiable: `additionalProperties: false` (the only supported
+        # object mode) rejects any key not listed in `properties`, so the
+        # required key can never be present. No seen bit is emitted for such a
+        # key, so the generated parser would silently accept every document.
+        # Follow the other unsupported constructs and reject at generation time.
+        dangling = sorted(required - set(props))
+        if dangling:
+            quoted = ", ".join(f"'{k}'" for k in dangling)
+            if len(dangling) == 1:
+                raise GenError(
+                    f"required key {quoted} has no matching properties entry"
+                )
+            raise GenError(
+                f"required keys {quoted} have no matching properties entries"
+            )
         seen_cpp = set()
         for key, sub in props.items():
             ty, nullable = self._unpack(self.build_type(sub, key))
@@ -1058,10 +1080,10 @@ private:
       ++trim;
     }}
     int64_t finalExp = exp - fracDigits + trim;
-    if (finalExp < 0) return false;
     size_t leadingZeros = 0;
     while (leadingZeros < digits.size() && digits[leadingZeros] == '0') ++leadingZeros;
     if (leadingZeros == digits.size()) {{ out = 0; return true; }}
+    if (finalExp < 0) return false;
     if (leadingZeros > 0) digits.erase(0, leadingZeros);
 
     constexpr uint64_t kMaxNeg = 9223372036854775808ULL;

include/weaseljson.h

@@ -36,8 +36,10 @@ enum WeaselJsonStatus {
   WeaselJson_AGAIN,
   /** Invalid json */
   WeaselJson_REJECT,
-  /** json is too deeply nested */
+  /** json is too deeply nested for the parser's stack size */
   WeaselJson_OVERFLOW,
+  /** Tried to call parse on a null parser */
+  WeaselJson_NULL,
 };
 
 typedef struct WeaselJsonParser WeaselJsonParser;
@@ -48,10 +50,12 @@ enum WeaselJsonFlags {
 };
 
 /** Create a parser. Increasing stack size increases memory usage but also
- * increases the depth of nested json accepted. `callbacks` and `userdata` must
- * outlive the returned parser. Returns null if there's insufficient available
- * memory, or if `stackSize` is negative or too small to hold a minimal
- * document. */
+ * increases the depth of nested json accepted. Whether a document is accepted,
+ * rejected or reported as too deeply nested depends only on its bytes and on
+ * `stackSize`, not on how the bytes are split across WeaselJsonParser_parse
+ * calls. `callbacks` and `userdata` must outlive the returned parser. Returns
+ * null if there's insufficient available memory, or if `stackSize` is negative
+ * or too small to hold a minimal document. */
 WeaselJsonParser *WeaselJsonParser_create(int stackSize,
                                           const WeaselJsonCallbacks *callbacks,
                                           void *userdata, int flags);
@@ -65,7 +69,7 @@ void WeaselJsonParser_destroy(WeaselJsonParser *parser);
 /** Incrementally parse `len` more bytes starting at `buf`. `buf` may be
  * modified. Call with `len` 0 to indicate end of data. `buf` may be null if
  * `len` is 0. `len` must not be negative; a negative length is treated as a
- * rejected input. */
+ * rejected input. Returns WeaselJson_NULL if parser is null */
 WeaselJsonStatus WeaselJsonParser_parse(WeaselJsonParser *parser, char *buf,
                                         int len);
 

src/fuzz.cpp

@@ -5,12 +5,12 @@
 
 #include <simdjson.h>
 
-std::pair<std::string, WeaselJsonStatus> runStreaming(std::string copy,
-                                                      int stride) {
+std::pair<std::string, WeaselJsonStatus>
+runStreaming(std::string copy, int stackSize, int stride) {
   SerializeState state;
   auto c = serializeCallbacks();
   std::unique_ptr<WeaselJsonParser, decltype(&WeaselJsonParser_destroy)> parser{
-      WeaselJsonParser_create(1024, &c, &state,
+      WeaselJsonParser_create(stackSize, &c, &state,
                               copy.size() % 2 == 0 ? WeaselJsonRaw : 0),
       WeaselJsonParser_destroy};
   if (stride == 0) {
@@ -31,11 +31,12 @@ std::pair<std::string, WeaselJsonStatus> runStreaming(std::string copy,
   return {state.result, s};
 }
 
-std::pair<std::string, WeaselJsonStatus> runBatch(std::string copy) {
+std::pair<std::string, WeaselJsonStatus> runBatch(std::string copy,
+                                                  int stackSize) {
   SerializeState state;
   auto c = serializeCallbacks();
   std::unique_ptr<WeaselJsonParser, decltype(&WeaselJsonParser_destroy)> parser{
-      WeaselJsonParser_create(1 << 20, &c, &state,
+      WeaselJsonParser_create(stackSize, &c, &state,
                               copy.size() % 2 == 0 ? WeaselJsonRaw : 0),
       WeaselJsonParser_destroy};
   auto s = WeaselJsonParser_parse(parser.get(), copy.data(), copy.size());
@@ -47,11 +48,11 @@ std::pair<std::string, WeaselJsonStatus> runBatch(std::string copy) {
 }
 
 std::pair<std::string, WeaselJsonStatus> runPrefix(std::string copy,
-                                                   int prefix) {
+                                                   int stackSize, int prefix) {
   SerializeState state;
   auto c = serializeCallbacks();
   std::unique_ptr<WeaselJsonParser, decltype(&WeaselJsonParser_destroy)> parser{
-      WeaselJsonParser_create(1024, &c, &state,
+      WeaselJsonParser_create(stackSize, &c, &state,
                               copy.size() % 2 == 0 ? WeaselJsonRaw : 0),
       WeaselJsonParser_destroy};
   auto s = WeaselJsonParser_parse(parser.get(), copy.data(), prefix);
@@ -67,55 +68,54 @@ std::pair<std::string, WeaselJsonStatus> runPrefix(std::string copy,
   return {state.result, s};
 }
 
-void testStreaming(std::string const &json) {
-  auto batch = runBatch(json);
+void checkStreamingResult(
+    int stackSize, char const *kind,
+    std::pair<std::string, WeaselJsonStatus> const &got,
+    std::pair<std::string, WeaselJsonStatus> const &batch) {
+  if (got.second == WeaselJson_AGAIN) {
+    abort();
+  }
+  // The status must be identical however the document is chunked; the events
+  // reported for a document that failed to parse need not match.
+  bool ok = got.second == batch.second &&
+            (got.second != WeaselJson_OK || got.first == batch.first);
+  if (!ok) {
+    printf("%s (stackSize %d): %d, %s\n", kind, stackSize, (int)got.second,
+           got.first.c_str());
+    printf("batch (stackSize %d): %d, %s\n", stackSize, (int)batch.second,
+           batch.first.c_str());
+    abort();
+  }
+}
+
+void testStreaming(std::string const &json, int stackSize) {
+  auto batch = runBatch(json, stackSize);
   if (batch.second == WeaselJson_AGAIN) {
     abort();
   }
   for (int stride = 1; stride < 16; ++stride) {
-    auto streaming = runStreaming(json, stride);
-    if (streaming != batch) {
-      if (streaming.second == WeaselJson_AGAIN) {
-        abort();
-      }
-      bool streamingOk = streaming.second == WeaselJson_OK;
-      bool batchOk = batch.second == WeaselJson_OK;
-      if (streamingOk == batchOk && !batchOk) {
-        // It's ok if the processed data doesn't match if parsing failed
-      } else {
-        printf("streaming: %s, %s\n",
-               streaming.second == WeaselJson_OK ? "accept" : "reject",
-               streaming.first.c_str());
-        printf("batch: %s, %s\n",
-               batch.second == WeaselJson_OK ? "accept" : "reject",
-               batch.first.c_str());
-        abort();
-      }
-    }
+    checkStreamingResult(stackSize, "streaming",
+                         runStreaming(json, stackSize, stride), batch);
     if (int(json.size()) > stride) {
-      auto prefix = runPrefix(json, stride);
-      if (prefix != batch) {
-        if (prefix.second == WeaselJson_AGAIN) {
-          abort();
-        }
-        bool prefixOk = prefix.second == WeaselJson_OK;
-        bool batchOk = batch.second == WeaselJson_OK;
-        if (prefixOk == batchOk && !batchOk) {
-          // It's ok if the processed data doesn't match if parsing failed
-        } else {
-          printf("prefix: %s, %s\n",
-                 prefix.second == WeaselJson_OK ? "accept" : "reject",
-                 prefix.first.c_str());
-          printf("batch: %s, %s\n",
-                 batch.second == WeaselJson_OK ? "accept" : "reject",
-                 batch.first.c_str());
-          abort();
-        }
-      }
+      checkStreamingResult(stackSize, "prefix",
+                           runPrefix(json, stackSize, stride), batch);
     }
   }
 }
 
+void testStreaming(std::string const &json) {
+  // Whether a document parses must not depend on the streaming pattern: the
+  // parser hands the tokenizer progress for a literal or a \uXXXX escape that
+  // straddles a chunk boundary to the pushdown stack, so a stackSize picked for
+  // the document's nesting depth used to overflow for some chunkings only.
+  // Sweep stack sizes around the minimum (3 is the smallest accepted by
+  // WeaselJsonParser_create) so that the property is exercised, then a large
+  // one for documents that need the room.
+  for (int stackSize : {3, 4, 5, 6, 7, 8, 12, 1024}) {
+    testStreaming(json, stackSize);
+  }
+}
+
 void compareWithSimdjson(std::string const &json) {
   WeaselJsonStatus ours;
   {

src/lib.cpp

@@ -13,8 +13,7 @@ WeaselJsonParser_create(int stackSize, const WeaselJsonCallbacks *callbacks,
   if (stackSize < 0) { // avoid (size_t)stackSize wrapping to a huge allocation
     return nullptr;
   }
-  auto *buf =
-      malloc(sizeof(Parser3) + (size_t)stackSize * sizeof(*Parser3::stackPtr));
+  auto *buf = malloc(Parser3::allocationSize(stackSize));
   if (buf == nullptr) {
     return nullptr;
   }
@@ -46,6 +45,9 @@ WeaselJsonParser_destroy(WeaselJsonParser *parser) {
 
 __attribute__((visibility("default"))) WeaselJsonStatus
 WeaselJsonParser_parse(WeaselJsonParser *parser, char *buf, int len) {
+  if (parser == nullptr) [[unlikely]] {
+    return WeaselJson_NULL;
+  }
   return ((Parser3 *)parser)->parse(buf, len);
 }
 }

src/parser3.h

@@ -72,6 +72,30 @@ enum Symbol : uint8_t {
   T_BACKSLASH,
   N_SYMBOL_COUNT, // Must be last
 };
+
+// Some symbols are on the stack only because a literal (`true`/`false`/`null`)
+// or a `\uXXXX` escape straddles the end of a chunk. The whole-buffer path
+// consumes those characters inline and pushes nothing, so these symbols must
+// not spend the caller's nesting budget: otherwise the stackSize a document
+// needs would depend on where the parse() boundaries happen to fall (issue
+// #71). WeaselJsonParser_create() allocates this many slots beyond `stackSize`
+// to hold them, and Parser3::pushTransient() pushes them there.
+//
+// All the pushes of such symbols, with their sizes:
+//   n_value()                    3, 4, 3  (true, false, null)
+//   n_string2()                  1        (N_STRING_FOLLOWING_ESCAPE)
+//   n_string_following_escape()  4        (the four hex digits)
+//   t_hex2()                     6        (utf-16 surrogate pair)
+//
+// Upper bound on how many of them are on the stack at once: 6. These are
+// pushdown symbols, so the ones that are on the stack are consumed before
+// anything else is pushed; and none of those continuations pushes while
+// another's symbols are still pending, because each pops the symbol it took
+// before pushing (the literal ones pop N_VALUE before n_value() pushes, and the
+// rest pop themselves). The number of pending symbols is therefore bounded by
+// the largest single push, the six symbols that start a surrogate pair.
+inline constexpr int kTransientSymbols = 6;
+
 struct Parser3 {
   Parser3(const WeaselJsonCallbacks *callbacks, void *userdata, int stackSize,
           int flags)
@@ -80,6 +104,14 @@ struct Parser3 {
     reset();
   }
 
+  // Size of the allocation that holds a parser with a nesting budget of
+  // `stackSize` symbols: the parser itself, that budget, and the reserve for
+  // transient tokenizer progress symbols.
+  static constexpr size_t allocationSize(int stackSize) {
+    return sizeof(Parser3) +
+           (size_t(stackSize) + kTransientSymbols) * sizeof(Symbol);
+  }
+
   [[nodiscard]] WeaselJsonStatus parse(char *buf, int len);
 
   void flushNumber(bool done, char *buf) {
@@ -114,9 +146,12 @@ struct Parser3 {
   }
 
   // Pushing symbols onto the stack roughly corresponds with "parse these
-  // nonterminals/terminals in order"
-  [[nodiscard]] WeaselJsonStatus push(std::initializer_list<Symbol> symbols) {
-    if (stackEnd - stackPtr < ptrdiff_t(symbols.size())) [[unlikely]] {
+  // nonterminals/terminals in order". `end` is the first slot the symbols may
+  // not occupy.
+  [[nodiscard]] static WeaselJsonStatus
+  pushOnto(Symbol *&stackPtr, Symbol *end,
+           std::initializer_list<Symbol> symbols) {
+    if (end - stackPtr < ptrdiff_t(symbols.size())) [[unlikely]] {
       return WeaselJson_OVERFLOW;
     }
     for (int i = symbols.size() - 1; i >= 0; --i) {
@@ -125,6 +160,23 @@ struct Parser3 {
     return WeaselJson_OK;
   }
 
+  // Push symbols that model the nesting of the document. This is what
+  // WeaselJson_OVERFLOW reports, so it's bounded by the caller's stackSize.
+  [[nodiscard]] WeaselJsonStatus push(std::initializer_list<Symbol> symbols) {
+    return pushOnto(stackPtr, stackEnd, symbols);
+  }
+
+  // Push tokenizer progress symbols for a literal or escape that straddles a
+  // chunk boundary. Unlike push(), these may grow past `stackEnd` into the
+  // reserve, so they don't spend the caller's nesting budget and can never make
+  // the parser overflow on input it would have accepted in one piece. See
+  // kTransientSymbols for why the reserve is big enough to always succeed.
+  [[nodiscard]] WeaselJsonStatus
+  pushTransient(std::initializer_list<Symbol> symbols) {
+    assert(ptrdiff_t(symbols.size()) <= kTransientSymbols);
+    return pushOnto(stackPtr, stackEnd + kTransientSymbols, symbols);
+  }
+
   [[nodiscard]] Symbol top() const {
     assert(!empty());
     return *(stackPtr - 1);
@@ -139,7 +191,7 @@ struct Parser3 {
     stackPtr = stack();
     std::ignore = push({N_VALUE, N_WHITESPACE, T_EOF});
     inKey = false;
-    rejected = false;
+    terminalStatus = WeaselJson_OK;
     utf8Codepoint = 0;
     utf16Surrogate = 0;
     minCodepoint = 0;
@@ -162,7 +214,7 @@ struct Parser3 {
   NumDfa numDfa;
   Utf8Dfa strDfa;
   bool inKey = false;
-  bool rejected = false;
+  WeaselJsonStatus terminalStatus = WeaselJson_OK;
 
 #ifndef HAS_MUSTTAIL
   char *stashBufForTrampoline;
@@ -376,7 +428,7 @@ inline PRESERVE_NONE ContinuationStatus n_value(Parser3 *self, char *buf,
         return WeaselJson_REJECT;
       }
     } else {
-      if (auto s = self->push({T_R, T_U, N_TRUE})) {
+      if (auto s = self->pushTransient({T_R, T_U, N_TRUE})) {
         return s;
       }
     }
@@ -392,7 +444,7 @@ inline PRESERVE_NONE ContinuationStatus n_value(Parser3 *self, char *buf,
         return WeaselJson_REJECT;
       }
     } else {
-      if (auto s = self->push({T_A, T_L, T_S, N_FALSE})) {
+      if (auto s = self->pushTransient({T_A, T_L, T_S, N_FALSE})) {
         return s;
       }
     }
@@ -408,7 +460,7 @@ inline PRESERVE_NONE ContinuationStatus n_value(Parser3 *self, char *buf,
         return WeaselJson_REJECT;
       }
     } else {
-      if (auto s = self->push({T_U, T_L, N_NULL})) {
+      if (auto s = self->pushTransient({T_U, T_L, N_NULL})) {
         return s;
       }
     }
@@ -609,7 +661,7 @@ inline PRESERVE_NONE ContinuationStatus n_string2(Parser3 *self, char *buf,
   case '\\':
     ++buf;
     if (bufEnd - buf < /*strlen("u0000\\u0000")*/ 11) {
-      if (auto s = self->push({N_STRING_FOLLOWING_ESCAPE})) {
+      if (auto s = self->pushTransient({N_STRING_FOLLOWING_ESCAPE})) {
         return s;
       }
       if (buf == bufEnd) {
@@ -650,7 +702,7 @@ inline PRESERVE_NONE ContinuationStatus n_string2(Parser3 *self, char *buf,
             self->writeBuf[0] = (0b00000111 & codepoint) | 0b11110000;
             self->writeBuf += 4;
           }
-        } else if (0xdc00 <= codepoint && codepoint <= 0xdfff) {
+        } else if (0xdc00 <= codepoint && codepoint <= 0xdfff) [[unlikely]] {
           return WeaselJson_REJECT;
         } else {
           if (!(self->flags & WeaselJsonRaw)) {
@@ -718,7 +770,7 @@ n_string_following_escape(Parser3 *self, char *buf, char *bufEnd) {
     ++buf;
     self->utf8Codepoint = 0;
     self->pop();
-    if (auto s = self->push({T_HEX, T_HEX, T_HEX, T_HEX2})) {
+    if (auto s = self->pushTransient({T_HEX, T_HEX, T_HEX, T_HEX2})) {
       return s;
     }
     break;
@@ -800,8 +852,8 @@ inline PRESERVE_NONE ContinuationStatus t_hex2(Parser3 *self, char *buf,
       self->utf16Surrogate = self->utf8Codepoint;
       self->utf8Codepoint = 0;
       self->pop();
-      if (auto s =
-              self->push({T_BACKSLASH, T_U2, T_HEX, T_HEX, T_HEX, T_HEX3})) {
+      if (auto s = self->pushTransient(
+              {T_BACKSLASH, T_U2, T_HEX, T_HEX, T_HEX, T_HEX3})) {
         return s;
       }
       if (buf == bufEnd) {
@@ -1070,12 +1122,12 @@ constexpr inline struct ContinuationTable {
 inline WeaselJsonStatus Parser3::parse(char *buf, int len) {
   this->dataBegin = this->writeBuf = buf;
 
-  if (this->rejected) [[unlikely]] {
-    return WeaselJson_REJECT;
+  if (this->terminalStatus != WeaselJson_OK) [[unlikely]] {
+    return this->terminalStatus;
   }
 
   if (len < 0) [[unlikely]] {
-    this->rejected = true;
+    this->terminalStatus = WeaselJson_REJECT;
     return WeaselJson_REJECT;
   }
 
@@ -1085,8 +1137,8 @@ inline WeaselJsonStatus Parser3::parse(char *buf, int len) {
   // range.
   ContinuationStatus status =
       symbolTables.continuations[top()](this, buf, buf + len);
-  if (status == WeaselJson_REJECT) {
-    this->rejected = true;
+  if (status > WeaselJson_AGAIN) {
+    this->terminalStatus = WeaselJsonStatus(status);
   }
   return WeaselJsonStatus(status);
 #else
@@ -1095,8 +1147,8 @@ inline WeaselJsonStatus Parser3::parse(char *buf, int len) {
   while ((result = symbolTables.continuations[top()](
               this, stashBufForTrampoline, buf + len)) == kBounce)
     ;
-  if (result == WeaselJson_REJECT) {
-    this->rejected = true;
+  if (result > WeaselJson_AGAIN) {
+    this->terminalStatus = WeaselJsonStatus(result);
   }
   return WeaselJsonStatus(result);
 #endif

src/test.cpp

@@ -4,8 +4,10 @@
 #include <cstring>
 #include <fstream>
 
+#include <algorithm>
 #include <limits>
 #include <string>
+#include <vector>
 
 #include <doctest.h>
 #include <nanobench.h>
@@ -152,6 +154,47 @@ void testStreaming(std::string const &json) {
   CHECK(streaming.result == batch.result);
 }
 
+// Feed `doc` to a fresh parser with the given stackSize, cutting it into chunks
+// at the offsets in `cuts` (ascending, each > 0 and < doc.size(); an empty
+// `cuts` feeds the whole document at once), and return the final status.
+WeaselJsonStatus parseChunked(std::string const &doc, int stackSize,
+                              std::vector<size_t> const &cuts) {
+  auto c = noopCallbacks();
+  std::string copy = doc;
+  auto *parser = WeaselJsonParser_create(stackSize, &c, nullptr, 0);
+  REQUIRE(parser != nullptr);
+  WeaselJsonStatus status = WeaselJson_AGAIN;
+  size_t pos = 0;
+  for (size_t cut : cuts) {
+    if (status != WeaselJson_AGAIN) {
+      break;
+    }
+    if (cut > pos && cut < copy.size()) {
+      status =
+          WeaselJsonParser_parse(parser, copy.data() + pos, int(cut - pos));
+      pos = cut;
+    }
+  }
+  if (status == WeaselJson_AGAIN && pos < copy.size()) {
+    status = WeaselJsonParser_parse(parser, copy.data() + pos,
+                                    int(copy.size() - pos));
+  }
+  if (status == WeaselJson_AGAIN) {
+    status = WeaselJsonParser_parse(parser, nullptr, 0);
+  }
+  WeaselJsonParser_destroy(parser);
+  return status;
+}
+
+// The offsets to cut `doc` into chunks of `stride` bytes.
+std::vector<size_t> stridedCuts(std::string const &doc, size_t stride) {
+  std::vector<size_t> cuts;
+  for (size_t cut = stride; cut < doc.size(); cut += stride) {
+    cuts.push_back(cut);
+  }
+  return cuts;
+}
+
 } // namespace
 
 TEST_CASE("parser3") {
@@ -202,6 +245,111 @@ TEST_CASE("parser3") {
   }
 }
 
+TEST_CASE("overflow state is sticky") {
+  auto c = noopCallbacks();
+  // stackSize 3 is exactly big enough to hold reset()'s bootstrap, but too
+  // small for nested arrays. Overflows must be terminal like rejects: a later
+  // end-of-data call must never report OK for an incomplete document.
+  auto *parser = WeaselJsonParser_create(3, &c, nullptr, 0);
+  REQUIRE(parser != nullptr);
+
+  std::string doc = "[[";
+  REQUIRE(WeaselJsonParser_parse(parser, doc.data(), doc.size()) ==
+          WeaselJson_OVERFLOW);
+
+  // After overflow, the end-of-data call must not return OK.
+  REQUIRE(WeaselJsonParser_parse(parser, nullptr, 0) != WeaselJson_OK);
+  // It should keep reporting a terminal failure.
+  REQUIRE(WeaselJsonParser_parse(parser, nullptr, 0) != WeaselJson_OK);
+
+  // Further data chunks must also stay terminal.
+  std::string more = "]]";
+  REQUIRE(WeaselJsonParser_parse(parser, more.data(), more.size()) !=
+          WeaselJson_OK);
+
+  WeaselJsonParser_destroy(parser);
+}
+
+TEST_CASE("overflow is sticky for nested objects") {
+  auto c = noopCallbacks();
+  auto *parser = WeaselJsonParser_create(4, &c, nullptr, 0);
+  REQUIRE(parser != nullptr);
+
+  std::string doc = "{\"a\":{ \"a\":";
+  REQUIRE(WeaselJsonParser_parse(parser, doc.data(), doc.size()) ==
+          WeaselJson_OVERFLOW);
+  REQUIRE(WeaselJsonParser_parse(parser, nullptr, 0) != WeaselJson_OK);
+
+  WeaselJsonParser_destroy(parser);
+}
+
+TEST_CASE("reset clears overflow state") {
+  auto c = noopCallbacks();
+  auto *parser = WeaselJsonParser_create(3, &c, nullptr, 0);
+  REQUIRE(parser != nullptr);
+
+  std::string doc = "[[";
+  REQUIRE(WeaselJsonParser_parse(parser, doc.data(), doc.size()) ==
+          WeaselJson_OVERFLOW);
+  // After reset the parser should accept a minimal document again.
+  WeaselJsonParser_reset(parser);
+  std::string copy = "1";
+  REQUIRE(WeaselJsonParser_parse(parser, copy.data(), copy.size()) ==
+          WeaselJson_AGAIN);
+  REQUIRE(WeaselJsonParser_parse(parser, nullptr, 0) == WeaselJson_OK);
+
+  WeaselJsonParser_destroy(parser);
+}
+
+TEST_CASE("acceptance is independent of the chunking") {
+  // Literals and \uXXXX escapes that straddle the end of a chunk are tokenized
+  // by pushing their not-yet-matched symbols onto the pushdown stack, so they
+  // used to spend the caller's nesting budget: a document could be accepted
+  // when fed whole and reported as WeaselJson_OVERFLOW when streamed in smaller
+  // pieces. The decision must depend only on the document and on `stackSize`.
+  std::vector<std::string> docs = {
+      "1",
+      "true",
+      "false",
+      "null",
+      "[true]",
+      "[true,false,null]",
+      "\"\\u00e9abcdefgh\"",
+      // Needs the full transient reserve: a surrogate pair split across chunks
+      // pushes six symbols at once.
+      "\"\\ud801\\udc37\"",
+      "[\"\\u00e9abcdefgh\"]",
+      "{\"\\ud801\\udc37\":[true,{\"x\":null}]}",
+      "[[[[[[[[[[[[true]]]]]]]]]]]]",
+      // Whether an invalid document is rejected must not depend on the chunking
+      // either.
+      "tru",
+      "truX",
+      "[[[[x",
+      "\"\\ud801x\"",
+  };
+  for (auto const &doc : docs) {
+    CAPTURE(doc);
+    // 3 is the smallest stackSize WeaselJsonParser_create accepts, so this
+    // sweeps from "everything overflows" to "everything fits".
+    for (int stackSize = 3; stackSize <= 16; ++stackSize) {
+      CAPTURE(stackSize);
+      auto whole = parseChunked(doc, stackSize, {});
+      // Every way to split the document in two, so that every literal and
+      // escape in `docs` straddles a chunk boundary at some point.
+      for (size_t cut = 1; cut < doc.size(); ++cut) {
+        CAPTURE(cut);
+        REQUIRE(parseChunked(doc, stackSize, {cut}) == whole);
+      }
+      for (size_t stride = 1; stride <= 12; ++stride) {
+        CAPTURE(stride);
+        REQUIRE(parseChunked(doc, stackSize, stridedCuts(doc, stride)) ==
+                whole);
+      }
+    }
+  }
+}
+
 TEST_CASE("rejected state is sticky") {
   auto c = noopCallbacks();
   auto *parser = WeaselJsonParser_create(1024, &c, nullptr, 0);
@@ -274,6 +422,10 @@ TEST_CASE("parse rejects negative length") {
   WeaselJsonParser_destroy(parser);
 }
 
+TEST_CASE("Calling parse with nullptr doesn't crash") {
+  REQUIRE(WeaselJsonParser_parse(nullptr, nullptr, 0) == WeaselJson_NULL);
+}
+
 TEST_CASE("streaming") { testStreaming(json); }
 
 TEST_CASE("reset clears inKey and transient state") {

src/validate.cpp

@@ -33,6 +33,9 @@ int main(int argc, char **argv) {
     case WeaselJson_REJECT:
     case WeaselJson_OVERFLOW:
       return 1;
+    case WeaselJson_NULL:
+      fprintf(stderr, "parse called with a null parser\n");
+      return 1;
     }
     if (l == 0) {
       return 1;

weaseljson.py

@@ -30,6 +30,7 @@ class WeaselJsonStatus(enum.Enum):
     AGAIN = 1
     REJECT = 2
     OVERFLOW = 3
+    NULL = 4
 
 
 class WeaselJsonCallbacksBase: