Merge pull request 'schemagen: fix nullable root types (#13)' (#29) from weaselbot/weaseljson:weaselbot/issue-13 into main

Reviewed-on: weaselab/weaseljson#29

.gitignore

@@ -1,2 +1,6 @@
 build
 .cache
+contrib/schemagen/gen.h
+contrib/schemagen/big.h
+contrib/schemagen/test_gen
+contrib/schemagen/test_big

contrib/schemagen/CMakeLists.txt

@@ -12,6 +12,8 @@ add_test(
 set(SCHEMAGEN_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/weaseljson_schemagen.py)
 set(EXAMPLE_SCHEMA ${CMAKE_CURRENT_SOURCE_DIR}/example.schema.json)
 set(GEN_H ${CMAKE_CURRENT_BINARY_DIR}/gen.h)
+set(BIG_SCHEMA ${CMAKE_CURRENT_SOURCE_DIR}/big.schema.json)
+set(BIG_H ${CMAKE_CURRENT_BINARY_DIR}/big.h)
 
 add_custom_command(
   OUTPUT ${GEN_H}
@@ -20,7 +22,15 @@ add_custom_command(
   DEPENDS ${SCHEMAGEN_SCRIPT} ${EXAMPLE_SCHEMA}
   COMMENT "Generating gen.h from example.schema.json")
 
+add_custom_command(
+  OUTPUT ${BIG_H}
+  COMMAND ${Python3_EXECUTABLE} ${SCHEMAGEN_SCRIPT} ${BIG_SCHEMA} -o ${BIG_H}
+          --namespace big_schema
+  DEPENDS ${SCHEMAGEN_SCRIPT} ${BIG_SCHEMA}
+  COMMENT "Generating big.h from big.schema.json")
+
 add_custom_target(schemagen_gen_h DEPENDS ${GEN_H})
+add_custom_target(schemagen_big_h DEPENDS ${BIG_H})
 
 add_executable(schemagen_example ${CMAKE_CURRENT_SOURCE_DIR}/test_gen.cpp)
 target_include_directories(schemagen_example
@@ -29,7 +39,67 @@ target_link_libraries(schemagen_example PRIVATE ${PROJECT_NAME})
 target_compile_options(schemagen_example PRIVATE -Wno-switch-enum)
 add_dependencies(schemagen_example schemagen_gen_h)
 
+add_executable(schemagen_big ${CMAKE_CURRENT_SOURCE_DIR}/test_big.cpp)
+target_include_directories(schemagen_big PRIVATE include
+                                                 ${CMAKE_CURRENT_BINARY_DIR})
+target_link_libraries(schemagen_big PRIVATE ${PROJECT_NAME})
+target_compile_options(schemagen_big PRIVATE -Wno-switch-enum)
+add_dependencies(schemagen_big schemagen_big_h)
+
+set(NULLABLE_OBJECT_SCHEMA
+    ${CMAKE_CURRENT_SOURCE_DIR}/nullable_object.schema.json)
+set(NULLABLE_STRING_SCHEMA
+    ${CMAKE_CURRENT_SOURCE_DIR}/nullable_string.schema.json)
+set(NULLABLE_ARRAY_SCHEMA
+    ${CMAKE_CURRENT_SOURCE_DIR}/nullable_array.schema.json)
+set(NULLABLE_OBJECT_H ${CMAKE_CURRENT_BINARY_DIR}/nullable_object.h)
+set(NULLABLE_STRING_H ${CMAKE_CURRENT_BINARY_DIR}/nullable_string.h)
+set(NULLABLE_ARRAY_H ${CMAKE_CURRENT_BINARY_DIR}/nullable_array.h)
+
+add_custom_command(
+  OUTPUT ${NULLABLE_OBJECT_H}
+  COMMAND ${Python3_EXECUTABLE} ${SCHEMAGEN_SCRIPT} ${NULLABLE_OBJECT_SCHEMA} -o
+          ${NULLABLE_OBJECT_H} --namespace nullable_object
+  DEPENDS ${SCHEMAGEN_SCRIPT} ${NULLABLE_OBJECT_SCHEMA}
+  COMMENT "Generating nullable_object.h")
+
+add_custom_command(
+  OUTPUT ${NULLABLE_STRING_H}
+  COMMAND ${Python3_EXECUTABLE} ${SCHEMAGEN_SCRIPT} ${NULLABLE_STRING_SCHEMA} -o
+          ${NULLABLE_STRING_H} --namespace nullable_string
+  DEPENDS ${SCHEMAGEN_SCRIPT} ${NULLABLE_STRING_SCHEMA}
+  COMMENT "Generating nullable_string.h")
+
+add_custom_command(
+  OUTPUT ${NULLABLE_ARRAY_H}
+  COMMAND ${Python3_EXECUTABLE} ${SCHEMAGEN_SCRIPT} ${NULLABLE_ARRAY_SCHEMA} -o
+          ${NULLABLE_ARRAY_H} --namespace nullable_array
+  DEPENDS ${SCHEMAGEN_SCRIPT} ${NULLABLE_ARRAY_SCHEMA}
+  COMMENT "Generating nullable_array.h")
+
+add_custom_target(
+  schemagen_nullable_h DEPENDS ${NULLABLE_OBJECT_H} ${NULLABLE_STRING_H}
+                               ${NULLABLE_ARRAY_H})
+
+add_executable(schemagen_nullable_root
+               ${CMAKE_CURRENT_SOURCE_DIR}/test_nullable_root.cpp)
+target_include_directories(schemagen_nullable_root
+                           PRIVATE include ${CMAKE_CURRENT_BINARY_DIR})
+target_link_libraries(schemagen_nullable_root PRIVATE ${PROJECT_NAME})
+target_compile_options(schemagen_nullable_root PRIVATE -Wno-switch-enum)
+add_dependencies(schemagen_nullable_root schemagen_nullable_h)
+
 add_test(
   NAME schemagen_example
   COMMAND schemagen_example
   WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+
+add_test(
+  NAME schemagen_big
+  COMMAND schemagen_big
+  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+
+add_test(
+  NAME schemagen_nullable_root
+  COMMAND schemagen_nullable_root
+  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})

contrib/schemagen/README.md

@@ -73,3 +73,28 @@ union `type` lists other than `["T", "null"]`, non-string enums, and remote
   cannot hold partial digits.
 - `test_gen.cpp` generates from `example.schema.json` and exercises the parser
   byte-by-byte (covering chunked strings/numbers), plus the rejection cases.
+
+## Testing
+
+The schemagen tests are registered with CTest and run as part of the default
+`ctest` invocation from the build directory:
+
+```sh
+cmake -S . -B build
+make -C build -j "$(nproc)"
+cd build
+ctest --output-on-failure
+```
+
+For local development you can also use the convenience script:
+
+```sh
+cd contrib/schemagen
+./run_tests.sh
+```
+
+Both regenerate the example parser (`gen.h`) and a regression parser with 40
+required properties (`big.h`), compile `test_gen.cpp` and `test_big.cpp`, and run
+them. `test_big.cpp` specifically covers issue #3: it checks that a 40-property
+object accepts all fields, rejects a missing field at index 32, and rejects
+duplicate keys around the 32-bit boundary.

contrib/schemagen/big.schema.json

@@ -0,0 +1,168 @@
+{
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "p0": {
+      "type": "string"
+    },
+    "p1": {
+      "type": "string"
+    },
+    "p2": {
+      "type": "string"
+    },
+    "p3": {
+      "type": "string"
+    },
+    "p4": {
+      "type": "string"
+    },
+    "p5": {
+      "type": "string"
+    },
+    "p6": {
+      "type": "string"
+    },
+    "p7": {
+      "type": "string"
+    },
+    "p8": {
+      "type": "string"
+    },
+    "p9": {
+      "type": "string"
+    },
+    "p10": {
+      "type": "string"
+    },
+    "p11": {
+      "type": "string"
+    },
+    "p12": {
+      "type": "string"
+    },
+    "p13": {
+      "type": "string"
+    },
+    "p14": {
+      "type": "string"
+    },
+    "p15": {
+      "type": "string"
+    },
+    "p16": {
+      "type": "string"
+    },
+    "p17": {
+      "type": "string"
+    },
+    "p18": {
+      "type": "string"
+    },
+    "p19": {
+      "type": "string"
+    },
+    "p20": {
+      "type": "string"
+    },
+    "p21": {
+      "type": "string"
+    },
+    "p22": {
+      "type": "string"
+    },
+    "p23": {
+      "type": "string"
+    },
+    "p24": {
+      "type": "string"
+    },
+    "p25": {
+      "type": "string"
+    },
+    "p26": {
+      "type": "string"
+    },
+    "p27": {
+      "type": "string"
+    },
+    "p28": {
+      "type": "string"
+    },
+    "p29": {
+      "type": "string"
+    },
+    "p30": {
+      "type": "string"
+    },
+    "p31": {
+      "type": "string"
+    },
+    "p32": {
+      "type": "string"
+    },
+    "p33": {
+      "type": "string"
+    },
+    "p34": {
+      "type": "string"
+    },
+    "p35": {
+      "type": "string"
+    },
+    "p36": {
+      "type": "string"
+    },
+    "p37": {
+      "type": "string"
+    },
+    "p38": {
+      "type": "string"
+    },
+    "p39": {
+      "type": "string"
+    }
+  },
+  "required": [
+    "p0",
+    "p1",
+    "p2",
+    "p3",
+    "p4",
+    "p5",
+    "p6",
+    "p7",
+    "p8",
+    "p9",
+    "p10",
+    "p11",
+    "p12",
+    "p13",
+    "p14",
+    "p15",
+    "p16",
+    "p17",
+    "p18",
+    "p19",
+    "p20",
+    "p21",
+    "p22",
+    "p23",
+    "p24",
+    "p25",
+    "p26",
+    "p27",
+    "p28",
+    "p29",
+    "p30",
+    "p31",
+    "p32",
+    "p33",
+    "p34",
+    "p35",
+    "p36",
+    "p37",
+    "p38",
+    "p39"
+  ]
+}

contrib/schemagen/example.schema.json

@@ -71,6 +71,24 @@
     },
     "tree": {
       "$ref": "#/$defs/Node"
+    },
+    "nullable_hobbies": {
+      "type": "array",
+      "items": {
+        "type": [
+          "string",
+          "null"
+        ]
+      }
+    },
+    "nullable_scores": {
+      "type": "array",
+      "items": {
+        "type": [
+          "integer",
+          "null"
+        ]
+      }
     }
   },
   "$defs": {

contrib/schemagen/nullable_array.schema.json

@@ -0,0 +1,9 @@
+{
+  "type": [
+    "array",
+    "null"
+  ],
+  "items": {
+    "type": "integer"
+  }
+}

contrib/schemagen/nullable_object.schema.json

@@ -0,0 +1,15 @@
+{
+  "type": [
+    "object",
+    "null"
+  ],
+  "additionalProperties": false,
+  "required": [
+    "x"
+  ],
+  "properties": {
+    "x": {
+      "type": "string"
+    }
+  }
+}

contrib/schemagen/nullable_string.schema.json

@@ -0,0 +1,6 @@
+{
+  "type": [
+    "string",
+    "null"
+  ]
+}

contrib/schemagen/run_tests.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+# Build and run the schemagen regression tests.
+# Expects weaseljson to be built at ../../build (the default CMake build dir).
+set -euo pipefail
+
+cd "$(dirname "$0")"
+
+python3 weaseljson_schemagen.py example.schema.json -o gen.h --namespace weasel_schema
+python3 weaseljson_schemagen.py big.schema.json -o big.h --namespace big_schema
+
+g++ -std=c++20 -I../../include -I. test_gen.cpp \
+    -L../../build -lweaseljson \
+    -Wl,-rpath,'$ORIGIN'/../../build \
+    -o test_gen
+
+g++ -std=c++20 -I../../include -I. test_big.cpp \
+    -L../../build -lweaseljson \
+    -Wl,-rpath,'$ORIGIN'/../../build \
+    -o test_big
+
+./test_gen
+./test_big

contrib/schemagen/test_big.cpp

@@ -0,0 +1,140 @@
+// Regression test for issue #3: objects with more than 32 properties.
+#include <cassert>
+#include <cstdio>
+#include <string>
+
+#include "big.h"
+
+using namespace big_schema;
+
+static WeaselJsonStatus parseStrided(RootBuilder &b, std::string in) {
+  for (size_t i = 0; i < in.size(); ++i) {
+    char c = in[i];
+    WeaselJsonStatus s = b.feed(&c, 1);
+    if (s != WeaselJson_AGAIN)
+      return s;
+  }
+  return b.finish();
+}
+
+static int failures = 0;
+#define CHECK(cond)                                                            \
+  do {                                                                         \
+    if (!(cond)) {                                                             \
+      printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond);                   \
+      ++failures;                                                              \
+    }                                                                          \
+  } while (0)
+
+static void expectReject(const std::string &json, const char *what) {
+  RootBuilder b;
+  WeaselJsonStatus s = parseStrided(b, json);
+  if (s == WeaselJson_REJECT) {
+    printf("ok   reject: %s\n", what);
+  } else {
+    printf("FAIL expected reject (%s) got status %d for: %s\n", what, s,
+           json.c_str());
+    ++failures;
+  }
+}
+
+static std::string all40() {
+  std::string json = "{";
+  for (int i = 0; i < 40; ++i) {
+    if (i)
+      json += ",";
+    json += "\"p" + std::to_string(i) + "\":\"v" + std::to_string(i) + "\"";
+  }
+  json += "}";
+  return json;
+}
+
+int main() {
+  // All 40 required fields present -> accepted, values land in right slots.
+  {
+    RootBuilder b;
+    std::string json = all40();
+    WeaselJsonStatus s = parseStrided(b, json);
+    CHECK(s == WeaselJson_OK);
+    if (s == WeaselJson_OK) {
+      Root r = b.take();
+      CHECK(r.p0 == "v0");
+      CHECK(r.p31 == "v31");
+      CHECK(r.p32 == "v32");
+      CHECK(r.p39 == "v39");
+      printf("ok   all 40 required fields accepted\n");
+    }
+  }
+
+  // Missing field p32 (index 32) must be detected, not aliased to bit 0.
+  {
+    std::string json = all40();
+    // Remove the p32 entry: find its substring and erase it.
+    std::string entry = "\"p32\":\"v32\"";
+    size_t pos = json.find(entry);
+    assert(pos != std::string::npos);
+    // Remove the trailing comma before it if present, or the leading comma
+    // after it.
+    if (pos > 0 && json[pos - 1] == ',') {
+      json.erase(pos - 1, entry.size() + 1);
+    } else if (pos + entry.size() < json.size() &&
+               json[pos + entry.size()] == ',') {
+      json.erase(pos, entry.size() + 1);
+    } else {
+      json.erase(pos, entry.size());
+    }
+    expectReject(json, "missing required field p32 (index 32)");
+  }
+
+  // Missing p0 still rejected.
+  {
+    std::string json = all40();
+    std::string entry = "\"p0\":\"v0\"";
+    size_t pos = json.find(entry);
+    assert(pos != std::string::npos);
+    if (json[pos + entry.size()] == ',') {
+      json.erase(pos, entry.size() + 1);
+    } else {
+      json.erase(pos, entry.size());
+    }
+    expectReject(json, "missing required field p0");
+  }
+
+  // Duplicate keys p0 and p32 must both be detected.
+  {
+    std::string json = all40();
+    // Insert p32 again right after the existing p32 entry.
+    std::string entry = "\"p32\":\"v32\"";
+    size_t pos = json.find(entry);
+    assert(pos != std::string::npos);
+    json.insert(pos + entry.size(), ",\"p32\":\"dup\"");
+    expectReject(json, "duplicate key p32 (index 32)");
+  }
+
+  // Duplicate keys p0 and p31 cover boundary bits.
+  {
+    std::string json = all40();
+    std::string entry = "\"p0\":\"v0\"";
+    size_t pos = json.find(entry);
+    assert(pos != std::string::npos);
+    json.insert(pos + entry.size(), ",\"p0\":\"dup\"");
+    expectReject(json, "duplicate key p0");
+  }
+
+  // Duplicate p31/p32 around the 32-bit boundary.
+  {
+    std::string json = all40();
+    std::string entry = "\"p31\":\"v31\"";
+    size_t pos = json.find(entry);
+    assert(pos != std::string::npos);
+    json.insert(pos + entry.size(), ",\"p31\":\"dup\"");
+    expectReject(json, "duplicate key p31");
+  }
+
+  if (failures == 0) {
+    printf("\nALL TESTS PASSED\n");
+    return 0;
+  }
+  printf("\n%d FAILURE(S)\n", failures);
+  return 1;
+}

contrib/schemagen/test_gen.cpp

@@ -86,7 +86,9 @@ int main() {
         "value": 1,
         "children": [ { "value": 2 }, { "value": 3 } ],
         "next": { "value": 99 }
-      }
+      },
+      "nullable_hobbies": [null, "math", null, "lace", null],
+      "nullable_scores": [null, 10, null, 20, null]
     })";
     RootBuilder b;
     WeaselJsonStatus s = parseStrided(b, json);
@@ -116,6 +118,16 @@ int main() {
       CHECK(r.tree && r.tree->children && (*r.tree->children)[0].value == 2);
       CHECK(r.tree && r.tree->children && (*r.tree->children)[1].value == 3);
       CHECK(r.tree && r.tree->next && r.tree->next->value == 99);
+      CHECK(r.nullable_hobbies.has_value() && r.nullable_hobbies->size() == 5);
+      CHECK(r.nullable_hobbies && !(*r.nullable_hobbies)[0] &&
+            (*r.nullable_hobbies)[1] && *(*r.nullable_hobbies)[1] == "math" &&
+            !(*r.nullable_hobbies)[2] && (*r.nullable_hobbies)[3] &&
+            *(*r.nullable_hobbies)[3] == "lace" && !(*r.nullable_hobbies)[4]);
+      CHECK(r.nullable_scores.has_value() && r.nullable_scores->size() == 5);
+      CHECK(r.nullable_scores && !(*r.nullable_scores)[0] &&
+            (*r.nullable_scores)[1] && *(*r.nullable_scores)[1] == 10 &&
+            !(*r.nullable_scores)[2] && (*r.nullable_scores)[3] &&
+            *(*r.nullable_scores)[3] == 20 && !(*r.nullable_scores)[4]);
       printf("ok   happy path (byte-strided)\n");
     }
   }

contrib/schemagen/test_nullable_root.cpp

@@ -0,0 +1,158 @@
+// Regression test for issue #13: nullable root types.
+#include <cassert>
+#include <cstdio>
+#include <string>
+
+#include "nullable_array.h"
+#include "nullable_object.h"
+#include "nullable_string.h"
+
+static int failures = 0;
+#define CHECK(cond)                                                            \
+  do {                                                                         \
+    if (!(cond)) {                                                             \
+      printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond);                   \
+      ++failures;                                                              \
+    }                                                                          \
+  } while (0)
+
+static WeaselJsonStatus parseStrided(nullable_object::RootBuilder &b,
+                                     std::string in) {
+  for (size_t i = 0; i < in.size(); ++i) {
+    char c = in[i];
+    WeaselJsonStatus s = b.feed(&c, 1);
+    if (s != WeaselJson_AGAIN)
+      return s;
+  }
+  return b.finish();
+}
+
+static WeaselJsonStatus parseStrided(nullable_string::RootBuilder &b,
+                                     std::string in) {
+  for (size_t i = 0; i < in.size(); ++i) {
+    char c = in[i];
+    WeaselJsonStatus s = b.feed(&c, 1);
+    if (s != WeaselJson_AGAIN)
+      return s;
+  }
+  return b.finish();
+}
+
+static WeaselJsonStatus parseStrided(nullable_array::RootBuilder &b,
+                                     std::string in) {
+  for (size_t i = 0; i < in.size(); ++i) {
+    char c = in[i];
+    WeaselJsonStatus s = b.feed(&c, 1);
+    if (s != WeaselJson_AGAIN)
+      return s;
+  }
+  return b.finish();
+}
+
+static void expectReject(nullable_object::RootBuilder &b, std::string in,
+                         const char *what) {
+  WeaselJsonStatus s = parseStrided(b, in);
+  if (s == WeaselJson_REJECT) {
+    printf("ok   reject: %s\n", what);
+  } else {
+    printf("FAIL expected reject (%s) got status %d for: %s\n", what, s,
+           in.c_str());
+    ++failures;
+  }
+}
+
+int main() {
+  // ---- nullable root object: valid document ----
+  {
+    nullable_object::RootBuilder b;
+    WeaselJsonStatus s = parseStrided(b, R"({"x":"hello"})");
+    CHECK(s == WeaselJson_OK);
+    if (s == WeaselJson_OK) {
+      nullable_object::Root r = b.take();
+      CHECK(r.has_value());
+      CHECK(r->x == "hello");
+      printf("ok   nullable root object accepts object\n");
+    }
+  }
+
+  // ---- nullable root object: null document ----
+  {
+    nullable_object::RootBuilder b;
+    WeaselJsonStatus s = parseStrided(b, "null");
+    CHECK(s == WeaselJson_OK);
+    if (s == WeaselJson_OK) {
+      nullable_object::Root r = b.take();
+      CHECK(!r.has_value());
+      printf("ok   nullable root object accepts null\n");
+    }
+  }
+
+  // ---- nullable root object: schema checks still run ----
+  {
+    nullable_object::RootBuilder b;
+    expectReject(b, R"({"x":"hello","extra":1})",
+                 "unknown key in strict nullable root object");
+  }
+  {
+    nullable_object::RootBuilder b;
+    expectReject(b, R"({})", "missing required field in nullable root object");
+  }
+
+  // ---- nullable root string: valid value ----
+  {
+    nullable_string::RootBuilder b;
+    WeaselJsonStatus s = parseStrided(b, R"("hello")");
+    CHECK(s == WeaselJson_OK);
+    if (s == WeaselJson_OK) {
+      nullable_string::Root r = b.take();
+      CHECK(r.has_value());
+      CHECK(*r == "hello");
+      printf("ok   nullable root string accepts string\n");
+    }
+  }
+
+  // ---- nullable root string: null value ----
+  {
+    nullable_string::RootBuilder b;
+    WeaselJsonStatus s = parseStrided(b, "null");
+    CHECK(s == WeaselJson_OK);
+    if (s == WeaselJson_OK) {
+      nullable_string::Root r = b.take();
+      CHECK(!r.has_value());
+      printf("ok   nullable root string accepts null\n");
+    }
+  }
+
+  // ---- nullable root array: valid value ----
+  {
+    nullable_array::RootBuilder b;
+    WeaselJsonStatus s = parseStrided(b, "[1,2,3]");
+    CHECK(s == WeaselJson_OK);
+    if (s == WeaselJson_OK) {
+      nullable_array::Root r = b.take();
+      CHECK(r.has_value());
+      CHECK(r->size() == 3);
+      CHECK((*r)[0] == 1 && (*r)[1] == 2 && (*r)[2] == 3);
+      printf("ok   nullable root array accepts array\n");
+    }
+  }
+
+  // ---- nullable root array: null value ----
+  {
+    nullable_array::RootBuilder b;
+    WeaselJsonStatus s = parseStrided(b, "null");
+    CHECK(s == WeaselJson_OK);
+    if (s == WeaselJson_OK) {
+      nullable_array::Root r = b.take();
+      CHECK(!r.has_value());
+      printf("ok   nullable root array accepts null\n");
+    }
+  }
+
+  if (failures == 0) {
+    printf("\nALL TESTS PASSED\n");
+    return 0;
+  }
+  printf("\n%d FAILURE(S)\n", failures);
+  return 1;
+}

contrib/schemagen/test_schemagen.py

@@ -49,5 +49,43 @@ class SchemagenEnumTest(unittest.TestCase):
         self.assertIn("enum class Role : int { admin, user, guest };", stdout)
 
 
+class SchemagenKeywordTest(unittest.TestCase):
+    def run_schemagen(self, schema, args=None):
+        """Run schemagen on a schema dict. Returns (returncode, stdout, stderr)."""
+        with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as fp:
+            json.dump(schema, fp)
+            schema_path = fp.name
+        try:
+            cmd = [sys.executable, SCRIPT, schema_path]
+            if args:
+                cmd.extend(args)
+            result = subprocess.run(cmd, capture_output=True, text=True, check=False)
+            return result.returncode, result.stdout, result.stderr
+        finally:
+            os.unlink(schema_path)
+
+    def test_cpp20_keywords_are_sanitized(self):
+        """C++20 keywords used as JSON property names must be suffixed."""
+        keywords = [
+            "concept",
+            "consteval",
+            "constinit",
+            "co_await",
+            "co_return",
+            "co_yield",
+            "requires",
+            "module",
+            "import",
+        ]
+        schema = {"type": "object", "properties": {}}
+        for kw in keywords:
+            schema["properties"][kw] = {"type": "string"}
+        rc, stdout, stderr = self.run_schemagen(schema)
+        self.assertEqual(rc, 0, msg=stderr)
+        for kw in keywords:
+            self.assertIn(f"std::optional<std::string> {kw}_;", stdout)
+            self.assertNotIn(f"std::optional<std::string> {kw};", stdout)
+
+
 if __name__ == "__main__":
     unittest.main()

contrib/schemagen/weaseljson_schemagen.py

@@ -133,9 +133,15 @@ _CPP_KEYWORDS = {
     "catch",
     "char",
     "class",
+    "concept",
     "const",
+    "consteval",
+    "constinit",
     "constexpr",
     "continue",
+    "co_await",
+    "co_return",
+    "co_yield",
     "decltype",
     "default",
     "delete",
@@ -152,9 +158,11 @@ _CPP_KEYWORDS = {
     "friend",
     "goto",
     "if",
+    "import",
     "inline",
     "int",
     "long",
+    "module",
     "namespace",
     "new",
     "not",
@@ -166,6 +174,7 @@ _CPP_KEYWORDS = {
     "public",
     "register",
     "return",
+    "requires",
     "short",
     "signed",
     "sizeof",
@@ -484,6 +493,18 @@ class Emitter:
         self.root_ty, self.root_nullable = Builder._unpack(
             self.b.build_type(self.b.root_schema, "Root")
         )
+
+        # A nullable root object would otherwise produce
+        #   using Root = std::optional<Root>;
+        # which conflicts with the struct named Root.  Rename the inner struct.
+        if isinstance(self.root_ty, TObj) and self.root_nullable:
+            old_name = self.root_ty.name
+            new_name = self.b.unique_name("RootInner")
+            obj = self.b.objects.pop(old_name)
+            obj.name = new_name
+            self.b.objects[new_name] = obj
+            self.root_ty = TObj(new_name)
+
         break_cycles(self.b.objects)
 
         # register all array kinds (walk every field + root)
@@ -573,9 +594,7 @@ namespace {ns} {{"""
     def _kind_enum(self):
         kinds = list(self.b.objects.keys())
         kinds += [n for n, _ in self.arr_types]
-        root_is_container = (
-            isinstance(self.root_ty, (TObj, TArr)) and not self.root_nullable
-        )
+        root_is_container = isinstance(self.root_ty, (TObj, TArr))
         if not root_is_container:
             kinds.append("RootScalar")
         kinds.append("Skip")
@@ -584,9 +603,9 @@ namespace {ns} {{"""
 
     def _root_info(self):
         """Return (root_cat, root_container_kind_or_None)."""
-        if isinstance(self.root_ty, TObj) and not self.root_nullable:
+        if isinstance(self.root_ty, TObj):
             return ("Obj", f"Kind::{self.root_ty.name}")
-        if isinstance(self.root_ty, TArr) and not self.root_nullable:
+        if isinstance(self.root_ty, TArr):
             return ("Arr", f"Kind::{self.arr_kind(self.root_ty)}")
         return (self.cat(self.root_ty), None)
 
@@ -656,7 +675,12 @@ namespace {ns} {{"""
             lines.append("    }")
         root_cat, root_kind = self._root_info()
         if root_kind is None:
-            lines.append("    case Kind::RootScalar: return &result_;")
+            if self.root_nullable:
+                lines.append(
+                    "    case Kind::RootScalar: { if (!result_) result_.emplace(); return &*result_; }"
+                )
+            else:
+                lines.append("    case Kind::RootScalar: return &result_;")
         lines.append("    default: return nullptr;")
         lines.append("    }")
         lines.append("  }")
@@ -690,19 +714,35 @@ namespace {ns} {{"""
         lines.append("  }")
         return "\n".join(lines)
 
-    def _bitmask(self, obj, pred):
-        mask = 0
-        for i, fld in enumerate(obj.fields):
-            if pred(fld):
-                mask |= 1 << i
-        return mask
+    def _fieldcount(self):
+        lines = ["  int fieldCount(Kind k) const {", "    switch (k) {"]
+        for name, obj in self.b.objects.items():
+            lines.append(f"    case Kind::{name}: return {len(obj.fields)};")
+        lines.append("    default: return 0;")
+        lines.append("    }")
+        lines.append("  }")
+        return "\n".join(lines)
 
     def _reqmask(self):
-        lines = ["  uint32_t requiredMask(Kind k) const {", "    switch (k) {"]
+        lines = [
+            "  const std::vector<uint64_t> &requiredMask(Kind k) const {",
+            "    switch (k) {",
+        ]
         for name, obj in self.b.objects.items():
-            m = self._bitmask(obj, lambda f: f.required)
-            lines.append(f"    case Kind::{name}: return {hex(m)}u;")
-        lines.append("    default: return 0u;")
+            words = (len(obj.fields) + 63) // 64
+            req = [0] * words
+            for i, fld in enumerate(obj.fields):
+                if fld.required:
+                    req[i >> 6] |= 1 << (i & 63)
+            init = ", ".join(f"{hex(w)}u" for w in req) if words else ""
+            lines.append(f"    case Kind::{name}: {{")
+            lines.append(f"      static const std::vector<uint64_t> m = {{ {init} }};")
+            lines.append("      return m;")
+            lines.append("    }")
+        lines.append("    default: {")
+        lines.append("      static const std::vector<uint64_t> empty;")
+        lines.append("      return empty;")
+        lines.append("    }")
         lines.append("    }")
         lines.append("  }")
         return "\n".join(lines)
@@ -718,6 +758,34 @@ namespace {ns} {{"""
             "  }"
         )
 
+    def _is_array_kind(self):
+        arrs = [n for n, _ in self.arr_types]
+        if not arrs:
+            return "  bool isArrayKind(Kind) const { return false; }"
+        cases = " ".join(f"case Kind::{n}:" for n in arrs)
+        return (
+            "  bool isArrayKind(Kind k) const {\n"
+            f"    switch (k) {{ {cases} return true; default: return false; }}\n"
+            "  }"
+        )
+
+    def _append_null(self):
+        lines = [
+            "  void appendNull(Frame &f) {",
+            "    switch (f.kind) {",
+        ]
+        for name, tarr in self.arr_types:
+            vectype = self.base_cpp(tarr)
+            lines.append(f"    case Kind::{name}: {{")
+            lines.append(f"      auto *v = ({vectype} *)f.dest;")
+            lines.append("      v->emplace_back();")
+            lines.append("      return;")
+            lines.append("    }")
+        lines.append("    default: return;")
+        lines.append("    }")
+        lines.append("  }")
+        return "\n".join(lines)
+
     def _is_strict(self):
         strict = [n for n, o in self.b.objects.items() if o.strict]
         if not strict:
@@ -754,7 +822,17 @@ namespace {ns} {{"""
         lines = ["    if (error_) return;"]
         if root_kind is not None and root_cat == event_cat:
             lines.append("    if (stack_.empty()) {")
-            lines.append(f"      stack_.push_back(Frame{{{root_kind}, &result_}});")
+            if self.root_nullable:
+                lines.append("      result_.emplace();")
+                dest = "&*result_"
+            else:
+                dest = "&result_"
+            lines.append(f"      stack_.push_back(Frame{{{root_kind}, {dest}}});")
+            if event_cat == "Obj":
+                lines.append("      {")
+                lines.append(f"        int n = fieldCount({root_kind});")
+                lines.append("        stack_.back().seen.assign((n + 63) / 64, 0);")
+                lines.append("      }")
             lines.append("      return;")
             lines.append("    }")
         else:
@@ -770,6 +848,10 @@ namespace {ns} {{"""
         lines.append(f"    if (si.cat != Cat::{event_cat}) {{ reject(); return; }}")
         lines.append("    void *p = engage(f, true);")
         lines.append("    stack_.push_back(Frame{si.child, p});")
+        lines.append("    if (isObjectKind(si.child)) {")
+        lines.append("      int n = fieldCount(si.child);")
+        lines.append("      stack_.back().seen.assign((n + 63) / 64, 0);")
+        lines.append("    }")
         return "\n".join(lines)
 
     def _builder(self, order):
@@ -777,6 +859,9 @@ namespace {ns} {{"""
         root_cat, root_kind = self._root_info()
         begin_obj = self._begin_container("Obj", root_kind)
         begin_arr = self._begin_container("Arr", root_kind)
+        null_at_root = (
+            "done_ = true; return;" if self.root_nullable else "reject(); return;"
+        )
 
         return f"""
 class RootBuilder {{
@@ -821,8 +906,8 @@ private:
   struct Frame {{
     Kind kind;
     void *dest;
-    int field = -1;       // object: selected field (-1 want key, -2 skip)
-    uint32_t seen = 0;    // bitmask of populated fields
+    int field = -1;              // object: selected field (-1 want key, -2 skip)
+    std::vector<uint64_t> seen;  // populated field bitset (object frames)
   }};
   static constexpr int kWantKey = -1;
   static constexpr int kSkip = -2;
@@ -857,7 +942,7 @@ private:
     scratch_.clear();
 {("    if (p.kind == Kind::RootScalar) { stack_.pop_back(); done_ = true; return; }" if root_kind is None else "")}
     if (isObjectKind(p.kind)) {{
-      if (p.field >= 0) p.seen |= (1u << p.field);
+      if (p.field >= 0) p.seen[p.field >> 6] |= (1ull << (p.field & 63));
       p.field = kWantKey;
     }}
   }}
@@ -867,16 +952,19 @@ private:
   }}
   void cbEndObject() {{
     if (error_) return;
-    Frame f = stack_.back();
+    Frame &f = stack_.back();
     if (f.kind == Kind::Skip) {{
       stack_.pop_back();
       if (stack_.empty() || stack_.back().kind != Kind::Skip) valueComplete();
       return;
     }}
-    if (isObjectKind(f.kind) &&
-        (f.seen & requiredMask(f.kind)) != requiredMask(f.kind)) {{
-      reject();
-      return;
+    if (isObjectKind(f.kind)) {{
+      const auto &req = requiredMask(f.kind);
+      bool missing = false;
+      for (size_t i = 0; i < req.size(); ++i) {{
+        if ((f.seen[i] & req[i]) != req[i]) {{ missing = true; break; }}
+      }}
+      if (missing) {{ reject(); return; }}
     }}
     stack_.pop_back();
     valueComplete();
@@ -909,7 +997,9 @@ private:
       f.field = kSkip;
       return;
     }}
-    if (f.seen & (1u << idx)) {{ reject(); return; }}  // duplicate key
+    if (f.seen[idx >> 6] & (1ull << (idx & 63))) {{
+      reject(); return;  // duplicate key
+    }}
     f.field = idx;
   }}
 
@@ -992,13 +1082,21 @@ private:
     valueComplete();
   }}
 
+{self._is_array_kind()}
+
+{self._append_null()}
+
   void cbNull() {{
     if (error_) return;
+    if (stack_.empty()) {{
+      {null_at_root}
+    }}
     Frame &f = stack_.back();
     if (f.kind == Kind::Skip) return;
     SlotInfo si = slotInfoG(f);
     if (si.cat == Cat::Skip) {{ valueComplete(); return; }}
     if (!si.nullable) {{ reject(); return; }}
+    if (isArrayKind(f.kind)) appendNull(f);  // keep null array elements
     valueComplete();  // leave optional empty / pointer null
   }}
 
@@ -1029,6 +1127,8 @@ private:
 
 {self._matchkey()}
 
+{self._fieldcount()}
+
 {self._reqmask()}
 
 {self._is_object_kind()}

include/weaseljson.h

@@ -64,7 +64,8 @@ void WeaselJsonParser_destroy(WeaselJsonParser *parser);
 
 /** Incrementally parse `len` more bytes starting at `buf`. `buf` may be
  * modified. Call with `len` 0 to indicate end of data. `buf` may be null if
- * `len` is 0 */
+ * `len` is 0. `len` must not be negative; a negative length is treated as a
+ * rejected input. */
 WeaselJsonStatus WeaselJsonParser_parse(WeaselJsonParser *parser, char *buf,
                                         int len);
 

src/parser3.h

@@ -138,6 +138,13 @@ struct Parser3 {
   void reset() {
     stackPtr = stack();
     std::ignore = push({N_VALUE, N_WHITESPACE, T_EOF});
+    inKey = false;
+    rejected = false;
+    utf8Codepoint = 0;
+    utf16Surrogate = 0;
+    minCodepoint = 0;
+    numDfa.reset();
+    strDfa.reset();
   }
 
   // Used for flushing pending data with on_*_data callbacks
@@ -155,6 +162,7 @@ struct Parser3 {
   NumDfa numDfa;
   Utf8Dfa strDfa;
   bool inKey = false;
+  bool rejected = false;
 
 #ifndef HAS_MUSTTAIL
   char *stashBufForTrampoline;
@@ -1056,18 +1064,34 @@ constexpr inline struct ContinuationTable {
 inline WeaselJsonStatus Parser3::parse(char *buf, int len) {
   this->dataBegin = this->writeBuf = buf;
 
+  if (this->rejected) [[unlikely]] {
+    return WeaselJson_REJECT;
+  }
+
+  if (len < 0) [[unlikely]] {
+    this->rejected = true;
+    return WeaselJson_REJECT;
+  }
+
 #ifdef HAS_MUSTTAIL
   // The continuation returns a value in 0..3 here (kBounce is only used by the
   // no-musttail trampoline below), so the conversion back to the enum is in
   // range.
-  return WeaselJsonStatus(
-      symbolTables.continuations[top()](this, buf, buf + len));
+  ContinuationStatus status =
+      symbolTables.continuations[top()](this, buf, buf + len);
+  if (status == WeaselJson_REJECT) {
+    this->rejected = true;
+  }
+  return WeaselJsonStatus(status);
 #else
   this->stashBufForTrampoline = buf;
   ContinuationStatus result;
   while ((result = symbolTables.continuations[top()](
               this, stashBufForTrampoline, buf + len)) == kBounce)
     ;
+  if (result == WeaselJson_REJECT) {
+    this->rejected = true;
+  }
   return WeaselJsonStatus(result);
 #endif
 }

src/test.cpp

@@ -202,6 +202,31 @@ TEST_CASE("parser3") {
   }
 }
 
+TEST_CASE("rejected state is sticky") {
+  auto c = noopCallbacks();
+  auto *parser = WeaselJsonParser_create(1024, &c, nullptr, 0);
+  REQUIRE(parser != nullptr);
+
+  // Feed a valid prefix, then an invalid byte. The parser must reject.
+  std::string chunk1 = "1";
+  REQUIRE(WeaselJsonParser_parse(parser, chunk1.data(), chunk1.size()) ==
+          WeaselJson_AGAIN);
+  std::string chunk2 = "x";
+  REQUIRE(WeaselJsonParser_parse(parser, chunk2.data(), chunk2.size()) ==
+          WeaselJson_REJECT);
+
+  // After a reject, every later call must also reject, including the EOF
+  // finish call that would otherwise return OK.
+  REQUIRE(WeaselJsonParser_parse(parser, nullptr, 0) == WeaselJson_REJECT);
+
+  // Further data chunks must also stay rejected.
+  std::string chunk3 = " ";
+  REQUIRE(WeaselJsonParser_parse(parser, chunk3.data(), chunk3.size()) ==
+          WeaselJson_REJECT);
+
+  WeaselJsonParser_destroy(parser);
+}
+
 TEST_CASE("create rejects too-small stack") {
   auto c = noopCallbacks();
   // The parser needs room for the bootstrap symbols pushed by reset(); a stack
@@ -221,8 +246,63 @@ TEST_CASE("create rejects too-small stack") {
   WeaselJsonParser_destroy(parser);
 }
 
+TEST_CASE("parse rejects negative length") {
+  auto c = noopCallbacks();
+  auto *parser = WeaselJsonParser_create(1024, &c, nullptr, 0);
+  REQUIRE(parser != nullptr);
+
+  // A negative length must not cause pointer arithmetic UB. It should be
+  // rejected, and the rejected state should remain sticky.
+  char buf[10] = "hello";
+  REQUIRE(WeaselJsonParser_parse(parser, buf, -1) == WeaselJson_REJECT);
+  REQUIRE(WeaselJsonParser_parse(parser, nullptr, 0) == WeaselJson_REJECT);
+
+  WeaselJsonParser_destroy(parser);
+}
+
 TEST_CASE("streaming") { testStreaming(json); }
 
+TEST_CASE("reset clears inKey and transient state") {
+  struct State {
+    std::string stringData;
+    std::string keyData;
+  } state;
+  auto c = noopCallbacks();
+  c.on_string_data = +[](void *p, const char *buf, int len, int /*done*/) {
+    ((State *)p)->stringData.append(buf, len);
+  };
+  c.on_key_data = +[](void *p, const char *buf, int len, int /*done*/) {
+    ((State *)p)->keyData.append(buf, len);
+  };
+
+  auto *parser = WeaselJsonParser_create(1024, &c, &state, 0);
+  REQUIRE(parser != nullptr);
+
+  {
+    std::string chunk = "{\"ab";
+    REQUIRE(WeaselJsonParser_parse(parser, chunk.data(), chunk.size()) ==
+            WeaselJson_AGAIN);
+  }
+
+  // Reset mid-key: the next top-level string must be delivered as a string,
+  // not appended to the aborted key.
+  WeaselJsonParser_reset(parser);
+  state.stringData.clear();
+  state.keyData.clear();
+
+  {
+    std::string chunk = "\"hello\"";
+    REQUIRE(WeaselJsonParser_parse(parser, chunk.data(), chunk.size()) ==
+            WeaselJson_AGAIN);
+  }
+  REQUIRE(WeaselJsonParser_parse(parser, nullptr, 0) == WeaselJson_OK);
+
+  CHECK(state.stringData == "hello");
+  CHECK(state.keyData.empty());
+
+  WeaselJsonParser_destroy(parser);
+}
+
 void doTestUnescapingUtf8(std::string const &escaped,
                           std::string const &expected, int stride, int flags) {
   CAPTURE(escaped);

test_python_bindings.py

@@ -84,7 +84,19 @@ def test_mixed_values():
     assert recorder.events.count("null") == 1
 
 
+def test_create_rejects_too_small_stack():
+    for stack_size in (-1, 0, 1, 2):
+        try:
+            parser = weaseljson.WeaselJsonParser(Recorder(), stackSize=stack_size)
+        except ValueError:
+            continue
+        # If creation unexpectedly succeeds, close it cleanly and fail the test.
+        parser.close()
+        raise AssertionError(f"expected ValueError for stackSize={stack_size}")
+
+
 if __name__ == "__main__":
     test_object_keys_routed_correctly()
     test_mixed_values()
+    test_create_rejects_too_small_stack()
     print("python bindings ok")

weaseljson.py

@@ -117,12 +117,24 @@ class WeaselJsonParser:
             self.voidp_callbacks,
             0,
         )
+        if self.p is None:
+            raise ValueError(
+                "WeaselJsonParser_create returned NULL; "
+                "check stackSize (must be positive and large enough) "
+                "and available memory"
+            )
+
+    def _check_open(self):
+        if self.p is None:
+            raise RuntimeError("parser has been closed or creation failed")
 
     def parse(self, data: bytes) -> WeaselJsonStatus:
+        self._check_open()
         buf = (ctypes.c_ubyte * len(data)).from_buffer(bytearray(data))
         return self._lib.WeaselJsonParser_parse(self.p, buf, len(data))
 
     def reset(self):
+        self._check_open()
         self._lib.WeaselJsonParser_reset(self.p)
 
     def __enter__(self):