ci: register schemagen regression tests with ctest

Remove the standalone "Run schemagen tests" workflow step and instead
add the big-schema regression test to contrib/schemagen/CMakeLists.txt
so ctest picks it up alongside schemagen_example. Update README to
document both `ctest` and the convenience `./run_tests.sh`.

Closes #3

.gitea/workflows/ci.yml

@@ -59,11 +59,6 @@ jobs:
           cd build
           ctest --output-on-failure -j "$(nproc)" --timeout 90
 
-      - name: Run schemagen tests
-        run: |
-          cd contrib/schemagen
-          ./run_tests.sh
-
       - name: Package
         if: matrix.upload
         run: |

CMakeLists.txt

@@ -163,6 +163,15 @@ target_link_libraries(mytest PRIVATE ${PROJECT_NAME} doctest nanobench simdjson)
 target_compile_options(mytest PRIVATE ${TEST_FLAGS})
 doctest_discover_tests(mytest WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
 
+find_package(Python3 COMPONENTS Interpreter)
+if(Python3_Interpreter_FOUND)
+  add_test(NAME python_bindings
+           COMMAND ${Python3_EXECUTABLE}
+                   ${CMAKE_CURRENT_SOURCE_DIR}/test_python_bindings.py)
+endif()
+
+add_subdirectory(contrib/schemagen)
+
 include(CMakePushCheckState)
 include(CheckCXXCompilerFlag)
 cmake_push_check_state()

contrib/schemagen/CMakeLists.txt

@@ -0,0 +1,57 @@
+# Tests for contrib/schemagen
+
+if(NOT Python3_Interpreter_FOUND)
+  return()
+endif()
+
+add_test(
+  NAME schemagen_python_tests
+  COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_schemagen.py
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
+
+set(SCHEMAGEN_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/weaseljson_schemagen.py)
+set(EXAMPLE_SCHEMA ${CMAKE_CURRENT_SOURCE_DIR}/example.schema.json)
+set(GEN_H ${CMAKE_CURRENT_BINARY_DIR}/gen.h)
+set(BIG_SCHEMA ${CMAKE_CURRENT_SOURCE_DIR}/big.schema.json)
+set(BIG_H ${CMAKE_CURRENT_BINARY_DIR}/big.h)
+
+add_custom_command(
+  OUTPUT ${GEN_H}
+  COMMAND ${Python3_EXECUTABLE} ${SCHEMAGEN_SCRIPT} ${EXAMPLE_SCHEMA} -o
+          ${GEN_H} --namespace weasel_schema
+  DEPENDS ${SCHEMAGEN_SCRIPT} ${EXAMPLE_SCHEMA}
+  COMMENT "Generating gen.h from example.schema.json")
+
+add_custom_command(
+  OUTPUT ${BIG_H}
+  COMMAND ${Python3_EXECUTABLE} ${SCHEMAGEN_SCRIPT} ${BIG_SCHEMA} -o ${BIG_H}
+          --namespace big_schema
+  DEPENDS ${SCHEMAGEN_SCRIPT} ${BIG_SCHEMA}
+  COMMENT "Generating big.h from big.schema.json")
+
+add_custom_target(schemagen_gen_h DEPENDS ${GEN_H})
+add_custom_target(schemagen_big_h DEPENDS ${BIG_H})
+
+add_executable(schemagen_example ${CMAKE_CURRENT_SOURCE_DIR}/test_gen.cpp)
+target_include_directories(schemagen_example
+                           PRIVATE include ${CMAKE_CURRENT_BINARY_DIR})
+target_link_libraries(schemagen_example PRIVATE ${PROJECT_NAME})
+target_compile_options(schemagen_example PRIVATE -Wno-switch-enum)
+add_dependencies(schemagen_example schemagen_gen_h)
+
+add_executable(schemagen_big ${CMAKE_CURRENT_SOURCE_DIR}/test_big.cpp)
+target_include_directories(schemagen_big PRIVATE include
+                                                 ${CMAKE_CURRENT_BINARY_DIR})
+target_link_libraries(schemagen_big PRIVATE ${PROJECT_NAME})
+target_compile_options(schemagen_big PRIVATE -Wno-switch-enum)
+add_dependencies(schemagen_big schemagen_big_h)
+
+add_test(
+  NAME schemagen_example
+  COMMAND schemagen_example
+  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+
+add_test(
+  NAME schemagen_big
+  COMMAND schemagen_big
+  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})

contrib/schemagen/README.md

@@ -76,15 +76,25 @@ union `type` lists other than `["T", "null"]`, non-string enums, and remote
 
 ## Testing
 
-After building weaseljson (e.g. `cmake -S . -B build && make -C build`), run:
+The schemagen tests are registered with CTest and run as part of the default
+`ctest` invocation from the build directory:
+
+```sh
+cmake -S . -B build
+make -C build -j "$(nproc)"
+cd build
+ctest --output-on-failure
+```
+
+For local development you can also use the convenience script:
 
 ```sh
 cd contrib/schemagen
 ./run_tests.sh
 ```
 
-This regenerates the example parser (`gen.h`) and a regression parser with 40
-required properties (`big.h`), compiles `test_gen.cpp` and `test_big.cpp`, and
-runs both. `test_big.cpp` specifically covers issue #3: it checks that a
-40-property object accepts all fields, rejects a missing field at index 32, and
-rejects duplicate keys around the 32-bit boundary.
+Both regenerate the example parser (`gen.h`) and a regression parser with 40
+required properties (`big.h`), compile `test_gen.cpp` and `test_big.cpp`, and run
+them. `test_big.cpp` specifically covers issue #3: it checks that a 40-property
+object accepts all fields, rejects a missing field at index 32, and rejects
+duplicate keys around the 32-bit boundary.

contrib/schemagen/example.schema.json

@@ -71,6 +71,24 @@
     },
     "tree": {
       "$ref": "#/$defs/Node"
+    },
+    "nullable_hobbies": {
+      "type": "array",
+      "items": {
+        "type": [
+          "string",
+          "null"
+        ]
+      }
+    },
+    "nullable_scores": {
+      "type": "array",
+      "items": {
+        "type": [
+          "integer",
+          "null"
+        ]
+      }
     }
   },
   "$defs": {

contrib/schemagen/test_gen.cpp

@@ -86,7 +86,9 @@ int main() {
         "value": 1,
         "children": [ { "value": 2 }, { "value": 3 } ],
         "next": { "value": 99 }
-      }
+      },
+      "nullable_hobbies": [null, "math", null, "lace", null],
+      "nullable_scores": [null, 10, null, 20, null]
     })";
     RootBuilder b;
     WeaselJsonStatus s = parseStrided(b, json);
@@ -116,6 +118,16 @@ int main() {
       CHECK(r.tree && r.tree->children && (*r.tree->children)[0].value == 2);
       CHECK(r.tree && r.tree->children && (*r.tree->children)[1].value == 3);
       CHECK(r.tree && r.tree->next && r.tree->next->value == 99);
+      CHECK(r.nullable_hobbies.has_value() && r.nullable_hobbies->size() == 5);
+      CHECK(r.nullable_hobbies && !(*r.nullable_hobbies)[0] &&
+            (*r.nullable_hobbies)[1] && *(*r.nullable_hobbies)[1] == "math" &&
+            !(*r.nullable_hobbies)[2] && (*r.nullable_hobbies)[3] &&
+            *(*r.nullable_hobbies)[3] == "lace" && !(*r.nullable_hobbies)[4]);
+      CHECK(r.nullable_scores.has_value() && r.nullable_scores->size() == 5);
+      CHECK(r.nullable_scores && !(*r.nullable_scores)[0] &&
+            (*r.nullable_scores)[1] && *(*r.nullable_scores)[1] == 10 &&
+            !(*r.nullable_scores)[2] && (*r.nullable_scores)[3] &&
+            *(*r.nullable_scores)[3] == 20 && !(*r.nullable_scores)[4]);
       printf("ok   happy path (byte-strided)\n");
     }
   }

contrib/schemagen/test_schemagen.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+"""Tests for weaseljson_schemagen.py."""
+
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+
+SCRIPT = os.path.join(os.path.dirname(__file__), "weaseljson_schemagen.py")
+
+
+class SchemagenEnumTest(unittest.TestCase):
+    def run_schemagen(self, schema, args=None):
+        """Run schemagen on a schema dict. Returns (returncode, stdout, stderr)."""
+        with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as fp:
+            json.dump(schema, fp)
+            schema_path = fp.name
+        try:
+            cmd = [sys.executable, SCRIPT, schema_path]
+            if args:
+                cmd.extend(args)
+            result = subprocess.run(cmd, capture_output=True, text=True, check=False)
+            return result.returncode, result.stdout, result.stderr
+        finally:
+            os.unlink(schema_path)
+
+    def test_colliding_enum_values_deduplicate(self):
+        schema = {
+            "type": "object",
+            "properties": {"role": {"enum": ["foo-bar", "foo_bar"]}},
+        }
+        rc, stdout, stderr = self.run_schemagen(schema)
+        self.assertEqual(rc, 0, msg=stderr)
+        self.assertIn("enum class Role : int { foo_bar, foo_bar_1 };", stdout)
+        self.assertIn(
+            'static constexpr const char *Role_names[] = { "foo-bar", "foo_bar" };',
+            stdout,
+        )
+
+    def test_distinct_enum_values_generate(self):
+        schema = {
+            "type": "object",
+            "properties": {"role": {"enum": ["admin", "user", "guest"]}},
+        }
+        rc, stdout, stderr = self.run_schemagen(schema)
+        self.assertEqual(rc, 0, msg=stderr)
+        self.assertIn("enum class Role : int { admin, user, guest };", stdout)
+
+
+if __name__ == "__main__":
+    unittest.main()

contrib/schemagen/weaseljson_schemagen.py

@@ -86,6 +86,34 @@ def sanitize(name, fallback="x"):
     return s
 
 
+def unique_enum_identifiers(values):
+    """Return a list of sanitized C++ identifiers, one per input value.
+
+    Distinct JSON enum values may sanitize to the same C++ token (e.g.
+    "foo-bar" and "foo_bar" both become "foo_bar").  This helper appends a
+    numeric suffix to later collisions so the generated `enum class` stays
+    valid while preserving the original order and therefore the index-to-value
+    mapping used at parse time.
+    """
+    used = set()
+    out = []
+    for v in values:
+        base = sanitize(v)
+        if base not in used:
+            used.add(base)
+            out.append(base)
+            continue
+        n = 1
+        while True:
+            candidate = f"{base}_{n}"
+            if candidate not in used:
+                used.add(candidate)
+                out.append(candidate)
+                break
+            n += 1
+    return out
+
+
 def camel(name):
     parts = [p for p in name.replace("-", " ").replace("_", " ").split(" ") if p]
     if not parts:
@@ -508,7 +536,7 @@ namespace {ns} {{"""
             return ""
         out = []
         for e in self.b.enums.values():
-            vals = ", ".join(sanitize(v) for v in e.values)
+            vals = ", ".join(unique_enum_identifiers(e.values))
             out.append(f"enum class {e.name} : int {{ {vals} }};")
         return "\n".join(out) + "\n"
 
@@ -706,6 +734,34 @@ namespace {ns} {{"""
             "  }"
         )
 
+    def _is_array_kind(self):
+        arrs = [n for n, _ in self.arr_types]
+        if not arrs:
+            return "  bool isArrayKind(Kind) const { return false; }"
+        cases = " ".join(f"case Kind::{n}:" for n in arrs)
+        return (
+            "  bool isArrayKind(Kind k) const {\n"
+            f"    switch (k) {{ {cases} return true; default: return false; }}\n"
+            "  }"
+        )
+
+    def _append_null(self):
+        lines = [
+            "  void appendNull(Frame &f) {",
+            "    switch (f.kind) {",
+        ]
+        for name, tarr in self.arr_types:
+            vectype = self.base_cpp(tarr)
+            lines.append(f"    case Kind::{name}: {{")
+            lines.append(f"      auto *v = ({vectype} *)f.dest;")
+            lines.append("      v->emplace_back();")
+            lines.append("      return;")
+            lines.append("    }")
+        lines.append("    default: return;")
+        lines.append("    }")
+        lines.append("  }")
+        return "\n".join(lines)
+
     def _is_strict(self):
         strict = [n for n, o in self.b.objects.items() if o.strict]
         if not strict:
@@ -994,6 +1050,10 @@ private:
     valueComplete();
   }}
 
+{self._is_array_kind()}
+
+{self._append_null()}
+
   void cbNull() {{
     if (error_) return;
     Frame &f = stack_.back();
@@ -1001,6 +1061,7 @@ private:
     SlotInfo si = slotInfoG(f);
     if (si.cat == Cat::Skip) {{ valueComplete(); return; }}
     if (!si.nullable) {{ reject(); return; }}
+    if (isArrayKind(f.kind)) appendNull(f);  // keep null array elements
     valueComplete();  // leave optional empty / pointer null
   }}
 

src/parser3.h

@@ -138,6 +138,12 @@ struct Parser3 {
   void reset() {
     stackPtr = stack();
     std::ignore = push({N_VALUE, N_WHITESPACE, T_EOF});
+    inKey = false;
+    utf8Codepoint = 0;
+    utf16Surrogate = 0;
+    minCodepoint = 0;
+    numDfa.reset();
+    strDfa.reset();
   }
 
   // Used for flushing pending data with on_*_data callbacks

src/test.cpp

@@ -223,6 +223,47 @@ TEST_CASE("create rejects too-small stack") {
 
 TEST_CASE("streaming") { testStreaming(json); }
 
+TEST_CASE("reset clears inKey and transient state") {
+  struct State {
+    std::string stringData;
+    std::string keyData;
+  } state;
+  auto c = noopCallbacks();
+  c.on_string_data = +[](void *p, const char *buf, int len, int /*done*/) {
+    ((State *)p)->stringData.append(buf, len);
+  };
+  c.on_key_data = +[](void *p, const char *buf, int len, int /*done*/) {
+    ((State *)p)->keyData.append(buf, len);
+  };
+
+  auto *parser = WeaselJsonParser_create(1024, &c, &state, 0);
+  REQUIRE(parser != nullptr);
+
+  {
+    std::string chunk = "{\"ab";
+    REQUIRE(WeaselJsonParser_parse(parser, chunk.data(), chunk.size()) ==
+            WeaselJson_AGAIN);
+  }
+
+  // Reset mid-key: the next top-level string must be delivered as a string,
+  // not appended to the aborted key.
+  WeaselJsonParser_reset(parser);
+  state.stringData.clear();
+  state.keyData.clear();
+
+  {
+    std::string chunk = "\"hello\"";
+    REQUIRE(WeaselJsonParser_parse(parser, chunk.data(), chunk.size()) ==
+            WeaselJson_AGAIN);
+  }
+  REQUIRE(WeaselJsonParser_parse(parser, nullptr, 0) == WeaselJson_OK);
+
+  CHECK(state.stringData == "hello");
+  CHECK(state.keyData.empty());
+
+  WeaselJsonParser_destroy(parser);
+}
+
 void doTestUnescapingUtf8(std::string const &escaped,
                           std::string const &expected, int stride, int flags) {
   CAPTURE(escaped);

test_python_bindings.py

@@ -0,0 +1,90 @@
+import json
+
+import weaseljson
+
+
+class Recorder(weaseljson.WeaselJsonCallbacksBase):
+    def __init__(self):
+        self.keys = []
+        self.strings = []
+        self.numbers = []
+        self.events = []
+        self._current = bytearray()
+
+    def _flush(self, target, data, done):
+        self._current.extend(data)
+        if done:
+            target.append(bytes(self._current))
+            self._current = bytearray()
+
+    def on_begin_object(self):
+        self.events.append("begin_object")
+
+    def on_end_object(self):
+        self.events.append("end_object")
+
+    def on_begin_array(self):
+        self.events.append("begin_array")
+
+    def on_end_array(self):
+        self.events.append("end_array")
+
+    def on_key_data(self, data, done):
+        self._flush(self.keys, data, done)
+
+    def on_string_data(self, data, done):
+        self._flush(self.strings, data, done)
+
+    def on_number_data(self, data, done):
+        self._flush(self.numbers, data, done)
+
+    def on_true_literal(self):
+        self.events.append("true")
+
+    def on_false_literal(self):
+        self.events.append("false")
+
+    def on_null_literal(self):
+        self.events.append("null")
+
+
+def parse_all(parser, data):
+    for i in range(len(data)):
+        status = parser.parse(data[i : i + 1])
+        if status != weaseljson.WeaselJsonStatus.AGAIN:
+            return status
+    return parser.parse(b"")
+
+
+def test_object_keys_routed_correctly():
+    recorder = Recorder()
+    with weaseljson.WeaselJsonParser(recorder) as parser:
+        status = parse_all(
+            parser, json.dumps({"hello": "world", "foo": "bar"}).encode()
+        )
+
+    assert status == weaseljson.WeaselJsonStatus.OK, status
+    assert recorder.keys == [b"hello", b"foo"], recorder.keys
+    assert recorder.strings == [b"world", b"bar"], recorder.strings
+
+
+def test_mixed_values():
+    recorder = Recorder()
+    with weaseljson.WeaselJsonParser(recorder) as parser:
+        status = parse_all(
+            parser,
+            json.dumps({"answer": 42, "yes": True, "no": False, "nil": None}).encode(),
+        )
+
+    assert status == weaseljson.WeaselJsonStatus.OK, status
+    assert recorder.keys == [b"answer", b"yes", b"no", b"nil"], recorder.keys
+    assert recorder.numbers == [b"42"], recorder.numbers
+    assert recorder.events.count("true") == 1
+    assert recorder.events.count("false") == 1
+    assert recorder.events.count("null") == 1
+
+
+if __name__ == "__main__":
+    test_object_keys_routed_correctly()
+    test_mixed_values()
+    print("python bindings ok")

weaseljson.py

@@ -15,6 +15,7 @@ class WeaselJsonCallbacks(ctypes.Structure):
         ("on_begin_object", event_callback),
         ("on_end_object", event_callback),
         ("on_string_data", data_callback),
+        ("on_key_data", data_callback),
         ("on_begin_array", event_callback),
         ("on_end_array", event_callback),
         ("on_number_data", data_callback),
@@ -41,6 +42,9 @@ class WeaselJsonCallbacksBase:
     def on_string_data(self, data, done):
         pass
 
+    def on_key_data(self, data, done):
+        pass
+
     def on_begin_array(self):
         pass
 
@@ -151,6 +155,12 @@ def on_string_data(p, buf, len, done):
     self.on_string_data(bytes(ctypes.string_at(buf, len)), bool(done))
 
 
+@ctypes.CFUNCTYPE(None, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_int)
+def on_key_data(p, buf, len, done):
+    self = ctypes.cast(p, ctypes.POINTER(ctypes.py_object)).contents.value
+    self.on_key_data(bytes(ctypes.string_at(buf, len)), bool(done))
+
+
 @ctypes.CFUNCTYPE(None, ctypes.c_void_p)
 def on_begin_array(p):
     self = ctypes.cast(p, ctypes.POINTER(ctypes.py_object)).contents.value
@@ -191,6 +201,7 @@ c_callbacks = WeaselJsonCallbacks(
     on_begin_object,
     on_end_object,
     on_string_data,
+    on_key_data,
     on_begin_array,
     on_end_array,
     on_number_data,
@@ -206,14 +217,15 @@ class MyCallbacks(WeaselJsonCallbacksBase):
         print(data)
 
 
-with WeaselJsonParser(MyCallbacks()) as parser:
-    raw = json.dumps({"hello": "world", "foo": 42}).encode()
-    i = 0
-    stride = 1
-    while True:
-        slice = raw[i : i + stride]
-        s = parser.parse(slice)
-        if s != WeaselJsonStatus.AGAIN:
-            break
-        i += stride
-    print(s)
+if __name__ == "__main__":
+    with WeaselJsonParser(MyCallbacks()) as parser:
+        raw = json.dumps({"hello": "world", "foo": 42}).encode()
+        i = 0
+        stride = 1
+        while True:
+            slice = raw[i : i + stride]
+            s = parser.parse(slice)
+            if s != WeaselJsonStatus.AGAIN:
+                break
+            i += stride
+        print(s)