schemagen: deduplicate enum constants that collide after sanitization

Distinct JSON enum values can sanitize to the same C++ identifier
(e.g. "foo-bar" and "foo_bar" both become `foo_bar`), producing an
invalid `enum class` with duplicate constants.

Add `unique_enum_identifiers()` which appends a numeric suffix to
later collisions while preserving enum declaration order, so the
index-to-JSON-value mapping used by the generated parser stays intact.

Also add contrib/schemagen/test_schemagen.py and wire the schemagen
Python tests plus the existing example.schema.json/test_gen.cpp example
into ctest via CMakeLists.txt.

Closes #4

CMakeLists.txt

@@ -163,6 +163,45 @@ target_link_libraries(mytest PRIVATE ${PROJECT_NAME} doctest nanobench simdjson)
 target_compile_options(mytest PRIVATE ${TEST_FLAGS})
 doctest_discover_tests(mytest WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
 
+find_package(Python3 COMPONENTS Interpreter)
+if(Python3_Interpreter_FOUND)
+  add_test(NAME python_bindings
+           COMMAND ${Python3_EXECUTABLE}
+                   ${CMAKE_CURRENT_SOURCE_DIR}/test_python_bindings.py)
+endif()
+
+# schemagen tests
+add_test(
+  NAME schemagen_python_tests
+  COMMAND python3 contrib/schemagen/test_schemagen.py
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
+
+set(SCHEMAGEN_DIR ${CMAKE_SOURCE_DIR}/contrib/schemagen)
+set(SCHEMAGEN_SCRIPT ${SCHEMAGEN_DIR}/weaseljson_schemagen.py)
+set(EXAMPLE_SCHEMA ${SCHEMAGEN_DIR}/example.schema.json)
+set(GEN_H ${CMAKE_BINARY_DIR}/gen.h)
+
+add_custom_command(
+  OUTPUT ${GEN_H}
+  COMMAND python3 ${SCHEMAGEN_SCRIPT} ${EXAMPLE_SCHEMA} -o ${GEN_H} --namespace
+          weasel_schema
+  DEPENDS ${SCHEMAGEN_SCRIPT} ${EXAMPLE_SCHEMA}
+  COMMENT "Generating gen.h from example.schema.json")
+
+add_custom_target(schemagen_gen_h DEPENDS ${GEN_H})
+
+add_executable(schemagen_example ${SCHEMAGEN_DIR}/test_gen.cpp)
+target_include_directories(schemagen_example PRIVATE include
+                                                     ${CMAKE_BINARY_DIR})
+target_link_libraries(schemagen_example PRIVATE ${PROJECT_NAME})
+target_compile_options(schemagen_example PRIVATE -Wno-switch-enum)
+add_dependencies(schemagen_example schemagen_gen_h)
+
+add_test(
+  NAME schemagen_example
+  COMMAND schemagen_example
+  WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+
 include(CMakePushCheckState)
 include(CheckCXXCompilerFlag)
 cmake_push_check_state()

contrib/schemagen/test_schemagen.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+"""Tests for weaseljson_schemagen.py."""
+
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+
+SCRIPT = os.path.join(os.path.dirname(__file__), "weaseljson_schemagen.py")
+
+
+class SchemagenEnumTest(unittest.TestCase):
+    def run_schemagen(self, schema, args=None):
+        """Run schemagen on a schema dict. Returns (returncode, stdout, stderr)."""
+        with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as fp:
+            json.dump(schema, fp)
+            schema_path = fp.name
+        try:
+            cmd = [sys.executable, SCRIPT, schema_path]
+            if args:
+                cmd.extend(args)
+            result = subprocess.run(cmd, capture_output=True, text=True, check=False)
+            return result.returncode, result.stdout, result.stderr
+        finally:
+            os.unlink(schema_path)
+
+    def test_colliding_enum_values_deduplicate(self):
+        schema = {
+            "type": "object",
+            "properties": {"role": {"enum": ["foo-bar", "foo_bar"]}},
+        }
+        rc, stdout, stderr = self.run_schemagen(schema)
+        self.assertEqual(rc, 0, msg=stderr)
+        self.assertIn("enum class Role : int { foo_bar, foo_bar_1 };", stdout)
+        self.assertIn(
+            'static constexpr const char *Role_names[] = { "foo-bar", "foo_bar" };',
+            stdout,
+        )
+
+    def test_distinct_enum_values_generate(self):
+        schema = {
+            "type": "object",
+            "properties": {"role": {"enum": ["admin", "user", "guest"]}},
+        }
+        rc, stdout, stderr = self.run_schemagen(schema)
+        self.assertEqual(rc, 0, msg=stderr)
+        self.assertIn("enum class Role : int { admin, user, guest };", stdout)
+
+
+if __name__ == "__main__":
+    unittest.main()

contrib/schemagen/weaseljson_schemagen.py

@@ -86,6 +86,34 @@ def sanitize(name, fallback="x"):
     return s
 
 
+def unique_enum_identifiers(values):
+    """Return a list of sanitized C++ identifiers, one per input value.
+
+    Distinct JSON enum values may sanitize to the same C++ token (e.g.
+    "foo-bar" and "foo_bar" both become "foo_bar").  This helper appends a
+    numeric suffix to later collisions so the generated `enum class` stays
+    valid while preserving the original order and therefore the index-to-value
+    mapping used at parse time.
+    """
+    used = set()
+    out = []
+    for v in values:
+        base = sanitize(v)
+        if base not in used:
+            used.add(base)
+            out.append(base)
+            continue
+        n = 1
+        while True:
+            candidate = f"{base}_{n}"
+            if candidate not in used:
+                used.add(candidate)
+                out.append(candidate)
+                break
+            n += 1
+    return out
+
+
 def camel(name):
     parts = [p for p in name.replace("-", " ").replace("_", " ").split(" ") if p]
     if not parts:
@@ -508,7 +536,7 @@ namespace {ns} {{"""
             return ""
         out = []
         for e in self.b.enums.values():
-            vals = ", ".join(sanitize(v) for v in e.values)
+            vals = ", ".join(unique_enum_identifiers(e.values))
             out.append(f"enum class {e.name} : int {{ {vals} }};")
         return "\n".join(out) + "\n"
 

test_python_bindings.py

@@ -0,0 +1,90 @@
+import json
+
+import weaseljson
+
+
+class Recorder(weaseljson.WeaselJsonCallbacksBase):
+    def __init__(self):
+        self.keys = []
+        self.strings = []
+        self.numbers = []
+        self.events = []
+        self._current = bytearray()
+
+    def _flush(self, target, data, done):
+        self._current.extend(data)
+        if done:
+            target.append(bytes(self._current))
+            self._current = bytearray()
+
+    def on_begin_object(self):
+        self.events.append("begin_object")
+
+    def on_end_object(self):
+        self.events.append("end_object")
+
+    def on_begin_array(self):
+        self.events.append("begin_array")
+
+    def on_end_array(self):
+        self.events.append("end_array")
+
+    def on_key_data(self, data, done):
+        self._flush(self.keys, data, done)
+
+    def on_string_data(self, data, done):
+        self._flush(self.strings, data, done)
+
+    def on_number_data(self, data, done):
+        self._flush(self.numbers, data, done)
+
+    def on_true_literal(self):
+        self.events.append("true")
+
+    def on_false_literal(self):
+        self.events.append("false")
+
+    def on_null_literal(self):
+        self.events.append("null")
+
+
+def parse_all(parser, data):
+    for i in range(len(data)):
+        status = parser.parse(data[i : i + 1])
+        if status != weaseljson.WeaselJsonStatus.AGAIN:
+            return status
+    return parser.parse(b"")
+
+
+def test_object_keys_routed_correctly():
+    recorder = Recorder()
+    with weaseljson.WeaselJsonParser(recorder) as parser:
+        status = parse_all(
+            parser, json.dumps({"hello": "world", "foo": "bar"}).encode()
+        )
+
+    assert status == weaseljson.WeaselJsonStatus.OK, status
+    assert recorder.keys == [b"hello", b"foo"], recorder.keys
+    assert recorder.strings == [b"world", b"bar"], recorder.strings
+
+
+def test_mixed_values():
+    recorder = Recorder()
+    with weaseljson.WeaselJsonParser(recorder) as parser:
+        status = parse_all(
+            parser,
+            json.dumps({"answer": 42, "yes": True, "no": False, "nil": None}).encode(),
+        )
+
+    assert status == weaseljson.WeaselJsonStatus.OK, status
+    assert recorder.keys == [b"answer", b"yes", b"no", b"nil"], recorder.keys
+    assert recorder.numbers == [b"42"], recorder.numbers
+    assert recorder.events.count("true") == 1
+    assert recorder.events.count("false") == 1
+    assert recorder.events.count("null") == 1
+
+
+if __name__ == "__main__":
+    test_object_keys_routed_correctly()
+    test_mixed_values()
+    print("python bindings ok")

weaseljson.py

@@ -15,6 +15,7 @@ class WeaselJsonCallbacks(ctypes.Structure):
         ("on_begin_object", event_callback),
         ("on_end_object", event_callback),
         ("on_string_data", data_callback),
+        ("on_key_data", data_callback),
         ("on_begin_array", event_callback),
         ("on_end_array", event_callback),
         ("on_number_data", data_callback),
@@ -41,6 +42,9 @@ class WeaselJsonCallbacksBase:
     def on_string_data(self, data, done):
         pass
 
+    def on_key_data(self, data, done):
+        pass
+
     def on_begin_array(self):
         pass
 
@@ -151,6 +155,12 @@ def on_string_data(p, buf, len, done):
     self.on_string_data(bytes(ctypes.string_at(buf, len)), bool(done))
 
 
+@ctypes.CFUNCTYPE(None, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_int)
+def on_key_data(p, buf, len, done):
+    self = ctypes.cast(p, ctypes.POINTER(ctypes.py_object)).contents.value
+    self.on_key_data(bytes(ctypes.string_at(buf, len)), bool(done))
+
+
 @ctypes.CFUNCTYPE(None, ctypes.c_void_p)
 def on_begin_array(p):
     self = ctypes.cast(p, ctypes.POINTER(ctypes.py_object)).contents.value
@@ -191,6 +201,7 @@ c_callbacks = WeaselJsonCallbacks(
     on_begin_object,
     on_end_object,
     on_string_data,
+    on_key_data,
     on_begin_array,
     on_end_array,
     on_number_data,
@@ -206,6 +217,7 @@ class MyCallbacks(WeaselJsonCallbacksBase):
         print(data)
 
 
+if __name__ == "__main__":
     with WeaselJsonParser(MyCallbacks()) as parser:
         raw = json.dumps({"hello": "world", "foo": 42}).encode()
         i = 0