schemagen: move test configuration into contrib/schemagen/CMakeLists.txt

Addresses review feedback: keep the schemagen-specific CMake rules close to the tool instead of inline in the top-level CMakeLists.txt. The subdirectory file is added from the root and guarded by the same Python3 availability check that was already in use.
schemagen: deduplicate enum constants that collide after sanitization
2026-06-18 15:26:55 -04:00 · 2026-06-18 11:02:47 -04:00 · 2026-06-18 15:01:55 +00:00
12 changed files with 258 additions and 417 deletions
@@ -59,11 +59,6 @@ jobs:
          cd build
          ctest --output-on-failure -j "$(nproc)" --timeout 90
      - name: Run schemagen tests
        run: |
          cd contrib/schemagen
          ./run_tests.sh
      - name: Package
        if: matrix.upload
        run: |
@@ -1,6 +1,2 @@
 build
 .cache
 contrib/schemagen/gen.h
 contrib/schemagen/big.h
 contrib/schemagen/test_gen
 contrib/schemagen/test_big
@@ -163,6 +163,15 @@ target_link_libraries(mytest PRIVATE ${PROJECT_NAME} doctest nanobench simdjson)
 target_compile_options(mytest PRIVATE ${TEST_FLAGS})
 doctest_discover_tests(mytest WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
 find_package(Python3 COMPONENTS Interpreter)
 if(Python3_Interpreter_FOUND)
  add_test(NAME python_bindings
           COMMAND ${Python3_EXECUTABLE}
                   ${CMAKE_CURRENT_SOURCE_DIR}/test_python_bindings.py)
 endif()
 add_subdirectory(contrib/schemagen)
 include(CMakePushCheckState)
 include(CheckCXXCompilerFlag)
 cmake_push_check_state()
@@ -0,0 +1,35 @@
 # Tests for contrib/schemagen
 if(NOT Python3_Interpreter_FOUND)
  return()
 endif()
 add_test(
  NAME schemagen_python_tests
  COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_schemagen.py
  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
 set(SCHEMAGEN_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/weaseljson_schemagen.py)
 set(EXAMPLE_SCHEMA ${CMAKE_CURRENT_SOURCE_DIR}/example.schema.json)
 set(GEN_H ${CMAKE_CURRENT_BINARY_DIR}/gen.h)
 add_custom_command(
  OUTPUT ${GEN_H}
  COMMAND ${Python3_EXECUTABLE} ${SCHEMAGEN_SCRIPT} ${EXAMPLE_SCHEMA} -o
          ${GEN_H} --namespace weasel_schema
  DEPENDS ${SCHEMAGEN_SCRIPT} ${EXAMPLE_SCHEMA}
  COMMENT "Generating gen.h from example.schema.json")
 add_custom_target(schemagen_gen_h DEPENDS ${GEN_H})
 add_executable(schemagen_example ${CMAKE_CURRENT_SOURCE_DIR}/test_gen.cpp)
 target_include_directories(schemagen_example
                           PRIVATE include ${CMAKE_CURRENT_BINARY_DIR})
 target_link_libraries(schemagen_example PRIVATE ${PROJECT_NAME})
 target_compile_options(schemagen_example PRIVATE -Wno-switch-enum)
 add_dependencies(schemagen_example schemagen_gen_h)
 add_test(
  NAME schemagen_example
  COMMAND schemagen_example
  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
@@ -73,18 +73,3 @@ union `type` lists other than `["T", "null"]`, non-string enums, and remote
  cannot hold partial digits.
 - `test_gen.cpp` generates from `example.schema.json` and exercises the parser
  byte-by-byte (covering chunked strings/numbers), plus the rejection cases.
 ## Testing
 After building weaseljson (e.g. `cmake -S . -B build && make -C build`), run:
 ```sh
 cd contrib/schemagen
 ./run_tests.sh
 ```
 This regenerates the example parser (`gen.h`) and a regression parser with 40
 required properties (`big.h`), compiles `test_gen.cpp` and `test_big.cpp`, and
 runs both. `test_big.cpp` specifically covers issue #3: it checks that a
 40-property object accepts all fields, rejects a missing field at index 32, and
 rejects duplicate keys around the 32-bit boundary.
@@ -1,168 +0,0 @@
 {
  "type": "object",
  "additionalProperties": false,
  "properties": {
    "p0": {
      "type": "string"
    },
    "p1": {
      "type": "string"
    },
    "p2": {
      "type": "string"
    },
    "p3": {
      "type": "string"
    },
    "p4": {
      "type": "string"
    },
    "p5": {
      "type": "string"
    },
    "p6": {
      "type": "string"
    },
    "p7": {
      "type": "string"
    },
    "p8": {
      "type": "string"
    },
    "p9": {
      "type": "string"
    },
    "p10": {
      "type": "string"
    },
    "p11": {
      "type": "string"
    },
    "p12": {
      "type": "string"
    },
    "p13": {
      "type": "string"
    },
    "p14": {
      "type": "string"
    },
    "p15": {
      "type": "string"
    },
    "p16": {
      "type": "string"
    },
    "p17": {
      "type": "string"
    },
    "p18": {
      "type": "string"
    },
    "p19": {
      "type": "string"
    },
    "p20": {
      "type": "string"
    },
    "p21": {
      "type": "string"
    },
    "p22": {
      "type": "string"
    },
    "p23": {
      "type": "string"
    },
    "p24": {
      "type": "string"
    },
    "p25": {
      "type": "string"
    },
    "p26": {
      "type": "string"
    },
    "p27": {
      "type": "string"
    },
    "p28": {
      "type": "string"
    },
    "p29": {
      "type": "string"
    },
    "p30": {
      "type": "string"
    },
    "p31": {
      "type": "string"
    },
    "p32": {
      "type": "string"
    },
    "p33": {
      "type": "string"
    },
    "p34": {
      "type": "string"
    },
    "p35": {
      "type": "string"
    },
    "p36": {
      "type": "string"
    },
    "p37": {
      "type": "string"
    },
    "p38": {
      "type": "string"
    },
    "p39": {
      "type": "string"
    }
  },
  "required": [
    "p0",
    "p1",
    "p2",
    "p3",
    "p4",
    "p5",
    "p6",
    "p7",
    "p8",
    "p9",
    "p10",
    "p11",
    "p12",
    "p13",
    "p14",
    "p15",
    "p16",
    "p17",
    "p18",
    "p19",
    "p20",
    "p21",
    "p22",
    "p23",
    "p24",
    "p25",
    "p26",
    "p27",
    "p28",
    "p29",
    "p30",
    "p31",
    "p32",
    "p33",
    "p34",
    "p35",
    "p36",
    "p37",
    "p38",
    "p39"
  ]
 }
@@ -1,22 +0,0 @@
 #!/bin/bash
 # Build and run the schemagen regression tests.
 # Expects weaseljson to be built at ../../build (the default CMake build dir).
 set -euo pipefail
 cd "$(dirname "$0")"
 python3 weaseljson_schemagen.py example.schema.json -o gen.h --namespace weasel_schema
 python3 weaseljson_schemagen.py big.schema.json -o big.h --namespace big_schema
 g++ -std=c++20 -I../../include -I. test_gen.cpp \
    -L../../build -lweaseljson \
    -Wl,-rpath,'$ORIGIN'/../../build \
    -o test_gen
 g++ -std=c++20 -I../../include -I. test_big.cpp \
    -L../../build -lweaseljson \
    -Wl,-rpath,'$ORIGIN'/../../build \
    -o test_big
 ./test_gen
 ./test_big
@@ -1,140 +0,0 @@
 // Regression test for issue #3: objects with more than 32 properties.
 #include <cassert>
 #include <cstdio>
 #include <string>
 #include "big.h"
 using namespace big_schema;
 static WeaselJsonStatus parseStrided(RootBuilder &b, std::string in) {
  for (size_t i = 0; i < in.size(); ++i) {
    char c = in[i];
    WeaselJsonStatus s = b.feed(&c, 1);
    if (s != WeaselJson_AGAIN)
      return s;
  }
  return b.finish();
 }
 static int failures = 0;
 #define CHECK(cond)                                                            \
  do {                                                                         \
    if (!(cond)) {                                                             \
      printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond);                   \
      ++failures;                                                              \
    }                                                                          \
  } while (0)
 static void expectReject(const std::string &json, const char *what) {
  RootBuilder b;
  WeaselJsonStatus s = parseStrided(b, json);
  if (s == WeaselJson_REJECT) {
    printf("ok   reject: %s\n", what);
  } else {
    printf("FAIL expected reject (%s) got status %d for: %s\n", what, s,
           json.c_str());
    ++failures;
  }
 }
 static std::string all40() {
  std::string json = "{";
  for (int i = 0; i < 40; ++i) {
    if (i)
      json += ",";
    json += "\"p" + std::to_string(i) + "\":\"v" + std::to_string(i) + "\"";
  }
  json += "}";
  return json;
 }
 int main() {
  // All 40 required fields present -> accepted, values land in right slots.
  {
    RootBuilder b;
    std::string json = all40();
    WeaselJsonStatus s = parseStrided(b, json);
    CHECK(s == WeaselJson_OK);
    if (s == WeaselJson_OK) {
      Root r = b.take();
      CHECK(r.p0 == "v0");
      CHECK(r.p31 == "v31");
      CHECK(r.p32 == "v32");
      CHECK(r.p39 == "v39");
      printf("ok   all 40 required fields accepted\n");
    }
  }
  // Missing field p32 (index 32) must be detected, not aliased to bit 0.
  {
    std::string json = all40();
    // Remove the p32 entry: find its substring and erase it.
    std::string entry = "\"p32\":\"v32\"";
    size_t pos = json.find(entry);
    assert(pos != std::string::npos);
    // Remove the trailing comma before it if present, or the leading comma
    // after it.
    if (pos > 0 && json[pos - 1] == ',') {
      json.erase(pos - 1, entry.size() + 1);
    } else if (pos + entry.size() < json.size() &&
               json[pos + entry.size()] == ',') {
      json.erase(pos, entry.size() + 1);
    } else {
      json.erase(pos, entry.size());
    }
    expectReject(json, "missing required field p32 (index 32)");
  }
  // Missing p0 still rejected.
  {
    std::string json = all40();
    std::string entry = "\"p0\":\"v0\"";
    size_t pos = json.find(entry);
    assert(pos != std::string::npos);
    if (json[pos + entry.size()] == ',') {
      json.erase(pos, entry.size() + 1);
    } else {
      json.erase(pos, entry.size());
    }
    expectReject(json, "missing required field p0");
  }
  // Duplicate keys p0 and p32 must both be detected.
  {
    std::string json = all40();
    // Insert p32 again right after the existing p32 entry.
    std::string entry = "\"p32\":\"v32\"";
    size_t pos = json.find(entry);
    assert(pos != std::string::npos);
    json.insert(pos + entry.size(), ",\"p32\":\"dup\"");
    expectReject(json, "duplicate key p32 (index 32)");
  }
  // Duplicate keys p0 and p31 cover boundary bits.
  {
    std::string json = all40();
    std::string entry = "\"p0\":\"v0\"";
    size_t pos = json.find(entry);
    assert(pos != std::string::npos);
    json.insert(pos + entry.size(), ",\"p0\":\"dup\"");
    expectReject(json, "duplicate key p0");
  }
  // Duplicate p31/p32 around the 32-bit boundary.
  {
    std::string json = all40();
    std::string entry = "\"p31\":\"v31\"";
    size_t pos = json.find(entry);
    assert(pos != std::string::npos);
    json.insert(pos + entry.size(), ",\"p31\":\"dup\"");
    expectReject(json, "duplicate key p31");
  }
  if (failures == 0) {
    printf("\nALL TESTS PASSED\n");
    return 0;
  }
  printf("\n%d FAILURE(S)\n", failures);
  return 1;
 }
@@ -0,0 +1,53 @@
 #!/usr/bin/env python3
 """Tests for weaseljson_schemagen.py."""
 import json
 import os
 import subprocess
 import sys
 import tempfile
 import unittest
 SCRIPT = os.path.join(os.path.dirname(__file__), "weaseljson_schemagen.py")
 class SchemagenEnumTest(unittest.TestCase):
    def run_schemagen(self, schema, args=None):
        """Run schemagen on a schema dict. Returns (returncode, stdout, stderr)."""
        with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as fp:
            json.dump(schema, fp)
            schema_path = fp.name
        try:
            cmd = [sys.executable, SCRIPT, schema_path]
            if args:
                cmd.extend(args)
            result = subprocess.run(cmd, capture_output=True, text=True, check=False)
            return result.returncode, result.stdout, result.stderr
        finally:
            os.unlink(schema_path)
    def test_colliding_enum_values_deduplicate(self):
        schema = {
            "type": "object",
            "properties": {"role": {"enum": ["foo-bar", "foo_bar"]}},
        }
        rc, stdout, stderr = self.run_schemagen(schema)
        self.assertEqual(rc, 0, msg=stderr)
        self.assertIn("enum class Role : int { foo_bar, foo_bar_1 };", stdout)
        self.assertIn(
            'static constexpr const char *Role_names[] = { "foo-bar", "foo_bar" };',
            stdout,
        )
    def test_distinct_enum_values_generate(self):
        schema = {
            "type": "object",
            "properties": {"role": {"enum": ["admin", "user", "guest"]}},
        }
        rc, stdout, stderr = self.run_schemagen(schema)
        self.assertEqual(rc, 0, msg=stderr)
        self.assertIn("enum class Role : int { admin, user, guest };", stdout)
 if __name__ == "__main__":
    unittest.main()
@@ -86,6 +86,34 @@ def sanitize(name, fallback="x"):
    return s
 def unique_enum_identifiers(values):
    """Return a list of sanitized C++ identifiers, one per input value.
    Distinct JSON enum values may sanitize to the same C++ token (e.g.
    "foo-bar" and "foo_bar" both become "foo_bar").  This helper appends a
    numeric suffix to later collisions so the generated `enum class` stays
    valid while preserving the original order and therefore the index-to-value
    mapping used at parse time.
    """
    used = set()
    out = []
    for v in values:
        base = sanitize(v)
        if base not in used:
            used.add(base)
            out.append(base)
            continue
        n = 1
        while True:
            candidate = f"{base}_{n}"
            if candidate not in used:
                used.add(candidate)
                out.append(candidate)
                break
            n += 1
    return out
 def camel(name):
    parts = [p for p in name.replace("-", " ").replace("_", " ").split(" ") if p]
    if not parts:
@@ -508,7 +536,7 @@ namespace {ns} {{"""
            return ""
        out = []
        for e in self.b.enums.values():
-            vals = ", ".join(sanitize(v) for v in e.values)
+            vals = ", ".join(unique_enum_identifiers(e.values))
            out.append(f"enum class {e.name} : int {{ {vals} }};")
        return "\n".join(out) + "\n"
@@ -662,35 +690,19 @@ namespace {ns} {{"""
        lines.append("  }")
        return "\n".join(lines)
-    def _fieldcount(self):
+    def _bitmask(self, obj, pred):
-        lines = ["  int fieldCount(Kind k) const {", "    switch (k) {"]
+        mask = 0
-        for name, obj in self.b.objects.items():
+        for i, fld in enumerate(obj.fields):
-            lines.append(f"    case Kind::{name}: return {len(obj.fields)};")
+            if pred(fld):
-        lines.append("    default: return 0;")
+                mask |= 1 << i
-        lines.append("    }")
+        return mask
        lines.append("  }")
        return "\n".join(lines)
    def _reqmask(self):
-        lines = [
+        lines = ["  uint32_t requiredMask(Kind k) const {", "    switch (k) {"]
            "  const std::vector<uint64_t> &requiredMask(Kind k) const {",
            "    switch (k) {",
        ]
        for name, obj in self.b.objects.items():
-            words = (len(obj.fields) + 63) // 64
+            m = self._bitmask(obj, lambda f: f.required)
-            req = [0] * words
+            lines.append(f"    case Kind::{name}: return {hex(m)}u;")
-            for i, fld in enumerate(obj.fields):
+        lines.append("    default: return 0u;")
                if fld.required:
                    req[i >> 6] |= 1 << (i & 63)
            init = ", ".join(f"{hex(w)}u" for w in req) if words else ""
            lines.append(f"    case Kind::{name}: {{")
            lines.append(f"      static const std::vector<uint64_t> m = {{ {init} }};")
            lines.append("      return m;")
            lines.append("    }")
        lines.append("    default: {")
        lines.append("      static const std::vector<uint64_t> empty;")
        lines.append("      return empty;")
        lines.append("    }")
        lines.append("    }")
        lines.append("  }")
        return "\n".join(lines)
@@ -743,11 +755,6 @@ namespace {ns} {{"""
        if root_kind is not None and root_cat == event_cat:
            lines.append("    if (stack_.empty()) {")
            lines.append(f"      stack_.push_back(Frame{{{root_kind}, &result_}});")
            if event_cat == "Obj":
                lines.append("      {")
                lines.append(f"        int n = fieldCount({root_kind});")
                lines.append("        stack_.back().seen.assign((n + 63) / 64, 0);")
                lines.append("      }")
            lines.append("      return;")
            lines.append("    }")
        else:
@@ -763,10 +770,6 @@ namespace {ns} {{"""
        lines.append(f"    if (si.cat != Cat::{event_cat}) {{ reject(); return; }}")
        lines.append("    void *p = engage(f, true);")
        lines.append("    stack_.push_back(Frame{si.child, p});")
        lines.append("    if (isObjectKind(si.child)) {")
        lines.append("      int n = fieldCount(si.child);")
        lines.append("      stack_.back().seen.assign((n + 63) / 64, 0);")
        lines.append("    }")
        return "\n".join(lines)
    def _builder(self, order):
@@ -819,7 +822,7 @@ private:
    Kind kind;
    void *dest;
    int field = -1;       // object: selected field (-1 want key, -2 skip)
-    std::vector<uint64_t> seen;  // populated field bitset (object frames)
+    uint32_t seen = 0;    // bitmask of populated fields
  }};
  static constexpr int kWantKey = -1;
  static constexpr int kSkip = -2;
@@ -854,7 +857,7 @@ private:
    scratch_.clear();
 {("    if (p.kind == Kind::RootScalar) { stack_.pop_back(); done_ = true; return; }" if root_kind is None else "")}
    if (isObjectKind(p.kind)) {{
-      if (p.field >= 0) p.seen[p.field >> 6] |= (1ull << (p.field & 63));
+      if (p.field >= 0) p.seen |= (1u << p.field);
      p.field = kWantKey;
    }}
  }}
@@ -864,19 +867,16 @@ private:
  }}
  void cbEndObject() {{
    if (error_) return;
-    Frame &f = stack_.back();
+    Frame f = stack_.back();
    if (f.kind == Kind::Skip) {{
      stack_.pop_back();
      if (stack_.empty() || stack_.back().kind != Kind::Skip) valueComplete();
      return;
    }}
-    if (isObjectKind(f.kind)) {{
+    if (isObjectKind(f.kind) &&
-      const auto &req = requiredMask(f.kind);
+        (f.seen & requiredMask(f.kind)) != requiredMask(f.kind)) {{
-      bool missing = false;
+      reject();
-      for (size_t i = 0; i < req.size(); ++i) {{
+      return;
        if ((f.seen[i] & req[i]) != req[i]) {{ missing = true; break; }}
      }}
      if (missing) {{ reject(); return; }}
    }}
    stack_.pop_back();
    valueComplete();
@@ -909,9 +909,7 @@ private:
      f.field = kSkip;
      return;
    }}
-    if (f.seen[idx >> 6] & (1ull << (idx & 63))) {{
+    if (f.seen & (1u << idx)) {{ reject(); return; }}  // duplicate key
      reject(); return;  // duplicate key
    }}
    f.field = idx;
  }}
@@ -1031,8 +1029,6 @@ private:
 {self._matchkey()}
 {self._fieldcount()}
 {self._reqmask()}
 {self._is_object_kind()}
@@ -0,0 +1,90 @@
 import json
 import weaseljson
 class Recorder(weaseljson.WeaselJsonCallbacksBase):
    def __init__(self):
        self.keys = []
        self.strings = []
        self.numbers = []
        self.events = []
        self._current = bytearray()
    def _flush(self, target, data, done):
        self._current.extend(data)
        if done:
            target.append(bytes(self._current))
            self._current = bytearray()
    def on_begin_object(self):
        self.events.append("begin_object")
    def on_end_object(self):
        self.events.append("end_object")
    def on_begin_array(self):
        self.events.append("begin_array")
    def on_end_array(self):
        self.events.append("end_array")
    def on_key_data(self, data, done):
        self._flush(self.keys, data, done)
    def on_string_data(self, data, done):
        self._flush(self.strings, data, done)
    def on_number_data(self, data, done):
        self._flush(self.numbers, data, done)
    def on_true_literal(self):
        self.events.append("true")
    def on_false_literal(self):
        self.events.append("false")
    def on_null_literal(self):
        self.events.append("null")
 def parse_all(parser, data):
    for i in range(len(data)):
        status = parser.parse(data[i : i + 1])
        if status != weaseljson.WeaselJsonStatus.AGAIN:
            return status
    return parser.parse(b"")
 def test_object_keys_routed_correctly():
    recorder = Recorder()
    with weaseljson.WeaselJsonParser(recorder) as parser:
        status = parse_all(
            parser, json.dumps({"hello": "world", "foo": "bar"}).encode()
        )
    assert status == weaseljson.WeaselJsonStatus.OK, status
    assert recorder.keys == [b"hello", b"foo"], recorder.keys
    assert recorder.strings == [b"world", b"bar"], recorder.strings
 def test_mixed_values():
    recorder = Recorder()
    with weaseljson.WeaselJsonParser(recorder) as parser:
        status = parse_all(
            parser,
            json.dumps({"answer": 42, "yes": True, "no": False, "nil": None}).encode(),
        )
    assert status == weaseljson.WeaselJsonStatus.OK, status
    assert recorder.keys == [b"answer", b"yes", b"no", b"nil"], recorder.keys
    assert recorder.numbers == [b"42"], recorder.numbers
    assert recorder.events.count("true") == 1
    assert recorder.events.count("false") == 1
    assert recorder.events.count("null") == 1
 if __name__ == "__main__":
    test_object_keys_routed_correctly()
    test_mixed_values()
    print("python bindings ok")
@@ -15,6 +15,7 @@ class WeaselJsonCallbacks(ctypes.Structure):
        ("on_begin_object", event_callback),
        ("on_end_object", event_callback),
        ("on_string_data", data_callback),
        ("on_key_data", data_callback),
        ("on_begin_array", event_callback),
        ("on_end_array", event_callback),
        ("on_number_data", data_callback),
@@ -41,6 +42,9 @@ class WeaselJsonCallbacksBase:
    def on_string_data(self, data, done):
        pass
    def on_key_data(self, data, done):
        pass
    def on_begin_array(self):
        pass
@@ -151,6 +155,12 @@ def on_string_data(p, buf, len, done):
    self.on_string_data(bytes(ctypes.string_at(buf, len)), bool(done))
@ctypes.CFUNCTYPE(None, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_int)
 def on_key_data(p, buf, len, done):
    self = ctypes.cast(p, ctypes.POINTER(ctypes.py_object)).contents.value
    self.on_key_data(bytes(ctypes.string_at(buf, len)), bool(done))
@ctypes.CFUNCTYPE(None, ctypes.c_void_p)
 def on_begin_array(p):
    self = ctypes.cast(p, ctypes.POINTER(ctypes.py_object)).contents.value
@@ -191,6 +201,7 @@ c_callbacks = WeaselJsonCallbacks(
    on_begin_object,
    on_end_object,
    on_string_data,
    on_key_data,
    on_begin_array,
    on_end_array,
    on_number_data,
@@ -206,6 +217,7 @@ class MyCallbacks(WeaselJsonCallbacksBase):
        print(data)
 if __name__ == "__main__":
    with WeaselJsonParser(MyCallbacks()) as parser:
        raw = json.dumps({"hello": "world", "foo": 42}).encode()
        i = 0