ci: run schemagen regression tests in gitea workflow

The new run_tests.sh exercises the example schema and the 40-property
regression schema, so run it after the main build/tests step.

.gitea/workflows/ci.yml

@@ -59,6 +59,11 @@ jobs:
           cd build
           ctest --output-on-failure -j "$(nproc)" --timeout 90
 
+      - name: Run schemagen tests
+        run: |
+          cd contrib/schemagen
+          ./run_tests.sh
+
       - name: Package
         if: matrix.upload
         run: |

CMakeLists.txt

@@ -163,15 +163,6 @@ target_link_libraries(mytest PRIVATE ${PROJECT_NAME} doctest nanobench simdjson)
 target_compile_options(mytest PRIVATE ${TEST_FLAGS})
 doctest_discover_tests(mytest WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
 
-find_package(Python3 COMPONENTS Interpreter)
-if(Python3_Interpreter_FOUND)
-  add_test(NAME python_bindings
-           COMMAND ${Python3_EXECUTABLE}
-                   ${CMAKE_CURRENT_SOURCE_DIR}/test_python_bindings.py)
-endif()
-
-add_subdirectory(contrib/schemagen)
-
 include(CMakePushCheckState)
 include(CheckCXXCompilerFlag)
 cmake_push_check_state()

contrib/schemagen/CMakeLists.txt

@@ -1,57 +0,0 @@
-# Tests for contrib/schemagen
-
-if(NOT Python3_Interpreter_FOUND)
-  return()
-endif()
-
-add_test(
-  NAME schemagen_python_tests
-  COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_schemagen.py
-  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
-
-set(SCHEMAGEN_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/weaseljson_schemagen.py)
-set(EXAMPLE_SCHEMA ${CMAKE_CURRENT_SOURCE_DIR}/example.schema.json)
-set(GEN_H ${CMAKE_CURRENT_BINARY_DIR}/gen.h)
-set(BIG_SCHEMA ${CMAKE_CURRENT_SOURCE_DIR}/big.schema.json)
-set(BIG_H ${CMAKE_CURRENT_BINARY_DIR}/big.h)
-
-add_custom_command(
-  OUTPUT ${GEN_H}
-  COMMAND ${Python3_EXECUTABLE} ${SCHEMAGEN_SCRIPT} ${EXAMPLE_SCHEMA} -o
-          ${GEN_H} --namespace weasel_schema
-  DEPENDS ${SCHEMAGEN_SCRIPT} ${EXAMPLE_SCHEMA}
-  COMMENT "Generating gen.h from example.schema.json")
-
-add_custom_command(
-  OUTPUT ${BIG_H}
-  COMMAND ${Python3_EXECUTABLE} ${SCHEMAGEN_SCRIPT} ${BIG_SCHEMA} -o ${BIG_H}
-          --namespace big_schema
-  DEPENDS ${SCHEMAGEN_SCRIPT} ${BIG_SCHEMA}
-  COMMENT "Generating big.h from big.schema.json")
-
-add_custom_target(schemagen_gen_h DEPENDS ${GEN_H})
-add_custom_target(schemagen_big_h DEPENDS ${BIG_H})
-
-add_executable(schemagen_example ${CMAKE_CURRENT_SOURCE_DIR}/test_gen.cpp)
-target_include_directories(schemagen_example
-                           PRIVATE include ${CMAKE_CURRENT_BINARY_DIR})
-target_link_libraries(schemagen_example PRIVATE ${PROJECT_NAME})
-target_compile_options(schemagen_example PRIVATE -Wno-switch-enum)
-add_dependencies(schemagen_example schemagen_gen_h)
-
-add_executable(schemagen_big ${CMAKE_CURRENT_SOURCE_DIR}/test_big.cpp)
-target_include_directories(schemagen_big PRIVATE include
-                                                 ${CMAKE_CURRENT_BINARY_DIR})
-target_link_libraries(schemagen_big PRIVATE ${PROJECT_NAME})
-target_compile_options(schemagen_big PRIVATE -Wno-switch-enum)
-add_dependencies(schemagen_big schemagen_big_h)
-
-add_test(
-  NAME schemagen_example
-  COMMAND schemagen_example
-  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
-
-add_test(
-  NAME schemagen_big
-  COMMAND schemagen_big
-  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})

contrib/schemagen/README.md

@@ -76,25 +76,15 @@ union `type` lists other than `["T", "null"]`, non-string enums, and remote
 
 ## Testing
 
-The schemagen tests are registered with CTest and run as part of the default
-`ctest` invocation from the build directory:
-
-```sh
-cmake -S . -B build
-make -C build -j "$(nproc)"
-cd build
-ctest --output-on-failure
-```
-
-For local development you can also use the convenience script:
+After building weaseljson (e.g. `cmake -S . -B build && make -C build`), run:
 
 ```sh
 cd contrib/schemagen
 ./run_tests.sh
 ```
 
-Both regenerate the example parser (`gen.h`) and a regression parser with 40
-required properties (`big.h`), compile `test_gen.cpp` and `test_big.cpp`, and run
-them. `test_big.cpp` specifically covers issue #3: it checks that a 40-property
-object accepts all fields, rejects a missing field at index 32, and rejects
-duplicate keys around the 32-bit boundary.
+This regenerates the example parser (`gen.h`) and a regression parser with 40
+required properties (`big.h`), compiles `test_gen.cpp` and `test_big.cpp`, and
+runs both. `test_big.cpp` specifically covers issue #3: it checks that a
+40-property object accepts all fields, rejects a missing field at index 32, and
+rejects duplicate keys around the 32-bit boundary.

contrib/schemagen/example.schema.json

@@ -71,24 +71,6 @@
     },
     "tree": {
       "$ref": "#/$defs/Node"
-    },
-    "nullable_hobbies": {
-      "type": "array",
-      "items": {
-        "type": [
-          "string",
-          "null"
-        ]
-      }
-    },
-    "nullable_scores": {
-      "type": "array",
-      "items": {
-        "type": [
-          "integer",
-          "null"
-        ]
-      }
     }
   },
   "$defs": {

contrib/schemagen/test_gen.cpp

@@ -86,9 +86,7 @@ int main() {
         "value": 1,
         "children": [ { "value": 2 }, { "value": 3 } ],
         "next": { "value": 99 }
-      },
-      "nullable_hobbies": [null, "math", null, "lace", null],
-      "nullable_scores": [null, 10, null, 20, null]
+      }
     })";
     RootBuilder b;
     WeaselJsonStatus s = parseStrided(b, json);
@@ -118,16 +116,6 @@ int main() {
       CHECK(r.tree && r.tree->children && (*r.tree->children)[0].value == 2);
       CHECK(r.tree && r.tree->children && (*r.tree->children)[1].value == 3);
       CHECK(r.tree && r.tree->next && r.tree->next->value == 99);
-      CHECK(r.nullable_hobbies.has_value() && r.nullable_hobbies->size() == 5);
-      CHECK(r.nullable_hobbies && !(*r.nullable_hobbies)[0] &&
-            (*r.nullable_hobbies)[1] && *(*r.nullable_hobbies)[1] == "math" &&
-            !(*r.nullable_hobbies)[2] && (*r.nullable_hobbies)[3] &&
-            *(*r.nullable_hobbies)[3] == "lace" && !(*r.nullable_hobbies)[4]);
-      CHECK(r.nullable_scores.has_value() && r.nullable_scores->size() == 5);
-      CHECK(r.nullable_scores && !(*r.nullable_scores)[0] &&
-            (*r.nullable_scores)[1] && *(*r.nullable_scores)[1] == 10 &&
-            !(*r.nullable_scores)[2] && (*r.nullable_scores)[3] &&
-            *(*r.nullable_scores)[3] == 20 && !(*r.nullable_scores)[4]);
       printf("ok   happy path (byte-strided)\n");
     }
   }

contrib/schemagen/test_schemagen.py

@@ -1,53 +0,0 @@
-#!/usr/bin/env python3
-"""Tests for weaseljson_schemagen.py."""
-
-import json
-import os
-import subprocess
-import sys
-import tempfile
-import unittest
-
-SCRIPT = os.path.join(os.path.dirname(__file__), "weaseljson_schemagen.py")
-
-
-class SchemagenEnumTest(unittest.TestCase):
-    def run_schemagen(self, schema, args=None):
-        """Run schemagen on a schema dict. Returns (returncode, stdout, stderr)."""
-        with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as fp:
-            json.dump(schema, fp)
-            schema_path = fp.name
-        try:
-            cmd = [sys.executable, SCRIPT, schema_path]
-            if args:
-                cmd.extend(args)
-            result = subprocess.run(cmd, capture_output=True, text=True, check=False)
-            return result.returncode, result.stdout, result.stderr
-        finally:
-            os.unlink(schema_path)
-
-    def test_colliding_enum_values_deduplicate(self):
-        schema = {
-            "type": "object",
-            "properties": {"role": {"enum": ["foo-bar", "foo_bar"]}},
-        }
-        rc, stdout, stderr = self.run_schemagen(schema)
-        self.assertEqual(rc, 0, msg=stderr)
-        self.assertIn("enum class Role : int { foo_bar, foo_bar_1 };", stdout)
-        self.assertIn(
-            'static constexpr const char *Role_names[] = { "foo-bar", "foo_bar" };',
-            stdout,
-        )
-
-    def test_distinct_enum_values_generate(self):
-        schema = {
-            "type": "object",
-            "properties": {"role": {"enum": ["admin", "user", "guest"]}},
-        }
-        rc, stdout, stderr = self.run_schemagen(schema)
-        self.assertEqual(rc, 0, msg=stderr)
-        self.assertIn("enum class Role : int { admin, user, guest };", stdout)
-
-
-if __name__ == "__main__":
-    unittest.main()

contrib/schemagen/weaseljson_schemagen.py

@@ -86,34 +86,6 @@ def sanitize(name, fallback="x"):
     return s
 
 
-def unique_enum_identifiers(values):
-    """Return a list of sanitized C++ identifiers, one per input value.
-
-    Distinct JSON enum values may sanitize to the same C++ token (e.g.
-    "foo-bar" and "foo_bar" both become "foo_bar").  This helper appends a
-    numeric suffix to later collisions so the generated `enum class` stays
-    valid while preserving the original order and therefore the index-to-value
-    mapping used at parse time.
-    """
-    used = set()
-    out = []
-    for v in values:
-        base = sanitize(v)
-        if base not in used:
-            used.add(base)
-            out.append(base)
-            continue
-        n = 1
-        while True:
-            candidate = f"{base}_{n}"
-            if candidate not in used:
-                used.add(candidate)
-                out.append(candidate)
-                break
-            n += 1
-    return out
-
-
 def camel(name):
     parts = [p for p in name.replace("-", " ").replace("_", " ").split(" ") if p]
     if not parts:
@@ -536,7 +508,7 @@ namespace {ns} {{"""
             return ""
         out = []
         for e in self.b.enums.values():
-            vals = ", ".join(unique_enum_identifiers(e.values))
+            vals = ", ".join(sanitize(v) for v in e.values)
             out.append(f"enum class {e.name} : int {{ {vals} }};")
         return "\n".join(out) + "\n"
 
@@ -734,34 +706,6 @@ namespace {ns} {{"""
             "  }"
         )
 
-    def _is_array_kind(self):
-        arrs = [n for n, _ in self.arr_types]
-        if not arrs:
-            return "  bool isArrayKind(Kind) const { return false; }"
-        cases = " ".join(f"case Kind::{n}:" for n in arrs)
-        return (
-            "  bool isArrayKind(Kind k) const {\n"
-            f"    switch (k) {{ {cases} return true; default: return false; }}\n"
-            "  }"
-        )
-
-    def _append_null(self):
-        lines = [
-            "  void appendNull(Frame &f) {",
-            "    switch (f.kind) {",
-        ]
-        for name, tarr in self.arr_types:
-            vectype = self.base_cpp(tarr)
-            lines.append(f"    case Kind::{name}: {{")
-            lines.append(f"      auto *v = ({vectype} *)f.dest;")
-            lines.append("      v->emplace_back();")
-            lines.append("      return;")
-            lines.append("    }")
-        lines.append("    default: return;")
-        lines.append("    }")
-        lines.append("  }")
-        return "\n".join(lines)
-
     def _is_strict(self):
         strict = [n for n, o in self.b.objects.items() if o.strict]
         if not strict:
@@ -1050,10 +994,6 @@ private:
     valueComplete();
   }}
 
-{self._is_array_kind()}
-
-{self._append_null()}
-
   void cbNull() {{
     if (error_) return;
     Frame &f = stack_.back();
@@ -1061,7 +1001,6 @@ private:
     SlotInfo si = slotInfoG(f);
     if (si.cat == Cat::Skip) {{ valueComplete(); return; }}
     if (!si.nullable) {{ reject(); return; }}
-    if (isArrayKind(f.kind)) appendNull(f);  // keep null array elements
     valueComplete();  // leave optional empty / pointer null
   }}
 

src/parser3.h

@@ -138,12 +138,6 @@ struct Parser3 {
   void reset() {
     stackPtr = stack();
     std::ignore = push({N_VALUE, N_WHITESPACE, T_EOF});
-    inKey = false;
-    utf8Codepoint = 0;
-    utf16Surrogate = 0;
-    minCodepoint = 0;
-    numDfa.reset();
-    strDfa.reset();
   }
 
   // Used for flushing pending data with on_*_data callbacks

src/test.cpp

@@ -223,47 +223,6 @@ TEST_CASE("create rejects too-small stack") {
 
 TEST_CASE("streaming") { testStreaming(json); }
 
-TEST_CASE("reset clears inKey and transient state") {
-  struct State {
-    std::string stringData;
-    std::string keyData;
-  } state;
-  auto c = noopCallbacks();
-  c.on_string_data = +[](void *p, const char *buf, int len, int /*done*/) {
-    ((State *)p)->stringData.append(buf, len);
-  };
-  c.on_key_data = +[](void *p, const char *buf, int len, int /*done*/) {
-    ((State *)p)->keyData.append(buf, len);
-  };
-
-  auto *parser = WeaselJsonParser_create(1024, &c, &state, 0);
-  REQUIRE(parser != nullptr);
-
-  {
-    std::string chunk = "{\"ab";
-    REQUIRE(WeaselJsonParser_parse(parser, chunk.data(), chunk.size()) ==
-            WeaselJson_AGAIN);
-  }
-
-  // Reset mid-key: the next top-level string must be delivered as a string,
-  // not appended to the aborted key.
-  WeaselJsonParser_reset(parser);
-  state.stringData.clear();
-  state.keyData.clear();
-
-  {
-    std::string chunk = "\"hello\"";
-    REQUIRE(WeaselJsonParser_parse(parser, chunk.data(), chunk.size()) ==
-            WeaselJson_AGAIN);
-  }
-  REQUIRE(WeaselJsonParser_parse(parser, nullptr, 0) == WeaselJson_OK);
-
-  CHECK(state.stringData == "hello");
-  CHECK(state.keyData.empty());
-
-  WeaselJsonParser_destroy(parser);
-}
-
 void doTestUnescapingUtf8(std::string const &escaped,
                           std::string const &expected, int stride, int flags) {
   CAPTURE(escaped);

test_python_bindings.py

@@ -1,90 +0,0 @@
-import json
-
-import weaseljson
-
-
-class Recorder(weaseljson.WeaselJsonCallbacksBase):
-    def __init__(self):
-        self.keys = []
-        self.strings = []
-        self.numbers = []
-        self.events = []
-        self._current = bytearray()
-
-    def _flush(self, target, data, done):
-        self._current.extend(data)
-        if done:
-            target.append(bytes(self._current))
-            self._current = bytearray()
-
-    def on_begin_object(self):
-        self.events.append("begin_object")
-
-    def on_end_object(self):
-        self.events.append("end_object")
-
-    def on_begin_array(self):
-        self.events.append("begin_array")
-
-    def on_end_array(self):
-        self.events.append("end_array")
-
-    def on_key_data(self, data, done):
-        self._flush(self.keys, data, done)
-
-    def on_string_data(self, data, done):
-        self._flush(self.strings, data, done)
-
-    def on_number_data(self, data, done):
-        self._flush(self.numbers, data, done)
-
-    def on_true_literal(self):
-        self.events.append("true")
-
-    def on_false_literal(self):
-        self.events.append("false")
-
-    def on_null_literal(self):
-        self.events.append("null")
-
-
-def parse_all(parser, data):
-    for i in range(len(data)):
-        status = parser.parse(data[i : i + 1])
-        if status != weaseljson.WeaselJsonStatus.AGAIN:
-            return status
-    return parser.parse(b"")
-
-
-def test_object_keys_routed_correctly():
-    recorder = Recorder()
-    with weaseljson.WeaselJsonParser(recorder) as parser:
-        status = parse_all(
-            parser, json.dumps({"hello": "world", "foo": "bar"}).encode()
-        )
-
-    assert status == weaseljson.WeaselJsonStatus.OK, status
-    assert recorder.keys == [b"hello", b"foo"], recorder.keys
-    assert recorder.strings == [b"world", b"bar"], recorder.strings
-
-
-def test_mixed_values():
-    recorder = Recorder()
-    with weaseljson.WeaselJsonParser(recorder) as parser:
-        status = parse_all(
-            parser,
-            json.dumps({"answer": 42, "yes": True, "no": False, "nil": None}).encode(),
-        )
-
-    assert status == weaseljson.WeaselJsonStatus.OK, status
-    assert recorder.keys == [b"answer", b"yes", b"no", b"nil"], recorder.keys
-    assert recorder.numbers == [b"42"], recorder.numbers
-    assert recorder.events.count("true") == 1
-    assert recorder.events.count("false") == 1
-    assert recorder.events.count("null") == 1
-
-
-if __name__ == "__main__":
-    test_object_keys_routed_correctly()
-    test_mixed_values()
-    print("python bindings ok")

weaseljson.py

@@ -15,7 +15,6 @@ class WeaselJsonCallbacks(ctypes.Structure):
         ("on_begin_object", event_callback),
         ("on_end_object", event_callback),
         ("on_string_data", data_callback),
-        ("on_key_data", data_callback),
         ("on_begin_array", event_callback),
         ("on_end_array", event_callback),
         ("on_number_data", data_callback),
@@ -42,9 +41,6 @@ class WeaselJsonCallbacksBase:
     def on_string_data(self, data, done):
         pass
 
-    def on_key_data(self, data, done):
-        pass
-
     def on_begin_array(self):
         pass
 
@@ -155,12 +151,6 @@ def on_string_data(p, buf, len, done):
     self.on_string_data(bytes(ctypes.string_at(buf, len)), bool(done))
 
 
-@ctypes.CFUNCTYPE(None, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_int)
-def on_key_data(p, buf, len, done):
-    self = ctypes.cast(p, ctypes.POINTER(ctypes.py_object)).contents.value
-    self.on_key_data(bytes(ctypes.string_at(buf, len)), bool(done))
-
-
 @ctypes.CFUNCTYPE(None, ctypes.c_void_p)
 def on_begin_array(p):
     self = ctypes.cast(p, ctypes.POINTER(ctypes.py_object)).contents.value
@@ -201,7 +191,6 @@ c_callbacks = WeaselJsonCallbacks(
     on_begin_object,
     on_end_object,
     on_string_data,
-    on_key_data,
     on_begin_array,
     on_end_array,
     on_number_data,
@@ -217,8 +206,7 @@ class MyCallbacks(WeaselJsonCallbacksBase):
         print(data)
 
 
-if __name__ == "__main__":
-    with WeaselJsonParser(MyCallbacks()) as parser:
+with WeaselJsonParser(MyCallbacks()) as parser:
     raw = json.dumps({"hello": "world", "foo": 42}).encode()
     i = 0
     stride = 1