schemagen: support objects with more than 32 properties

Replace the 32-bit `uint32_t seen` bitmask with a `std::vector<uint64_t>`
bitset sized to the actual field count.  `requiredMask` is now a vector of
the same word count, and required-field checks use per-word masking so that
optional fields do not cause false rejections.

Adds big.schema.json + test_big.cpp regression tests covering the issue
reproducer (40 required properties, missing/duplicate at index 32), plus
run_tests.sh to exercise both test_gen.cpp and test_big.cpp.

Fixes #3

.gitignore

@@ -1,2 +1,6 @@
 build
 .cache
+contrib/schemagen/gen.h
+contrib/schemagen/big.h
+contrib/schemagen/test_gen
+contrib/schemagen/test_big

contrib/schemagen/README.md

@@ -73,3 +73,18 @@ union `type` lists other than `["T", "null"]`, non-string enums, and remote
   cannot hold partial digits.
 - `test_gen.cpp` generates from `example.schema.json` and exercises the parser
   byte-by-byte (covering chunked strings/numbers), plus the rejection cases.
+
+## Testing
+
+After building weaseljson (e.g. `cmake -S . -B build && make -C build`), run:
+
+```sh
+cd contrib/schemagen
+./run_tests.sh
+```
+
+This regenerates the example parser (`gen.h`) and a regression parser with 40
+required properties (`big.h`), compiles `test_gen.cpp` and `test_big.cpp`, and
+runs both. `test_big.cpp` specifically covers issue #3: it checks that a
+40-property object accepts all fields, rejects a missing field at index 32, and
+rejects duplicate keys around the 32-bit boundary.

contrib/schemagen/big.schema.json

@@ -0,0 +1,168 @@
+{
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "p0": {
+      "type": "string"
+    },
+    "p1": {
+      "type": "string"
+    },
+    "p2": {
+      "type": "string"
+    },
+    "p3": {
+      "type": "string"
+    },
+    "p4": {
+      "type": "string"
+    },
+    "p5": {
+      "type": "string"
+    },
+    "p6": {
+      "type": "string"
+    },
+    "p7": {
+      "type": "string"
+    },
+    "p8": {
+      "type": "string"
+    },
+    "p9": {
+      "type": "string"
+    },
+    "p10": {
+      "type": "string"
+    },
+    "p11": {
+      "type": "string"
+    },
+    "p12": {
+      "type": "string"
+    },
+    "p13": {
+      "type": "string"
+    },
+    "p14": {
+      "type": "string"
+    },
+    "p15": {
+      "type": "string"
+    },
+    "p16": {
+      "type": "string"
+    },
+    "p17": {
+      "type": "string"
+    },
+    "p18": {
+      "type": "string"
+    },
+    "p19": {
+      "type": "string"
+    },
+    "p20": {
+      "type": "string"
+    },
+    "p21": {
+      "type": "string"
+    },
+    "p22": {
+      "type": "string"
+    },
+    "p23": {
+      "type": "string"
+    },
+    "p24": {
+      "type": "string"
+    },
+    "p25": {
+      "type": "string"
+    },
+    "p26": {
+      "type": "string"
+    },
+    "p27": {
+      "type": "string"
+    },
+    "p28": {
+      "type": "string"
+    },
+    "p29": {
+      "type": "string"
+    },
+    "p30": {
+      "type": "string"
+    },
+    "p31": {
+      "type": "string"
+    },
+    "p32": {
+      "type": "string"
+    },
+    "p33": {
+      "type": "string"
+    },
+    "p34": {
+      "type": "string"
+    },
+    "p35": {
+      "type": "string"
+    },
+    "p36": {
+      "type": "string"
+    },
+    "p37": {
+      "type": "string"
+    },
+    "p38": {
+      "type": "string"
+    },
+    "p39": {
+      "type": "string"
+    }
+  },
+  "required": [
+    "p0",
+    "p1",
+    "p2",
+    "p3",
+    "p4",
+    "p5",
+    "p6",
+    "p7",
+    "p8",
+    "p9",
+    "p10",
+    "p11",
+    "p12",
+    "p13",
+    "p14",
+    "p15",
+    "p16",
+    "p17",
+    "p18",
+    "p19",
+    "p20",
+    "p21",
+    "p22",
+    "p23",
+    "p24",
+    "p25",
+    "p26",
+    "p27",
+    "p28",
+    "p29",
+    "p30",
+    "p31",
+    "p32",
+    "p33",
+    "p34",
+    "p35",
+    "p36",
+    "p37",
+    "p38",
+    "p39"
+  ]
+}

contrib/schemagen/run_tests.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+# Build and run the schemagen regression tests.
+# Expects weaseljson to be built at ../../build (the default CMake build dir).
+set -euo pipefail
+
+cd "$(dirname "$0")"
+
+python3 weaseljson_schemagen.py example.schema.json -o gen.h --namespace weasel_schema
+python3 weaseljson_schemagen.py big.schema.json -o big.h --namespace big_schema
+
+g++ -std=c++20 -I../../include -I. test_gen.cpp \
+    -L../../build -lweaseljson \
+    -Wl,-rpath,'$ORIGIN'/../../build \
+    -o test_gen
+
+g++ -std=c++20 -I../../include -I. test_big.cpp \
+    -L../../build -lweaseljson \
+    -Wl,-rpath,'$ORIGIN'/../../build \
+    -o test_big
+
+./test_gen
+./test_big

contrib/schemagen/test_big.cpp

@@ -0,0 +1,140 @@
+// Regression test for issue #3: objects with more than 32 properties.
+#include <cassert>
+#include <cstdio>
+#include <string>
+
+#include "big.h"
+
+using namespace big_schema;
+
+static WeaselJsonStatus parseStrided(RootBuilder &b, std::string in) {
+  for (size_t i = 0; i < in.size(); ++i) {
+    char c = in[i];
+    WeaselJsonStatus s = b.feed(&c, 1);
+    if (s != WeaselJson_AGAIN)
+      return s;
+  }
+  return b.finish();
+}
+
+static int failures = 0;
+#define CHECK(cond)                                                            \
+  do {                                                                         \
+    if (!(cond)) {                                                             \
+      printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond);                   \
+      ++failures;                                                              \
+    }                                                                          \
+  } while (0)
+
+static void expectReject(const std::string &json, const char *what) {
+  RootBuilder b;
+  WeaselJsonStatus s = parseStrided(b, json);
+  if (s == WeaselJson_REJECT) {
+    printf("ok   reject: %s\n", what);
+  } else {
+    printf("FAIL expected reject (%s) got status %d for: %s\n", what, s,
+           json.c_str());
+    ++failures;
+  }
+}
+
+static std::string all40() {
+  std::string json = "{";
+  for (int i = 0; i < 40; ++i) {
+    if (i)
+      json += ",";
+    json += "\"p" + std::to_string(i) + "\":\"v" + std::to_string(i) + "\"";
+  }
+  json += "}";
+  return json;
+}
+
+int main() {
+  // All 40 required fields present -> accepted, values land in right slots.
+  {
+    RootBuilder b;
+    std::string json = all40();
+    WeaselJsonStatus s = parseStrided(b, json);
+    CHECK(s == WeaselJson_OK);
+    if (s == WeaselJson_OK) {
+      Root r = b.take();
+      CHECK(r.p0 == "v0");
+      CHECK(r.p31 == "v31");
+      CHECK(r.p32 == "v32");
+      CHECK(r.p39 == "v39");
+      printf("ok   all 40 required fields accepted\n");
+    }
+  }
+
+  // Missing field p32 (index 32) must be detected, not aliased to bit 0.
+  {
+    std::string json = all40();
+    // Remove the p32 entry: find its substring and erase it.
+    std::string entry = "\"p32\":\"v32\"";
+    size_t pos = json.find(entry);
+    assert(pos != std::string::npos);
+    // Remove the trailing comma before it if present, or the leading comma
+    // after it.
+    if (pos > 0 && json[pos - 1] == ',') {
+      json.erase(pos - 1, entry.size() + 1);
+    } else if (pos + entry.size() < json.size() &&
+               json[pos + entry.size()] == ',') {
+      json.erase(pos, entry.size() + 1);
+    } else {
+      json.erase(pos, entry.size());
+    }
+    expectReject(json, "missing required field p32 (index 32)");
+  }
+
+  // Missing p0 still rejected.
+  {
+    std::string json = all40();
+    std::string entry = "\"p0\":\"v0\"";
+    size_t pos = json.find(entry);
+    assert(pos != std::string::npos);
+    if (json[pos + entry.size()] == ',') {
+      json.erase(pos, entry.size() + 1);
+    } else {
+      json.erase(pos, entry.size());
+    }
+    expectReject(json, "missing required field p0");
+  }
+
+  // Duplicate keys p0 and p32 must both be detected.
+  {
+    std::string json = all40();
+    // Insert p32 again right after the existing p32 entry.
+    std::string entry = "\"p32\":\"v32\"";
+    size_t pos = json.find(entry);
+    assert(pos != std::string::npos);
+    json.insert(pos + entry.size(), ",\"p32\":\"dup\"");
+    expectReject(json, "duplicate key p32 (index 32)");
+  }
+
+  // Duplicate keys p0 and p31 cover boundary bits.
+  {
+    std::string json = all40();
+    std::string entry = "\"p0\":\"v0\"";
+    size_t pos = json.find(entry);
+    assert(pos != std::string::npos);
+    json.insert(pos + entry.size(), ",\"p0\":\"dup\"");
+    expectReject(json, "duplicate key p0");
+  }
+
+  // Duplicate p31/p32 around the 32-bit boundary.
+  {
+    std::string json = all40();
+    std::string entry = "\"p31\":\"v31\"";
+    size_t pos = json.find(entry);
+    assert(pos != std::string::npos);
+    json.insert(pos + entry.size(), ",\"p31\":\"dup\"");
+    expectReject(json, "duplicate key p31");
+  }
+
+  if (failures == 0) {
+    printf("\nALL TESTS PASSED\n");
+    return 0;
+  }
+  printf("\n%d FAILURE(S)\n", failures);
+  return 1;
+}

contrib/schemagen/weaseljson_schemagen.py

@@ -690,19 +690,35 @@ namespace {ns} {{"""
         lines.append("  }")
         return "\n".join(lines)
 
-    def _bitmask(self, obj, pred):
+    def _fieldcount(self):
-        mask = 0
+        lines = ["  int fieldCount(Kind k) const {", "    switch (k) {"]
-        for i, fld in enumerate(obj.fields):
+        for name, obj in self.b.objects.items():
-            if pred(fld):
+            lines.append(f"    case Kind::{name}: return {len(obj.fields)};")
-                mask |= 1 << i
+        lines.append("    default: return 0;")
-        return mask
+        lines.append("    }")
+        lines.append("  }")
+        return "\n".join(lines)
 
     def _reqmask(self):
-        lines = ["  uint32_t requiredMask(Kind k) const {", "    switch (k) {"]
+        lines = [
+            "  const std::vector<uint64_t> &requiredMask(Kind k) const {",
+            "    switch (k) {",
+        ]
         for name, obj in self.b.objects.items():
-            m = self._bitmask(obj, lambda f: f.required)
+            words = (len(obj.fields) + 63) // 64
-            lines.append(f"    case Kind::{name}: return {hex(m)}u;")
+            req = [0] * words
-        lines.append("    default: return 0u;")
+            for i, fld in enumerate(obj.fields):
+                if fld.required:
+                    req[i >> 6] |= 1 << (i & 63)
+            init = ", ".join(f"{hex(w)}u" for w in req) if words else ""
+            lines.append(f"    case Kind::{name}: {{")
+            lines.append(f"      static const std::vector<uint64_t> m = {{ {init} }};")
+            lines.append("      return m;")
+            lines.append("    }")
+        lines.append("    default: {")
+        lines.append("      static const std::vector<uint64_t> empty;")
+        lines.append("      return empty;")
+        lines.append("    }")
         lines.append("    }")
         lines.append("  }")
         return "\n".join(lines)
@@ -783,6 +799,11 @@ namespace {ns} {{"""
         if root_kind is not None and root_cat == event_cat:
             lines.append("    if (stack_.empty()) {")
             lines.append(f"      stack_.push_back(Frame{{{root_kind}, &result_}});")
+            if event_cat == "Obj":
+                lines.append("      {")
+                lines.append(f"        int n = fieldCount({root_kind});")
+                lines.append("        stack_.back().seen.assign((n + 63) / 64, 0);")
+                lines.append("      }")
             lines.append("      return;")
             lines.append("    }")
         else:
@@ -798,6 +819,10 @@ namespace {ns} {{"""
         lines.append(f"    if (si.cat != Cat::{event_cat}) {{ reject(); return; }}")
         lines.append("    void *p = engage(f, true);")
         lines.append("    stack_.push_back(Frame{si.child, p});")
+        lines.append("    if (isObjectKind(si.child)) {")
+        lines.append("      int n = fieldCount(si.child);")
+        lines.append("      stack_.back().seen.assign((n + 63) / 64, 0);")
+        lines.append("    }")
         return "\n".join(lines)
 
     def _builder(self, order):
@@ -850,7 +875,7 @@ private:
     Kind kind;
     void *dest;
     int field = -1;              // object: selected field (-1 want key, -2 skip)
-    uint32_t seen = 0;    // bitmask of populated fields
+    std::vector<uint64_t> seen;  // populated field bitset (object frames)
   }};
   static constexpr int kWantKey = -1;
   static constexpr int kSkip = -2;
@@ -885,7 +910,7 @@ private:
     scratch_.clear();
 {("    if (p.kind == Kind::RootScalar) { stack_.pop_back(); done_ = true; return; }" if root_kind is None else "")}
     if (isObjectKind(p.kind)) {{
-      if (p.field >= 0) p.seen |= (1u << p.field);
+      if (p.field >= 0) p.seen[p.field >> 6] |= (1ull << (p.field & 63));
       p.field = kWantKey;
     }}
   }}
@@ -895,16 +920,19 @@ private:
   }}
   void cbEndObject() {{
     if (error_) return;
-    Frame f = stack_.back();
+    Frame &f = stack_.back();
     if (f.kind == Kind::Skip) {{
       stack_.pop_back();
       if (stack_.empty() || stack_.back().kind != Kind::Skip) valueComplete();
       return;
     }}
-    if (isObjectKind(f.kind) &&
+    if (isObjectKind(f.kind)) {{
-        (f.seen & requiredMask(f.kind)) != requiredMask(f.kind)) {{
+      const auto &req = requiredMask(f.kind);
-      reject();
+      bool missing = false;
-      return;
+      for (size_t i = 0; i < req.size(); ++i) {{
+        if ((f.seen[i] & req[i]) != req[i]) {{ missing = true; break; }}
+      }}
+      if (missing) {{ reject(); return; }}
     }}
     stack_.pop_back();
     valueComplete();
@@ -937,7 +965,9 @@ private:
       f.field = kSkip;
       return;
     }}
-    if (f.seen & (1u << idx)) {{ reject(); return; }}  // duplicate key
+    if (f.seen[idx >> 6] & (1ull << (idx & 63))) {{
+      reject(); return;  // duplicate key
+    }}
     f.field = idx;
   }}
 
@@ -1062,6 +1092,8 @@ private:
 
 {self._matchkey()}
 
+{self._fieldcount()}
+
 {self._reqmask()}
 
 {self._is_object_kind()}