From ca474e3f99c255546d0363f08f74c1f2911bb4be Mon Sep 17 00:00:00 2001 From: Weaselbot Date: Thu, 18 Jun 2026 10:30:58 -0400 Subject: [PATCH] schemagen: support objects with more than 32 properties Replace the 32-bit `uint32_t seen` bitmask with a `std::vector` bitset sized to the actual field count. `requiredMask` is now a vector of the same word count, and required-field checks use per-word masking so that optional fields do not cause false rejections. Adds big.schema.json + test_big.cpp regression tests covering the issue reproducer (40 required properties, missing/duplicate at index 32), plus run_tests.sh to exercise both test_gen.cpp and test_big.cpp. Fixes #3 --- .gitignore | 4 + contrib/schemagen/README.md | 15 ++ contrib/schemagen/big.schema.json | 168 ++++++++++++++++++++++ contrib/schemagen/run_tests.sh | 22 +++ contrib/schemagen/test_big.cpp | 140 ++++++++++++++++++ contrib/schemagen/weaseljson_schemagen.py | 70 ++++++--- 6 files changed, 400 insertions(+), 19 deletions(-) create mode 100644 contrib/schemagen/big.schema.json create mode 100755 contrib/schemagen/run_tests.sh create mode 100644 contrib/schemagen/test_big.cpp diff --git a/.gitignore b/.gitignore index 9785597..4d337fe 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,6 @@ build .cache +contrib/schemagen/gen.h +contrib/schemagen/big.h +contrib/schemagen/test_gen +contrib/schemagen/test_big diff --git a/contrib/schemagen/README.md b/contrib/schemagen/README.md index 5418333..7c96b6f 100644 --- a/contrib/schemagen/README.md +++ b/contrib/schemagen/README.md @@ -73,3 +73,18 @@ union `type` lists other than `["T", "null"]`, non-string enums, and remote cannot hold partial digits. - `test_gen.cpp` generates from `example.schema.json` and exercises the parser byte-by-byte (covering chunked strings/numbers), plus the rejection cases. + +## Testing + +After building weaseljson (e.g. `cmake -S . -B build && make -C build`), run: + +```sh +cd contrib/schemagen +./run_tests.sh +``` + +This regenerates the example parser (`gen.h`) and a regression parser with 40 +required properties (`big.h`), compiles `test_gen.cpp` and `test_big.cpp`, and +runs both. `test_big.cpp` specifically covers issue #3: it checks that a +40-property object accepts all fields, rejects a missing field at index 32, and +rejects duplicate keys around the 32-bit boundary. diff --git a/contrib/schemagen/big.schema.json b/contrib/schemagen/big.schema.json new file mode 100644 index 0000000..9474fbb --- /dev/null +++ b/contrib/schemagen/big.schema.json @@ -0,0 +1,168 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "p0": { + "type": "string" + }, + "p1": { + "type": "string" + }, + "p2": { + "type": "string" + }, + "p3": { + "type": "string" + }, + "p4": { + "type": "string" + }, + "p5": { + "type": "string" + }, + "p6": { + "type": "string" + }, + "p7": { + "type": "string" + }, + "p8": { + "type": "string" + }, + "p9": { + "type": "string" + }, + "p10": { + "type": "string" + }, + "p11": { + "type": "string" + }, + "p12": { + "type": "string" + }, + "p13": { + "type": "string" + }, + "p14": { + "type": "string" + }, + "p15": { + "type": "string" + }, + "p16": { + "type": "string" + }, + "p17": { + "type": "string" + }, + "p18": { + "type": "string" + }, + "p19": { + "type": "string" + }, + "p20": { + "type": "string" + }, + "p21": { + "type": "string" + }, + "p22": { + "type": "string" + }, + "p23": { + "type": "string" + }, + "p24": { + "type": "string" + }, + "p25": { + "type": "string" + }, + "p26": { + "type": "string" + }, + "p27": { + "type": "string" + }, + "p28": { + "type": "string" + }, + "p29": { + "type": "string" + }, + "p30": { + "type": "string" + }, + "p31": { + "type": "string" + }, + "p32": { + "type": "string" + }, + "p33": { + "type": "string" + }, + "p34": { + "type": "string" + }, + "p35": { + "type": "string" + }, + "p36": { + "type": "string" + }, + "p37": { + "type": "string" + }, + "p38": { + "type": "string" + }, + "p39": { + "type": "string" + } + }, + "required": [ + "p0", + "p1", + "p2", + "p3", + "p4", + "p5", + "p6", + "p7", + "p8", + "p9", + "p10", + "p11", + "p12", + "p13", + "p14", + "p15", + "p16", + "p17", + "p18", + "p19", + "p20", + "p21", + "p22", + "p23", + "p24", + "p25", + "p26", + "p27", + "p28", + "p29", + "p30", + "p31", + "p32", + "p33", + "p34", + "p35", + "p36", + "p37", + "p38", + "p39" + ] +} diff --git a/contrib/schemagen/run_tests.sh b/contrib/schemagen/run_tests.sh new file mode 100755 index 0000000..8577141 --- /dev/null +++ b/contrib/schemagen/run_tests.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Build and run the schemagen regression tests. +# Expects weaseljson to be built at ../../build (the default CMake build dir). +set -euo pipefail + +cd "$(dirname "$0")" + +python3 weaseljson_schemagen.py example.schema.json -o gen.h --namespace weasel_schema +python3 weaseljson_schemagen.py big.schema.json -o big.h --namespace big_schema + +g++ -std=c++20 -I../../include -I. test_gen.cpp \ + -L../../build -lweaseljson \ + -Wl,-rpath,'$ORIGIN'/../../build \ + -o test_gen + +g++ -std=c++20 -I../../include -I. test_big.cpp \ + -L../../build -lweaseljson \ + -Wl,-rpath,'$ORIGIN'/../../build \ + -o test_big + +./test_gen +./test_big diff --git a/contrib/schemagen/test_big.cpp b/contrib/schemagen/test_big.cpp new file mode 100644 index 0000000..0cb258a --- /dev/null +++ b/contrib/schemagen/test_big.cpp @@ -0,0 +1,140 @@ +// Regression test for issue #3: objects with more than 32 properties. +#include +#include +#include + +#include "big.h" + +using namespace big_schema; + +static WeaselJsonStatus parseStrided(RootBuilder &b, std::string in) { + for (size_t i = 0; i < in.size(); ++i) { + char c = in[i]; + WeaselJsonStatus s = b.feed(&c, 1); + if (s != WeaselJson_AGAIN) + return s; + } + return b.finish(); +} + +static int failures = 0; +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + ++failures; \ + } \ + } while (0) + +static void expectReject(const std::string &json, const char *what) { + RootBuilder b; + WeaselJsonStatus s = parseStrided(b, json); + if (s == WeaselJson_REJECT) { + printf("ok reject: %s\n", what); + } else { + printf("FAIL expected reject (%s) got status %d for: %s\n", what, s, + json.c_str()); + ++failures; + } +} + +static std::string all40() { + std::string json = "{"; + for (int i = 0; i < 40; ++i) { + if (i) + json += ","; + json += "\"p" + std::to_string(i) + "\":\"v" + std::to_string(i) + "\""; + } + json += "}"; + return json; +} + +int main() { + // All 40 required fields present -> accepted, values land in right slots. + { + RootBuilder b; + std::string json = all40(); + WeaselJsonStatus s = parseStrided(b, json); + CHECK(s == WeaselJson_OK); + if (s == WeaselJson_OK) { + Root r = b.take(); + CHECK(r.p0 == "v0"); + CHECK(r.p31 == "v31"); + CHECK(r.p32 == "v32"); + CHECK(r.p39 == "v39"); + printf("ok all 40 required fields accepted\n"); + } + } + + // Missing field p32 (index 32) must be detected, not aliased to bit 0. + { + std::string json = all40(); + // Remove the p32 entry: find its substring and erase it. + std::string entry = "\"p32\":\"v32\""; + size_t pos = json.find(entry); + assert(pos != std::string::npos); + // Remove the trailing comma before it if present, or the leading comma + // after it. + if (pos > 0 && json[pos - 1] == ',') { + json.erase(pos - 1, entry.size() + 1); + } else if (pos + entry.size() < json.size() && + json[pos + entry.size()] == ',') { + json.erase(pos, entry.size() + 1); + } else { + json.erase(pos, entry.size()); + } + expectReject(json, "missing required field p32 (index 32)"); + } + + // Missing p0 still rejected. + { + std::string json = all40(); + std::string entry = "\"p0\":\"v0\""; + size_t pos = json.find(entry); + assert(pos != std::string::npos); + if (json[pos + entry.size()] == ',') { + json.erase(pos, entry.size() + 1); + } else { + json.erase(pos, entry.size()); + } + expectReject(json, "missing required field p0"); + } + + // Duplicate keys p0 and p32 must both be detected. + { + std::string json = all40(); + // Insert p32 again right after the existing p32 entry. + std::string entry = "\"p32\":\"v32\""; + size_t pos = json.find(entry); + assert(pos != std::string::npos); + json.insert(pos + entry.size(), ",\"p32\":\"dup\""); + expectReject(json, "duplicate key p32 (index 32)"); + } + + // Duplicate keys p0 and p31 cover boundary bits. + { + std::string json = all40(); + std::string entry = "\"p0\":\"v0\""; + size_t pos = json.find(entry); + assert(pos != std::string::npos); + json.insert(pos + entry.size(), ",\"p0\":\"dup\""); + expectReject(json, "duplicate key p0"); + } + + // Duplicate p31/p32 around the 32-bit boundary. + { + std::string json = all40(); + std::string entry = "\"p31\":\"v31\""; + size_t pos = json.find(entry); + assert(pos != std::string::npos); + json.insert(pos + entry.size(), ",\"p31\":\"dup\""); + expectReject(json, "duplicate key p31"); + } + + if (failures == 0) { + printf("\nALL TESTS PASSED\n"); + return 0; + } + printf("\n%d FAILURE(S)\n", failures); + return 1; +} diff --git a/contrib/schemagen/weaseljson_schemagen.py b/contrib/schemagen/weaseljson_schemagen.py index d0cbcf0..1a7bab9 100644 --- a/contrib/schemagen/weaseljson_schemagen.py +++ b/contrib/schemagen/weaseljson_schemagen.py @@ -690,19 +690,35 @@ namespace {ns} {{""" lines.append(" }") return "\n".join(lines) - def _bitmask(self, obj, pred): - mask = 0 - for i, fld in enumerate(obj.fields): - if pred(fld): - mask |= 1 << i - return mask + def _fieldcount(self): + lines = [" int fieldCount(Kind k) const {", " switch (k) {"] + for name, obj in self.b.objects.items(): + lines.append(f" case Kind::{name}: return {len(obj.fields)};") + lines.append(" default: return 0;") + lines.append(" }") + lines.append(" }") + return "\n".join(lines) def _reqmask(self): - lines = [" uint32_t requiredMask(Kind k) const {", " switch (k) {"] + lines = [ + " const std::vector &requiredMask(Kind k) const {", + " switch (k) {", + ] for name, obj in self.b.objects.items(): - m = self._bitmask(obj, lambda f: f.required) - lines.append(f" case Kind::{name}: return {hex(m)}u;") - lines.append(" default: return 0u;") + words = (len(obj.fields) + 63) // 64 + req = [0] * words + for i, fld in enumerate(obj.fields): + if fld.required: + req[i >> 6] |= 1 << (i & 63) + init = ", ".join(f"{hex(w)}u" for w in req) if words else "" + lines.append(f" case Kind::{name}: {{") + lines.append(f" static const std::vector m = {{ {init} }};") + lines.append(" return m;") + lines.append(" }") + lines.append(" default: {") + lines.append(" static const std::vector empty;") + lines.append(" return empty;") + lines.append(" }") lines.append(" }") lines.append(" }") return "\n".join(lines) @@ -783,6 +799,11 @@ namespace {ns} {{""" if root_kind is not None and root_cat == event_cat: lines.append(" if (stack_.empty()) {") lines.append(f" stack_.push_back(Frame{{{root_kind}, &result_}});") + if event_cat == "Obj": + lines.append(" {") + lines.append(f" int n = fieldCount({root_kind});") + lines.append(" stack_.back().seen.assign((n + 63) / 64, 0);") + lines.append(" }") lines.append(" return;") lines.append(" }") else: @@ -798,6 +819,10 @@ namespace {ns} {{""" lines.append(f" if (si.cat != Cat::{event_cat}) {{ reject(); return; }}") lines.append(" void *p = engage(f, true);") lines.append(" stack_.push_back(Frame{si.child, p});") + lines.append(" if (isObjectKind(si.child)) {") + lines.append(" int n = fieldCount(si.child);") + lines.append(" stack_.back().seen.assign((n + 63) / 64, 0);") + lines.append(" }") return "\n".join(lines) def _builder(self, order): @@ -849,8 +874,8 @@ private: struct Frame {{ Kind kind; void *dest; - int field = -1; // object: selected field (-1 want key, -2 skip) - uint32_t seen = 0; // bitmask of populated fields + int field = -1; // object: selected field (-1 want key, -2 skip) + std::vector seen; // populated field bitset (object frames) }}; static constexpr int kWantKey = -1; static constexpr int kSkip = -2; @@ -885,7 +910,7 @@ private: scratch_.clear(); {(" if (p.kind == Kind::RootScalar) { stack_.pop_back(); done_ = true; return; }" if root_kind is None else "")} if (isObjectKind(p.kind)) {{ - if (p.field >= 0) p.seen |= (1u << p.field); + if (p.field >= 0) p.seen[p.field >> 6] |= (1ull << (p.field & 63)); p.field = kWantKey; }} }} @@ -895,16 +920,19 @@ private: }} void cbEndObject() {{ if (error_) return; - Frame f = stack_.back(); + Frame &f = stack_.back(); if (f.kind == Kind::Skip) {{ stack_.pop_back(); if (stack_.empty() || stack_.back().kind != Kind::Skip) valueComplete(); return; }} - if (isObjectKind(f.kind) && - (f.seen & requiredMask(f.kind)) != requiredMask(f.kind)) {{ - reject(); - return; + if (isObjectKind(f.kind)) {{ + const auto &req = requiredMask(f.kind); + bool missing = false; + for (size_t i = 0; i < req.size(); ++i) {{ + if ((f.seen[i] & req[i]) != req[i]) {{ missing = true; break; }} + }} + if (missing) {{ reject(); return; }} }} stack_.pop_back(); valueComplete(); @@ -937,7 +965,9 @@ private: f.field = kSkip; return; }} - if (f.seen & (1u << idx)) {{ reject(); return; }} // duplicate key + if (f.seen[idx >> 6] & (1ull << (idx & 63))) {{ + reject(); return; // duplicate key + }} f.field = idx; }} @@ -1062,6 +1092,8 @@ private: {self._matchkey()} +{self._fieldcount()} + {self._reqmask()} {self._is_object_kind()}