weaselbot/weaseljson
1
Fork 0
forked from weaselab/weaseljson
Code Pull Requests Activity

schemagen: support objects with more than 32 properties

Browse Source 
Replace the 32-bit `uint32_t seen` bitmask with a `std::vector<uint64_t>`
bitset sized to the actual field count.  `requiredMask` is now a vector of
the same word count, and required-field checks use per-word masking so that
optional fields do not cause false rejections.

Adds big.schema.json + test_big.cpp regression tests covering the issue
reproducer (40 required properties, missing/duplicate at index 32), plus
run_tests.sh to exercise both test_gen.cpp and test_big.cpp.

Fixes #3
This commit is contained in:
weaselbot
2026-06-18 16:31:26 -04:00
parent 7c1c18fe6f
commit ca474e3f99
6 changed files with 400 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+4
View File
@@ -1,2 +1,6 @@
build
.cache
contrib/schemagen/gen.h
contrib/schemagen/big.h
contrib/schemagen/test_gen
contrib/schemagen/test_big
contrib/schemagen/README.md
+15
View File
@@ -73,3 +73,18 @@ union `type` lists other than `["T", "null"]`, non-string enums, and remote
cannot hold partial digits.
- `test_gen.cpp` generates from `example.schema.json` and exercises the parser
byte-by-byte (covering chunked strings/numbers), plus the rejection cases.
## Testing
After building weaseljson (e.g. `cmake -S . -B build && make -C build`), run:
```sh
cd contrib/schemagen
./run_tests.sh
```
This regenerates the example parser (`gen.h`) and a regression parser with 40
required properties (`big.h`), compiles `test_gen.cpp` and `test_big.cpp`, and
runs both. `test_big.cpp` specifically covers issue #3: it checks that a
40-property object accepts all fields, rejects a missing field at index 32, and
rejects duplicate keys around the 32-bit boundary.
contrib/schemagen/big.schema.json
+168
View File
@@ -0,0 +1,168 @@
{
"type": "object",
"additionalProperties": false,
"properties": {
"p0": {
"type": "string"
},
"p1": {
"type": "string"
},
"p2": {
"type": "string"
},
"p3": {
"type": "string"
},
"p4": {
"type": "string"
},
"p5": {
"type": "string"
},
"p6": {
"type": "string"
},
"p7": {
"type": "string"
},
"p8": {
"type": "string"
},
"p9": {
"type": "string"
},
"p10": {
"type": "string"
},
"p11": {
"type": "string"
},
"p12": {
"type": "string"
},
"p13": {
"type": "string"
},
"p14": {
"type": "string"
},
"p15": {
"type": "string"
},
"p16": {
"type": "string"
},
"p17": {
"type": "string"
},
"p18": {
"type": "string"
},
"p19": {
"type": "string"
},
"p20": {
"type": "string"
},
"p21": {
"type": "string"
},
"p22": {
"type": "string"
},
"p23": {
"type": "string"
},
"p24": {
"type": "string"
},
"p25": {
"type": "string"
},
"p26": {
"type": "string"
},
"p27": {
"type": "string"
},
"p28": {
"type": "string"
},
"p29": {
"type": "string"
},
"p30": {
"type": "string"
},
"p31": {
"type": "string"
},
"p32": {
"type": "string"
},
"p33": {
"type": "string"
},
"p34": {
"type": "string"
},
"p35": {
"type": "string"
},
"p36": {
"type": "string"
},
"p37": {
"type": "string"
},
"p38": {
"type": "string"
},
"p39": {
"type": "string"
}
},
"required": [
"p0",
"p1",
"p2",
"p3",
"p4",
"p5",
"p6",
"p7",
"p8",
"p9",
"p10",
"p11",
"p12",
"p13",
"p14",
"p15",
"p16",
"p17",
"p18",
"p19",
"p20",
"p21",
"p22",
"p23",
"p24",
"p25",
"p26",
"p27",
"p28",
"p29",
"p30",
"p31",
"p32",
"p33",
"p34",
"p35",
"p36",
"p37",
"p38",
"p39"
]
}
contrib/schemagen/run_tests.sh
Executable
+22
View File
@@ -0,0 +1,22 @@
#!/bin/bash
# Build and run the schemagen regression tests.
# Expects weaseljson to be built at ../../build (the default CMake build dir).
set -euo pipefail
cd "$(dirname "$0")"
python3 weaseljson_schemagen.py example.schema.json -o gen.h --namespace weasel_schema
python3 weaseljson_schemagen.py big.schema.json -o big.h --namespace big_schema
g++ -std=c++20 -I../../include -I. test_gen.cpp \
-L../../build -lweaseljson \
-Wl,-rpath,'$ORIGIN'/../../build \
-o test_gen
g++ -std=c++20 -I../../include -I. test_big.cpp \
-L../../build -lweaseljson \
-Wl,-rpath,'$ORIGIN'/../../build \
-o test_big
./test_gen
./test_big
contrib/schemagen/test_big.cpp
+140
View File
@@ -0,0 +1,140 @@
// Regression test for issue #3: objects with more than 32 properties.
#include <cassert>
#include <cstdio>
#include <string>
#include "big.h"
using namespace big_schema;
static WeaselJsonStatus parseStrided(RootBuilder &b, std::string in) {
for (size_t i = 0; i < in.size(); ++i) {
char c = in[i];
WeaselJsonStatus s = b.feed(&c, 1);
if (s != WeaselJson_AGAIN)
return s;
}
return b.finish();
}
static int failures = 0;
#define CHECK(cond) \
do { \
if (!(cond)) { \
printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
++failures; \
} \
} while (0)
static void expectReject(const std::string &json, const char *what) {
RootBuilder b;
WeaselJsonStatus s = parseStrided(b, json);
if (s == WeaselJson_REJECT) {
printf("ok reject: %s\n", what);
} else {
printf("FAIL expected reject (%s) got status %d for: %s\n", what, s,
json.c_str());
++failures;
}
}
static std::string all40() {
std::string json = "{";
for (int i = 0; i < 40; ++i) {
if (i)
json += ",";
json += "\"p" + std::to_string(i) + "\":\"v" + std::to_string(i) + "\"";
}
json += "}";
return json;
}
int main() {
// All 40 required fields present -> accepted, values land in right slots.
{
RootBuilder b;
std::string json = all40();
WeaselJsonStatus s = parseStrided(b, json);
CHECK(s == WeaselJson_OK);
if (s == WeaselJson_OK) {
Root r = b.take();
CHECK(r.p0 == "v0");
CHECK(r.p31 == "v31");
CHECK(r.p32 == "v32");
CHECK(r.p39 == "v39");
printf("ok all 40 required fields accepted\n");
}
}
// Missing field p32 (index 32) must be detected, not aliased to bit 0.
{
std::string json = all40();
// Remove the p32 entry: find its substring and erase it.
std::string entry = "\"p32\":\"v32\"";
size_t pos = json.find(entry);
assert(pos != std::string::npos);
// Remove the trailing comma before it if present, or the leading comma
// after it.
if (pos > 0 && json[pos - 1] == ',') {
json.erase(pos - 1, entry.size() + 1);
} else if (pos + entry.size() < json.size() &&
json[pos + entry.size()] == ',') {
json.erase(pos, entry.size() + 1);
} else {
json.erase(pos, entry.size());
}
expectReject(json, "missing required field p32 (index 32)");
}
// Missing p0 still rejected.
{
std::string json = all40();
std::string entry = "\"p0\":\"v0\"";
size_t pos = json.find(entry);
assert(pos != std::string::npos);
if (json[pos + entry.size()] == ',') {
json.erase(pos, entry.size() + 1);
} else {
json.erase(pos, entry.size());
}
expectReject(json, "missing required field p0");
}
// Duplicate keys p0 and p32 must both be detected.
{
std::string json = all40();
// Insert p32 again right after the existing p32 entry.
std::string entry = "\"p32\":\"v32\"";
size_t pos = json.find(entry);
assert(pos != std::string::npos);
json.insert(pos + entry.size(), ",\"p32\":\"dup\"");
expectReject(json, "duplicate key p32 (index 32)");
}
// Duplicate keys p0 and p31 cover boundary bits.
{
std::string json = all40();
std::string entry = "\"p0\":\"v0\"";
size_t pos = json.find(entry);
assert(pos != std::string::npos);
json.insert(pos + entry.size(), ",\"p0\":\"dup\"");
expectReject(json, "duplicate key p0");
}
// Duplicate p31/p32 around the 32-bit boundary.
{
std::string json = all40();
std::string entry = "\"p31\":\"v31\"";
size_t pos = json.find(entry);
assert(pos != std::string::npos);
json.insert(pos + entry.size(), ",\"p31\":\"dup\"");
expectReject(json, "duplicate key p31");
}
if (failures == 0) {
printf("\nALL TESTS PASSED\n");
return 0;
}
printf("\n%d FAILURE(S)\n", failures);
return 1;
}
contrib/schemagen/weaseljson_schemagen.py
+51 -19
View File
@@ -690,19 +690,35 @@ namespace {ns} {{"""
lines.append(" }")
return "\n".join(lines)
def _bitmask(self, obj, pred):
mask = 0
for i, fld in enumerate(obj.fields):
if pred(fld):
mask |= 1 << i
return mask
def _fieldcount(self):
lines = [" int fieldCount(Kind k) const {", " switch (k) {"]
for name, obj in self.b.objects.items():
lines.append(f" case Kind::{name}: return {len(obj.fields)};")
lines.append(" default: return 0;")
lines.append(" }")
lines.append(" }")
return "\n".join(lines)
def _reqmask(self):
lines = [" uint32_t requiredMask(Kind k) const {", " switch (k) {"]
lines = [
" const std::vector<uint64_t> &requiredMask(Kind k) const {",
" switch (k) {",
]
for name, obj in self.b.objects.items():
m = self._bitmask(obj, lambda f: f.required)
lines.append(f" case Kind::{name}: return {hex(m)}u;")
lines.append(" default: return 0u;")
words = (len(obj.fields) + 63) // 64
req = [0] * words
for i, fld in enumerate(obj.fields):
if fld.required:
req[i >> 6] |= 1 << (i & 63)
init = ", ".join(f"{hex(w)}u" for w in req) if words else ""
lines.append(f" case Kind::{name}: {{")
lines.append(f" static const std::vector<uint64_t> m = {{ {init} }};")
lines.append(" return m;")
lines.append(" }")
lines.append(" default: {")
lines.append(" static const std::vector<uint64_t> empty;")
lines.append(" return empty;")
lines.append(" }")
lines.append(" }")
lines.append(" }")
return "\n".join(lines)
@@ -783,6 +799,11 @@ namespace {ns} {{"""
if root_kind is not None and root_cat == event_cat:
lines.append(" if (stack_.empty()) {")
lines.append(f" stack_.push_back(Frame{{{root_kind}, &result_}});")
if event_cat == "Obj":
lines.append(" {")
lines.append(f" int n = fieldCount({root_kind});")
lines.append(" stack_.back().seen.assign((n + 63) / 64, 0);")
lines.append(" }")
lines.append(" return;")
lines.append(" }")
else:
@@ -798,6 +819,10 @@ namespace {ns} {{"""
lines.append(f" if (si.cat != Cat::{event_cat}) {{ reject(); return; }}")
lines.append(" void *p = engage(f, true);")
lines.append(" stack_.push_back(Frame{si.child, p});")
lines.append(" if (isObjectKind(si.child)) {")
lines.append(" int n = fieldCount(si.child);")
lines.append(" stack_.back().seen.assign((n + 63) / 64, 0);")
lines.append(" }")
return "\n".join(lines)
def _builder(self, order):
@@ -849,8 +874,8 @@ private:
struct Frame {{
Kind kind;
void *dest;
int field = -1; // object: selected field (-1 want key, -2 skip)
uint32_t seen = 0; // bitmask of populated fields
int field = -1; // object: selected field (-1 want key, -2 skip)
std::vector<uint64_t> seen; // populated field bitset (object frames)
}};
static constexpr int kWantKey = -1;
static constexpr int kSkip = -2;
@@ -885,7 +910,7 @@ private:
scratch_.clear();
{(" if (p.kind == Kind::RootScalar) { stack_.pop_back(); done_ = true; return; }" if root_kind is None else "")}
if (isObjectKind(p.kind)) {{
if (p.field >= 0) p.seen |= (1u << p.field);
if (p.field >= 0) p.seen[p.field >> 6] |= (1ull << (p.field & 63));
p.field = kWantKey;
}}
}}
@@ -895,16 +920,19 @@ private:
}}
void cbEndObject() {{
if (error_) return;
Frame f = stack_.back();
Frame &f = stack_.back();
if (f.kind == Kind::Skip) {{
stack_.pop_back();
if (stack_.empty() || stack_.back().kind != Kind::Skip) valueComplete();
return;
}}
if (isObjectKind(f.kind) &&
(f.seen & requiredMask(f.kind)) != requiredMask(f.kind)) {{
reject();
return;
if (isObjectKind(f.kind)) {{
const auto &req = requiredMask(f.kind);
bool missing = false;
for (size_t i = 0; i < req.size(); ++i) {{
if ((f.seen[i] & req[i]) != req[i]) {{ missing = true; break; }}
}}
if (missing) {{ reject(); return; }}
}}
stack_.pop_back();
valueComplete();
@@ -937,7 +965,9 @@ private:
f.field = kSkip;
return;
}}
if (f.seen & (1u << idx)) {{ reject(); return; }} // duplicate key
if (f.seen[idx >> 6] & (1ull << (idx & 63))) {{
reject(); return; // duplicate key
}}
f.field = idx;
}}
@@ -1062,6 +1092,8 @@ private:
{self._matchkey()}
{self._fieldcount()}
{self._reqmask()}
{self._is_object_kind()}