weaselbot/weaseljson
1
Fork 0
forked from weaselab/weaseljson
Code Pull Requests Activity

schemagen: parse integer fallback exactly for decimal/exponent forms

Browse Source 
Replace the double-based fallback in generated integer slots with a
string-to-int64 parser that handles decimal points and exponents
without losing precision near the int64 boundaries.

The old path used std::from_chars<double> and compared against
±9223372036854775808.0, which rounds the int64 max and min so that
valid values are rejected and out-of-range negatives are accepted.

The new helper:
- Parses sign, integer part, optional fraction, and optional exponent.
- Strips trailing zeros to cancel fractional places.
- Rejects non-integral values and overflow using exact uint64_t
  arithmetic.

Adds regression tests covering root integer and object-field integer
boundary values, including the cases from issue #19.
This commit is contained in:
weaselbot
2026-06-23 18:36:53 -04:00
parent ceb4bc1041
commit 38079cc278
2 changed files with 285 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
contrib/schemagen/test_schemagen.py
+196
View File
@@ -8,6 +8,7 @@ import shutil
import subprocess
import sys
import tempfile
import textwrap
import unittest
SCRIPT = os.path.join(os.path.dirname(__file__), "weaseljson_schemagen.py")
@@ -242,5 +243,200 @@ class SchemagenAdditionalPropertiesTest(unittest.TestCase):
self.assertEqual(rc, 0, msg=stderr)
class SchemagenIntegerBoundaryTest(unittest.TestCase):
"""Regression tests for issue #19: integer slot parsing near int64 boundaries."""
def setUp(self):
self.repo_root = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
)
self.include_dir = os.path.join(self.repo_root, "include")
self.lib_src = os.path.join(self.repo_root, "src", "lib.cpp")
self.compiler = shutil.which("c++")
def _compile_harness(self, tmpdir, schema, harness):
schema_path = os.path.join(tmpdir, "schema.json")
with open(schema_path, "w") as fp:
json.dump(schema, fp)
header_path = os.path.join(tmpdir, "gen.h")
result = subprocess.run(
[
sys.executable,
SCRIPT,
schema_path,
"-o",
header_path,
"--namespace",
"test_schema",
],
capture_output=True,
text=True,
check=False,
)
self.assertEqual(result.returncode, 0, msg=result.stderr)
lib_obj = os.path.join(tmpdir, "lib.o")
comp_lib = subprocess.run(
[
self.compiler,
"-std=c++20",
"-I",
self.include_dir,
"-I",
os.path.join(self.repo_root, "third_party", "include"),
"-I",
os.path.join(self.repo_root, "third_party", "valgrind"),
"-c",
self.lib_src,
"-o",
lib_obj,
],
capture_output=True,
text=True,
check=False,
)
self.assertEqual(comp_lib.returncode, 0, msg=comp_lib.stderr)
cpp_path = os.path.join(tmpdir, "test.cpp")
with open(cpp_path, "w") as fp:
fp.write(harness)
exe_path = os.path.join(tmpdir, "test")
comp = subprocess.run(
[
self.compiler,
"-std=c++20",
"-I",
self.include_dir,
"-I",
tmpdir,
lib_obj,
cpp_path,
"-o",
exe_path,
],
capture_output=True,
text=True,
check=False,
)
self.assertEqual(comp.returncode, 0, msg=comp.stderr)
run = subprocess.run([exe_path], capture_output=True, text=True, check=False)
self.assertEqual(run.returncode, 0, msg=run.stdout + run.stderr)
def _build_cases_array(self, name, entries):
lines = [
f" struct {name}Case {{ const char *s; WeaselJsonStatus expected; long long v; }};"
]
lines.append(f" {name}Case {name}_cases[] = {{")
for s, exp, v in entries:
val = f"{v}LL" if isinstance(v, int) else v
lines.append(f' {{ R"({s})", {exp}, {val} }},')
lines.append(" };")
return "\n".join(lines)
def test_integer_boundary_root(self):
if not self.compiler:
self.skipTest("C++ compiler not available")
cases = [
("9223372036854775807.0", "WeaselJson_OK", 9223372036854775807),
("9223372036854775807", "WeaselJson_OK", 9223372036854775807),
("9223372036854775806.0", "WeaselJson_OK", 9223372036854775806),
("9223372036854775808", "WeaselJson_REJECT", 0),
("-9223372036854775808", "WeaselJson_OK", "-9223372036854775807LL - 1"),
("-9223372036854775808.0", "WeaselJson_OK", "-9223372036854775807LL - 1"),
("-9223372036854775809", "WeaselJson_REJECT", 0),
("1e3", "WeaselJson_OK", 1000),
("2.0", "WeaselJson_OK", 2),
("0.001", "WeaselJson_REJECT", 0),
("1e-3", "WeaselJson_REJECT", 0),
("1000e-3", "WeaselJson_OK", 1),
("100.0e-2", "WeaselJson_OK", 1),
("123.0", "WeaselJson_OK", 123),
("9e18", "WeaselJson_OK", 9000000000000000000),
("10e18", "WeaselJson_REJECT", 0),
]
cases_src = self._build_cases_array("root", cases)
harness = textwrap.dedent(
f"""
#include "gen.h"
#include <cstdio>
#include <cstring>
{cases_src}
int main() {{
for (const auto \u0026c : root_cases) {{
test_schema::RootBuilder b;
char buf[512];
std::strncpy(buf, c.s, sizeof(buf) - 1);
buf[sizeof(buf) - 1] = '\\0';
WeaselJsonStatus st = b.feed(buf, std::strlen(buf));
st = b.finish();
if (st != c.expected) {{
std::printf("root case %s expected %d got %d\\n", c.s, c.expected, st);
return 1;
}}
if (st == WeaselJson_OK \u0026\u0026 b.take() != c.v) {{
std::printf("root case %s value mismatch\\n", c.s);
return 2;
}}
}}
return 0;
}}
"""
)
with tempfile.TemporaryDirectory() as tmpdir:
self._compile_harness(tmpdir, {"type": "integer"}, harness)
def test_integer_boundary_object_field(self):
if not self.compiler:
self.skipTest("C++ compiler not available")
cases = [
('{"age":9223372036854775807.0}', "WeaselJson_OK", 9223372036854775807),
('{"age":-9223372036854775809}', "WeaselJson_REJECT", 0),
('{"age":1e3}', "WeaselJson_OK", 1000),
('{"age":2.0}', "WeaselJson_OK", 2),
('{"age":0.001}', "WeaselJson_REJECT", 0),
(
'{"age":-9223372036854775808.0}',
"WeaselJson_OK",
"-9223372036854775807LL - 1",
),
]
cases_src = self._build_cases_array("object", cases)
harness = textwrap.dedent(
f"""
#include "gen.h"
#include <cstdio>
#include <cstring>
{cases_src}
int main() {{
for (const auto \u0026c : object_cases) {{
test_schema::RootBuilder b;
char buf[512];
std::strncpy(buf, c.s, sizeof(buf) - 1);
buf[sizeof(buf) - 1] = '\\0';
WeaselJsonStatus st = b.feed(buf, std::strlen(buf));
st = b.finish();
if (st != c.expected) {{
std::printf("object case %s expected %d got %d\\n", c.s, c.expected, st);
return 3;
}}
if (st == WeaselJson_OK \u0026\u0026 b.take().age != c.v) {{
std::printf("object case %s value mismatch\\n", c.s);
return 4;
}}
}}
return 0;
}}
"""
)
schema = {
"type": "object",
"properties": {"age": {"type": "integer"}},
"required": ["age"],
}
with tempfile.TemporaryDirectory() as tmpdir:
self._compile_harness(tmpdir, schema, harness)
if __name__ == "__main__":
unittest.main()
contrib/schemagen/weaseljson_schemagen.py
+89 -10
View File
@@ -929,6 +929,90 @@ private:
void reject() {{ error_ = true; }}
// Parse a JSON number text as int64_t. Accepts optional decimal point and
// exponent only when the mathematical value is an integer that fits in a
// signed 64-bit range.
static bool parseJsonInt64(const char *b, const char *e, int64_t &out) {{
const char *p = b;
bool neg = false;
if (p < e) {{
if (*p == '-') {{ neg = true; ++p; }}
else if (*p == '+') return false;
}}
const char *intStart = p;
while (p < e && *p >= '0' && *p <= '9') ++p;
const char *intEnd = p;
int fracDigits = 0;
const char *fracStart = p;
if (p < e && *p == '.') {{
++p;
fracStart = p;
while (p < e && *p >= '0' && *p <= '9') {{ ++p; ++fracDigits; }}
if (fracStart == p) return false;
}}
int64_t exp = 0;
bool expNeg = false;
if (p < e && (*p == 'e' || *p == 'E')) {{
++p;
if (p < e && (*p == '-' || *p == '+')) {{ expNeg = (*p == '-'); ++p; }}
if (p == e || *p < '0' || *p > '9') return false;
while (p < e && *p >= '0' && *p <= '9') {{
int digit = *p - '0';
if (exp <= (INT64_MAX - digit) / 10) exp = exp * 10 + digit;
else exp = INT64_MAX;
++p;
}}
if (expNeg) exp = -exp;
}}
if (p != e) return false;
if (intStart == intEnd) return false;
std::string digits;
digits.reserve((intEnd - intStart) + fracDigits);
for (const char *q = intStart; q < intEnd; ++q) digits.push_back(*q);
for (int i = 0; i < fracDigits; ++i) digits.push_back(fracStart[i]);
int64_t trim = 0;
while (!digits.empty() && digits.back() == '0') {{
digits.pop_back();
++trim;
}}
int64_t finalExp = exp - fracDigits + trim;
if (finalExp < 0) return false;
while (!digits.empty() && digits.front() == '0') digits.erase(digits.begin());
if (digits.empty()) {{ out = 0; return true; }}
constexpr uint64_t kMaxNeg = 9223372036854775808ULL;
constexpr uint64_t kMaxPos = 9223372036854775807ULL;
const uint64_t limit = neg ? kMaxNeg : kMaxPos;
if (finalExp > 19) return false;
int64_t maxSig = 19 - finalExp;
uint64_t mag = 0;
int64_t sigDigits = 0;
for (char ch : digits) {{
uint64_t d = static_cast<uint64_t>(ch - '0');
if (sigDigits >= maxSig) return false;
if (mag > (limit - d) / 10) return false;
mag = mag * 10 + d;
++sigDigits;
}}
for (int64_t i = 0; i < finalExp; ++i) {{
if (mag > limit / 10) return false;
mag *= 10;
}}
if (mag > limit) return false;
if (neg) {{
if (mag == kMaxNeg) {{
out = INT64_MIN;
}} else {{
out = -static_cast<int64_t>(mag);
}}
}} else {{
out = static_cast<int64_t>(mag);
}}
return true;
}}
// Wrap the generated slotInfo() with the generic key state.
SlotInfo slotInfoG(const Frame &f) {{
if (isObjectKind(f.kind)) {{
@@ -1038,16 +1122,11 @@ private:
*(int64_t *)p = v; // plain integer literal: parsed exactly
}} else {{
// JSON Schema "integer" accepts any number with no fractional part,
// including exponent/decimal forms like 1e3 or 2.0. Parse those as a
// double and require an integral value within int64 range.
double d = 0;
auto rd = std::from_chars(b, e, d);
if (rd.ec != std::errc() || rd.ptr != e || d != std::trunc(d) ||
d < -9223372036854775808.0 || d >= 9223372036854775808.0) {{
reject();
return;
}}
*(int64_t *)p = (int64_t)d;
// including exponent/decimal forms like 1e3 or 2.0. Parse those
// exactly as int64 when the value is integral and in range.
int64_t v2 = 0;
if (!parseJsonInt64(b, e, v2)) {{ reject(); return; }}
*(int64_t *)p = v2;
}}
}} else {{
double v = 0;