forked from weaselab/weaseljson
schemagen: parse integer fallback exactly for decimal/exponent forms
Replace the double-based fallback in generated integer slots with a string-to-int64 parser that handles decimal points and exponents without losing precision near the int64 boundaries. The old path used std::from_chars<double> and compared against ±9223372036854775808.0, which rounds the int64 max and min so that valid values are rejected and out-of-range negatives are accepted. The new helper: - Parses sign, integer part, optional fraction, and optional exponent. - Strips trailing zeros to cancel fractional places. - Rejects non-integral values and overflow using exact uint64_t arithmetic. Adds regression tests covering root integer and object-field integer boundary values, including the cases from issue #19.
This commit is contained in:
@@ -8,6 +8,7 @@ import shutil
|
|||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
|
import textwrap
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
SCRIPT = os.path.join(os.path.dirname(__file__), "weaseljson_schemagen.py")
|
SCRIPT = os.path.join(os.path.dirname(__file__), "weaseljson_schemagen.py")
|
||||||
@@ -242,5 +243,200 @@ class SchemagenAdditionalPropertiesTest(unittest.TestCase):
|
|||||||
self.assertEqual(rc, 0, msg=stderr)
|
self.assertEqual(rc, 0, msg=stderr)
|
||||||
|
|
||||||
|
|
||||||
|
class SchemagenIntegerBoundaryTest(unittest.TestCase):
|
||||||
|
"""Regression tests for issue #19: integer slot parsing near int64 boundaries."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.repo_root = os.path.dirname(
|
||||||
|
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||||
|
)
|
||||||
|
self.include_dir = os.path.join(self.repo_root, "include")
|
||||||
|
self.lib_src = os.path.join(self.repo_root, "src", "lib.cpp")
|
||||||
|
self.compiler = shutil.which("c++")
|
||||||
|
|
||||||
|
def _compile_harness(self, tmpdir, schema, harness):
|
||||||
|
schema_path = os.path.join(tmpdir, "schema.json")
|
||||||
|
with open(schema_path, "w") as fp:
|
||||||
|
json.dump(schema, fp)
|
||||||
|
header_path = os.path.join(tmpdir, "gen.h")
|
||||||
|
result = subprocess.run(
|
||||||
|
[
|
||||||
|
sys.executable,
|
||||||
|
SCRIPT,
|
||||||
|
schema_path,
|
||||||
|
"-o",
|
||||||
|
header_path,
|
||||||
|
"--namespace",
|
||||||
|
"test_schema",
|
||||||
|
],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
self.assertEqual(result.returncode, 0, msg=result.stderr)
|
||||||
|
|
||||||
|
lib_obj = os.path.join(tmpdir, "lib.o")
|
||||||
|
comp_lib = subprocess.run(
|
||||||
|
[
|
||||||
|
self.compiler,
|
||||||
|
"-std=c++20",
|
||||||
|
"-I",
|
||||||
|
self.include_dir,
|
||||||
|
"-I",
|
||||||
|
os.path.join(self.repo_root, "third_party", "include"),
|
||||||
|
"-I",
|
||||||
|
os.path.join(self.repo_root, "third_party", "valgrind"),
|
||||||
|
"-c",
|
||||||
|
self.lib_src,
|
||||||
|
"-o",
|
||||||
|
lib_obj,
|
||||||
|
],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
self.assertEqual(comp_lib.returncode, 0, msg=comp_lib.stderr)
|
||||||
|
|
||||||
|
cpp_path = os.path.join(tmpdir, "test.cpp")
|
||||||
|
with open(cpp_path, "w") as fp:
|
||||||
|
fp.write(harness)
|
||||||
|
|
||||||
|
exe_path = os.path.join(tmpdir, "test")
|
||||||
|
comp = subprocess.run(
|
||||||
|
[
|
||||||
|
self.compiler,
|
||||||
|
"-std=c++20",
|
||||||
|
"-I",
|
||||||
|
self.include_dir,
|
||||||
|
"-I",
|
||||||
|
tmpdir,
|
||||||
|
lib_obj,
|
||||||
|
cpp_path,
|
||||||
|
"-o",
|
||||||
|
exe_path,
|
||||||
|
],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
self.assertEqual(comp.returncode, 0, msg=comp.stderr)
|
||||||
|
run = subprocess.run([exe_path], capture_output=True, text=True, check=False)
|
||||||
|
self.assertEqual(run.returncode, 0, msg=run.stdout + run.stderr)
|
||||||
|
|
||||||
|
def _build_cases_array(self, name, entries):
|
||||||
|
lines = [
|
||||||
|
f" struct {name}Case {{ const char *s; WeaselJsonStatus expected; long long v; }};"
|
||||||
|
]
|
||||||
|
lines.append(f" {name}Case {name}_cases[] = {{")
|
||||||
|
for s, exp, v in entries:
|
||||||
|
val = f"{v}LL" if isinstance(v, int) else v
|
||||||
|
lines.append(f' {{ R"({s})", {exp}, {val} }},')
|
||||||
|
lines.append(" };")
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
def test_integer_boundary_root(self):
|
||||||
|
if not self.compiler:
|
||||||
|
self.skipTest("C++ compiler not available")
|
||||||
|
cases = [
|
||||||
|
("9223372036854775807.0", "WeaselJson_OK", 9223372036854775807),
|
||||||
|
("9223372036854775807", "WeaselJson_OK", 9223372036854775807),
|
||||||
|
("9223372036854775806.0", "WeaselJson_OK", 9223372036854775806),
|
||||||
|
("9223372036854775808", "WeaselJson_REJECT", 0),
|
||||||
|
("-9223372036854775808", "WeaselJson_OK", "-9223372036854775807LL - 1"),
|
||||||
|
("-9223372036854775808.0", "WeaselJson_OK", "-9223372036854775807LL - 1"),
|
||||||
|
("-9223372036854775809", "WeaselJson_REJECT", 0),
|
||||||
|
("1e3", "WeaselJson_OK", 1000),
|
||||||
|
("2.0", "WeaselJson_OK", 2),
|
||||||
|
("0.001", "WeaselJson_REJECT", 0),
|
||||||
|
("1e-3", "WeaselJson_REJECT", 0),
|
||||||
|
("1000e-3", "WeaselJson_OK", 1),
|
||||||
|
("100.0e-2", "WeaselJson_OK", 1),
|
||||||
|
("123.0", "WeaselJson_OK", 123),
|
||||||
|
("9e18", "WeaselJson_OK", 9000000000000000000),
|
||||||
|
("10e18", "WeaselJson_REJECT", 0),
|
||||||
|
]
|
||||||
|
cases_src = self._build_cases_array("root", cases)
|
||||||
|
harness = textwrap.dedent(
|
||||||
|
f"""
|
||||||
|
#include "gen.h"
|
||||||
|
#include <cstdio>
|
||||||
|
#include <cstring>
|
||||||
|
{cases_src}
|
||||||
|
int main() {{
|
||||||
|
for (const auto \u0026c : root_cases) {{
|
||||||
|
test_schema::RootBuilder b;
|
||||||
|
char buf[512];
|
||||||
|
std::strncpy(buf, c.s, sizeof(buf) - 1);
|
||||||
|
buf[sizeof(buf) - 1] = '\\0';
|
||||||
|
WeaselJsonStatus st = b.feed(buf, std::strlen(buf));
|
||||||
|
st = b.finish();
|
||||||
|
if (st != c.expected) {{
|
||||||
|
std::printf("root case %s expected %d got %d\\n", c.s, c.expected, st);
|
||||||
|
return 1;
|
||||||
|
}}
|
||||||
|
if (st == WeaselJson_OK \u0026\u0026 b.take() != c.v) {{
|
||||||
|
std::printf("root case %s value mismatch\\n", c.s);
|
||||||
|
return 2;
|
||||||
|
}}
|
||||||
|
}}
|
||||||
|
return 0;
|
||||||
|
}}
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
self._compile_harness(tmpdir, {"type": "integer"}, harness)
|
||||||
|
|
||||||
|
def test_integer_boundary_object_field(self):
|
||||||
|
if not self.compiler:
|
||||||
|
self.skipTest("C++ compiler not available")
|
||||||
|
cases = [
|
||||||
|
('{"age":9223372036854775807.0}', "WeaselJson_OK", 9223372036854775807),
|
||||||
|
('{"age":-9223372036854775809}', "WeaselJson_REJECT", 0),
|
||||||
|
('{"age":1e3}', "WeaselJson_OK", 1000),
|
||||||
|
('{"age":2.0}', "WeaselJson_OK", 2),
|
||||||
|
('{"age":0.001}', "WeaselJson_REJECT", 0),
|
||||||
|
(
|
||||||
|
'{"age":-9223372036854775808.0}',
|
||||||
|
"WeaselJson_OK",
|
||||||
|
"-9223372036854775807LL - 1",
|
||||||
|
),
|
||||||
|
]
|
||||||
|
cases_src = self._build_cases_array("object", cases)
|
||||||
|
harness = textwrap.dedent(
|
||||||
|
f"""
|
||||||
|
#include "gen.h"
|
||||||
|
#include <cstdio>
|
||||||
|
#include <cstring>
|
||||||
|
{cases_src}
|
||||||
|
int main() {{
|
||||||
|
for (const auto \u0026c : object_cases) {{
|
||||||
|
test_schema::RootBuilder b;
|
||||||
|
char buf[512];
|
||||||
|
std::strncpy(buf, c.s, sizeof(buf) - 1);
|
||||||
|
buf[sizeof(buf) - 1] = '\\0';
|
||||||
|
WeaselJsonStatus st = b.feed(buf, std::strlen(buf));
|
||||||
|
st = b.finish();
|
||||||
|
if (st != c.expected) {{
|
||||||
|
std::printf("object case %s expected %d got %d\\n", c.s, c.expected, st);
|
||||||
|
return 3;
|
||||||
|
}}
|
||||||
|
if (st == WeaselJson_OK \u0026\u0026 b.take().age != c.v) {{
|
||||||
|
std::printf("object case %s value mismatch\\n", c.s);
|
||||||
|
return 4;
|
||||||
|
}}
|
||||||
|
}}
|
||||||
|
return 0;
|
||||||
|
}}
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
schema = {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {"age": {"type": "integer"}},
|
||||||
|
"required": ["age"],
|
||||||
|
}
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
self._compile_harness(tmpdir, schema, harness)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -929,6 +929,90 @@ private:
|
|||||||
|
|
||||||
void reject() {{ error_ = true; }}
|
void reject() {{ error_ = true; }}
|
||||||
|
|
||||||
|
// Parse a JSON number text as int64_t. Accepts optional decimal point and
|
||||||
|
// exponent only when the mathematical value is an integer that fits in a
|
||||||
|
// signed 64-bit range.
|
||||||
|
static bool parseJsonInt64(const char *b, const char *e, int64_t &out) {{
|
||||||
|
const char *p = b;
|
||||||
|
bool neg = false;
|
||||||
|
if (p < e) {{
|
||||||
|
if (*p == '-') {{ neg = true; ++p; }}
|
||||||
|
else if (*p == '+') return false;
|
||||||
|
}}
|
||||||
|
const char *intStart = p;
|
||||||
|
while (p < e && *p >= '0' && *p <= '9') ++p;
|
||||||
|
const char *intEnd = p;
|
||||||
|
int fracDigits = 0;
|
||||||
|
const char *fracStart = p;
|
||||||
|
if (p < e && *p == '.') {{
|
||||||
|
++p;
|
||||||
|
fracStart = p;
|
||||||
|
while (p < e && *p >= '0' && *p <= '9') {{ ++p; ++fracDigits; }}
|
||||||
|
if (fracStart == p) return false;
|
||||||
|
}}
|
||||||
|
int64_t exp = 0;
|
||||||
|
bool expNeg = false;
|
||||||
|
if (p < e && (*p == 'e' || *p == 'E')) {{
|
||||||
|
++p;
|
||||||
|
if (p < e && (*p == '-' || *p == '+')) {{ expNeg = (*p == '-'); ++p; }}
|
||||||
|
if (p == e || *p < '0' || *p > '9') return false;
|
||||||
|
while (p < e && *p >= '0' && *p <= '9') {{
|
||||||
|
int digit = *p - '0';
|
||||||
|
if (exp <= (INT64_MAX - digit) / 10) exp = exp * 10 + digit;
|
||||||
|
else exp = INT64_MAX;
|
||||||
|
++p;
|
||||||
|
}}
|
||||||
|
if (expNeg) exp = -exp;
|
||||||
|
}}
|
||||||
|
if (p != e) return false;
|
||||||
|
if (intStart == intEnd) return false;
|
||||||
|
|
||||||
|
std::string digits;
|
||||||
|
digits.reserve((intEnd - intStart) + fracDigits);
|
||||||
|
for (const char *q = intStart; q < intEnd; ++q) digits.push_back(*q);
|
||||||
|
for (int i = 0; i < fracDigits; ++i) digits.push_back(fracStart[i]);
|
||||||
|
int64_t trim = 0;
|
||||||
|
while (!digits.empty() && digits.back() == '0') {{
|
||||||
|
digits.pop_back();
|
||||||
|
++trim;
|
||||||
|
}}
|
||||||
|
int64_t finalExp = exp - fracDigits + trim;
|
||||||
|
if (finalExp < 0) return false;
|
||||||
|
while (!digits.empty() && digits.front() == '0') digits.erase(digits.begin());
|
||||||
|
if (digits.empty()) {{ out = 0; return true; }}
|
||||||
|
|
||||||
|
constexpr uint64_t kMaxNeg = 9223372036854775808ULL;
|
||||||
|
constexpr uint64_t kMaxPos = 9223372036854775807ULL;
|
||||||
|
const uint64_t limit = neg ? kMaxNeg : kMaxPos;
|
||||||
|
|
||||||
|
if (finalExp > 19) return false;
|
||||||
|
int64_t maxSig = 19 - finalExp;
|
||||||
|
uint64_t mag = 0;
|
||||||
|
int64_t sigDigits = 0;
|
||||||
|
for (char ch : digits) {{
|
||||||
|
uint64_t d = static_cast<uint64_t>(ch - '0');
|
||||||
|
if (sigDigits >= maxSig) return false;
|
||||||
|
if (mag > (limit - d) / 10) return false;
|
||||||
|
mag = mag * 10 + d;
|
||||||
|
++sigDigits;
|
||||||
|
}}
|
||||||
|
for (int64_t i = 0; i < finalExp; ++i) {{
|
||||||
|
if (mag > limit / 10) return false;
|
||||||
|
mag *= 10;
|
||||||
|
}}
|
||||||
|
if (mag > limit) return false;
|
||||||
|
if (neg) {{
|
||||||
|
if (mag == kMaxNeg) {{
|
||||||
|
out = INT64_MIN;
|
||||||
|
}} else {{
|
||||||
|
out = -static_cast<int64_t>(mag);
|
||||||
|
}}
|
||||||
|
}} else {{
|
||||||
|
out = static_cast<int64_t>(mag);
|
||||||
|
}}
|
||||||
|
return true;
|
||||||
|
}}
|
||||||
|
|
||||||
// Wrap the generated slotInfo() with the generic key state.
|
// Wrap the generated slotInfo() with the generic key state.
|
||||||
SlotInfo slotInfoG(const Frame &f) {{
|
SlotInfo slotInfoG(const Frame &f) {{
|
||||||
if (isObjectKind(f.kind)) {{
|
if (isObjectKind(f.kind)) {{
|
||||||
@@ -1038,16 +1122,11 @@ private:
|
|||||||
*(int64_t *)p = v; // plain integer literal: parsed exactly
|
*(int64_t *)p = v; // plain integer literal: parsed exactly
|
||||||
}} else {{
|
}} else {{
|
||||||
// JSON Schema "integer" accepts any number with no fractional part,
|
// JSON Schema "integer" accepts any number with no fractional part,
|
||||||
// including exponent/decimal forms like 1e3 or 2.0. Parse those as a
|
// including exponent/decimal forms like 1e3 or 2.0. Parse those
|
||||||
// double and require an integral value within int64 range.
|
// exactly as int64 when the value is integral and in range.
|
||||||
double d = 0;
|
int64_t v2 = 0;
|
||||||
auto rd = std::from_chars(b, e, d);
|
if (!parseJsonInt64(b, e, v2)) {{ reject(); return; }}
|
||||||
if (rd.ec != std::errc() || rd.ptr != e || d != std::trunc(d) ||
|
*(int64_t *)p = v2;
|
||||||
d < -9223372036854775808.0 || d >= 9223372036854775808.0) {{
|
|
||||||
reject();
|
|
||||||
return;
|
|
||||||
}}
|
|
||||||
*(int64_t *)p = (int64_t)d;
|
|
||||||
}}
|
}}
|
||||||
}} else {{
|
}} else {{
|
||||||
double v = 0;
|
double v = 0;
|
||||||
|
|||||||
Reference in New Issue
Block a user