From 681892107f5dd99fd65c25972121d8deae70ed41 Mon Sep 17 00:00:00 2001 From: Weaselbot Date: Tue, 30 Jun 2026 12:26:10 -0400 Subject: [PATCH] schemagen: escape control characters in generated C++ string literals Fixes #35. Add a helper to escape C++ string literals so that JSON control characters (\n, \r, \t, and other bytes below 0x20) are emitted as escape sequences instead of raw bytes. Use it for: - field comments that include the JSON property key - object key comparison literals in matchKey() - enum name arrays Also add regression tests that generate and syntax-check headers for schemas containing newlines and other control characters in property keys and enum values. --- contrib/schemagen/test_schemagen.py | 95 +++++++++++++++++++++++ contrib/schemagen/weaseljson_schemagen.py | 47 +++++++++-- 2 files changed, 136 insertions(+), 6 deletions(-) diff --git a/contrib/schemagen/test_schemagen.py b/contrib/schemagen/test_schemagen.py index 4fdd0ae..96fff26 100644 --- a/contrib/schemagen/test_schemagen.py +++ b/contrib/schemagen/test_schemagen.py @@ -518,5 +518,100 @@ class SchemagenIntegerBoundaryTest(unittest.TestCase): self._compile_harness(tmpdir, {"type": "integer"}, harness) +class SchemagenStringEscapeTest(unittest.TestCase): + """Regression tests for issue #35: control characters in string literals.""" + + def setUp(self): + self.repo_root = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) + self.include_dir = os.path.join(self.repo_root, "include") + self.compiler = shutil.which("c++") + + def generate_and_compile(self, schema): + """Run schemagen on schema and syntax-check the resulting header.""" + with tempfile.TemporaryDirectory() as tmpdir: + schema_path = os.path.join(tmpdir, "schema.json") + with open(schema_path, "w") as fp: + json.dump(schema, fp) + header_path = os.path.join(tmpdir, "gen.h") + cmd = [ + sys.executable, + SCRIPT, + schema_path, + "-o", + header_path, + "--namespace", + "test_schema", + ] + result = subprocess.run(cmd, capture_output=True, text=True, check=False) + self.assertEqual(result.returncode, 0, msg=result.stderr) + + if self.compiler: + cpp_path = os.path.join(tmpdir, "test.cpp") + with open(cpp_path, "w") as fp: + fp.write( + '#include "gen.h"\n' + "int main() {\n" + " test_schema::RootBuilder b;\n" + " test_schema::Root r = b.take();\n" + " (void)r;\n" + "}\n" + ) + comp = subprocess.run( + [ + self.compiler, + "-std=c++20", + "-fsyntax-only", + "-I", + self.include_dir, + "-I", + tmpdir, + cpp_path, + ], + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(comp.returncode, 0, msg=comp.stderr) + + with open(header_path) as fp: + return fp.read() + + def test_newline_in_property_key(self): + """A JSON key containing a newline must become a valid C++ literal.""" + schema = {"type": "object", "properties": {"a\nb": {"type": "string"}}} + out = self.generate_and_compile(schema) + self.assertIn('// "a\\nb"', out) + self.assertIn('if (key == "a\\nb") return 0;', out) + + def test_newline_in_enum_value(self): + """An enum value containing a newline must become a valid C++ literal.""" + schema = {"type": "object", "properties": {"x": {"enum": ["a\nb"]}}} + out = self.generate_and_compile(schema) + self.assertIn('static constexpr const char *X_names[] = { "a\\nb" };', out) + + def test_mixed_control_chars_in_enum_value(self): + """Mixed control characters in an enum value must be escaped.""" + schema = { + "type": "object", + "properties": { + "x": {"enum": ["x\ny\rz\tw\vq\x00\x01"]}, + }, + } + out = self.generate_and_compile(schema) + self.assertIn( + 'static constexpr const char *X_names[] = { "x\\ny\\rz\\tw\\vq\\u0000\\u0001" };', + out, + ) + + def test_backslash_and_quote_still_escaped(self): + """Existing escaping for backslash and double quote must remain correct.""" + schema = {"type": "object", "properties": {'a"b\\c': {"type": "string"}}} + out = self.generate_and_compile(schema) + self.assertIn('// "a\\"b\\\\c"', out) + self.assertIn('if (key == "a\\"b\\\\c") return 0;', out) + + if __name__ == "__main__": unittest.main() diff --git a/contrib/schemagen/weaseljson_schemagen.py b/contrib/schemagen/weaseljson_schemagen.py index c64e514..25bc12c 100644 --- a/contrib/schemagen/weaseljson_schemagen.py +++ b/contrib/schemagen/weaseljson_schemagen.py @@ -18,6 +18,43 @@ import keyword import sys +def _escape_cpp_string(s): + """Return *s* escaped for use inside a C++ double-quoted string literal. + + JSON strings may contain control characters; emitting them verbatim into + generated C++ source breaks tokenization. This helper escapes backslashes + and double quotes, maps common control characters to their short escape + sequences, and uses universal character names (\\u00XX) for any other + character below 0x20. + """ + out = [] + for ch in s: + cp = ord(ch) + if cp == 0x09: + out.append("\\t") + elif cp == 0x0A: + out.append("\\n") + elif cp == 0x0B: + out.append("\\v") + elif cp == 0x0C: + out.append("\\f") + elif cp == 0x0D: + out.append("\\r") + elif cp == 0x08: + out.append("\\b") + elif cp == 0x07: + out.append("\\a") + elif cp < 0x20: + out.append(f"\\u{cp:04X}") + elif ch == "\\": + out.append("\\\\") + elif ch == '"': + out.append('\\"') + else: + out.append(ch) + return "".join(out) + + class GenError(Exception): pass @@ -618,7 +655,8 @@ namespace {ns} {{""" and f.ty.kind in ("int", "dbl", "bool") ): init = " = 0" if f.ty.kind != "bool" else " = false" - out.append(f' {store} {f.cpp}{init}; // "{f.key}"') + esc_key = _escape_cpp_string(f.key) + out.append(f' {store} {f.cpp}{init}; // "{esc_key}"') out.append("};") out.append("") return "\n".join(out) @@ -746,7 +784,7 @@ namespace {ns} {{""" for name, obj in self.b.objects.items(): lines.append(f" case Kind::{name}:") for i, fld in enumerate(obj.fields): - esc = fld.key.replace("\\", "\\\\").replace('"', '\\"') + esc = _escape_cpp_string(fld.key) lines.append(f' if (key == "{esc}") return {i};') lines.append(" return -1;") lines.append(" default: return -1;") @@ -829,10 +867,7 @@ namespace {ns} {{""" def _enum_name_arrays(self): out = [] for e in self.b.enums.values(): - lits = ", ".join( - '"' + v.replace("\\", "\\\\").replace('"', '\\"') + '"' - for v in e.values - ) + lits = ", ".join('"' + _escape_cpp_string(v) + '"' for v in e.values) out.append( f" static constexpr const char *{e.name}_names[] = {{ {lits} }};" )