Merge pull request 'schemagen: escape control characters in generated C++ string literals' (#48) from weaselbot/weaseljson:weaselbot/issue-35 into main

Reviewed-on: weaselab/weaseljson#48

contrib/schemagen/test_schemagen.py

@@ -518,5 +518,100 @@ class SchemagenIntegerBoundaryTest(unittest.TestCase):
             self._compile_harness(tmpdir, {"type": "integer"}, harness)
 
 
+class SchemagenStringEscapeTest(unittest.TestCase):
+    """Regression tests for issue #35: control characters in string literals."""
+
+    def setUp(self):
+        self.repo_root = os.path.dirname(
+            os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+        )
+        self.include_dir = os.path.join(self.repo_root, "include")
+        self.compiler = shutil.which("c++")
+
+    def generate_and_compile(self, schema):
+        """Run schemagen on schema and syntax-check the resulting header."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            schema_path = os.path.join(tmpdir, "schema.json")
+            with open(schema_path, "w") as fp:
+                json.dump(schema, fp)
+            header_path = os.path.join(tmpdir, "gen.h")
+            cmd = [
+                sys.executable,
+                SCRIPT,
+                schema_path,
+                "-o",
+                header_path,
+                "--namespace",
+                "test_schema",
+            ]
+            result = subprocess.run(cmd, capture_output=True, text=True, check=False)
+            self.assertEqual(result.returncode, 0, msg=result.stderr)
+
+            if self.compiler:
+                cpp_path = os.path.join(tmpdir, "test.cpp")
+                with open(cpp_path, "w") as fp:
+                    fp.write(
+                        '#include "gen.h"\n'
+                        "int main() {\n"
+                        "  test_schema::RootBuilder b;\n"
+                        "  test_schema::Root r = b.take();\n"
+                        "  (void)r;\n"
+                        "}\n"
+                    )
+                comp = subprocess.run(
+                    [
+                        self.compiler,
+                        "-std=c++20",
+                        "-fsyntax-only",
+                        "-I",
+                        self.include_dir,
+                        "-I",
+                        tmpdir,
+                        cpp_path,
+                    ],
+                    capture_output=True,
+                    text=True,
+                    check=False,
+                )
+                self.assertEqual(comp.returncode, 0, msg=comp.stderr)
+
+            with open(header_path) as fp:
+                return fp.read()
+
+    def test_newline_in_property_key(self):
+        """A JSON key containing a newline must become a valid C++ literal."""
+        schema = {"type": "object", "properties": {"a\nb": {"type": "string"}}}
+        out = self.generate_and_compile(schema)
+        self.assertIn('// "a\\nb"', out)
+        self.assertIn('if (key == "a\\nb") return 0;', out)
+
+    def test_newline_in_enum_value(self):
+        """An enum value containing a newline must become a valid C++ literal."""
+        schema = {"type": "object", "properties": {"x": {"enum": ["a\nb"]}}}
+        out = self.generate_and_compile(schema)
+        self.assertIn('static constexpr const char *X_names[] = { "a\\nb" };', out)
+
+    def test_mixed_control_chars_in_enum_value(self):
+        """Mixed control characters in an enum value must be escaped."""
+        schema = {
+            "type": "object",
+            "properties": {
+                "x": {"enum": ["x\ny\rz\tw\vq\x00\x01"]},
+            },
+        }
+        out = self.generate_and_compile(schema)
+        self.assertIn(
+            'static constexpr const char *X_names[] = { "x\\ny\\rz\\tw\\vq\\u0000\\u0001" };',
+            out,
+        )
+
+    def test_backslash_and_quote_still_escaped(self):
+        """Existing escaping for backslash and double quote must remain correct."""
+        schema = {"type": "object", "properties": {'a"b\\c': {"type": "string"}}}
+        out = self.generate_and_compile(schema)
+        self.assertIn('// "a\\"b\\\\c"', out)
+        self.assertIn('if (key == "a\\"b\\\\c") return 0;', out)
+
+
 if __name__ == "__main__":
     unittest.main()

contrib/schemagen/weaseljson_schemagen.py

@@ -18,6 +18,43 @@ import keyword
 import sys
 
 
+def _escape_cpp_string(s):
+    """Return *s* escaped for use inside a C++ double-quoted string literal.
+
+    JSON strings may contain control characters; emitting them verbatim into
+    generated C++ source breaks tokenization.  This helper escapes backslashes
+    and double quotes, maps common control characters to their short escape
+    sequences, and uses universal character names (\\u00XX) for any other
+    character below 0x20.
+    """
+    out = []
+    for ch in s:
+        cp = ord(ch)
+        if cp == 0x09:
+            out.append("\\t")
+        elif cp == 0x0A:
+            out.append("\\n")
+        elif cp == 0x0B:
+            out.append("\\v")
+        elif cp == 0x0C:
+            out.append("\\f")
+        elif cp == 0x0D:
+            out.append("\\r")
+        elif cp == 0x08:
+            out.append("\\b")
+        elif cp == 0x07:
+            out.append("\\a")
+        elif cp < 0x20:
+            out.append(f"\\u{cp:04X}")
+        elif ch == "\\":
+            out.append("\\\\")
+        elif ch == '"':
+            out.append('\\"')
+        else:
+            out.append(ch)
+    return "".join(out)
+
+
 class GenError(Exception):
     pass
 
@@ -618,7 +655,8 @@ namespace {ns} {{"""
                     and f.ty.kind in ("int", "dbl", "bool")
                 ):
                     init = " = 0" if f.ty.kind != "bool" else " = false"
-                out.append(f'  {store} {f.cpp}{init};  // "{f.key}"')
+                esc_key = _escape_cpp_string(f.key)
+                out.append(f'  {store} {f.cpp}{init};  // "{esc_key}"')
             out.append("};")
             out.append("")
         return "\n".join(out)
@@ -746,7 +784,7 @@ namespace {ns} {{"""
         for name, obj in self.b.objects.items():
             lines.append(f"    case Kind::{name}:")
             for i, fld in enumerate(obj.fields):
-                esc = fld.key.replace("\\", "\\\\").replace('"', '\\"')
+                esc = _escape_cpp_string(fld.key)
                 lines.append(f'      if (key == "{esc}") return {i};')
             lines.append("      return -1;")
         lines.append("    default: return -1;")
@@ -829,10 +867,7 @@ namespace {ns} {{"""
     def _enum_name_arrays(self):
         out = []
         for e in self.b.enums.values():
-            lits = ", ".join(
-                '"' + v.replace("\\", "\\\\").replace('"', '\\"') + '"'
-                for v in e.values
-            )
+            lits = ", ".join('"' + _escape_cpp_string(v) + '"' for v in e.values)
             out.append(
                 f"  static constexpr const char *{e.name}_names[] = {{ {lits} }};"
             )