Merge pull request 'Reset transient parser state in Parser3::reset' (#10) from weaselbot/weaseljson:weaselbot/issue-2 into main

Reviewed-on: weaselab/weaseljson#10

contrib/schemagen/example.schema.json

@@ -71,6 +71,24 @@
     },
     "tree": {
       "$ref": "#/$defs/Node"
+    },
+    "nullable_hobbies": {
+      "type": "array",
+      "items": {
+        "type": [
+          "string",
+          "null"
+        ]
+      }
+    },
+    "nullable_scores": {
+      "type": "array",
+      "items": {
+        "type": [
+          "integer",
+          "null"
+        ]
+      }
     }
   },
   "$defs": {

contrib/schemagen/test_gen.cpp

@@ -86,7 +86,9 @@ int main() {
         "value": 1,
         "children": [ { "value": 2 }, { "value": 3 } ],
         "next": { "value": 99 }
-      }
+      },
+      "nullable_hobbies": [null, "math", null, "lace", null],
+      "nullable_scores": [null, 10, null, 20, null]
     })";
     RootBuilder b;
     WeaselJsonStatus s = parseStrided(b, json);
@@ -116,6 +118,16 @@ int main() {
       CHECK(r.tree && r.tree->children && (*r.tree->children)[0].value == 2);
       CHECK(r.tree && r.tree->children && (*r.tree->children)[1].value == 3);
       CHECK(r.tree && r.tree->next && r.tree->next->value == 99);
+      CHECK(r.nullable_hobbies.has_value() && r.nullable_hobbies->size() == 5);
+      CHECK(r.nullable_hobbies && !(*r.nullable_hobbies)[0] &&
+            (*r.nullable_hobbies)[1] && *(*r.nullable_hobbies)[1] == "math" &&
+            !(*r.nullable_hobbies)[2] && (*r.nullable_hobbies)[3] &&
+            *(*r.nullable_hobbies)[3] == "lace" && !(*r.nullable_hobbies)[4]);
+      CHECK(r.nullable_scores.has_value() && r.nullable_scores->size() == 5);
+      CHECK(r.nullable_scores && !(*r.nullable_scores)[0] &&
+            (*r.nullable_scores)[1] && *(*r.nullable_scores)[1] == 10 &&
+            !(*r.nullable_scores)[2] && (*r.nullable_scores)[3] &&
+            *(*r.nullable_scores)[3] == 20 && !(*r.nullable_scores)[4]);
       printf("ok   happy path (byte-strided)\n");
     }
   }

contrib/schemagen/weaseljson_schemagen.py

@@ -718,6 +718,34 @@ namespace {ns} {{"""
             "  }"
         )
 
+    def _is_array_kind(self):
+        arrs = [n for n, _ in self.arr_types]
+        if not arrs:
+            return "  bool isArrayKind(Kind) const { return false; }"
+        cases = " ".join(f"case Kind::{n}:" for n in arrs)
+        return (
+            "  bool isArrayKind(Kind k) const {\n"
+            f"    switch (k) {{ {cases} return true; default: return false; }}\n"
+            "  }"
+        )
+
+    def _append_null(self):
+        lines = [
+            "  void appendNull(Frame &f) {",
+            "    switch (f.kind) {",
+        ]
+        for name, tarr in self.arr_types:
+            vectype = self.base_cpp(tarr)
+            lines.append(f"    case Kind::{name}: {{")
+            lines.append(f"      auto *v = ({vectype} *)f.dest;")
+            lines.append("      v->emplace_back();")
+            lines.append("      return;")
+            lines.append("    }")
+        lines.append("    default: return;")
+        lines.append("    }")
+        lines.append("  }")
+        return "\n".join(lines)
+
     def _is_strict(self):
         strict = [n for n, o in self.b.objects.items() if o.strict]
         if not strict:
@@ -992,6 +1020,10 @@ private:
     valueComplete();
   }}
 
+{self._is_array_kind()}
+
+{self._append_null()}
+
   void cbNull() {{
     if (error_) return;
     Frame &f = stack_.back();
@@ -999,6 +1031,7 @@ private:
     SlotInfo si = slotInfoG(f);
     if (si.cat == Cat::Skip) {{ valueComplete(); return; }}
     if (!si.nullable) {{ reject(); return; }}
+    if (isArrayKind(f.kind)) appendNull(f);  // keep null array elements
     valueComplete();  // leave optional empty / pointer null
   }}
 

src/parser3.h

@@ -138,6 +138,12 @@ struct Parser3 {
   void reset() {
     stackPtr = stack();
     std::ignore = push({N_VALUE, N_WHITESPACE, T_EOF});
+    inKey = false;
+    utf8Codepoint = 0;
+    utf16Surrogate = 0;
+    minCodepoint = 0;
+    numDfa.reset();
+    strDfa.reset();
   }
 
   // Used for flushing pending data with on_*_data callbacks

src/test.cpp

@@ -223,6 +223,47 @@ TEST_CASE("create rejects too-small stack") {
 
 TEST_CASE("streaming") { testStreaming(json); }
 
+TEST_CASE("reset clears inKey and transient state") {
+  struct State {
+    std::string stringData;
+    std::string keyData;
+  } state;
+  auto c = noopCallbacks();
+  c.on_string_data = +[](void *p, const char *buf, int len, int /*done*/) {
+    ((State *)p)->stringData.append(buf, len);
+  };
+  c.on_key_data = +[](void *p, const char *buf, int len, int /*done*/) {
+    ((State *)p)->keyData.append(buf, len);
+  };
+
+  auto *parser = WeaselJsonParser_create(1024, &c, &state, 0);
+  REQUIRE(parser != nullptr);
+
+  {
+    std::string chunk = "{\"ab";
+    REQUIRE(WeaselJsonParser_parse(parser, chunk.data(), chunk.size()) ==
+            WeaselJson_AGAIN);
+  }
+
+  // Reset mid-key: the next top-level string must be delivered as a string,
+  // not appended to the aborted key.
+  WeaselJsonParser_reset(parser);
+  state.stringData.clear();
+  state.keyData.clear();
+
+  {
+    std::string chunk = "\"hello\"";
+    REQUIRE(WeaselJsonParser_parse(parser, chunk.data(), chunk.size()) ==
+            WeaselJson_AGAIN);
+  }
+  REQUIRE(WeaselJsonParser_parse(parser, nullptr, 0) == WeaselJson_OK);
+
+  CHECK(state.stringData == "hello");
+  CHECK(state.keyData.empty());
+
+  WeaselJsonParser_destroy(parser);
+}
+
 void doTestUnescapingUtf8(std::string const &escaped,
                           std::string const &expected, int stride, int flags) {
   CAPTURE(escaped);