From 3cedd66d744c87e0c368376255079507b610ccfe Mon Sep 17 00:00:00 2001
From: Andrew Noyes <andrew@weaselab.dev>
Date: Thu, 15 May 2025 20:33:56 -0400
Subject: [PATCH] Add whitespace nonterminal

---
 src/test.cpp | 53 +++++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 40 insertions(+), 13 deletions(-)

diff --git a/src/test.cpp b/src/test.cpp
index 8ec530b..837ef38 100644
--- a/src/test.cpp
+++ b/src/test.cpp
@@ -157,6 +157,7 @@ enum Symbol : int8_t {
   N_OBJECT_VALUE_OR_END,
   N_ARRAY_MAYBE_CONTINUE,
   N_OBJECT_MAYBE_CONTINUE,
+  N_WHITESPACE,
   N_PAST_END, // Must be last nonterminal
 };
 
@@ -178,6 +179,7 @@ static const char *symbolNames[N_PAST_END] = {
     "N_OBJECT_VALUE_OR_END",
     "N_ARRAY_MAYBE_CONTINUE",
     "N_OBJECT_MAYBE_CONTINUE",
+    "N_WHITESPACE",
 };
 
 constexpr static struct Tables {
@@ -437,7 +439,7 @@ private:
 struct Parser2 {
   Parser2(const Callbacks *callbacks, void *data)
       : callbacks(callbacks), data(data) {
-    std::ignore = push({N_VALUE});
+    std::ignore = push({N_WHITESPACE, N_VALUE});
   }
 
   void prime(char *buf, int len) {
@@ -481,7 +483,9 @@ private:
         break;
       }
     }
-    callbacks->on_number_data(data, bufBefore, buf - bufBefore);
+    if (buf != bufBefore) {
+      callbacks->on_number_data(data, bufBefore, buf - bufBefore);
+    }
     if (len() == 0) {
       return S_AGAIN;
     }
@@ -517,7 +521,6 @@ private:
     if (self->empty()) {
       return S_OK;
     }
-    self->maybeSkipWs();
     if (self->len() == 0) {
       return S_AGAIN;
     }
@@ -545,7 +548,7 @@ private:
       ++self->buf;
       self->callbacks->on_begin_object(self->data);
       self->pop();
-      if (Status s = self->push({N_OBJECT_VALUE_OR_END})) {
+      if (Status s = self->push({N_WHITESPACE, N_OBJECT_VALUE_OR_END})) {
         return s;
       }
       break;
@@ -553,7 +556,7 @@ private:
       ++self->buf;
       self->callbacks->on_begin_array(self->data);
       self->pop();
-      if (Status s = self->push({N_ARRAY_VALUE_OR_END})) {
+      if (Status s = self->push({N_WHITESPACE, N_ARRAY_VALUE_OR_END})) {
         return s;
       }
       break;
@@ -604,7 +607,8 @@ private:
       MUSTTAIL return keepGoing(self);
     } else {
       self->pop();
-      if (Status s = self->push({N_VALUE, N_ARRAY_MAYBE_CONTINUE})) {
+      if (Status s =
+              self->push({N_VALUE, N_WHITESPACE, N_ARRAY_MAYBE_CONTINUE})) {
         return s;
       }
       MUSTTAIL return keepGoing(self);
@@ -617,10 +621,12 @@ private:
       self->callbacks->on_end_object(self->data);
       MUSTTAIL return keepGoing(self);
     } else if (*self->buf == '"') {
+      self->callbacks->on_begin_string(self->data);
       ++self->buf;
       self->pop();
-      if (Status s = self->push(
-              {N_STRING, T_COLON, N_VALUE, N_OBJECT_MAYBE_CONTINUE})) {
+      if (Status s =
+              self->push({N_STRING, N_WHITESPACE, T_COLON, N_WHITESPACE,
+                          N_VALUE, N_WHITESPACE, N_OBJECT_MAYBE_CONTINUE})) {
         return s;
       }
       MUSTTAIL return keepGoing(self);
@@ -631,7 +637,8 @@ private:
     if (*self->buf == ',') {
       ++self->buf;
       self->pop();
-      if (Status s = self->push({N_VALUE, N_ARRAY_MAYBE_CONTINUE})) {
+      if (Status s = self->push(
+              {N_WHITESPACE, N_VALUE, N_WHITESPACE, N_ARRAY_MAYBE_CONTINUE})) {
         return s;
       }
       MUSTTAIL return keepGoing(self);
@@ -647,8 +654,9 @@ private:
     if (*self->buf == ',') {
       ++self->buf;
       self->pop();
-      if (Status s = self->push({T_DUBQUOTE, N_STRING, T_COLON, N_VALUE,
-                                 N_OBJECT_MAYBE_CONTINUE})) {
+      if (Status s = self->push({N_WHITESPACE, T_DUBQUOTE, N_STRING,
+                                 N_WHITESPACE, T_COLON, N_WHITESPACE, N_VALUE,
+                                 N_WHITESPACE, N_OBJECT_MAYBE_CONTINUE})) {
         return s;
       }
       MUSTTAIL return keepGoing(self);
@@ -691,8 +699,24 @@ private:
     }
     return S_REJECT;
   }
+  static Status dubquote(Parser2 *self) {
+    if (*self->buf++ == '"') {
+      self->callbacks->on_begin_string(self->data);
+      self->pop();
+      MUSTTAIL return keepGoing(self);
+    }
+    return S_REJECT;
+  }
+  static Status whitespace(Parser2 *self) {
+    self->maybeSkipWs();
+    if (self->len() == 0) {
+      return S_AGAIN;
+    }
+    self->pop();
+    MUSTTAIL return keepGoing(self);
+  }
 
-  static constexpr continuation table[N_PAST_END] = {
+  static constexpr continuation table[] = {
       /*T_COLON*/ singleChar<':'>,
       /*T_TRUE*/ finishTrue,
       /*T_FALSE*/ finishFalse,
@@ -702,7 +726,7 @@ private:
       /*T_A*/ singleChar<'a'>,
       /*T_L*/ singleChar<'l'>,
       /*T_S*/ singleChar<'s'>,
-      /*T_DUBQUOTE*/ singleChar<'"'>,
+      /*T_DUBQUOTE*/ dubquote,
       /*N_STRING*/ string,
       /*N_NUMBER*/ number,
       /*N_VALUE*/ value,
@@ -710,8 +734,11 @@ private:
       /*N_OBJECT_VALUE_OR_END*/ objectOrEnd,
       /*N_ARRAY_MAYBE_CONTINUE*/ arrayContinue,
       /*N_OBJECT_MAYBE_CONTINUE*/ objectContinue,
+      /*N_WHITESPACE*/ whitespace,
   };
 
+  static_assert(sizeof(table) / sizeof(table[0]) == N_PAST_END);
+
   char *buf = nullptr;
   char *bufEnd = nullptr;
   int len() const { return bufEnd - buf; }