weaselab/weaseljson
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix issue with fuzz test

Browse Source 
Previously an implementation could never call on_begin_number or
on_end_number and still pass
This commit is contained in:
Andrew Noyes
2025-05-19 15:58:15 -04:00
parent 2cb9bc8ad2
commit be8187b2fd
4 changed files with 111 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
src/callbacks.h
View File

@@ -1,7 +1,11 @@
#pragma once #pragma once
#include "weaseljson.h" #include <cstdint>
#include <cstdio> #include <cstdio>
#include <string>
#include <vector>
#include "weaseljson.h"
inline Callbacks printCallbacks() { inline Callbacks printCallbacks() {
Callbacks result; Callbacks result;
@@ -42,3 +46,96 @@ inline Callbacks noopCallbacks() {
result.on_null_literal = +[](void *) {}; result.on_null_literal = +[](void *) {};
return result; return result;
} }
struct SerializeState {
bool isKey = false;
struct Cursor {
int64_t index;
bool isObject;
};
std::string result;
void on_begin_value() {
if (!stack.empty()) {
auto &back = stack.back();
if (back.isObject && back.index % 2 == 0 && back.index > 0) {
result.append(",");
}
if (back.isObject && back.index % 2 == 1 && back.index > 0) {
result.append(":");
}
if (!back.isObject && back.index > 0) {
result.append(",");
}
++back.index;
}
}
std::vector<Cursor> stack;
};
inline Callbacks serializeCallbacks() {
Callbacks result;
result.on_begin_object = +[](void *p) {
auto *state = (SerializeState *)p;
state->on_begin_value();
state->stack.push_back({0, true});
state->result.append("{");
};
result.on_end_object = +[](void *p) {
auto *state = (SerializeState *)p;
state->stack.pop_back();
state->result.append("}");
};
result.on_begin_string = +[](void *p) {
auto *state = (SerializeState *)p;
state->on_begin_value();
state->result.append("<");
};
result.on_string_data = +[](void *p, const char *buf, int len) {
auto *state = (SerializeState *)p;
state->result.append(std::string(buf, len));
};
result.on_end_string = +[](void *p) {
auto *state = (SerializeState *)p;
state->result.append(">");
};
result.on_begin_array = +[](void *p) {
auto *state = (SerializeState *)p;
state->on_begin_value();
state->stack.push_back({0, false});
state->result.append("[");
};
result.on_end_array = +[](void *p) {
auto *state = (SerializeState *)p;
state->stack.pop_back();
state->result.append("]");
};
result.on_begin_number = +[](void *p) {
auto *state = (SerializeState *)p;
state->on_begin_value();
state->result.append("(");
};
result.on_number_data = +[](void *p, const char *buf, int len) {
auto *state = (SerializeState *)p;
state->result.append(std::string(buf, len));
};
result.on_end_number = +[](void *p) {
auto *state = (SerializeState *)p;
state->result.append(")");
};
result.on_true_literal = +[](void *p) {
auto *state = (SerializeState *)p;
state->on_begin_value();
state->result.append("true");
};
result.on_false_literal = +[](void *p) {
auto *state = (SerializeState *)p;
state->on_begin_value();
state->result.append("false");
};
result.on_null_literal = +[](void *p) {
auto *state = (SerializeState *)p;
state->on_begin_value();
state->result.append("null");
};
return result;
}

13
src/fuzz.cpp
View File

@@ -1,12 +1,11 @@
#include "callbacks.h" #include "callbacks.h"
#include "minify.h"
#include "parser3.h" #include "parser3.h"
#include <simdjson.h> #include <simdjson.h>
std::pair<std::string, parser3::Status> runStreaming(std::string copy) { std::pair<std::string, parser3::Status> runStreaming(std::string copy) {
MinifyState state; SerializeState state;
auto c = minifyCallbacks(); auto c = serializeCallbacks();
parser3::Parser3 parser(&c, &state); parser3::Parser3 parser(&c, &state);
for (int i = 0; i < copy.size(); ++i) { for (int i = 0; i < copy.size(); ++i) {
auto s = parser.parse(copy.data() + i, 1); auto s = parser.parse(copy.data() + i, 1);
@@ -22,8 +21,8 @@ std::pair<std::string, parser3::Status> runStreaming(std::string copy) {
} }
std::pair<std::string, parser3::Status> runBatch(std::string copy) { std::pair<std::string, parser3::Status> runBatch(std::string copy) {
MinifyState state; SerializeState state;
auto c = minifyCallbacks(); auto c = serializeCallbacks();
parser3::Parser3 parser(&c, &state); parser3::Parser3 parser(&c, &state);
auto s = parser.parse(copy.data(), copy.size()); auto s = parser.parse(copy.data(), copy.size());
if (s != parser3::S_AGAIN) { if (s != parser3::S_AGAIN) {
@@ -39,6 +38,10 @@ std::pair<std::string, parser3::Status> runBatch(std::string copy) {
void testStreaming(std::string const &json) { void testStreaming(std::string const &json) {
auto streaming = runStreaming(json); auto streaming = runStreaming(json);
auto batch = runBatch(json); auto batch = runBatch(json);
if (streaming.second == parser3::S_OK) {
printf("streaming: %s\n", streaming.first.c_str());
printf("batch: %s\n", batch.first.c_str());
}
if (streaming != batch) { if (streaming != batch) {
if (streaming.second == batch.second && streaming.second != parser3::S_OK) { if (streaming.second == batch.second && streaming.second != parser3::S_OK) {
// It's ok if the processed data doesn't match if parsing failed // It's ok if the processed data doesn't match if parsing failed

95
src/minify.h
View File

@@ -1,95 +0,0 @@
#pragma once
#include "weaseljson.h"
#include <cstdint>
#include <string>
#include <vector>
struct MinifyState {
bool isKey = false;
struct Cursor {
int64_t index;
bool isObject;
};
std::string result;
void on_begin_value() {
if (!stack.empty()) {
auto &back = stack.back();
if (back.isObject && back.index % 2 == 0 && back.index > 0) {
result.append(",");
}
if (back.isObject && back.index % 2 == 1 && back.index > 0) {
result.append(":");
}
if (!back.isObject && back.index > 0) {
result.append(",");
}
++back.index;
}
}
std::vector<Cursor> stack;
};
inline Callbacks minifyCallbacks() {
Callbacks result;
result.on_begin_object = +[](void *p) {
auto *state = (MinifyState *)p;
state->on_begin_value();
state->stack.push_back({0, true});
state->result.append("{");
};
result.on_end_object = +[](void *p) {
auto *state = (MinifyState *)p;
state->stack.pop_back();
state->result.append("}");
};
result.on_begin_string = +[](void *p) {
auto *state = (MinifyState *)p;
state->on_begin_value();
state->result.append("\"");
};
result.on_string_data = +[](void *p, const char *buf, int len) {
auto *state = (MinifyState *)p;
state->result.append(std::string(buf, len));
};
result.on_end_string = +[](void *p) {
auto *state = (MinifyState *)p;
state->result.append("\"");
};
result.on_begin_array = +[](void *p) {
auto *state = (MinifyState *)p;
state->on_begin_value();
state->stack.push_back({0, false});
state->result.append("[");
};
result.on_end_array = +[](void *p) {
auto *state = (MinifyState *)p;
state->stack.pop_back();
state->result.append("]");
};
result.on_begin_number = +[](void *p) {
auto *state = (MinifyState *)p;
state->on_begin_value();
};
result.on_number_data = +[](void *p, const char *buf, int len) {
auto *state = (MinifyState *)p;
state->result.append(std::string(buf, len));
};
result.on_end_number = +[](void *) {};
result.on_true_literal = +[](void *p) {
auto *state = (MinifyState *)p;
state->on_begin_value();
state->result.append("true");
};
result.on_false_literal = +[](void *p) {
auto *state = (MinifyState *)p;
state->on_begin_value();
state->result.append("false");
};
result.on_null_literal = +[](void *p) {
auto *state = (MinifyState *)p;
state->on_begin_value();
state->result.append("null");
};
return result;
}

11
src/test.cpp
View File

@@ -10,7 +10,6 @@
#include <simdjson.h> #include <simdjson.h>
#include "callbacks.h" #include "callbacks.h"
#include "minify.h"
#include "parser3.h" #include "parser3.h"
// This is the JSON grammar in McKeeman Form. // This is the JSON grammar in McKeeman Form.
@@ -147,9 +146,9 @@ const std::string json = R"({
})"; })";
void testStreaming(std::string const &json) { void testStreaming(std::string const &json) {
MinifyState streaming; SerializeState streaming;
MinifyState batch; SerializeState batch;
auto c = minifyCallbacks(); auto c = serializeCallbacks();
{ {
auto copy = json; auto copy = json;
parser3::Parser3 parser(&c, &streaming); parser3::Parser3 parser(&c, &streaming);
@@ -170,8 +169,8 @@ void testStreaming(std::string const &json) {
} // namespace } // namespace
TEST_CASE("parser3") { TEST_CASE("parser3") {
Callbacks c = minifyCallbacks(); Callbacks c = serializeCallbacks();
MinifyState state; SerializeState state;
{ {
auto copy = json; auto copy = json;
parser3::Parser3 parser(&c, &state); parser3::Parser3 parser(&c, &state);