Skip to content

[FLINK-40225][table] Reject trailing tokens in JSON functions #28815

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
VasShabu wants to merge 4 commits into
base: master
Choose a base branch
from
Open

[FLINK-40225][table] Reject trailing tokens in JSON functions #28815

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6fd4f4d
[FLINK-40225][table] Fixed the Leading token bug + fixed invalid json…
VasShabu Jul 23, 2026
52455a9
[FLINK-40225][table] Added tests to IS_JSON to ensure correct parseing
VasShabu Jul 24, 2026
cc3cc72
[Flink-40225][table] fixed/added more testcases
VasShabu Jul 29, 2026
987f8b4
[FLINK-40225][table] Fix CI test cases issues/InvalidJSON
VasShabu Jul 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 26 additions & 1 deletion ...rc/test/java/org/apache/flink/table/planner/functions/JsonAggregationFunctionsITCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,18 @@ public Stream<TestSpec> getTestCaseSpecs() {
Arrays.asList(
Row.of(1, "{\"A\":1,\"B\":3}", 3),
Row.of(2, "{\"A\":2,\"C\":5}", 5))),
TestSpec.forFunction(BuiltInFunctionDefinitions.JSON_OBJECTAGG_NULL_ON_NULL)
.withDescription("Trailing Garbage After A Valid JSON Value")
.withSource(
ROW(STRING(), STRING()),
Collections.singletonList(Row.ofKind(INSERT, "A", "{\"a\":1} x")))
.testResult(
source -> "SELECT JSON_OBJECTAGG(f0 VALUE f1) FROM " + source,
TableApiAggSpec.select(
jsonObjectAgg(JsonOnNull.NULL, $("f0"), $("f1"))),
ROW(VARCHAR(2000).notNull()),
ROW(STRING().notNull()),
Collections.singletonList(Row.of("{\"A\":\"{\\\"a\\\":1} x\"}"))),

// JSON_ARRAYAGG
TestSpec.forFunction(BuiltInFunctionDefinitions.JSON_ARRAYAGG_ABSENT_ON_NULL)
Expand Down Expand Up @@ -338,6 +350,19 @@ public Stream<TestSpec> getTestCaseSpecs() {
+ source
+ " GROUP BY TUMBLE(f1, INTERVAL '5' SECOND)",
ROW(VARCHAR(2000).notNull()),
Arrays.asList(Row.of("[\"A\",\"B\"]"), Row.of("[\"C\"]"))));
Arrays.asList(Row.of("[\"A\",\"B\"]"), Row.of("[\"C\"]"))),

// Trailing garbage after a valid JSON value
TestSpec.forFunction(BuiltInFunctionDefinitions.JSON_ARRAYAGG_ABSENT_ON_NULL)
.withDescription("Trailing Garbage After A Valid JSON Value")
.withSource(
ROW(STRING()),
Collections.singletonList(Row.ofKind(INSERT, "{\"a\":1} x")))
.testResult(
source -> "SELECT JSON_ARRAYAGG(f0) FROM " + source,
TableApiAggSpec.select(jsonArrayAgg(JsonOnNull.ABSENT, $("f0"))),
ROW(VARCHAR(2000).notNull()),
ROW(STRING().notNull()),
Collections.singletonList(Row.of("[\"{\\\"a\\\":1} x\"]"))));
}
}
108 changes: 90 additions & 18 deletions ...e-planner/src/test/java/org/apache/flink/table/planner/functions/JsonFunctionsITCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,8 +101,8 @@ Stream<TestSetSpec> getTestSetSpecs() {
private static TestSetSpec jsonExistsSpec() {
final String jsonValue = getJsonFromResource("/json/json-exists.json");
return TestSetSpec.forFunction(BuiltInFunctionDefinitions.JSON_EXISTS)
.onFieldsWithData(jsonValue)
.andDataTypes(STRING())
.onFieldsWithData(jsonValue, "{\"a\":1} x")
.andDataTypes(STRING(), STRING())

// NULL
.testResult(
Expand Down Expand Up @@ -175,14 +175,17 @@ private static TestSetSpec jsonExistsSpec() {
.testTableApiRuntimeError(
$("f0").jsonExists("strict $.invalid", JsonExistsOnError.ERROR),
TableRuntimeException.class,
"No results for path: $['invalid']");
"No results for path: $['invalid']")

// Trailing garbage after a valid JSON value
.testResult($("f1").jsonExists("$.a"), "JSON_EXISTS(f1, '$.a')", false, BOOLEAN());
}

private static TestSetSpec jsonValueSpec() {
final String jsonValue = getJsonFromResource("/json/json-value.json");
return TestSetSpec.forFunction(BuiltInFunctionDefinitions.JSON_VALUE)
.onFieldsWithData(jsonValue)
.andDataTypes(STRING())
.onFieldsWithData(jsonValue, "{\"a\":1} x")
.andDataTypes(STRING(), STRING())

// NULL and invalid types
.testResult(
Expand Down Expand Up @@ -295,6 +298,9 @@ private static TestSetSpec jsonValueSpec() {
"right",
STRING())

// Trailing garbage after a valid JSON value
.testResult($("f1").jsonValue("$.a"), "JSON_VALUE(f1, '$.a')", null, STRING())

// Multiple JSON_VALUE calls on the same input should reuse parsed JSON
.testSqlResult(
"JSON_VALUE(f0, '$.type'), JSON_VALUE(f0, '$.age')",
Expand Down Expand Up @@ -407,7 +413,12 @@ private static List<TestSetSpec> isJsonSpec() {
$("f0").isJson(JsonType.OBJECT),
"f0 IS JSON OBJECT",
true,
BOOLEAN()));
BOOLEAN()),
// Trailing garbage after a valid JSON value
TestSetSpec.forFunction(BuiltInFunctionDefinitions.IS_JSON)

@gustavodemorais gustavodemorais Jul 28, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since the same mapper now affects every JSON function, could we add a trailing-garbage case for JSON_VALUE/JSON_QUERY/JSON_EXISTS as well (e.g. {"a":1} x)? Right now a future revert of the flag would only fail the IS JSON specs

@gustavodemorais gustavodemorais Jul 28, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You have multiple TestSetSpec, but you could group them under one single TestSetSpec as the tests above do

Suggested change
TestSetSpec.forFunction(BuiltInFunctionDefinitions.IS_JSON)
TestSetSpec.forFunction(BuiltInFunctionDefinitions.IS_JSON)
.onFieldsWithData("{} dsfsd", "true randomGarbage", "null, randomGarbage")

@VasShabu VasShabu Jul 28, 2026

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sounds good

.onFieldsWithData("{\"a\":1} x")
.andDataTypes(STRING())
.testResult($("f0").isJson(), "f0 IS JSON", false, BOOLEAN()));
}

private static List<TestSetSpec> jsonQuerySpec() {
Expand Down Expand Up @@ -632,7 +643,14 @@ private static List<TestSetSpec> jsonQuerySpec() {
.testTableApiRuntimeError(
$("f0").jsonQuery("strict $.err10", WITHOUT_ARRAY, NULL, ERROR),
TableRuntimeException.class,
"No results for path"));
"No results for path"),
// Trailing garbage after a valid JSON value. The path must select an object,
// because a scalar would return NULL even if the trailing garbage was accepted.
TestSetSpec.forFunction(BuiltInFunctionDefinitions.JSON_QUERY)
.onFieldsWithData("{\"a\":{\"b\":1}} x")
.andDataTypes(STRING())
.testResult(
$("f0").jsonQuery("$.a"), "JSON_QUERY(f0, '$.a')", null, STRING()));
}

private static List<TestSetSpec> jsonStringSpec() {
Expand Down Expand Up @@ -758,7 +776,17 @@ private static List<TestSetSpec> jsonStringSpec() {
jsonString($("f0")),
"JSON_STRING(f0)",
"{\"field\\ttab\":\"val4\",\"field\\nline\":\"val3\",\"field\\rreturn\":\"val5\",\"field\\\"quote\":\"val1\",\"field\\\\slash\":\"val2\"}",
STRING().notNull()));
STRING().notNull()),
// Trailing garbage after a valid JSON value. JSON_STRING serializes its argument
// instead of parsing it, so the garbage is kept as part of the string.
TestSetSpec.forFunction(BuiltInFunctionDefinitions.JSON_STRING)
.onFieldsWithData("{\"a\":1} x")
.andDataTypes(STRING())
.testResult(
jsonString($("f0")),
"JSON_STRING(f0)",
"\"{\\\"a\\\":1} x\"",
STRING()));
}

private static List<TestSetSpec> parseJsonSpec() {
Expand Down Expand Up @@ -1184,7 +1212,19 @@ private static List<TestSetSpec> jsonObjectSpec() {
jsonObject(JsonOnNull.NULL, "testRow", $("f0")),
"JSON_OBJECT(KEY 'testRow' VALUE f0 NULL ON NULL)",
"{\"testRow\":{\"field\\ttab\":\"val4\",\"field\\nline\":\"val3\",\"field\\rreturn\":\"val5\",\"field\\\"quote\":\"val1\",\"field\\\\slash\":\"val2\"}}",
STRING().notNull()));
STRING().notNull()),
// Trailing garbage after a valid JSON value
TestSetSpec.forFunction(BuiltInFunctionDefinitions.JSON_OBJECT)
.onFieldsWithData("{\"a\":1} x")
.andDataTypes(STRING())
.testSqlRuntimeError(
"JSON_OBJECT(KEY 'K' VALUE JSON(f0))",
TableRuntimeException.class,
"Invalid JSON string in JSON(value) function")
.testTableApiRuntimeError(
jsonObject(JsonOnNull.NULL, "K", json($("f0"))),
TableRuntimeException.class,
"Invalid JSON string in JSON(value) function"));
}

private static List<TestSetSpec> jsonQuoteSpec() {
Expand Down Expand Up @@ -1256,7 +1296,17 @@ private static List<TestSetSpec> jsonQuoteSpec() {
"\"\\u2260 will be escaped\"",
STRING().notNull())
.testResult(
$("f8").jsonQuote(), "JSON_QUOTE(f8)", null, STRING().nullable()));
$("f8").jsonQuote(), "JSON_QUOTE(f8)", null, STRING().nullable()),
// Trailing garbage after a valid JSON value. JSON_QUOTE only escapes its input, so
// the garbage is preserved.
TestSetSpec.forFunction(BuiltInFunctionDefinitions.JSON_QUOTE)
.onFieldsWithData("{\"a\":1} x")
.andDataTypes(STRING())
.testResult(
$("f0").jsonQuote(),
"JSON_QUOTE(f0)",
"\"{\\\"a\\\":1} x\"",
STRING()));
}

private static List<TestSetSpec> jsonUnquoteSpecWithValidInput() {
Expand All @@ -1272,13 +1322,13 @@ private static List<TestSetSpec> jsonUnquoteSpecWithValidInput() {
TestSetSpec.forFunction(BuiltInFunctionDefinitions.JSON_UNQUOTE)
.onFieldsWithData(
"\"abc\"",
"\"[\"abc\"]\"",
"\"[\"\\u0041\"]\"",
"\"[\\\"abc\\\"]\"",
"\"[\\\"\\u0041\\\"]\"",
"\"\\u0041\"",
"\"[\"\\t\\u0032\"]\"",
"\"[\"This is a \\t test \\n with special characters: \\b \\f \\r \\u0041\"]\"",
"\"\"\"",
"\"\"\ufffa\"",
"\"[\\\"\\t\\u0032\\\"]\"",
"\"[\\\"This is a \\t test \\n with special characters: \\b \\f \\r \\u0041\\\"]\"",
"\"\\\"\"",
"\"\\\"\ufffa\"",
"\"a unicode \u2260\"",
"\"valid unicode literal \\uD801\\uDC00\"",
"[1,2,3]",
Expand Down Expand Up @@ -1478,7 +1528,17 @@ private static List<TestSetSpec> jsonUnquoteSpecWithInvalidInput() {
$("f10").jsonQuote(),
"JSON_UNQUOTE(f10)",
null,
STRING().nullable()));
STRING().nullable()),
// Trailing garbage after a valid JSON value. The leading '"abc"' must not be
// unquoted on its own, the input is passed through unchanged.
TestSetSpec.forFunction(BuiltInFunctionDefinitions.JSON_UNQUOTE)
.onFieldsWithData("\"abc\" \"def\"")
.andDataTypes(STRING())
.testResult(
$("f0").jsonUnquote(),
"JSON_UNQUOTE(f0)",
"\"abc\" \"def\"",
STRING()));
}

private static List<TestSetSpec> jsonArraySpec() {
Expand Down Expand Up @@ -1621,7 +1681,19 @@ private static List<TestSetSpec> jsonArraySpec() {
jsonArray(JsonOnNull.NULL, $("f0")),
"JSON_ARRAY(f0 NULL ON NULL)",
"[{\"field\\ttab\":\"val4\",\"field\\nline\":\"val3\",\"field\\rreturn\":\"val5\",\"field\\\"quote\":\"val1\",\"field\\\\slash\":\"val2\"}]",
STRING().notNull()));
STRING().notNull()),
// Trailing garbage after a valid JSON value
TestSetSpec.forFunction(BuiltInFunctionDefinitions.JSON_ARRAY)
.onFieldsWithData("{\"a\":1} x")
.andDataTypes(STRING())
.testSqlRuntimeError(
"JSON_ARRAY(JSON(f0))",
TableRuntimeException.class,
"Invalid JSON string in JSON(value) function")
.testTableApiRuntimeError(
jsonArray(JsonOnNull.NULL, json($("f0"))),
TableRuntimeException.class,
"Invalid JSON string in JSON(value) function"));
}

/** Pins the local-ref / common-sub-expression handling for JSON construction calls. */
Expand Down
6 changes: 4 additions & 2 deletions ...anner/src/test/scala/org/apache/flink/table/planner/expressions/ScalarFunctionsTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -724,9 +724,7 @@ class ScalarFunctionsTest extends ScalarTypesTestBase {
"This is a \t test \n with special characters: \b \f \r A"
)
testSqlApi("JSON_UNQUOTE('\"\"')", "")
testSqlApi("JSON_UNQUOTE('\"\"\"')", "\"")
testSqlApi("JSON_UNQUOTE('[]')", "[]")
testSqlApi("JSON_UNQUOTE('\"\"\\ufffa\"')", "\"\ufffa")
testSqlApi("JSON_UNQUOTE('{\"key\":1}')", "{\"key\":1}")
testSqlApi("JSON_UNQUOTE('true')", "true")
}
Expand All @@ -747,6 +745,10 @@ class ScalarFunctionsTest extends ScalarTypesTestBase {
def testJsonUnquoteWithInvalidInput(): Unit = {
testSqlApi("JSON_UNQUOTE('\"[1, 2, 3}')", "\"[1, 2, 3}")
testSqlApi("JSON_UNQUOTE('\"')", "\"")
// Quoted, but not a valid JSON string literal: the leading '""' is followed by a
// trailing token, so the input is passed through instead of being unquoted.
testSqlApi("JSON_UNQUOTE('\"\"\"')", "\"\"\"")
testSqlApi("JSON_UNQUOTE('\"\"\\ufffa\"')", "\"\"\\ufffa\"")
testSqlApi("JSON_UNQUOTE('[}')", "[}")
testSqlApi("JSON_UNQUOTE('1\"')", "1\"")
testSqlApi("JSON_UNQUOTE('[')", "[")
Expand Down
3 changes: 2 additions & 1 deletion ...nk-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/SqlJsonUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,8 @@ public class SqlJsonUtils {
private static final ObjectMapper MAPPER =
new ObjectMapper(JSON_FACTORY)
.configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true)
.configure(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS, true);
.configure(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS, true)
.configure(DeserializationFeature.FAIL_ON_TRAILING_TOKENS, true);
private static final Pattern JSON_PATH_BASE =
Pattern.compile(
"^\\s*(?<mode>strict|lax)\\s+(?<spec>.+)$",
Expand Down