Skip to content

Eliminate String intermediaries in JSON codecs #1685

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tsegismont wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Eliminate String intermediaries in JSON codecs #1685

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 24 additions & 13 deletions vertx-mssql-client/src/main/java/io/vertx/mssqlclient/impl/codec/DataType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,8 @@

package io.vertx.mssqlclient.impl.codec;

import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
import io.netty.buffer.Unpooled;
import io.netty.buffer.*;
import io.netty.handler.codec.EncoderException;
import io.netty.util.collection.IntObjectHashMap;
import io.netty.util.collection.IntObjectMap;
import io.vertx.core.buffer.Buffer;
Expand All @@ -22,6 +21,7 @@
import io.vertx.core.json.JsonArray;
import io.vertx.core.json.JsonObject;

import java.io.*;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.math.RoundingMode;
Expand Down Expand Up @@ -984,8 +984,9 @@ public Object decodeValue(ByteBuf byteBuf, TypeInfo typeInfo) {
if (isPLPNull(payloadLength)) {
return null;
}
String jsonString = readPLP(byteBuf).toString(StandardCharsets.UTF_16LE);
return Json.decodeValue(jsonString);
ByteBuf heapBuf = readPLP(byteBuf);
Reader reader = new InputStreamReader(new ByteBufInputStream(heapBuf), StandardCharsets.UTF_16LE);
return Json.decodeValue(reader);
}

@Override
Expand All @@ -995,14 +996,24 @@ public String paramDefinition(Object value) {

@Override
public void encodeParam(ByteBuf byteBuf, String name, boolean out, Object value) {
writeParamDescription(byteBuf, name, out, NVARCHAR.id);
String val = value.toString();
byteBuf.writeShortLE(0xFFFF);
writeCollation(byteBuf);
byteBuf.writeLongLE(val.length() * 2L);
byteBuf.writeIntLE(val.length() * 2);
byteBuf.writeCharSequence(val, StandardCharsets.UTF_16LE);
byteBuf.writeIntLE(0);
ByteBuf tmp = byteBuf.alloc().buffer();
try {
Writer writer = new OutputStreamWriter(new ByteBufOutputStream(tmp), StandardCharsets.UTF_16LE);
Json.encodeTo(value, writer);
writer.flush();
int byteLen = tmp.readableBytes();
writeParamDescription(byteBuf, name, out, NVARCHAR.id);
byteBuf.writeShortLE(0xFFFF);
writeCollation(byteBuf);
byteBuf.writeLongLE(byteLen);
byteBuf.writeIntLE(byteLen);
byteBuf.writeBytes(tmp);
byteBuf.writeIntLE(0);
} catch (IOException e) {
throw new EncoderException(e);
} finally {
tmp.release();
}
}
},

Expand Down
59 changes: 23 additions & 36 deletions vertx-mysql-client/src/main/java/io/vertx/mysqlclient/impl/datatype/DataTypeCodec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,23 +12,24 @@
package io.vertx.mysqlclient.impl.datatype;

import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufInputStream;
import io.netty.buffer.ByteBufOutputStream;
import io.netty.buffer.Unpooled;
import io.netty.handler.codec.DecoderException;
import io.vertx.core.buffer.Buffer;
import io.vertx.core.internal.buffer.BufferInternal;
import io.vertx.core.json.Json;
import io.vertx.core.json.JsonArray;
import io.vertx.core.json.JsonObject;
import io.vertx.mysqlclient.data.spatial.*;
import io.vertx.mysqlclient.data.spatial.Geometry;
import io.vertx.mysqlclient.impl.MySQLCollation;
import io.vertx.mysqlclient.impl.util.BufferUtils;
import io.vertx.core.buffer.Buffer;
import io.vertx.sqlclient.Tuple;
import io.vertx.sqlclient.data.Numeric;
import io.vertx.sqlclient.impl.Utils;

import java.math.BigInteger;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.LocalDate;
import java.time.LocalDateTime;
Expand Down Expand Up @@ -95,7 +96,7 @@ public static Object decodeText(DataType dataType, int collationId, ByteBuf buff
case TIMESTAMP:
return textDecodeDateTime(collationId, buffer, index, length);
case JSON:
return textDecodeJson(collationId, buffer, index, length);
return textDecodeJson(buffer, index, length);
case GEOMETRY:
return textDecodeGeometry(buffer, index, length);
case BINARY:
Expand Down Expand Up @@ -178,7 +179,7 @@ public static void encodeBinary(DataType dataType, Object value, Charset charset
case VARSTRING:
default:
if (value instanceof JsonObject || value instanceof JsonArray) {
binaryEncodeJson(value, buffer, charset);
binaryEncodeJson(value, buffer);
return;
} else if (value == Tuple.JSON_NULL) {
// we have to make JSON literal null send as a STRING data type
Expand Down Expand Up @@ -231,7 +232,7 @@ public static Object decodeBinary(DataType dataType, int collationId, ByteBuf bu
case TIMESTAMP:
return binaryDecodeDatetime(buffer);
case JSON:
return binaryDecodeJson(collationId, buffer);
return binaryDecodeJson(buffer);
case GEOMETRY:
return binaryDecodeGeometry(buffer);
case BINARY:
Expand Down Expand Up @@ -466,8 +467,15 @@ private static void binaryEncodeDatetime(LocalDateTime value, ByteBuf buffer) {
}
}

private static void binaryEncodeJson(Object value, ByteBuf buffer, Charset charset) {
BufferUtils.writeLengthEncodedString(buffer, Json.encode(value), charset);
private static void binaryEncodeJson(Object value, ByteBuf buffer) {
ByteBuf tmp = buffer.alloc().buffer();
try {
Json.encodeTo(value, new ByteBufOutputStream(tmp));
BufferUtils.writeLengthEncodedInteger(buffer, tmp.readableBytes());
buffer.writeBytes(tmp);
} finally {
tmp.release();
}
}

private static Byte binaryDecodeInt8(ByteBuf buffer) {
Expand Down Expand Up @@ -629,9 +637,9 @@ private static Duration binaryDecodeTime(ByteBuf buffer) {
}
}

private static Object binaryDecodeJson(int collationId, ByteBuf buffer) {
private static Object binaryDecodeJson(ByteBuf buffer) {
int length = (int) BufferUtils.readLengthEncodedInteger(buffer);
Object result = textDecodeJson(collationId, buffer, buffer.readerIndex(), length);
Object result = textDecodeJson(buffer, buffer.readerIndex(), length);
buffer.skipBytes(length);
return result;
}
Expand Down Expand Up @@ -743,33 +751,12 @@ private static LocalDateTime textDecodeDateTime(int collationId, ByteBuf buffer,
return LocalDateTime.parse(cs, DATETIME_FORMAT);
}

private static Object textDecodeJson(int collationId, ByteBuf buffer, int index, int length) {
Charset charset = StandardCharsets.UTF_8; // MySQL JSON data type will only be UTF-8 string
// Try to do without the intermediary String (?)
CharSequence cs = buffer.getCharSequence(index, length, charset);
Object value = null;
String s = cs.toString();
int pos = 0;
while (pos < s.length() && Character.isWhitespace(s.charAt(pos))) {
pos++;
}
if (pos == s.length()) {
return null;
} else if (s.charAt(pos) == '{') {
value = new JsonObject(s);
} else if (s.charAt(pos) == '[') {
value = new JsonArray(s);
} else {
Object o = Json.decodeValue(s);
if (o == null) {
return Tuple.JSON_NULL;
}
if (o instanceof Number || o instanceof Boolean || o instanceof String) {
return o;
}
return null;
private static Object textDecodeJson(ByteBuf buffer, int index, int length) {
Object o = Json.decodeValue(new ByteBufInputStream(buffer.slice(index, length)));
if (o == null) {
return Tuple.JSON_NULL;
}
return value;
return o;
}

private static Long decodeBit(ByteBuf buffer, int index, int length) {
Expand Down
8 changes: 4 additions & 4 deletions vertx-pg-client/src/main/java/io/vertx/pgclient/impl/codec/DataType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,10 +96,10 @@ public enum DataType {
MACADDR8(774, true, Object[].class, JDBCType.OTHER, DataTypeEstimator.UNSUPPORTED),
UUID(2950, true, UUID.class, JDBCType.OTHER, null, Tuple::getUUID, DataTypeEstimator.UUID),
UUID_ARRAY(2951, true, UUID[].class, JDBCType.OTHER, null, Tuple::getArrayOfUUIDs, DataTypeEstimator.UUID),
JSON(114, true, Object.class, JDBCType.OTHER, ParamExtractor::prepareJson, Tuple::getJson, DataTypeEstimator.JSON),
JSON_ARRAY(199, true, Object[].class, JDBCType.OTHER, ParamExtractor::prepareJson, Tuple::getArrayOfJsons, DataTypeEstimator.JSON),
JSONB(3802, true, Object.class, JDBCType.OTHER, ParamExtractor::prepareJson, Tuple::getJson, DataTypeEstimator.JSONB),
JSONB_ARRAY(3807, true, Object[].class, JDBCType.OTHER, ParamExtractor::prepareJson, Tuple::getArrayOfJsons, DataTypeEstimator.JSONB),
JSON(114, true, Object.class, JDBCType.OTHER, null, Tuple::getJson, DataTypeEstimator.JSON),
JSON_ARRAY(199, true, Object[].class, JDBCType.OTHER, null, Tuple::getArrayOfJsons, DataTypeEstimator.JSON),
JSONB(3802, true, Object.class, JDBCType.OTHER, null, Tuple::getJson, DataTypeEstimator.JSONB),
JSONB_ARRAY(3807, true, Object[].class, JDBCType.OTHER, null, Tuple::getArrayOfJsons, DataTypeEstimator.JSONB),
XML(142, true, Object.class, JDBCType.OTHER, DataTypeEstimator.UNSUPPORTED),
XML_ARRAY(143, true, Object[].class, JDBCType.OTHER, DataTypeEstimator.UNSUPPORTED),
POINT(600, true, Point.class, JDBCType.OTHER, DataTypeEstimator.POINT),
Expand Down
49 changes: 14 additions & 35 deletions vertx-pg-client/src/main/java/io/vertx/pgclient/impl/codec/DataTypeCodec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,17 +17,13 @@

package io.vertx.pgclient.impl.codec;

import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
import io.netty.buffer.Unpooled;
import io.netty.buffer.*;
import io.netty.handler.codec.DecoderException;
import io.vertx.core.buffer.Buffer;
import io.vertx.core.internal.buffer.BufferInternal;
import io.vertx.core.internal.logging.Logger;
import io.vertx.core.internal.logging.LoggerFactory;
import io.vertx.core.json.Json;
import io.vertx.core.json.JsonArray;
import io.vertx.core.json.JsonObject;
import io.vertx.pgclient.data.*;
import io.vertx.pgclient.impl.util.UTF8StringEndDetector;
import io.vertx.sqlclient.Tuple;
Expand Down Expand Up @@ -279,13 +275,13 @@ public static void encodeBinary(DataType id, Object value, ByteBuf buff) {
binaryEncodeArray((UUID[]) value, DataType.UUID, buff);
break;
case JSON:
binaryEncodeJSON((CharSequence) value, buff);
binaryEncodeJSON(value, buff);
break;
case JSON_ARRAY:
binaryEncodeArray((Object[]) value, DataType.JSON, buff);
break;
case JSONB:
binaryEncodeJSONB((CharSequence) value, buff);
binaryEncodeJSONB(value, buff);
break;
case JSONB_ARRAY:
binaryEncodeArray((Object[]) value, DataType.JSONB, buff);
Expand Down Expand Up @@ -1378,45 +1374,28 @@ private static Object binaryDecodeJSON(int index, int len, ByteBuf buff) {
return textDecodeJSONB(index, len, buff);
}

private static void binaryEncodeJSON(CharSequence value, ByteBuf buff) {
buff.writeCharSequence(value, StandardCharsets.UTF_8);
private static void binaryEncodeJSON(Object value, ByteBuf buff) {
if (value == Tuple.JSON_NULL) {
buff.writeCharSequence("null", StandardCharsets.UTF_8);
} else {
Json.encodeTo(value, new ByteBufOutputStream(buff));
}
}

private static Object textDecodeJSONB(int index, int len, ByteBuf buff) {

// Try to do without the intermediary String (?)
CharSequence cs = buff.getCharSequence(index, len, StandardCharsets.UTF_8);
Object value = null;
String s = cs.toString();
int pos = 0;
while (pos < s.length() && Character.isWhitespace(s.charAt(pos))) {
pos++;
}
if (pos == s.length()) {
return null;
} else if (s.charAt(pos) == '{') {
value = new JsonObject(s);
} else if (s.charAt(pos) == '[') {
value = new JsonArray(s);
} else {
Object o = Json.decodeValue(s);
if (o == null) {
return Tuple.JSON_NULL;
}
if (o instanceof Number || o instanceof Boolean || o instanceof String) {
return o;
}
return null;
Object o = Json.decodeValue(new ByteBufInputStream(buff.slice(index, len)));
if (o == null) {
return Tuple.JSON_NULL;
}
return value;
return o;
}

private static Object binaryDecodeJSONB(int index, int len, ByteBuf buff) {
// Skip 1 byte for version (which is 1)
return textDecodeJSONB(index + 1, len - 1, buff);
}

private static void binaryEncodeJSONB(CharSequence value, ByteBuf buff) {
private static void binaryEncodeJSONB(Object value, ByteBuf buff) {
buff.writeByte(1); // version
binaryEncodeJSON(value, buff);
}
Expand Down
11 changes: 7 additions & 4 deletions vertx-pg-client/src/main/java/io/vertx/pgclient/impl/codec/DataTypeEstimator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,11 @@ private static int estimateUnknown(String value) {
return estimateUTF8(value);
}

private static int estimateJSONB(String value) {
return 1 + estimateUTF8(value);
private static int estimateJSONB(Object value) {
if (value instanceof String) {
return 1 + estimateUTF8((String) value);
}
return 256;
}

private static int estimateNumeric(String value) {
Expand Down Expand Up @@ -144,15 +147,15 @@ static int estimate(int estimator, Object o) {
case DataTypeEstimator.INET:
return estimateInetOrCidr((Inet) o);
case DataTypeEstimator.JSONB:
return estimateJSONB((String) o);
return estimateJSONB(o);
case DataTypeEstimator.UNKNOWN:
return estimateUnknown((String) o);
case DataTypeEstimator.NUMERIC:
return estimateNumeric((String) o);
case DataTypeEstimator.NUMERIC_ARRAY:
return estimateNumericArray((Object[]) o);
case DataTypeEstimator.UTF8:
return estimateUTF8((String) o);
return o instanceof String ? estimateUTF8((String) o) : 256;
case DataTypeEstimator.BUFFER:
return estimateBuffer((Buffer) o);
case DataTypeEstimator.POLYGON:
Expand Down
14 changes: 0 additions & 14 deletions vertx-pg-client/src/main/java/io/vertx/pgclient/impl/codec/ParamExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@
*/
package io.vertx.pgclient.impl.codec;

import io.vertx.core.json.Json;
import io.vertx.sqlclient.Tuple;
import io.vertx.sqlclient.internal.TupleBase;

import java.util.Arrays;
Expand All @@ -31,22 +29,10 @@ static String extractUnknownType(TupleBase tuple, int pos) {

T get(TupleBase tuple, int idx);

private static String encodeJsonToString(Object value) {
if (value == Tuple.JSON_NULL) {
return "null";
} else {
return Json.encode(value);
}
}

static Object prepareUnknown(Object value) {
return String.valueOf(value);
}

static Object prepareJson(Object value) {
return encodeJsonToString(value);
}

static Object prepareNumeric(Object value) {
assert value instanceof Number;
return value.toString();
Expand Down