Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,13 @@

import com.fasterxml.jackson.databind.DatabindContext;
import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.exc.InvalidTypeIdException;
import com.fasterxml.jackson.databind.jsontype.NamedType;
import com.fasterxml.jackson.databind.jsontype.PolymorphicTypeValidator;
import com.fasterxml.jackson.databind.jsontype.impl.ClassNameIdResolver;
import com.fasterxml.jackson.databind.type.TypeFactory;
import com.fasterxml.jackson.databind.util.LRUMap;

/**
* {@link com.fasterxml.jackson.databind.jsontype.TypeIdResolver} for Avro type IDs embedded in schemas. Avro generally uses class names,
Expand All @@ -21,8 +23,25 @@ public class AvroTypeIdResolver extends ClassNameIdResolver
{
private static final long serialVersionUID = 1L;

// Maximum number of dots in type id for which nested-class probing is done:
// each one means one more class loading attempt for (possibly untrusted) id
final static int MAX_PROBE_DOTS = 32;

// Maximum length of type id for which nested-class probing is done:
// well above length of any real class name
final static int MAX_PROBE_LENGTH = 1024;

private final Map<String, Class<?>> _idTypes = new HashMap<>();

/**
* Cache of type ids (full names from Avro schema, with dots for nested classes) resolved
* into class names; ids for which no class is found are mapped to themselves.
* Since resolver is constructed for a mapper, so is this cache.
*
* @since 2.24
*/
private final LRUMap<String, String> _resolvedIds = new LRUMap<>(16, 100);

// 04-Jan-2021, tatu: Not sure why this is being kept around, not used as of
// 2.12; commenting out may be removed from 2.13
// private final Map<Class<?>, String> _typeIds = new HashMap<>();
Expand Down Expand Up @@ -56,6 +75,11 @@ protected JavaType _typeFromId(String id, DatabindContext ctxt) throws IOExcepti
}
// check if there's a specific type we should be using for this ID
Class<?> subType = _idTypes.get(id);
if (subType == null) {
// [dataformats-binary#908]: id from schema uses dots for nested classes too
id = _resolveNestedClassName(id, ctxt);
subType = _idTypes.get(id);
}
if (subType != null) {
id = _idFrom(null, subType, _typeFactory);
}
Expand All @@ -67,4 +91,100 @@ protected JavaType _typeFromId(String id, DatabindContext ctxt) throws IOExcepti
return null;
}
}

/**
* Type ids for records, enums and fixed types are full names from Avro schema, in which
* (since Apache Avro 1.9) nested classes are separated by dots, not dollar signs: this
* method finds the name of the actual class, trying given id first, then replacing dots
* with dollar signs, one by one, starting from the last one.
* Classes are looked up same way as when resolving type itself (no class initialization,
* same {@link ClassLoader}s, subtype validation), unlike with earlier probing by
* {@code AvroSchemaHelper}.
*
* @return Name of the class to use, if one found; {@code id} if not
*
* @since 2.24
*/
protected String _resolveNestedClassName(String id, DatabindContext ctxt)
{
if (!_shouldProbe(id)) {
return id;
}
String resolved = _resolvedIds.get(id);
if (resolved == null) {
// Canonical id of generic type (see `AvroReaderFactory`, for `EnumSet`)?
// Type parameters may be full names from schema as well
if (id.indexOf('<') > 0) {
resolved = _resolveGenericClassNames(id, ctxt);
} else {
resolved = _resolveClassName(id, _baseType, ctxt);
}
_resolvedIds.put(id, resolved);
}
return resolved;
}

private String _resolveGenericClassNames(String id, DatabindContext ctxt)
{
final JavaType paramBaseType = ctxt.constructType(Object.class);
final StringBuilder sb = new StringBuilder(id.length());
boolean first = true;
int start = 0;
for (int i = 0, len = id.length(); i < len; ++i) {
final char c = id.charAt(i);
if (c == '<' || c == '>' || c == ',') {
if (i > start) {
sb.append(_resolveClassName(id.substring(start, i),
first ? _baseType : paramBaseType, ctxt));
first = false;
}
sb.append(c);
start = i + 1;
}
}
return sb.append(id, start, id.length()).toString();
}

private String _resolveClassName(String id, JavaType baseType, DatabindContext ctxt)
{
final StringBuilder sb = new StringBuilder(id);
int lastDot = sb.length();
String candidate = id;
while (true) {
// Registered subtypes are known without class lookups
if (_idTypes.containsKey(candidate)) {
return candidate;
}
try {
if (ctxt.resolveAndValidateSubType(baseType, candidate, _subTypeValidator) != null) {
return candidate;
}
} catch (JsonMappingException e) {
// Class exists but is not valid (not allowed, not a subtype): no point in
// trying others; caller gets to handle the problem
return candidate;
}
lastDot = sb.lastIndexOf(".", lastDot);
if (lastDot == -1) { // no more dots, nothing found
return id;
}
sb.setCharAt(lastDot, '$');
candidate = sb.toString();
}
}

// Package-visible for tests
static boolean _shouldProbe(String id) {
if (id.length() > MAX_PROBE_LENGTH) {
return false;
}
int count = 0;
for (int ix = id.indexOf('.'); ix >= 0; ix = id.indexOf('.', ix + 1)) {
if (++count > MAX_PROBE_DOTS) {
return false;
}
}
// No dots, no nested-class alternatives
return (count > 0);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import com.fasterxml.jackson.databind.json.JsonMapper;
import com.fasterxml.jackson.databind.jsonFormatVisitors.JsonFormatTypes;
import com.fasterxml.jackson.databind.util.ClassUtil;
import com.fasterxml.jackson.databind.util.LRUMap;
import com.fasterxml.jackson.dataformat.avro.annotation.AvroNamespace;

public abstract class AvroSchemaHelper
Expand Down Expand Up @@ -354,12 +353,15 @@ public static String getTypeId(Schema schema) {
}

/**
* Returns the full name of a schema; This is similar to {@link Schema#getFullName()}, except that it properly handles namespaces for
* nested classes. (<code>package.name.ClassName$NestedClassName</code> instead of <code>package.name.ClassName$.NestedClassName</code>)
* <p>
* <b>WARNING!</b> This method has to probe for nested classes in order to resolve whether or not a schema references a top-level
* class or a nested class and return the corresponding name for each.
* </p>
* Returns the full name of a schema; This is similar to {@link Schema#getFullName()}, except that it properly handles namespaces
* that use dollar sign for nested classes (<code>package.name.ClassName$NestedClassName</code> instead of
* <code>package.name.ClassName$.NestedClassName</code>).
*<p>
* NOTE: since 2.24 this method no longer probes for existence of nested classes (which required loading
* classes named by possibly untrusted schema, using this module's {@link ClassLoader}): for namespaces that
* use dots for nested classes (Apache Avro 1.9 and later) plain dotted name
* (<code>package.name.ClassName.NestedClassName</code>) is returned. Resolution into actual nested class
* is done by {@code AvroTypeIdResolver} when deserializing, and by {@link #typeIdMatches} when serializing.
*/
public static String getFullName(Schema schema) {
final Schema.Type type = schema.getType();
Expand All @@ -380,16 +382,64 @@ public static String getFullName(Schema schema) {
if (namespace.charAt(len-1) == '$') {
return namespace + name;
}
// 19-Sep-2020, tatu: Due to very expensive contortions of lookups introduced
// in [dataformats-binary#195], attempts to resolve [dataformats-binary#219]
// isolated into separate class
return FullNameResolver.instance.resolve(namespace, name);
// [dataformats-binary#908]: no probing for nested classes
return namespace + '.' + name;

default:
return type.getName();
}
}

/**
* Method for checking whether given Java class has the type id of given schema:
* for named types (records, enums, fixed) this means that class name matches full name
* of the schema, allowing for nested classes being separated by either dollar sign
* (as in class name) or dot (as in namespaces of Apache Avro 1.9 and later).
* Does not load any classes.
*
* @since 2.24
*/
public static boolean typeIdMatches(Class<?> type, Schema schema) {
final String typeId = getTypeId(type);
switch (schema.getType()) {
case RECORD:
case ENUM:
case FIXED:
break;
default:
return typeId.equals(schema.getProp(AVRO_SCHEMA_PROP_CLASS));
}
final String name = schema.getName();
final String namespace = schema.getNamespace();
if (namespace == null || namespace.isEmpty()) {
return typeId.equals(name);
}
final int nsLen = namespace.length();
// Namespace either ends with '$', or is separated from name by '.' ('$' in class name)
int nameStart = nsLen;
if (namespace.charAt(nsLen-1) != '$') {
++nameStart;
}
if ((typeId.length() != nameStart + name.length()) || !typeId.endsWith(name)) {
return false;
}
if (nameStart > nsLen) {
final char sep = typeId.charAt(nsLen);
if (sep != '.' && sep != '$') {
return false;
}
}
for (int i = 0; i < nsLen; ++i) {
final char c = typeId.charAt(i);
final char nsc = namespace.charAt(i);
// Class name may have '$' where namespace has '.' (but not vice versa)
if ((c != nsc) && (c != '$' || nsc != '.')) {
return false;
}
}
return true;
}

public static JsonNode nullNode() {
return DEFAULT_VALUE_MAPPER.nullNode();
}
Expand Down Expand Up @@ -442,112 +492,4 @@ public static JsonNode parseDefaultValue(String defaultValue) throws JsonMapping
throw new JsonMappingException(null, "Failed to parse default value", e);
}
}

// Maximum number of dots in namespace for which nested-class probing is done
final static int MAX_NAMESPACE_DOTS = 32;

// Maximum length of full name (namespace, dot, name) for which nested-class
// probing is done: well above length of any real class name
final static int MAX_CLASS_NAME_LENGTH = 1024;

// Whether to probe for nested classes for given (possibly untrusted) schema
// name: each dot of namespace means one more class loading attempt, and both
// probed and cached names are retained, so limit both dots and length.
// Package-visible for tests
static boolean _shouldProbe(String namespace, String name) {
if ((namespace.length() + 1 + name.length()) > MAX_CLASS_NAME_LENGTH) {
return false;
}
int count = 0;
for (int ix = namespace.indexOf('.'); ix >= 0; ix = namespace.indexOf('.', ix + 1)) {
if (++count > MAX_NAMESPACE_DOTS) {
return false;
}
}
return true;
}

// @since 2.11.3
private final static class FullNameResolver {
private final LRUMap<FullNameKey, String> SCHEMA_NAME_CACHE = new LRUMap<>(80, 800);

public final static FullNameResolver instance = new FullNameResolver();

public String resolve(final String namespace, final String name) {
// No real class has name this long or nested this deep: skip
// probing (and caching) altogether
if (!_shouldProbe(namespace, name)) {
return namespace + '.' + name;
}
final FullNameKey cacheKey = new FullNameKey(namespace, name);
String schemaName = SCHEMA_NAME_CACHE.get(cacheKey);

if (schemaName == null) {
schemaName = _resolve(cacheKey);
SCHEMA_NAME_CACHE.put(cacheKey, schemaName);
}
return schemaName;
}

private static String _resolve(FullNameKey key) {
// 28-Feb-2020: [dataformats-binary#195] somewhat complicated logic of trying
// to support differences between avro-lib 1.8 and 1.9...
// Check if this is a nested class
// 19-Sep-2020, tatu: This is a horrible, horribly inefficient and all-around
// wrong mechanism. To be abolished if possible.
// 23-Aug-2024:[dataformats-binary#167] Based on SpecificData::getClass
// from Apache Avro. Initially assume that namespace is a Java package
StringBuilder sb = new StringBuilder(key.nameWithSeparator('.'));
int lastDot = sb.length();
while (true) {
try {
// Try to resolve the class; only need to know it exists, so
// must not initialize it (name comes from possibly untrusted schema)
String className = sb.toString();
Class.forName(className, false, FullNameResolver.class.getClassLoader());
return className;
} catch (ClassNotFoundException e) {
// Class does not exist - perhaps last dot is actually a nested class.
// NOTE: `LinkageError`s (including `NoClassDefFoundError` for class
// that exists but cannot be loaded, e.g. due to missing superclass)
// are intentionally NOT caught: class may well exist, and fallback
// name would get retained by callers, masking the actual problem
lastDot = sb.lastIndexOf(".", lastDot);
if (lastDot == -1) {
// No more dots so we are unable to resolve, should we throw an exception?
return key.nameWithSeparator('.');
}
sb.setCharAt(lastDot, '$');
}
}
}
}

// @since 2.11.3
private final static class FullNameKey {
private final String _namespace, _name;
private final int _hashCode;

public FullNameKey(String namespace, String name) {
_namespace = namespace;
_name = name;
_hashCode = namespace.hashCode() + name.hashCode();
}

public String nameWithSeparator(char sep) {
return _namespace + sep + _name;
}

@Override
public int hashCode() { return _hashCode; }

@Override
public boolean equals(Object o) { // lgtm [java/unchecked-cast-in-equals]
if (o == this) return true;
if (o == null) return false;
// Only used internally don't bother with type checks
final FullNameKey other = (FullNameKey) o;
return other._name.equals(_name) && other._namespace.equals(_namespace);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -320,11 +320,10 @@ public static int resolveUnionIndex(Schema unionSchema, Object datum) {
// - ByteBuffer
// - Number wrappers (Integer, ...)

String typeId = AvroSchemaHelper.getTypeId(datum.getClass());
for (int i = 0, size = types.size(); i < size; ++i) {
Schema schema = types.get(i);
// Exact schema match?
if (typeId.equals(AvroSchemaHelper.getTypeId(schema))) {
if (AvroSchemaHelper.typeIdMatches(raw, schema)) {
return i;
}
}
Expand Down Expand Up @@ -369,11 +368,10 @@ public static Schema resolveUnionType(Schema unionSchema, Object datum) {
return types.get(_resolveMapIndex(unionSchema, types, datum));
}

String typeId = AvroSchemaHelper.getTypeId(datum.getClass());
for (int i = 0, size = types.size(); i < size; ++i) {
Schema schema = types.get(i);
// Exact schema match?
if (typeId.equals(AvroSchemaHelper.getTypeId(schema))) {
if (AvroSchemaHelper.typeIdMatches(raw, schema)) {
return schema;
}
}
Expand Down
Loading