From ecaaf5115e15fb1c2cff50150d134c8eb489d65d Mon Sep 17 00:00:00 2001 From: Anas Ismail Khan Date: Tue, 11 Aug 2026 20:41:23 +0500 Subject: [PATCH 1/2] bump to substrait v0.100.0 and implications --- .../java/io/substrait/extension/DefaultExtensionCatalog.java | 4 ++++ core/src/main/java/io/substrait/relation/Expand.java | 2 +- .../main/java/io/substrait/relation/ProtoRelConverter.java | 3 +++ .../main/java/io/substrait/relation/RelProtoConverter.java | 4 ++++ .../java/io/substrait/type/proto/ExtensionRoundtripTest.java | 1 + gradle/libs.versions.toml | 2 +- 6 files changed, 14 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/io/substrait/extension/DefaultExtensionCatalog.java b/core/src/main/java/io/substrait/extension/DefaultExtensionCatalog.java index 25dc76cad..6f5ecc852 100644 --- a/core/src/main/java/io/substrait/extension/DefaultExtensionCatalog.java +++ b/core/src/main/java/io/substrait/extension/DefaultExtensionCatalog.java @@ -63,6 +63,9 @@ public class DefaultExtensionCatalog { /** Identifier for extension types. */ public static final String EXTENSION_TYPES = "extension:io.substrait:extension_types"; + /** Extension identifier for unsigned integer types and functions. */ + public static final String UNSIGNED_INTEGERS = "extension:io.substrait:unsigned_integers"; + /** Default collection of built-in extensions loaded from YAML resources. */ public static final SimpleExtension.ExtensionCollection DEFAULT_COLLECTION = loadDefaultCollection(); @@ -93,6 +96,7 @@ private static SimpleExtension.ExtensionCollection loadDefaultCollection() { .collect(Collectors.toList()); defaultFiles.add("/substrait/extensions/extension_types.yaml"); + defaultFiles.add("/substrait/extensions/unsigned_integers.yaml"); return SimpleExtension.load(defaultFiles); } diff --git a/core/src/main/java/io/substrait/relation/Expand.java b/core/src/main/java/io/substrait/relation/Expand.java index 33af9033b..4573493e5 100644 --- a/core/src/main/java/io/substrait/relation/Expand.java +++ b/core/src/main/java/io/substrait/relation/Expand.java @@ -13,7 +13,7 @@ */ @Value.Enclosing @Value.Immutable -public abstract class Expand extends SingleInputRel { +public abstract class Expand extends SingleInputRel implements HasExtension { /** * Returns the fields produced by the expand operation. diff --git a/core/src/main/java/io/substrait/relation/ProtoRelConverter.java b/core/src/main/java/io/substrait/relation/ProtoRelConverter.java index ed7f08e2c..55b0e326f 100644 --- a/core/src/main/java/io/substrait/relation/ProtoRelConverter.java +++ b/core/src/main/java/io/substrait/relation/ProtoRelConverter.java @@ -962,6 +962,9 @@ protected Expand newExpand(ExpandRel rel) { .remap(optionalRelmap(rel.getCommon())) .hint(optionalHint(rel.getCommon())) .relAnchor(optionalRelAnchor(rel.getCommon())); + if (rel.hasAdvancedExtension()) { + builder.extension(protoExtensionConverter.fromProto(rel.getAdvancedExtension())); + } return builder.build(); } diff --git a/core/src/main/java/io/substrait/relation/RelProtoConverter.java b/core/src/main/java/io/substrait/relation/RelProtoConverter.java index 0096a872f..7333d22dc 100644 --- a/core/src/main/java/io/substrait/relation/RelProtoConverter.java +++ b/core/src/main/java/io/substrait/relation/RelProtoConverter.java @@ -814,6 +814,10 @@ public Rel visit(Expand expand, EmptyVisitationContext context) throws RuntimeEx "Consistent or Switching fields must be set for the Expand relation."); } }); + + expand + .getExtension() + .ifPresent(ae -> builder.setAdvancedExtension(extensionProtoConverter.toProto(ae))); return Rel.newBuilder().setExpand(builder).build(); } diff --git a/core/src/test/java/io/substrait/type/proto/ExtensionRoundtripTest.java b/core/src/test/java/io/substrait/type/proto/ExtensionRoundtripTest.java index 048c393e0..8b2c31dd9 100644 --- a/core/src/test/java/io/substrait/type/proto/ExtensionRoundtripTest.java +++ b/core/src/test/java/io/substrait/type/proto/ExtensionRoundtripTest.java @@ -260,6 +260,7 @@ void expand() { Expand.builder() .from(sb.expand(__ -> Collections.emptyList(), commonTable)) .commonExtension(commonExtension) + .extension(relExtension) .build(); verifyRoundTrip(rel); } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 7de44faba..ee6b87548 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -29,7 +29,7 @@ spark-3-5 = "3.5.4" spark-4-0 = "4.0.2" spotless = "8.9.0" # substrait-packaging artifacts, versioned by the Substrait spec release they are generated from. -substrait-packaging = "0.99.0" +substrait-packaging = "0.100.0" testcontainers = "2.0.5" validator = "3.0.6" From ccc465840072895abeb58ea115a6eb50e02f1dfc Mon Sep 17 00:00:00 2001 From: Anas Ismail Khan Date: Wed, 12 Aug 2026 15:21:00 +0500 Subject: [PATCH 2/2] add unsigned_integers to DialectGenerator.scala and generate spark_dialect.yaml --- spark/spark_dialect.yaml | 31 +++++++++++++++++++ .../spark/utils/DialectGenerator.scala | 1 + 2 files changed, 32 insertions(+) diff --git a/spark/spark_dialect.yaml b/spark/spark_dialect.yaml index 52c9cb47a..bc553228b 100644 --- a/spark/spark_dialect.yaml +++ b/spark/spark_dialect.yaml @@ -127,6 +127,7 @@ supported_relations: - "UNION_ALL" dependencies: rounding: "extension:io.substrait:functions_rounding" + unsigned_integers: "extension:io.substrait:unsigned_integers" comparison: "extension:io.substrait:functions_comparison" logarithmic: "extension:io.substrait:functions_logarithmic" datetime: "extension:io.substrait:functions_datetime" @@ -805,6 +806,36 @@ supported_aggregate_functions: - "pt" - "pts" - "ptstz" +- source: "unsigned_integers" + name: "max" + system_metadata: + name: "max" + notation: "FUNCTION" + supported_impls: + - "u!u16" + - "u!u32" + - "u!u64" + - "u!u8" +- source: "unsigned_integers" + name: "min" + system_metadata: + name: "min" + notation: "FUNCTION" + supported_impls: + - "u!u16" + - "u!u32" + - "u!u64" + - "u!u8" +- source: "unsigned_integers" + name: "sum" + system_metadata: + name: "sum" + notation: "FUNCTION" + supported_impls: + - "u!u16" + - "u!u32" + - "u!u64" + - "u!u8" supported_window_functions: - source: "arithmetic" name: "cume_dist" diff --git a/spark/src/main/scala/io/substrait/spark/utils/DialectGenerator.scala b/spark/src/main/scala/io/substrait/spark/utils/DialectGenerator.scala index 0df9776ee..6e11be743 100644 --- a/spark/src/main/scala/io/substrait/spark/utils/DialectGenerator.scala +++ b/spark/src/main/scala/io/substrait/spark/utils/DialectGenerator.scala @@ -63,6 +63,7 @@ class DialectGenerator { "extension:io.substrait:functions_rounding" -> "rounding", "extension:io.substrait:functions_rounding_decimal" -> "rounding_decimal", "extension:io.substrait:functions_string" -> "string", + "extension:io.substrait:unsigned_integers" -> "unsigned_integers", "extension:substrait:spark" -> "spark" )