From 92956bd56a89095f3244e105816bc35eac679b23 Mon Sep 17 00:00:00 2001 From: Christoph Mettler Date: Wed, 11 Mar 2026 13:29:06 +0100 Subject: [PATCH 1/9] Add source-generated Arrow serialization library Adds Apache.Arrow.Serialization with a Roslyn incremental source generator that emits compile-time Arrow schema derivation, serialization, and deserialization for types marked with [ArrowSerializable]. - Runtime library (Apache.Arrow.Serialization): attributes, helpers, IPC extension methods, reflection-based RecordBatchBuilder - Source generator (Apache.Arrow.Serialization.Generator): code emission for 31+ type mappings, polymorphism, custom converters, callbacks - Test suite: 197 tests covering all supported types and features - Integrated into solution, central package management, Apache 2.0 headers Co-Authored-By: Claude Opus 4.6 --- Apache.Arrow.sln | 51 + Directory.Packages.props | 2 + ...pache.Arrow.Serialization.Generator.csproj | 17 + .../ArrowSerializerGenerator.cs | 855 +++++ .../CodeEmitter.cs | 3115 +++++++++++++++++ .../JsonSchemaEmitter.cs | 195 ++ .../Models.cs | 151 + .../PolymorphicCodeEmitter.cs | 816 +++++ .../Apache.Arrow.Serialization.csproj | 20 + .../ArrowArrayHelper.cs | 409 +++ .../ArrowSerializerExtensions.cs | 137 + src/Apache.Arrow.Serialization/Attributes.cs | 173 + .../IArrowSerializer.cs | 70 + src/Apache.Arrow.Serialization/README.md | 620 ++++ .../RecordBatchBuilder.cs | 711 ++++ .../Apache.Arrow.Serialization.Tests.csproj | 26 + .../DiagnosticTests.cs | 255 ++ .../RecordBatchBuilderTests.cs | 400 +++ .../SerializationTests.cs | 2277 ++++++++++++ .../TestTypes.cs | 673 ++++ 20 files changed, 10973 insertions(+) create mode 100644 src/Apache.Arrow.Serialization.Generator/Apache.Arrow.Serialization.Generator.csproj create mode 100644 src/Apache.Arrow.Serialization.Generator/ArrowSerializerGenerator.cs create mode 100644 src/Apache.Arrow.Serialization.Generator/CodeEmitter.cs create mode 100644 src/Apache.Arrow.Serialization.Generator/JsonSchemaEmitter.cs create mode 100644 src/Apache.Arrow.Serialization.Generator/Models.cs create mode 100644 src/Apache.Arrow.Serialization.Generator/PolymorphicCodeEmitter.cs create mode 100644 src/Apache.Arrow.Serialization/Apache.Arrow.Serialization.csproj create mode 100644 src/Apache.Arrow.Serialization/ArrowArrayHelper.cs create mode 100644 src/Apache.Arrow.Serialization/ArrowSerializerExtensions.cs create mode 100644 src/Apache.Arrow.Serialization/Attributes.cs create mode 100644 src/Apache.Arrow.Serialization/IArrowSerializer.cs create mode 100644 src/Apache.Arrow.Serialization/README.md create mode 100644 src/Apache.Arrow.Serialization/RecordBatchBuilder.cs create mode 100644 test/Apache.Arrow.Serialization.Tests/Apache.Arrow.Serialization.Tests.csproj create mode 100644 test/Apache.Arrow.Serialization.Tests/DiagnosticTests.cs create mode 100644 test/Apache.Arrow.Serialization.Tests/RecordBatchBuilderTests.cs create mode 100644 test/Apache.Arrow.Serialization.Tests/SerializationTests.cs create mode 100644 test/Apache.Arrow.Serialization.Tests/TestTypes.cs diff --git a/Apache.Arrow.sln b/Apache.Arrow.sln index 26bd326c..310625a4 100644 --- a/Apache.Arrow.sln +++ b/Apache.Arrow.sln @@ -38,6 +38,16 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Apache.Arrow.Operations", " EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Apache.Arrow.Operations.Tests", "test\Apache.Arrow.Operations.Tests\Apache.Arrow.Operations.Tests.csproj", "{73EBE132-AE05-4C32-9525-515F4768156B}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Apache.Arrow.Serialization", "src\Apache.Arrow.Serialization\Apache.Arrow.Serialization.csproj", "{E0C418BE-DD55-4FB1-973E-272B142BAA9E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Apache.Arrow.Serialization.Generator", "src\Apache.Arrow.Serialization.Generator\Apache.Arrow.Serialization.Generator.csproj", "{FD8B13D7-16F4-4DBF-BB25-13EA5131EE03}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{0C88DD14-F956-CE84-757C-A364CCF449FC}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Apache.Arrow.Serialization.Tests", "test\Apache.Arrow.Serialization.Tests\Apache.Arrow.Serialization.Tests.csproj", "{3726633C-7093-40A1-8ABB-13A5CD64033A}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -264,10 +274,51 @@ Global {73EBE132-AE05-4C32-9525-515F4768156B}.Release|x64.Build.0 = Release|Any CPU {73EBE132-AE05-4C32-9525-515F4768156B}.Release|x86.ActiveCfg = Release|Any CPU {73EBE132-AE05-4C32-9525-515F4768156B}.Release|x86.Build.0 = Release|Any CPU + {E0C418BE-DD55-4FB1-973E-272B142BAA9E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E0C418BE-DD55-4FB1-973E-272B142BAA9E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E0C418BE-DD55-4FB1-973E-272B142BAA9E}.Debug|x64.ActiveCfg = Debug|Any CPU + {E0C418BE-DD55-4FB1-973E-272B142BAA9E}.Debug|x64.Build.0 = Debug|Any CPU + {E0C418BE-DD55-4FB1-973E-272B142BAA9E}.Debug|x86.ActiveCfg = Debug|Any CPU + {E0C418BE-DD55-4FB1-973E-272B142BAA9E}.Debug|x86.Build.0 = Debug|Any CPU + {E0C418BE-DD55-4FB1-973E-272B142BAA9E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E0C418BE-DD55-4FB1-973E-272B142BAA9E}.Release|Any CPU.Build.0 = Release|Any CPU + {E0C418BE-DD55-4FB1-973E-272B142BAA9E}.Release|x64.ActiveCfg = Release|Any CPU + {E0C418BE-DD55-4FB1-973E-272B142BAA9E}.Release|x64.Build.0 = Release|Any CPU + {E0C418BE-DD55-4FB1-973E-272B142BAA9E}.Release|x86.ActiveCfg = Release|Any CPU + {E0C418BE-DD55-4FB1-973E-272B142BAA9E}.Release|x86.Build.0 = Release|Any CPU + {FD8B13D7-16F4-4DBF-BB25-13EA5131EE03}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FD8B13D7-16F4-4DBF-BB25-13EA5131EE03}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FD8B13D7-16F4-4DBF-BB25-13EA5131EE03}.Debug|x64.ActiveCfg = Debug|Any CPU + {FD8B13D7-16F4-4DBF-BB25-13EA5131EE03}.Debug|x64.Build.0 = Debug|Any CPU + {FD8B13D7-16F4-4DBF-BB25-13EA5131EE03}.Debug|x86.ActiveCfg = Debug|Any CPU + {FD8B13D7-16F4-4DBF-BB25-13EA5131EE03}.Debug|x86.Build.0 = Debug|Any CPU + {FD8B13D7-16F4-4DBF-BB25-13EA5131EE03}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FD8B13D7-16F4-4DBF-BB25-13EA5131EE03}.Release|Any CPU.Build.0 = Release|Any CPU + {FD8B13D7-16F4-4DBF-BB25-13EA5131EE03}.Release|x64.ActiveCfg = Release|Any CPU + {FD8B13D7-16F4-4DBF-BB25-13EA5131EE03}.Release|x64.Build.0 = Release|Any CPU + {FD8B13D7-16F4-4DBF-BB25-13EA5131EE03}.Release|x86.ActiveCfg = Release|Any CPU + {FD8B13D7-16F4-4DBF-BB25-13EA5131EE03}.Release|x86.Build.0 = Release|Any CPU + {3726633C-7093-40A1-8ABB-13A5CD64033A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3726633C-7093-40A1-8ABB-13A5CD64033A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3726633C-7093-40A1-8ABB-13A5CD64033A}.Debug|x64.ActiveCfg = Debug|Any CPU + {3726633C-7093-40A1-8ABB-13A5CD64033A}.Debug|x64.Build.0 = Debug|Any CPU + {3726633C-7093-40A1-8ABB-13A5CD64033A}.Debug|x86.ActiveCfg = Debug|Any CPU + {3726633C-7093-40A1-8ABB-13A5CD64033A}.Debug|x86.Build.0 = Debug|Any CPU + {3726633C-7093-40A1-8ABB-13A5CD64033A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3726633C-7093-40A1-8ABB-13A5CD64033A}.Release|Any CPU.Build.0 = Release|Any CPU + {3726633C-7093-40A1-8ABB-13A5CD64033A}.Release|x64.ActiveCfg = Release|Any CPU + {3726633C-7093-40A1-8ABB-13A5CD64033A}.Release|x64.Build.0 = Release|Any CPU + {3726633C-7093-40A1-8ABB-13A5CD64033A}.Release|x86.ActiveCfg = Release|Any CPU + {3726633C-7093-40A1-8ABB-13A5CD64033A}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {E0C418BE-DD55-4FB1-973E-272B142BAA9E} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {FD8B13D7-16F4-4DBF-BB25-13EA5131EE03} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {3726633C-7093-40A1-8ABB-13A5CD64033A} = {0C88DD14-F956-CE84-757C-A364CCF449FC} + EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {FD0BB617-6031-4844-B99D-B331E335B572} EndGlobalSection diff --git a/Directory.Packages.props b/Directory.Packages.props index 70155384..a4a12e79 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -32,6 +32,8 @@ + + diff --git a/src/Apache.Arrow.Serialization.Generator/Apache.Arrow.Serialization.Generator.csproj b/src/Apache.Arrow.Serialization.Generator/Apache.Arrow.Serialization.Generator.csproj new file mode 100644 index 00000000..b0e58f98 --- /dev/null +++ b/src/Apache.Arrow.Serialization.Generator/Apache.Arrow.Serialization.Generator.csproj @@ -0,0 +1,17 @@ + + + + netstandard2.0 + enable + true + true + false + RS2008 + + + + + + + + diff --git a/src/Apache.Arrow.Serialization.Generator/ArrowSerializerGenerator.cs b/src/Apache.Arrow.Serialization.Generator/ArrowSerializerGenerator.cs new file mode 100644 index 00000000..27f04265 --- /dev/null +++ b/src/Apache.Arrow.Serialization.Generator/ArrowSerializerGenerator.cs @@ -0,0 +1,855 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; + +#nullable enable + +namespace Apache.Arrow.Serialization.Generator +{ + +[Generator(LanguageNames.CSharp)] +public class ArrowSerializerGenerator : IIncrementalGenerator +{ + private static readonly DiagnosticDescriptor NonPartialType = new DiagnosticDescriptor( + "ARROW001", "Type must be partial", + "[ArrowSerializable] type '{0}' must be declared as partial", + "ArrowSerialization", DiagnosticSeverity.Error, true); + + private static readonly DiagnosticDescriptor NoMatchingConstructor = new DiagnosticDescriptor( + "ARROW002", "No matching constructor for readonly fields", + "[ArrowSerializable] type '{0}' has readonly fields but no public constructor with matching parameters", + "ArrowSerialization", DiagnosticSeverity.Error, true); + + private static readonly DiagnosticDescriptor UnsupportedMemberType = new DiagnosticDescriptor( + "ARROW003", "Unsupported member type", + "Member '{0}' on type '{1}' has unsupported type '{2}'", + "ArrowSerialization", DiagnosticSeverity.Error, true); + + private static readonly DiagnosticDescriptor DuplicateFieldName = new DiagnosticDescriptor( + "ARROW004", "Duplicate Arrow field name", + "Members '{0}' and '{1}' on type '{2}' both map to Arrow field name '{3}'", + "ArrowSerialization", DiagnosticSeverity.Error, true); + + private static readonly DiagnosticDescriptor NonSettableProperty = new DiagnosticDescriptor( + "ARROW005", "Property is not settable", + "Property '{0}' on type '{1}' has no set or init accessor and cannot be deserialized", + "ArrowSerialization", DiagnosticSeverity.Error, true); + + private static readonly DiagnosticDescriptor ArrowAttributeOnIgnoredMember = new DiagnosticDescriptor( + "ARROW006", "Arrow attribute on non-serialized member", + "Member '{0}' on type '{1}' has Arrow attributes but is not serialized because it is {2}", + "ArrowSerialization", DiagnosticSeverity.Warning, true); + + public void Initialize(IncrementalGeneratorInitializationContext context) + { + var emitSchemaJson = context.AnalyzerConfigOptionsProvider.Select(static (options, _) => + { + options.GlobalOptions.TryGetValue("build_property.ArrowSerializerEmitSchemaJson", out var value); + return string.Equals(value, "true", StringComparison.OrdinalIgnoreCase); + }); + + var types = context.SyntaxProvider.ForAttributeWithMetadataName( + "Apache.Arrow.Serialization.ArrowSerializableAttribute", + predicate: static (node, _) => node is RecordDeclarationSyntax || node is ClassDeclarationSyntax || node is StructDeclarationSyntax, + transform: static (ctx, _) => GetTypeModel(ctx)) + .Where(static m => m != null) + .Select(static (m, _) => m!); + + var typesWithConfig = types.Combine(emitSchemaJson); + context.RegisterSourceOutput(typesWithConfig, static (spc, pair) => Execute(spc, pair.Left, pair.Right)); + + var polyTypes = context.SyntaxProvider.ForAttributeWithMetadataName( + "Apache.Arrow.Serialization.ArrowPolymorphicAttribute", + predicate: static (node, _) => node is RecordDeclarationSyntax || node is ClassDeclarationSyntax || node is StructDeclarationSyntax || node is InterfaceDeclarationSyntax, + transform: static (ctx, _) => GetPolymorphicModel(ctx)) + .Where(static m => m != null) + .Select(static (m, _) => m!); + + context.RegisterSourceOutput(polyTypes, static (spc, model) => ExecutePolymorphic(spc, model)); + } + + private static TypeModel? GetTypeModel(GeneratorAttributeSyntaxContext ctx) + { + if (!(ctx.TargetSymbol is INamedTypeSymbol typeSymbol)) + return null; + + var properties = new List(); + var ignoredMemberWarnings = new List(); + int declOrder = 0; + foreach (var member in typeSymbol.GetMembers()) + { + // Support both properties and fields + ITypeSymbol memberType; + string memberName; + bool isField = false; + if (member is IPropertySymbol prop) + { + string? skipReason = null; + if (prop.IsStatic) skipReason = "static"; + else if (prop.IsIndexer) skipReason = "an indexer"; + else if (prop.DeclaredAccessibility != Accessibility.Public) skipReason = "not public"; + + if (skipReason != null) + { + if (HasArrowAttributes(prop)) + ignoredMemberWarnings.Add(new DiagnosticInfo { Id = "ARROW006", Message = $"{prop.Name}\t{typeSymbol.Name}\t{skipReason}", IsError = false }); + continue; + } + memberType = prop.Type; + memberName = prop.Name; + } + else if (member is IFieldSymbol field) + { + if (field.IsImplicitlyDeclared) + continue; + + string? skipReason = null; + if (field.IsStatic || field.IsConst) skipReason = "static or const"; + else if (field.DeclaredAccessibility != Accessibility.Public) skipReason = "not public"; + + if (skipReason != null) + { + if (HasArrowAttributes(field)) + ignoredMemberWarnings.Add(new DiagnosticInfo { Id = "ARROW006", Message = $"{field.Name}\t{typeSymbol.Name}\t{skipReason}", IsError = false }); + continue; + } + memberType = field.Type; + memberName = field.Name; + isField = true; + } + else + { + continue; + } + + // Check for [ArrowIgnore] + bool isTransient = false; + string? arrowTypeName = null; + string? converterTypeName = null; + string? elementTypeOverride = null; + string? keyTypeOverride = null; + string? valueTypeOverride = null; + string? fieldName = null; + int order = int.MaxValue; + var propMetadata = new List>(); + + foreach (var attr in member.GetAttributes()) + { + var attrName = attr.AttributeClass?.ToDisplayString(); + if (attrName == "Apache.Arrow.Serialization.ArrowIgnoreAttribute") + { + isTransient = true; + } + else if (attrName == "Apache.Arrow.Serialization.ArrowTypeAttribute") + { + if (attr.ConstructorArguments.Length > 0) + arrowTypeName = attr.ConstructorArguments[0].Value as string; + foreach (var named in attr.NamedArguments) + { + if (named.Key == "Converter" && named.Value.Value is INamedTypeSymbol converterSym) + converterTypeName = converterSym.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat); + else if (named.Key == "ElementType" && named.Value.Value is string et) + elementTypeOverride = et; + else if (named.Key == "KeyType" && named.Value.Value is string kt) + keyTypeOverride = kt; + else if (named.Key == "ValueType" && named.Value.Value is string vt) + valueTypeOverride = vt; + } + } + else if (attrName == "Apache.Arrow.Serialization.ArrowFieldAttribute") + { + if (attr.ConstructorArguments.Length > 0) + fieldName = attr.ConstructorArguments[0].Value as string; + foreach (var named in attr.NamedArguments) + { + if (named.Key == "Order" && named.Value.Value is int o) + order = o; + } + } + else if (attrName == "Apache.Arrow.Serialization.ArrowMetadataAttribute") + { + if (attr.ConstructorArguments.Length >= 2) + { + var key = attr.ConstructorArguments[0].Value as string; + var val = attr.ConstructorArguments[1].Value as string; + if (key != null && val != null) + propMetadata.Add(new KeyValuePair(key, val)); + } + } + } + + if (isTransient) + continue; + + bool isNullable = memberType.NullableAnnotation == NullableAnnotation.Annotated; + if (isNullable && memberType is INamedTypeSymbol nullableType && nullableType.OriginalDefinition.SpecialType == SpecialType.System_Nullable_T) + { + memberType = nullableType.TypeArguments[0]; + } + + var typeInfo = AnalyzeType(memberType, isNullable); + if (converterTypeName != null) + typeInfo = new TypeInfo { Kind = TypeKind2.Custom, FullTypeName = typeInfo.FullTypeName, IsNullable = isNullable }; + else if (arrowTypeName != null) + typeInfo = typeInfo.WithOverride(arrowTypeName); + + // Apply element/key/value type overrides for collections + if (elementTypeOverride != null && typeInfo.ElementType != null) + typeInfo.ElementType = typeInfo.ElementType.WithOverride(elementTypeOverride); + if (keyTypeOverride != null && typeInfo.KeyType != null) + typeInfo.KeyType = typeInfo.KeyType.WithOverride(keyTypeOverride); + if (valueTypeOverride != null && typeInfo.ValueType != null) + typeInfo.ValueType = typeInfo.ValueType.WithOverride(valueTypeOverride); + + properties.Add(new PropertyModel + { + PropertyName = memberName, + FieldName = fieldName ?? memberName, + Order = order, + DeclOrder = declOrder++, + Type = typeInfo, + IsNullable = isNullable, + HasDefaultValue = isField ? false : HasDefaultValue((IPropertySymbol)member, typeSymbol), + Metadata = propMetadata, + ConverterTypeName = converterTypeName, + IsField = isField, + }); + } + + properties.Sort((a, b) => + { + int cmp = a.Order.CompareTo(b.Order); + return cmp != 0 ? cmp : a.DeclOrder.CompareTo(b.DeclOrder); + }); + + var typeMetadata = new List>(); + foreach (var attr in typeSymbol.GetAttributes()) + { + if (attr.AttributeClass?.ToDisplayString() == "Apache.Arrow.Serialization.ArrowMetadataAttribute" + && attr.ConstructorArguments.Length >= 2) + { + var key = attr.ConstructorArguments[0].Value as string; + var val = attr.ConstructorArguments[1].Value as string; + if (key != null && val != null) + typeMetadata.Add(new KeyValuePair(key, val)); + } + } + + // Determine if constructor-based deserialization is needed + // This is required when any member is a readonly field (can't use object initializer) + bool needsConstructor = false; + foreach (var p in properties) + { + if (p.IsField) { needsConstructor = true; break; } + } + + List? ctorParams = null; + if (needsConstructor) + { + ctorParams = ResolveConstructor(typeSymbol, properties); + } + + // Validate and collect diagnostics + var diagnostics = new List(ignoredMemberWarnings); + var typeName = typeSymbol.Name; + + // ARROW001: non-partial type + if (ctx.TargetNode is TypeDeclarationSyntax tds && !tds.Modifiers.Any(Microsoft.CodeAnalysis.CSharp.SyntaxKind.PartialKeyword)) + { + diagnostics.Add(new DiagnosticInfo { Id = "ARROW001", Message = typeName, IsError = true }); + } + + // ARROW002: readonly fields with no matching constructor + if (needsConstructor && ctorParams == null) + { + diagnostics.Add(new DiagnosticInfo { Id = "ARROW002", Message = typeName, IsError = true }); + } + + // ARROW003: unsupported member types + foreach (var p in properties) + { + if (p.Type.Kind == TypeKind2.Unknown) + diagnostics.Add(new DiagnosticInfo { Id = "ARROW003", Message = $"{p.PropertyName}\t{typeName}\t{p.Type.FullTypeName}", IsError = true }); + } + + // ARROW004: duplicate Arrow field names + var seenFields = new Dictionary(); + foreach (var p in properties) + { + if (seenFields.TryGetValue(p.FieldName, out var existingMember)) + diagnostics.Add(new DiagnosticInfo { Id = "ARROW004", Message = $"{existingMember}\t{p.PropertyName}\t{typeName}\t{p.FieldName}", IsError = true }); + else + seenFields[p.FieldName] = p.PropertyName; + } + + // ARROW005: non-settable properties (no set/init, not in constructor) + if (ctorParams == null) + { + foreach (var member in typeSymbol.GetMembers()) + { + if (!(member is IPropertySymbol ps) || ps.IsStatic || ps.IsIndexer) + continue; + if (ps.DeclaredAccessibility != Accessibility.Public) + continue; + if (ps.GetAttributes().Any(a => a.AttributeClass?.ToDisplayString() == "Apache.Arrow.Serialization.ArrowIgnoreAttribute")) + continue; + if (ps.SetMethod == null) + diagnostics.Add(new DiagnosticInfo { Id = "ARROW005", Message = $"{ps.Name}\t{typeName}", IsError = true }); + } + } + + return new TypeModel + { + Namespace = typeSymbol.ContainingNamespace.IsGlobalNamespace ? null : typeSymbol.ContainingNamespace.ToDisplayString(), + TypeName = typeSymbol.Name, + FullTypeName = typeSymbol.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat), + Properties = properties, + IsRecord = typeSymbol.IsRecord, + IsValueType = typeSymbol.IsValueType, + Metadata = typeMetadata, + HasArrowSerializableBase = HasArrowSerializableBaseType(typeSymbol), + ConstructorParams = ctorParams, + HasSerializationCallback = ImplementsInterface(typeSymbol, "Apache.Arrow.Serialization.IArrowSerializationCallback"), + Diagnostics = diagnostics, + }; + } + + private static bool HasArrowSerializableBaseType(INamedTypeSymbol typeSymbol) + { + var baseType = typeSymbol.BaseType; + while (baseType != null && baseType.SpecialType != SpecialType.System_Object) + { + foreach (var attr in baseType.GetAttributes()) + { + var attrName = attr.AttributeClass?.ToDisplayString(); + if (attrName == "Apache.Arrow.Serialization.ArrowSerializableAttribute" || + attrName == "Apache.Arrow.Serialization.ArrowPolymorphicAttribute") + return true; + } + baseType = baseType.BaseType; + } + return false; + } + + private static bool ImplementsInterface(INamedTypeSymbol typeSymbol, string interfaceFullName) + { + foreach (var iface in typeSymbol.AllInterfaces) + { + if (iface.ToDisplayString() == interfaceFullName) + return true; + } + return false; + } + + private static bool HasArrowAttributes(ISymbol member) + { + foreach (var attr in member.GetAttributes()) + { + var name = attr.AttributeClass?.ToDisplayString(); + if (name != null && name.StartsWith("Apache.Arrow.Serialization.Arrow")) + return true; + } + return false; + } + + private static bool HasDefaultValue(IPropertySymbol prop, INamedTypeSymbol containingType) + { + // Check if the property has a default in the constructor or an initializer + // For simplicity, we check if it's nullable or if there are constructors with optional params + if (prop.NullableAnnotation == NullableAnnotation.Annotated) + return true; + + foreach (var ctor in containingType.Constructors) + { + if (ctor.IsImplicitlyDeclared) continue; + foreach (var param in ctor.Parameters) + { + if (param.Name.Equals(prop.Name, StringComparison.OrdinalIgnoreCase) && param.HasExplicitDefaultValue) + return true; + } + } + + return false; + } + + /// + /// Find the best constructor for deserialization. Picks the constructor whose parameters + /// best match the serialized members (by name, case-insensitive). Prefers the constructor + /// with the most matching parameters. + /// + private static List? ResolveConstructor(INamedTypeSymbol typeSymbol, List properties) + { + var memberNames = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var p in properties) + memberNames.Add(p.PropertyName); + + IMethodSymbol? bestCtor = null; + int bestMatch = -1; + + foreach (var ctor in typeSymbol.Constructors) + { + if (ctor.IsStatic || ctor.IsImplicitlyDeclared) + continue; + if (ctor.DeclaredAccessibility != Accessibility.Public) + continue; + + // Count how many required params match members + int matchCount = 0; + bool allRequiredMatch = true; + foreach (var param in ctor.Parameters) + { + if (memberNames.Contains(param.Name)) + matchCount++; + else if (!param.HasExplicitDefaultValue) + { + allRequiredMatch = false; + break; + } + } + + if (allRequiredMatch && matchCount > bestMatch) + { + bestMatch = matchCount; + bestCtor = ctor; + } + } + + if (bestCtor == null) + return null; + + var result = new List(); + foreach (var param in bestCtor.Parameters) + { + result.Add(new ConstructorParamModel + { + Name = param.Name, + HasDefaultValue = param.HasExplicitDefaultValue, + }); + } + return result; + } + + private static TypeInfo AnalyzeType(ITypeSymbol type, bool isNullable) + { + // Unwrap Nullable for value types + if (type is INamedTypeSymbol nt && nt.OriginalDefinition.SpecialType == SpecialType.System_Nullable_T) + { + type = nt.TypeArguments[0]; + isNullable = true; + } + + var fullName = type.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat); + + // Check for enum + if (type.TypeKind == TypeKind.Enum) + { + return new TypeInfo + { + Kind = TypeKind2.Enum, + FullTypeName = fullName, + IsNullable = isNullable, + }; + } + + // Check for nested ArrowSerializable + foreach (var attr in type.GetAttributes()) + { + if (attr.AttributeClass?.ToDisplayString() == "Apache.Arrow.Serialization.ArrowSerializableAttribute") + { + return new TypeInfo + { + Kind = TypeKind2.NestedRecord, + FullTypeName = fullName, + IsNullable = isNullable, + }; + } + } + + // Check for collections + if (type is INamedTypeSymbol namedType) + { + var origDef = namedType.OriginalDefinition.ToDisplayString(); + + // List, IList, IReadOnlyList + if (origDef == "System.Collections.Generic.List" + || origDef == "System.Collections.Generic.IList" + || origDef == "System.Collections.Generic.IReadOnlyList" + || origDef == "System.Collections.Generic.IEnumerable" + || origDef == "System.Collections.Generic.ICollection" + || origDef == "System.Collections.Generic.IReadOnlyCollection") + { + var elemType = AnalyzeType(namedType.TypeArguments[0], false); + return new TypeInfo + { + Kind = TypeKind2.List, + FullTypeName = fullName, + IsNullable = isNullable, + ElementType = elemType, + }; + } + + // Dictionary, IReadOnlyDictionary + if (origDef == "System.Collections.Generic.Dictionary" + || origDef == "System.Collections.Generic.IDictionary" + || origDef == "System.Collections.Generic.IReadOnlyDictionary") + { + var keyType = AnalyzeType(namedType.TypeArguments[0], false); + var valueType = AnalyzeType(namedType.TypeArguments[1], false); + return new TypeInfo + { + Kind = TypeKind2.Dictionary, + FullTypeName = fullName, + IsNullable = isNullable, + KeyType = keyType, + ValueType = valueType, + }; + } + + // HashSet, ISet, IReadOnlySet + if (origDef == "System.Collections.Generic.HashSet" + || origDef == "System.Collections.Generic.ISet" + || origDef == "System.Collections.Generic.IReadOnlySet" + || origDef == "System.Collections.Frozen.FrozenSet") + { + var elemType = AnalyzeType(namedType.TypeArguments[0], false); + return new TypeInfo + { + Kind = TypeKind2.Set, + FullTypeName = fullName, + IsNullable = isNullable, + ElementType = elemType, + }; + } + } + + // Array T[] — but byte[] is Binary, not Array + if (type is IArrayTypeSymbol arrayType) + { + if (arrayType.ElementType.SpecialType == SpecialType.System_Byte) + { + return new TypeInfo + { + Kind = TypeKind2.Binary, + FullTypeName = fullName, + IsNullable = isNullable, + }; + } + + var elemType = AnalyzeType(arrayType.ElementType, false); + return new TypeInfo + { + Kind = TypeKind2.Array, + FullTypeName = fullName, + IsNullable = isNullable, + ElementType = elemType, + }; + } + + // Primitive types + var kind = fullName switch + { + "string" => TypeKind2.String, + "bool" => TypeKind2.Bool, + "byte" => TypeKind2.Byte, + "sbyte" => TypeKind2.SByte, + "short" => TypeKind2.Int16, + "ushort" => TypeKind2.UInt16, + "int" => TypeKind2.Int32, + "uint" => TypeKind2.UInt32, + "long" => TypeKind2.Int64, + "ulong" => TypeKind2.UInt64, + "float" => TypeKind2.Float, + "double" => TypeKind2.Double, + "decimal" => TypeKind2.Decimal, + "byte[]" => TypeKind2.Binary, + "System.ReadOnlyMemory" => TypeKind2.Binary, + "global::System.ReadOnlyMemory" => TypeKind2.Binary, + "System.DateTime" => TypeKind2.DateTime, + "global::System.DateTime" => TypeKind2.DateTime, + "System.DateTimeOffset" => TypeKind2.DateTimeOffset, + "global::System.DateTimeOffset" => TypeKind2.DateTimeOffset, + "System.DateOnly" => TypeKind2.DateOnly, + "global::System.DateOnly" => TypeKind2.DateOnly, + "System.TimeOnly" => TypeKind2.TimeOnly, + "global::System.TimeOnly" => TypeKind2.TimeOnly, + "System.TimeSpan" => TypeKind2.TimeSpan, + "global::System.TimeSpan" => TypeKind2.TimeSpan, + "System.Guid" => TypeKind2.Guid, + "global::System.Guid" => TypeKind2.Guid, + "System.Half" => TypeKind2.Half, + "global::System.Half" => TypeKind2.Half, + _ => TypeKind2.Unknown, + }; + + return new TypeInfo + { + Kind = kind, + FullTypeName = fullName, + IsNullable = isNullable, + }; + } + + private static void Execute(SourceProductionContext spc, TypeModel model, bool emitSchemaJson) + { + // Report diagnostics + foreach (var diag in model.Diagnostics) + { + var parts = diag.Message.Split('\t'); + var descriptor = diag.Id switch + { + "ARROW001" => NonPartialType, + "ARROW002" => NoMatchingConstructor, + "ARROW003" => UnsupportedMemberType, + "ARROW004" => DuplicateFieldName, + "ARROW005" => NonSettableProperty, + "ARROW006" => ArrowAttributeOnIgnoredMember, + _ => null, + }; + if (descriptor != null) + spc.ReportDiagnostic(Diagnostic.Create(descriptor, Location.None, parts)); + } + + // Don't emit code if there are errors + bool hasErrors = false; + foreach (var d in model.Diagnostics) + { + if (d.IsError) { hasErrors = true; break; } + } + if (hasErrors) return; + + var sb = new StringBuilder(); + var emitter = new CodeEmitter(sb, model); + emitter.Emit(); + spc.AddSource($"{model.TypeName}.ArrowSerializer.g.cs", sb.ToString()); + + if (emitSchemaJson) + { + var jsonSb = new StringBuilder(); + JsonSchemaEmitter.Emit(jsonSb, model); + spc.AddSource($"{model.TypeName}.ArrowSchemaJson.g.cs", jsonSb.ToString()); + } + } + + private static PolymorphicModel? GetPolymorphicModel(GeneratorAttributeSyntaxContext ctx) + { + if (!(ctx.TargetSymbol is INamedTypeSymbol typeSymbol)) + return null; + + // Read [ArrowPolymorphic] attribute + string discriminatorFieldName = "$type"; + foreach (var attr in typeSymbol.GetAttributes()) + { + if (attr.AttributeClass?.ToDisplayString() == "Apache.Arrow.Serialization.ArrowPolymorphicAttribute") + { + foreach (var named in attr.NamedArguments) + { + if (named.Key == "TypeDiscriminatorFieldName" && named.Value.Value is string s) + discriminatorFieldName = s; + } + } + } + + // Collect [ArrowDerivedType] attributes + var derivedTypes = new List(); + foreach (var attr in typeSymbol.GetAttributes()) + { + if (attr.AttributeClass?.ToDisplayString() != "Apache.Arrow.Serialization.ArrowDerivedTypeAttribute") + continue; + if (attr.ConstructorArguments.Length < 2) + continue; + + var derivedTypeSymbol = attr.ConstructorArguments[0].Value as INamedTypeSymbol; + var discriminator = attr.ConstructorArguments[1].Value as string; + if (derivedTypeSymbol == null || discriminator == null) + continue; + + var props = CollectProperties(derivedTypeSymbol); + derivedTypes.Add(new DerivedTypeInfo + { + TypeDiscriminator = discriminator, + FullTypeName = derivedTypeSymbol.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat), + TypeName = derivedTypeSymbol.Name, + Properties = props, + IsRecord = derivedTypeSymbol.IsRecord, + IsValueType = derivedTypeSymbol.IsValueType, + }); + } + + // Build union of all properties (deduplicated by FieldName, all nullable) + var unionProps = new List(); + var seenFields = new HashSet(); + foreach (var dt in derivedTypes) + { + foreach (var prop in dt.Properties) + { + if (seenFields.Add(prop.FieldName)) + { + // Make a nullable copy for the union schema + unionProps.Add(new PropertyModel + { + PropertyName = prop.PropertyName, + FieldName = prop.FieldName, + Order = prop.Order, + DeclOrder = prop.DeclOrder, + Type = prop.Type, + IsNullable = true, // always nullable in union schema + HasDefaultValue = true, + Metadata = prop.Metadata, + }); + } + } + } + + // Collect metadata + var typeMetadata = new List>(); + foreach (var attr in typeSymbol.GetAttributes()) + { + if (attr.AttributeClass?.ToDisplayString() == "Apache.Arrow.Serialization.ArrowMetadataAttribute" + && attr.ConstructorArguments.Length >= 2) + { + var key = attr.ConstructorArguments[0].Value as string; + var val = attr.ConstructorArguments[1].Value as string; + if (key != null && val != null) + typeMetadata.Add(new KeyValuePair(key, val)); + } + } + + return new PolymorphicModel + { + Namespace = typeSymbol.ContainingNamespace.IsGlobalNamespace ? null : typeSymbol.ContainingNamespace.ToDisplayString(), + TypeName = typeSymbol.Name, + FullTypeName = typeSymbol.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat), + TypeDiscriminatorFieldName = discriminatorFieldName, + IsInterface = typeSymbol.TypeKind == TypeKind.Interface, + IsRecord = typeSymbol.IsRecord, + DerivedTypes = derivedTypes, + UnionProperties = unionProps, + Metadata = typeMetadata, + }; + } + + /// + /// Collects properties from a type symbol (same logic as GetTypeModel but returns just the properties). + /// + private static List CollectProperties(INamedTypeSymbol typeSymbol) + { + var properties = new List(); + int declOrder = 0; + foreach (var member in typeSymbol.GetMembers()) + { + if (!(member is IPropertySymbol prop)) + continue; + if (prop.IsStatic || prop.IsIndexer) + continue; + if (prop.DeclaredAccessibility != Accessibility.Public) + continue; + + bool isTransient = false; + string? arrowTypeName = null; + string? converterTypeName2 = null; + string? fieldName = null; + int order = int.MaxValue; + var propMetadata = new List>(); + + foreach (var attr in prop.GetAttributes()) + { + var attrName = attr.AttributeClass?.ToDisplayString(); + if (attrName == "Apache.Arrow.Serialization.ArrowIgnoreAttribute") + isTransient = true; + else if (attrName == "Apache.Arrow.Serialization.ArrowTypeAttribute") + { + if (attr.ConstructorArguments.Length > 0) + arrowTypeName = attr.ConstructorArguments[0].Value as string; + foreach (var named in attr.NamedArguments) + { + if (named.Key == "Converter" && named.Value.Value is INamedTypeSymbol converterSym) + converterTypeName2 = converterSym.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat); + } + } + else if (attrName == "Apache.Arrow.Serialization.ArrowFieldAttribute") + { + if (attr.ConstructorArguments.Length > 0) + fieldName = attr.ConstructorArguments[0].Value as string; + foreach (var named in attr.NamedArguments) + { + if (named.Key == "Order" && named.Value.Value is int o) + order = o; + } + } + else if (attrName == "Apache.Arrow.Serialization.ArrowMetadataAttribute") + { + if (attr.ConstructorArguments.Length >= 2) + { + var key = attr.ConstructorArguments[0].Value as string; + var val = attr.ConstructorArguments[1].Value as string; + if (key != null && val != null) + propMetadata.Add(new KeyValuePair(key, val)); + } + } + } + + if (isTransient) + continue; + + var propType = prop.Type; + bool isNullable = propType.NullableAnnotation == NullableAnnotation.Annotated; + if (isNullable && propType is INamedTypeSymbol nullableType && nullableType.OriginalDefinition.SpecialType == SpecialType.System_Nullable_T) + propType = nullableType.TypeArguments[0]; + + var typeInfo = AnalyzeType(propType, isNullable); + if (converterTypeName2 != null) + typeInfo = new TypeInfo { Kind = TypeKind2.Custom, FullTypeName = typeInfo.FullTypeName, IsNullable = isNullable }; + else if (arrowTypeName != null) + typeInfo = typeInfo.WithOverride(arrowTypeName); + + properties.Add(new PropertyModel + { + PropertyName = prop.Name, + FieldName = fieldName ?? prop.Name, + Order = order, + DeclOrder = declOrder++, + Type = typeInfo, + IsNullable = isNullable, + HasDefaultValue = HasDefaultValue(prop, typeSymbol), + Metadata = propMetadata, + ConverterTypeName = converterTypeName2, + }); + } + + properties.Sort((a, b) => + { + int cmp = a.Order.CompareTo(b.Order); + return cmp != 0 ? cmp : a.DeclOrder.CompareTo(b.DeclOrder); + }); + + return properties; + } + + private static void ExecutePolymorphic(SourceProductionContext spc, PolymorphicModel model) + { + var sb = new StringBuilder(); + var emitter = new PolymorphicCodeEmitter(sb, model); + emitter.Emit(); + spc.AddSource($"{model.TypeName}.ArrowPolymorphic.g.cs", sb.ToString()); + } +} +} // namespace diff --git a/src/Apache.Arrow.Serialization.Generator/CodeEmitter.cs b/src/Apache.Arrow.Serialization.Generator/CodeEmitter.cs new file mode 100644 index 00000000..e515a58f --- /dev/null +++ b/src/Apache.Arrow.Serialization.Generator/CodeEmitter.cs @@ -0,0 +1,3115 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Text; + +#nullable enable + +namespace Apache.Arrow.Serialization.Generator +{ + +internal class CodeEmitter +{ + private readonly StringBuilder _sb; + private readonly TypeModel _model; + private int _indent; + + public CodeEmitter(StringBuilder sb, TypeModel model) + { + _sb = sb; + _model = model; + } + + public void Emit() + { + Line("// "); + Line("#nullable enable"); + Line("#pragma warning disable CS8629 // Nullable value type may be null"); + Line(); + Line("using System.Collections.Generic;"); + Line("using System.Linq;"); + Line("using Apache.Arrow;"); + Line("using Apache.Arrow.Arrays;"); + Line("using Apache.Arrow.Types;"); + Line("using Apache.Arrow.Serialization;"); + Line(); + + if (_model.Namespace != null) + { + Line($"namespace {_model.Namespace};"); + Line(); + } + + var typeKeyword = GetTypeKeyword(); + Line($"partial {typeKeyword} {_model.TypeName} : IArrowSerializer<{_model.TypeName}>"); + Line("{"); + _indent++; + + EmitSchemaField(); + Line(); + EmitSerialize(); + Line(); + EmitDeserialize(); + Line(); + EmitMultiRowSerialize(); + Line(); + EmitMultiRowDeserialize(); + + _indent--; + Line("}"); + } + + private void EmitSchemaField() + { + Line($"private static readonly Schema _arrowSchema = new Schema.Builder()"); + _indent++; + foreach (var prop in _model.Properties) + { + var arrowType = prop.ConverterTypeName != null + ? $"new {prop.ConverterTypeName}().ArrowType" + : GetArrowTypeExpression(prop.Type); + var nullable = prop.IsNullable ? "true" : "false"; + if (prop.Metadata.Count > 0) + { + var metadataEntries = string.Join(", ", + prop.Metadata.ConvertAll(kv => + $"new System.Collections.Generic.KeyValuePair(\"{Escape(kv.Key)}\", \"{Escape(kv.Value)}\")")); + Line($".Field(new Field(\"{prop.FieldName}\", {arrowType}, {nullable}, new[] {{ {metadataEntries} }}))"); + } + else + { + Line($".Field(new Field(\"{prop.FieldName}\", {arrowType}, {nullable}))"); + } + } + foreach (var kv in _model.Metadata) + { + Line($".Metadata(\"{Escape(kv.Key)}\", \"{Escape(kv.Value)}\")"); + } + Line(".Build();"); + _indent--; + Line(); + var newKw = _model.HasArrowSerializableBase ? "new " : ""; + Line($"public static {newKw}Schema ArrowSchema => _arrowSchema;"); + } + + private static string Escape(string s) => s.Replace("\\", "\\\\").Replace("\"", "\\\""); + + private void EmitSerialize() + { + Line($"public static RecordBatch ToRecordBatch({_model.TypeName} value)"); + Line("{"); + _indent++; + + if (_model.HasSerializationCallback) + { + Line("((Apache.Arrow.Serialization.IArrowSerializationCallback)value).OnBeforeSerialize();"); + Line(); + } + + // Build each column array + for (int i = 0; i < _model.Properties.Count; i++) + { + var prop = _model.Properties[i]; + EmitArrayBuilder(prop, i); + } + + // Build RecordBatch + Line(); + Line("return new RecordBatch(_arrowSchema, new IArrowArray[]"); + Line("{"); + _indent++; + for (int i = 0; i < _model.Properties.Count; i++) + { + var comma = i < _model.Properties.Count - 1 ? "," : ""; + Line($"arr_{i}{comma}"); + } + _indent--; + Line("}, 1);"); + + _indent--; + Line("}"); + } + + private void EmitArrayBuilder(PropertyModel prop, int index) + { + var varName = $"arr_{index}"; + var access = $"value.{prop.PropertyName}"; + + if (prop.IsNullable) + { + EmitNullableArrayBuilder(prop, index, varName, access); + return; + } + + switch (prop.Type.Kind) + { + case TypeKind2.String: + { + var si = GetStringArrayInfo(prop); + Line($"var {varName} = new {si.BuilderType}().Append({access}).Build();"); + break; + } + case TypeKind2.Bool: + { + var bl = GetBoolArrayInfo(prop); + Line($"var {varName} = new {bl.BuilderType}().Append({access}).Build();"); + break; + } + case TypeKind2.Byte: + Line($"var {varName} = new UInt8Array.Builder().Append({access}).Build();"); + break; + case TypeKind2.SByte: + Line($"var {varName} = new Int8Array.Builder().Append({access}).Build();"); + break; + case TypeKind2.Int16: + Line($"var {varName} = new Int16Array.Builder().Append({access}).Build();"); + break; + case TypeKind2.UInt16: + Line($"var {varName} = new UInt16Array.Builder().Append({access}).Build();"); + break; + case TypeKind2.Int32: + Line($"var {varName} = new Int32Array.Builder().Append({access}).Build();"); + break; + case TypeKind2.UInt32: + Line($"var {varName} = new UInt32Array.Builder().Append({access}).Build();"); + break; + case TypeKind2.Int64: + Line($"var {varName} = new Int64Array.Builder().Append({access}).Build();"); + break; + case TypeKind2.UInt64: + Line($"var {varName} = new UInt64Array.Builder().Append({access}).Build();"); + break; + case TypeKind2.Float: + Line($"var {varName} = new FloatArray.Builder().Append({access}).Build();"); + break; + case TypeKind2.Double: + Line($"var {varName} = new DoubleArray.Builder().Append({access}).Build();"); + break; + case TypeKind2.Binary: + { + var bi = GetBinaryArrayInfo(prop); + Line($"var {varName} = new {bi.BuilderType}().Append({BinaryToSpan(prop, access)}).Build();"); + break; + } + case TypeKind2.DateTime: + Line($"var {varName} = new TimestampArray.Builder({GetTimestampTypeExpr(prop)}).Append(Apache.Arrow.Serialization.ArrowArrayHelper.{TimestampConvertMethod(prop)}({access})).Build();"); + break; + case TypeKind2.DateTimeOffset: + Line($"var {varName} = new TimestampArray.Builder({GetTimestampTypeExpr(prop)}).Append(Apache.Arrow.Serialization.ArrowArrayHelper.{TimestampConvertMethod(prop)}({access})).Build();"); + break; + case TypeKind2.DateOnly: + { + var di = GetDateArrayInfo(prop); + Line($"var {varName} = new {di.BuilderType}().Append({access}).Build();"); + break; + } + case TypeKind2.TimeOnly: + { + var ti = GetTimeArrayInfo(prop); + Line($"var {varName} = new {ti.BuilderType}({ti.TypeExpr}).Append({access}).Build();"); + break; + } + case TypeKind2.TimeSpan: + Line($"var {varName} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildDurationArray({access});"); + break; + case TypeKind2.Decimal: + Line($"var {varName} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildDecimalArray({access});"); + break; + case TypeKind2.Guid: + Line($"var {varName} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildGuidArray({access});"); + break; + case TypeKind2.Half: + Line($"var {varName} = new HalfFloatArray.Builder().Append({access}).Build();"); + break; + case TypeKind2.Enum: + EmitEnumArrayBuilder(prop, index, varName, access); + break; + case TypeKind2.List: + case TypeKind2.Array: + EmitListArrayBuilder(prop, index, varName, access); + break; + case TypeKind2.Set: + EmitSetArrayBuilder(prop, index, varName, access); + break; + case TypeKind2.Dictionary: + EmitMapArrayBuilder(prop, index, varName, access); + break; + case TypeKind2.NestedRecord: + EmitNestedRecordSerializer(prop, index, varName, access); + break; + case TypeKind2.Custom: + Line($"var {varName} = new {prop.ConverterTypeName}().ToArray({access});"); + break; + default: + Line($"// TODO: Unsupported type {prop.Type.Kind} for {prop.PropertyName}"); + Line($"IArrowArray {varName} = new NullArray(1);"); + break; + } + } + + private void EmitNullableArrayBuilder(PropertyModel prop, int index, string varName, string access) + { + switch (prop.Type.Kind) + { + case TypeKind2.String: + { + var si = GetStringArrayInfo(prop); + Line($"var {varName} = new {si.BuilderType}().Append({access}).Build();"); + break; + } + case TypeKind2.Bool: + { + var bl = GetBoolArrayInfo(prop); + Line($"var b_{index} = new {bl.BuilderType}();"); + Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(v_{index}); else b_{index}.AppendNull();"); + Line($"var {varName} = b_{index}.Build();"); + break; + } + case TypeKind2.Int32: + Line($"var b_{index} = new Int32Array.Builder();"); + Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(v_{index}); else b_{index}.AppendNull();"); + Line($"var {varName} = b_{index}.Build();"); + break; + case TypeKind2.Int64: + Line($"var b_{index} = new Int64Array.Builder();"); + Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(v_{index}); else b_{index}.AppendNull();"); + Line($"var {varName} = b_{index}.Build();"); + break; + case TypeKind2.Float: + Line($"var b_{index} = new FloatArray.Builder();"); + Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(v_{index}); else b_{index}.AppendNull();"); + Line($"var {varName} = b_{index}.Build();"); + break; + case TypeKind2.Double: + Line($"var b_{index} = new DoubleArray.Builder();"); + Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(v_{index}); else b_{index}.AppendNull();"); + Line($"var {varName} = b_{index}.Build();"); + break; + case TypeKind2.DateTime: + Line($"var b_{index} = new TimestampArray.Builder({GetTimestampTypeExpr(prop)});"); + Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(Apache.Arrow.Serialization.ArrowArrayHelper.{TimestampConvertMethod(prop)}(v_{index})); else b_{index}.AppendNull();"); + Line($"var {varName} = b_{index}.Build();"); + break; + case TypeKind2.DateTimeOffset: + Line($"var b_{index} = new TimestampArray.Builder({GetTimestampTypeExpr(prop)});"); + Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(Apache.Arrow.Serialization.ArrowArrayHelper.{TimestampConvertMethod(prop)}(v_{index})); else b_{index}.AppendNull();"); + Line($"var {varName} = b_{index}.Build();"); + break; + case TypeKind2.DateOnly: + { + var di = GetDateArrayInfo(prop); + Line($"var b_{index} = new {di.BuilderType}();"); + Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(v_{index}); else b_{index}.AppendNull();"); + Line($"var {varName} = b_{index}.Build();"); + break; + } + case TypeKind2.TimeOnly: + { + var ti = GetTimeArrayInfo(prop); + Line($"var b_{index} = new {ti.BuilderType}({ti.TypeExpr});"); + Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(v_{index}); else b_{index}.AppendNull();"); + Line($"var {varName} = b_{index}.Build();"); + break; + } + case TypeKind2.TimeSpan: + Line($"var {varName} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildDurationArray({access});"); + break; + case TypeKind2.Decimal: + Line($"var {varName} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildDecimalArray({access});"); + break; + case TypeKind2.Guid: + Line($"var {varName} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildGuidArray({access});"); + break; + case TypeKind2.Half: + Line($"var b_{index} = new HalfFloatArray.Builder();"); + Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(v_{index}); else b_{index}.AppendNull();"); + Line($"var {varName} = b_{index}.Build();"); + break; + case TypeKind2.Enum: + EmitEnumArrayBuilder(prop, index, varName, access); + break; + case TypeKind2.NestedRecord: + EmitNestedRecordSerializer(prop, index, varName, access); + break; + case TypeKind2.Custom: + Line($"IArrowArray {varName};"); + Line($"if ({access} is {{ }} cv_{index})"); + Line($" {varName} = new {prop.ConverterTypeName}().ToArray(cv_{index});"); + Line($"else"); + Line($" {varName} = new NullArray(1);"); + break; + case TypeKind2.List: + case TypeKind2.Array: + case TypeKind2.Set: + { + Line($"IArrowArray {varName};"); + Line($"if ({access} != null)"); + Line("{"); + _indent++; + var tempVar = $"_nonnull_{index}"; + if (prop.Type.Kind == TypeKind2.Set) + EmitSetArrayBuilder(prop, index, tempVar, access); + else + EmitListArrayBuilder(prop, index, tempVar, access); + Line($"{varName} = {tempVar};"); + _indent--; + Line("}"); + Line("else"); + Line("{"); + _indent++; + Line($"{varName} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildNullArray(ArrowSchema.GetFieldByName(\"{prop.FieldName}\").DataType, 1);"); + _indent--; + Line("}"); + break; + } + case TypeKind2.Dictionary: + { + Line($"IArrowArray {varName};"); + Line($"if ({access} != null)"); + Line("{"); + _indent++; + var tempVar = $"_nonnull_{index}"; + EmitMapArrayBuilder(prop, index, tempVar, access); + Line($"{varName} = {tempVar};"); + _indent--; + Line("}"); + Line("else"); + Line("{"); + _indent++; + Line($"{varName} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildNullArray(ArrowSchema.GetFieldByName(\"{prop.FieldName}\").DataType, 1);"); + _indent--; + Line("}"); + break; + } + default: + // Fallback: use the non-nullable path (many builders accept null for ref types) + EmitArrayBuilder(new PropertyModel + { + PropertyName = prop.PropertyName, + FieldName = prop.FieldName, + Type = prop.Type, + IsNullable = false, + }, index); + break; + } + } + + private void EmitEnumArrayBuilder(PropertyModel prop, int index, string varName, string access) + { + // Dictionary-encoded string array: indices + dictionary + // For a single value, build a StringArray for the dictionary and Int16Array for indices + if (prop.IsNullable) + { + Line($"StringArray dict_{index};"); + Line($"Int16Array idx_{index};"); + Line($"if ({access} is {{ }} ev_{index})"); + Line("{"); + _indent++; + Line($"dict_{index} = new StringArray.Builder().Append(ev_{index}.ToString()).Build();"); + Line($"idx_{index} = new Int16Array.Builder().Append(0).Build();"); + _indent--; + Line("}"); + Line("else"); + Line("{"); + _indent++; + Line($"dict_{index} = new StringArray.Builder().Build();"); + Line($"idx_{index} = new Int16Array.Builder().AppendNull().Build();"); + _indent--; + Line("}"); + } + else + { + Line($"var dict_{index} = new StringArray.Builder().Append({access}.ToString()).Build();"); + Line($"var idx_{index} = new Int16Array.Builder().Append(0).Build();"); + } + Line($"var {varName} = new DictionaryArray(new DictionaryType(Int16Type.Default, StringType.Default, false), idx_{index}, dict_{index});"); + } + + private void EmitListArrayBuilder(PropertyModel prop, int index, string varName, string access) + { + var elemType = prop.Type.ElementType!; + if (NeedsManualListBuilder(elemType)) + { + EmitManualListArray(elemType, index, varName, access, prop.FieldName); + return; + } + var arrowElemType = GetArrowTypeExpression(elemType); + + Line($"var lb_{index} = new ListArray.Builder({arrowElemType});"); + Line($"lb_{index}.Append();"); + EmitListValuePopulation(elemType, $"lb_{index}", access, index); + Line($"var {varName} = lb_{index}.Build();"); + } + + private void EmitSetArrayBuilder(PropertyModel prop, int index, string varName, string access) + { + var elemType = prop.Type.ElementType!; + if (NeedsManualListBuilder(elemType)) + { + EmitManualListArray(elemType, index, varName, access, prop.FieldName); + return; + } + var arrowElemType = GetArrowTypeExpression(elemType); + + Line($"var lb_{index} = new ListArray.Builder({arrowElemType});"); + Line($"lb_{index}.Append();"); + EmitListValuePopulation(elemType, $"lb_{index}", access, index); + Line($"var {varName} = lb_{index}.Build();"); + } + + /// + /// Returns true for element types that cannot use ListArray.Builder because + /// ArrowArrayBuilderFactory.Build() does not support them (extension types like Guid). + /// + private static bool NeedsManualListBuilder(TypeInfo elemType) + { + return elemType.Kind == TypeKind2.Guid + || elemType.Kind == TypeKind2.Enum + || elemType.Kind == TypeKind2.NestedRecord; + } + + /// + /// Returns true for element types that need manual deserialization (can't use GetElementReader). + /// + private static bool NeedsManualListDeserialize(TypeInfo elemType) + { + return NeedsManualListBuilder(elemType) + || elemType.Kind == TypeKind2.List + || elemType.Kind == TypeKind2.Array + || elemType.Kind == TypeKind2.Dictionary; + } + + /// + /// Builds a single-row ListArray manually for element types unsupported by ArrowArrayBuilderFactory. + /// + private void EmitManualListArray(TypeInfo elemType, int index, string varName, string access, string fieldName) + { + var arrowElemType = GetArrowTypeExpression(elemType); + switch (elemType.Kind) + { + case TypeKind2.Guid: + Line($"var guidList_{index} = new System.Collections.Generic.List({access});"); + Line($"var guidValues_{index} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildGuidArray(guidList_{index});"); + Line($"var offsets_{index} = new ArrowBuffer.Builder();"); + Line($"offsets_{index}.Append(0);"); + Line($"offsets_{index}.Append(guidList_{index}.Count);"); + Line($"var {varName} = new ListArray(new ListType(new Field(\"item\", {arrowElemType}, false)), 1, offsets_{index}.Build(), guidValues_{index}, ArrowBuffer.Empty);"); + break; + case TypeKind2.Enum: + { + // Enum list elements: dictionary-encoded (matching Python schema) + Line($"var enumDict_{index} = new StringArray.Builder();"); + Line($"var enumIdx_{index} = new Int16Array.Builder();"); + Line($"var enumMap_{index} = new System.Collections.Generic.Dictionary();"); + Line($"foreach (var e_{index} in {access}) {{"); + Line($" var es_{index} = e_{index}.ToString();"); + Line($" if (!enumMap_{index}.TryGetValue(es_{index}, out var ei_{index})) {{ ei_{index} = (short)enumMap_{index}.Count; enumMap_{index}[es_{index}] = ei_{index}; }}"); + Line($" enumIdx_{index}.Append(ei_{index});"); + Line($"}}"); + Line($"foreach (var kv in enumMap_{index}.OrderBy(kv => kv.Value)) enumDict_{index}.Append(kv.Key);"); + Line($"var enumDictArr_{index} = new DictionaryArray(new DictionaryType(Int16Type.Default, StringType.Default, false), enumIdx_{index}.Build(), enumDict_{index}.Build());"); + Line($"var offsets_{index} = new ArrowBuffer.Builder();"); + Line($"offsets_{index}.Append(0);"); + Line($"offsets_{index}.Append(enumDictArr_{index}.Length);"); + Line($"var {varName} = new ListArray(ArrowSchema.GetFieldByName(\"{fieldName}\").DataType, 1, offsets_{index}.Build(), enumDictArr_{index}, ArrowBuffer.Empty);"); + break; + } + case TypeKind2.NestedRecord: + { + // Nested record list elements: serialize via child's ToRecordBatch, wrap in StructArray + var typeName = elemType.FullTypeName; + Line($"var nestedList_{index} = new System.Collections.Generic.List<{typeName}>({access});"); + Line($"var nestedBatch_{index} = {typeName}.ToRecordBatch((IReadOnlyList<{typeName}>)nestedList_{index});"); + Line($"var nestedCols_{index} = new IArrowArray[nestedBatch_{index}.ColumnCount];"); + Line($"for (int f = 0; f < nestedBatch_{index}.ColumnCount; f++) nestedCols_{index}[f] = nestedBatch_{index}.Column(f);"); + Line($"var structValues_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), nestedList_{index}.Count, nestedCols_{index}, ArrowBuffer.Empty);"); + Line($"var offsets_{index} = new ArrowBuffer.Builder();"); + Line($"offsets_{index}.Append(0);"); + Line($"offsets_{index}.Append(nestedList_{index}.Count);"); + Line($"var {varName} = new ListArray(new ListType(new Field(\"item\", new StructType({typeName}.ArrowSchema.FieldsList), false)), 1, offsets_{index}.Build(), structValues_{index}, ArrowBuffer.Empty);"); + break; + } + } + } + + private void EmitListValuePopulation(TypeInfo elemType, string builderVar, string access, int index, int depth = 0) + { + switch (elemType.Kind) + { + case TypeKind2.String: + { + var bt = GetArrayBuilderType(elemType); + Line($"foreach (var item_{index} in {access}) (({bt}){builderVar}.ValueBuilder).Append(item_{index});"); + break; + } + case TypeKind2.Byte: + Line($"foreach (var item_{index} in {access}) ((UInt8Array.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case TypeKind2.SByte: + Line($"foreach (var item_{index} in {access}) ((Int8Array.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case TypeKind2.Int16: + Line($"foreach (var item_{index} in {access}) ((Int16Array.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case TypeKind2.UInt16: + Line($"foreach (var item_{index} in {access}) ((UInt16Array.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case TypeKind2.Int32: + Line($"foreach (var item_{index} in {access}) ((Int32Array.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case TypeKind2.UInt32: + Line($"foreach (var item_{index} in {access}) ((UInt32Array.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case TypeKind2.Int64: + Line($"foreach (var item_{index} in {access}) ((Int64Array.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case TypeKind2.UInt64: + Line($"foreach (var item_{index} in {access}) ((UInt64Array.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case TypeKind2.Float: + Line($"foreach (var item_{index} in {access}) ((FloatArray.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case TypeKind2.Double: + Line($"foreach (var item_{index} in {access}) ((DoubleArray.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case TypeKind2.Half: + Line($"foreach (var item_{index} in {access}) ((HalfFloatArray.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case TypeKind2.Bool: + Line($"foreach (var item_{index} in {access}) ((BooleanArray.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case TypeKind2.Decimal: + Line($"foreach (var item_{index} in {access}) ((Decimal128Array.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case TypeKind2.DateTime: + { + var hasTz = TypeInfoHasTimezone(elemType); + var convert = hasTz ? "ToUtcDateTimeOffset" : "ToWallClockDateTimeOffset"; + Line($"foreach (var item_{index} in {access}) ((TimestampArray.Builder){builderVar}.ValueBuilder).Append(Apache.Arrow.Serialization.ArrowArrayHelper.{convert}(item_{index}));"); + break; + } + case TypeKind2.DateTimeOffset: + { + var hasTz = TypeInfoHasTimezone(elemType); + var convert = hasTz ? "ToUtcDateTimeOffset" : "ToWallClockDateTimeOffset"; + Line($"foreach (var item_{index} in {access}) ((TimestampArray.Builder){builderVar}.ValueBuilder).Append(Apache.Arrow.Serialization.ArrowArrayHelper.{convert}(item_{index}));"); + break; + } + case TypeKind2.DateOnly: + { + var arrayType = GetDateArrayBuilderType(elemType); + Line($"foreach (var item_{index} in {access}) (({arrayType}){builderVar}.ValueBuilder).Append(item_{index});"); + break; + } + case TypeKind2.TimeOnly: + { + var arrayType = GetTimeArrayBuilderType(elemType); + Line($"foreach (var item_{index} in {access}) (({arrayType}){builderVar}.ValueBuilder).Append(item_{index});"); + break; + } + case TypeKind2.TimeSpan: + Line($"foreach (var item_{index} in {access}) ((DurationArray.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case TypeKind2.Guid: + Line($"foreach (var item_{index} in {access}) ((GuidArray.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case TypeKind2.Binary: + Line($"foreach (var item_{index} in {access}) ((BinaryArray.Builder){builderVar}.ValueBuilder).Append((ReadOnlySpan)item_{index});"); + break; + case TypeKind2.Enum: + // For list of enums, we need to build dictionary-encoded values + // Simplified: store as string list for now + Line($"foreach (var item_{index} in {access}) ((StringArray.Builder){builderVar}.ValueBuilder).Append(item_{index}.ToString());"); + break; + case TypeKind2.List: + case TypeKind2.Array: + { + // Nested list: ValueBuilder is itself a ListArray.Builder + var innerElemType = elemType.ElementType!; + var innerBuilderCast = $"((ListArray.Builder){builderVar}.ValueBuilder)"; + var loopVar = depth == 0 ? $"inner_{index}" : $"inner_{index}_{depth}"; + Line($"foreach (var {loopVar} in {access})"); + Line("{"); + _indent++; + Line($"{innerBuilderCast}.Append();"); + EmitListValuePopulation(innerElemType, innerBuilderCast, loopVar, index, depth + 1); + _indent--; + Line("}"); + break; + } + case TypeKind2.Dictionary: + { + // Dict values inside a list: ValueBuilder is a MapArray.Builder + var keyType = elemType.KeyType!; + var valueType = elemType.ValueType!; + var innerMapBuilder = $"((MapArray.Builder){builderVar}.ValueBuilder)"; + var loopVar = depth == 0 ? $"dictItem_{index}" : $"dictItem_{index}_{depth}"; + var kvVar = depth == 0 ? $"mkv_{index}" : $"mkv_{index}_{depth}"; + Line($"foreach (var {loopVar} in {access})"); + Line("{"); + _indent++; + Line($"{innerMapBuilder}.Append();"); + Line($"foreach (var {kvVar} in {loopVar})"); + Line("{"); + _indent++; + EmitMapKeyAppend(keyType, innerMapBuilder, $"{kvVar}.Key", index, depth + 1); + EmitMapValueAppend(valueType, innerMapBuilder, $"{kvVar}.Value", index, depth + 1); + _indent--; + Line("}"); + _indent--; + Line("}"); + break; + } + default: + Line($"// TODO: List element type {elemType.Kind} not yet supported for population"); + break; + } + } + + /// + /// Returns true for value types that cannot use MapArray.Builder because + /// ArrowArrayBuilderFactory.Build() does not support them. + /// + private static bool NeedsManualMapBuilder(TypeInfo valueType) + { + return valueType.Kind == TypeKind2.Guid + || valueType.Kind == TypeKind2.Enum + || valueType.Kind == TypeKind2.NestedRecord; + } + + private void EmitMapArrayBuilder(PropertyModel prop, int index, string varName, string access) + { + var keyType = prop.Type.KeyType!; + var valueType = prop.Type.ValueType!; + + if (NeedsManualMapBuilder(valueType)) + { + EmitManualMapArray(prop, index, varName, access); + return; + } + + var mapType = GetArrowTypeExpression(prop.Type); + Line($"var mb_{index} = new MapArray.Builder({mapType});"); + Line($"mb_{index}.Append();"); + Line($"foreach (var kv_{index} in {access})"); + Line("{"); + _indent++; + EmitMapKeyAppend(keyType, $"mb_{index}", $"kv_{index}.Key", index); + EmitMapValueAppend(valueType, $"mb_{index}", $"kv_{index}.Value", index); + _indent--; + Line("}"); + Line($"var {varName} = mb_{index}.Build();"); + } + + /// + /// Builds a single-row MapArray manually for value types unsupported by ArrowArrayBuilderFactory. + /// + private void EmitManualMapArray(PropertyModel prop, int index, string varName, string access) + { + var keyType = prop.Type.KeyType!; + var valueType = prop.Type.ValueType!; + var mapType = GetArrowTypeExpression(prop.Type); + + // Build keys + var keyBuilderType = GetArrayBuilderType(keyType); + Line($"var mk_{index} = new {keyBuilderType}();"); + Line($"foreach (var kv_{index} in {access}) mk_{index}.Append(kv_{index}.Key);"); + Line($"var mkArr_{index} = mk_{index}.Build();"); + + // Build values based on type + switch (valueType.Kind) + { + case TypeKind2.Guid: + Line($"var mvItems_{index} = new System.Collections.Generic.List();"); + Line($"foreach (var kv_{index}v in {access}) mvItems_{index}.Add(kv_{index}v.Value);"); + Line($"var mvArr_{index} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildGuidArray(mvItems_{index});"); + break; + case TypeKind2.Enum: + { + Line($"var mvDict_{index} = new StringArray.Builder();"); + Line($"var mvIdx_{index} = new Int16Array.Builder();"); + Line($"var mvMap_{index} = new System.Collections.Generic.Dictionary();"); + Line($"foreach (var kv_{index}v in {access})"); + Line("{"); + _indent++; + Line($"var es_{index} = kv_{index}v.Value.ToString();"); + Line($"if (!mvMap_{index}.TryGetValue(es_{index}, out var ei_{index})) {{ ei_{index} = (short)mvMap_{index}.Count; mvMap_{index}[es_{index}] = ei_{index}; }}"); + Line($"mvIdx_{index}.Append(ei_{index});"); + _indent--; + Line("}"); + Line($"foreach (var kv in mvMap_{index}.OrderBy(kv => kv.Value)) mvDict_{index}.Append(kv.Key);"); + Line($"var mvArr_{index} = new DictionaryArray(new DictionaryType(Int16Type.Default, StringType.Default, false), mvIdx_{index}.Build(), mvDict_{index}.Build());"); + break; + } + case TypeKind2.NestedRecord: + { + var typeName = valueType.FullTypeName; + Line($"var mvItems_{index} = new System.Collections.Generic.List<{typeName}>();"); + Line($"foreach (var kv_{index}v in {access}) mvItems_{index}.Add(kv_{index}v.Value);"); + Line($"IArrowArray mvArr_{index};"); + Line($"if (mvItems_{index}.Count > 0)"); + Line("{"); + _indent++; + Line($"var nb_{index} = {typeName}.ToRecordBatch((IReadOnlyList<{typeName}>)mvItems_{index});"); + Line($"var ca_{index} = new IArrowArray[nb_{index}.ColumnCount];"); + Line($"for (int f = 0; f < nb_{index}.ColumnCount; f++) ca_{index}[f] = nb_{index}.Column(f);"); + Line($"mvArr_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), mvItems_{index}.Count, ca_{index}, ArrowBuffer.Empty);"); + _indent--; + Line("}"); + Line($"else mvArr_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), 0, System.Array.Empty(), ArrowBuffer.Empty);"); + break; + } + } + + // Construct MapArray manually: offsets, StructArray(keys, values), then MapArray + Line($"var mOffsets_{index} = new ArrowBuffer.Builder();"); + Line($"mOffsets_{index}.Append(0);"); + Line($"mOffsets_{index}.Append(mkArr_{index}.Length);"); + Line($"var mEntries_{index} = new StructArray(((MapType){mapType}).KeyValueType, mkArr_{index}.Length, new IArrowArray[] {{ mkArr_{index}, mvArr_{index} }}, ArrowBuffer.Empty);"); + Line($"var {varName} = new MapArray({mapType}, 1, mOffsets_{index}.Build(), mEntries_{index}, ArrowBuffer.Empty);"); + } + + private void EmitMapKeyAppend(TypeInfo keyType, string builderVar, string access, int index, int depth = 0) + { + var builderType = GetArrayBuilderType(keyType); + var appendExpr = GetMapAppendExpression(keyType, builderType, $"{builderVar}.KeyBuilder", access); + Line(appendExpr); + } + + private void EmitMapValueAppend(TypeInfo valueType, string builderVar, string access, int index, int depth = 0) + { + switch (valueType.Kind) + { + case TypeKind2.List: + case TypeKind2.Array: + case TypeKind2.Set: + { + // List value in map: ValueBuilder is a ListArray.Builder + var innerListBuilder = $"((ListArray.Builder){builderVar}.ValueBuilder)"; + Line($"{innerListBuilder}.Append();"); + EmitListValuePopulation(valueType.ElementType!, innerListBuilder, access, index, depth + 1); + break; + } + case TypeKind2.Dictionary: + { + // Nested map value: ValueBuilder is a MapArray.Builder + var innerMapBuilder = $"((MapArray.Builder){builderVar}.ValueBuilder)"; + var innerKvVar = depth == 0 ? $"mvkv_{index}" : $"mvkv_{index}_{depth}"; + Line($"{innerMapBuilder}.Append();"); + Line($"foreach (var {innerKvVar} in {access})"); + Line("{"); + _indent++; + EmitMapKeyAppend(valueType.KeyType!, innerMapBuilder, $"{innerKvVar}.Key", index, depth + 1); + EmitMapValueAppend(valueType.ValueType!, innerMapBuilder, $"{innerKvVar}.Value", index, depth + 1); + _indent--; + Line("}"); + break; + } + default: + { + var builderType = GetArrayBuilderType(valueType); + var appendExpr = GetMapAppendExpression(valueType, builderType, $"{builderVar}.ValueBuilder", access); + Line(appendExpr); + break; + } + } + } + + private string GetMapAppendExpression(TypeInfo type, string builderType, string builderAccess, string valueAccess) + { + switch (type.Kind) + { + case TypeKind2.DateTime: + case TypeKind2.DateTimeOffset: + { + var convert = TypeInfoHasTimezone(type) ? "ToUtcDateTimeOffset" : "ToWallClockDateTimeOffset"; + return $"(({builderType}){builderAccess}).Append(Apache.Arrow.Serialization.ArrowArrayHelper.{convert}({valueAccess}));"; + } + case TypeKind2.Binary: + return $"(({builderType}){builderAccess}).Append((ReadOnlySpan){valueAccess});"; + case TypeKind2.Enum: + return $"(({builderType}){builderAccess}).Append({valueAccess}.ToString());"; + default: + return $"(({builderType}){builderAccess}).Append({valueAccess});"; + } + } + + private void EmitNestedRecordSerializer(PropertyModel prop, int index, string varName, string access) + { + var typeName = prop.Type.FullTypeName; + + if (prop.IsNullable) + { + Line($"IArrowArray {varName};"); + Line($"if ({access} is {{ }} nv_{index})"); + Line("{"); + _indent++; + Line($"var nestedBatch_{index} = {typeName}.ToRecordBatch(nv_{index});"); + Line($"var nestedArrays_{index} = new IArrowArray[nestedBatch_{index}.ColumnCount];"); + Line($"for (int i = 0; i < nestedBatch_{index}.ColumnCount; i++) nestedArrays_{index}[i] = nestedBatch_{index}.Column(i);"); + Line($"{varName} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), 1, nestedArrays_{index}, ArrowBuffer.Empty);"); + _indent--; + Line("}"); + Line("else"); + Line("{"); + _indent++; + Line($"var nullArrays_{index} = new IArrowArray[{typeName}.ArrowSchema.FieldsList.Count];"); + Line($"for (int i = 0; i < nullArrays_{index}.Length; i++)"); + Line($" nullArrays_{index}[i] = Apache.Arrow.Serialization.ArrowArrayHelper.BuildNullArray({typeName}.ArrowSchema.FieldsList[i].DataType, 1);"); + Line($"var nullBitmap_{index} = new ArrowBuffer.BitmapBuilder();"); + Line($"nullBitmap_{index}.Append(false);"); + Line($"{varName} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), 1, nullArrays_{index}, nullBitmap_{index}.Build(), 1);"); + _indent--; + Line("}"); + } + else + { + Line($"var nestedBatch_{index} = {typeName}.ToRecordBatch({access});"); + Line($"var nestedArrays_{index} = new IArrowArray[nestedBatch_{index}.ColumnCount];"); + Line($"for (int i = 0; i < nestedBatch_{index}.ColumnCount; i++) nestedArrays_{index}[i] = nestedBatch_{index}.Column(i);"); + Line($"var {varName} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), 1, nestedArrays_{index}, ArrowBuffer.Empty);"); + } + } + + private void EmitDeserialize() + { + var newKw = _model.HasArrowSerializableBase ? "new " : ""; + Line($"public static {newKw}{_model.TypeName} FromRecordBatch(RecordBatch batch)"); + Line("{"); + _indent++; + + Line("if (batch.Length == 0)"); + Line(" throw new InvalidOperationException(\"Cannot deserialize from empty RecordBatch.\");"); + Line("if (batch.Length > 1)"); + Line(" throw new InvalidOperationException($\"Expected single-row RecordBatch, got {batch.Length} rows.\");"); + Line(); + + for (int i = 0; i < _model.Properties.Count; i++) + { + var prop = _model.Properties[i]; + EmitDeserializeProperty(prop, i); + } + + Line(); + if (_model.HasSerializationCallback) + { + EmitObjectConstruction("var result = ", ";"); + Line("((Apache.Arrow.Serialization.IArrowSerializationCallback)result).OnAfterDeserialize();"); + Line("return result;"); + } + else + { + EmitObjectConstruction("return ", ";"); + } + + _indent--; + Line("}"); + } + + /// + /// Emits either constructor call or object initializer to create an instance. + /// Uses constructor when ConstructorParams is set (field-based types), otherwise object initializer. + /// + private void EmitObjectConstruction(string prefix, string suffix) + { + if (_model.ConstructorParams != null) + { + // Constructor-based: new TypeName(paramName: prop_N, ...) + var propByName = new Dictionary(StringComparer.OrdinalIgnoreCase); + for (int i = 0; i < _model.Properties.Count; i++) + propByName[_model.Properties[i].PropertyName] = i; + + var args = new List(); + foreach (var param in _model.ConstructorParams) + { + if (propByName.TryGetValue(param.Name, out int idx)) + args.Add($"{param.Name}: prop_{idx}"); + else if (param.HasDefaultValue) + continue; // use default + } + + Line($"{prefix}new {_model.TypeName}({string.Join(", ", args)}){suffix}"); + } + else + { + // Object initializer: new TypeName { Prop = prop_N, ... } + Line($"{prefix}new {_model.TypeName}"); + Line("{"); + _indent++; + for (int i = 0; i < _model.Properties.Count; i++) + { + var prop = _model.Properties[i]; + var comma = i < _model.Properties.Count - 1 ? "," : ""; + Line($"{prop.PropertyName} = prop_{i}{comma}"); + } + _indent--; + Line($"}}{suffix}"); + } + } + + private void EmitDeserializeProperty(PropertyModel prop, int index) + { + var colAccess = $"batch.Column(\"{prop.FieldName}\")"; + + switch (prop.Type.Kind) + { + case TypeKind2.String: + { + var si = GetStringArrayInfo(prop); + if (prop.IsNullable) + Line($"var prop_{index} = (({colAccess}) is {si.ArrayType} sa_{index} && !sa_{index}.IsNull(0)) ? sa_{index}.GetString(0) : null;"); + else + Line($"var prop_{index} = (({si.ArrayType}){colAccess}).GetString(0)!;"); + break; + } + case TypeKind2.Bool: + { + var bl = GetBoolArrayInfo(prop); + EmitPrimitiveDeserialize(prop, index, colAccess, bl.ArrayType, "GetValue"); + break; + } + case TypeKind2.Byte: + EmitPrimitiveDeserialize(prop, index, colAccess, "UInt8Array", "GetValue"); + break; + case TypeKind2.SByte: + EmitPrimitiveDeserialize(prop, index, colAccess, "Int8Array", "GetValue"); + break; + case TypeKind2.Int16: + EmitPrimitiveDeserialize(prop, index, colAccess, "Int16Array", "GetValue"); + break; + case TypeKind2.UInt16: + EmitPrimitiveDeserialize(prop, index, colAccess, "UInt16Array", "GetValue"); + break; + case TypeKind2.Int32: + EmitPrimitiveDeserialize(prop, index, colAccess, "Int32Array", "GetValue"); + break; + case TypeKind2.UInt32: + EmitPrimitiveDeserialize(prop, index, colAccess, "UInt32Array", "GetValue"); + break; + case TypeKind2.Int64: + EmitPrimitiveDeserialize(prop, index, colAccess, "Int64Array", "GetValue"); + break; + case TypeKind2.UInt64: + EmitPrimitiveDeserialize(prop, index, colAccess, "UInt64Array", "GetValue"); + break; + case TypeKind2.Float: + EmitPrimitiveDeserialize(prop, index, colAccess, "FloatArray", "GetValue"); + break; + case TypeKind2.Double: + EmitPrimitiveDeserialize(prop, index, colAccess, "DoubleArray", "GetValue"); + break; + case TypeKind2.Binary: + { + var bi = GetBinaryArrayInfo(prop); + if (IsReadOnlyMemoryByte(prop)) + Line($"ReadOnlyMemory prop_{index} = (({bi.ArrayType}){colAccess}).GetBytes(0).ToArray();"); + else + Line($"var prop_{index} = (({bi.ArrayType}){colAccess}).GetBytes(0).ToArray();"); + break; + } + case TypeKind2.DateTime: + { + var dtProp = TimestampHasTimezone(prop) ? "UtcDateTime" : "DateTime"; + if (prop.IsNullable) + Line($"var prop_{index} = ((TimestampArray){colAccess}).IsNull(0) ? (System.DateTime?)null : ((TimestampArray){colAccess}).GetTimestamp(0)!.Value.{dtProp};"); + else + Line($"var prop_{index} = ((TimestampArray){colAccess}).GetTimestamp(0)!.Value.{dtProp};"); + break; + } + case TypeKind2.DateTimeOffset: + if (prop.IsNullable) + Line($"var prop_{index} = ((TimestampArray){colAccess}).IsNull(0) ? (System.DateTimeOffset?)null : ((TimestampArray){colAccess}).GetTimestamp(0)!.Value;"); + else + Line($"var prop_{index} = ((TimestampArray){colAccess}).GetTimestamp(0)!.Value;"); + break; + case TypeKind2.DateOnly: + { + var di = GetDateArrayInfo(prop); + if (prop.IsNullable) + Line($"var prop_{index} = (({di.ArrayType}){colAccess}).IsNull(0) ? (System.DateOnly?)null : (({di.ArrayType}){colAccess}).GetDateOnly(0);"); + else + Line($"var prop_{index} = (({di.ArrayType}){colAccess}).GetDateOnly(0)!.Value;"); + break; + } + case TypeKind2.TimeOnly: + { + var ti = GetTimeArrayInfo(prop); + if (prop.IsNullable) + Line($"var prop_{index} = (({ti.ArrayType}){colAccess}).GetTime(0);"); + else + Line($"var prop_{index} = (({ti.ArrayType}){colAccess}).GetTime(0)!.Value;"); + break; + } + case TypeKind2.TimeSpan: + if (prop.IsNullable) + Line($"var prop_{index} = ((DurationArray){colAccess}).IsNull(0) ? (System.TimeSpan?)null : Apache.Arrow.Serialization.ArrowArrayHelper.ReadDuration((DurationArray){colAccess}, 0);"); + else + Line($"var prop_{index} = Apache.Arrow.Serialization.ArrowArrayHelper.ReadDuration((DurationArray){colAccess}, 0);"); + break; + case TypeKind2.Decimal: + if (prop.IsNullable) + Line($"var prop_{index} = ((Decimal128Array){colAccess}).IsNull(0) ? (decimal?)null : ((Decimal128Array){colAccess}).GetValue(0).Value;"); + else + Line($"var prop_{index} = ((Decimal128Array){colAccess}).GetValue(0).Value;"); + break; + case TypeKind2.Guid: + if (prop.IsNullable) + Line($"var prop_{index} = ((GuidArray){colAccess}).GetGuid(0);"); + else + Line($"var prop_{index} = ((GuidArray){colAccess}).GetGuid(0)!.Value;"); + break; + case TypeKind2.Half: + EmitPrimitiveDeserialize(prop, index, colAccess, "HalfFloatArray", "GetValue"); + break; + case TypeKind2.Enum: + EmitEnumDeserialize(prop, index, colAccess); + break; + case TypeKind2.List: + case TypeKind2.Array: + EmitListDeserialize(prop, index, colAccess); + break; + case TypeKind2.Set: + EmitSetDeserialize(prop, index, colAccess); + break; + case TypeKind2.Dictionary: + EmitDictDeserialize(prop, index, colAccess); + break; + case TypeKind2.NestedRecord: + EmitNestedRecordDeserialize(prop, index, colAccess); + break; + case TypeKind2.Custom: + if (prop.IsNullable) + Line($"var prop_{index} = {colAccess}.IsNull(0) ? ({prop.Type.FullTypeName}?)null : new {prop.ConverterTypeName}().FromArray({colAccess}, 0);"); + else + Line($"var prop_{index} = new {prop.ConverterTypeName}().FromArray({colAccess}, 0);"); + break; + default: + Line($"object? prop_{index} = null; // TODO: Unsupported type {prop.Type.Kind}"); + break; + } + } + + private void EmitPrimitiveDeserialize(PropertyModel prop, int index, string colAccess, string arrayType, string getMethod) + { + if (prop.IsNullable) + Line($"var prop_{index} = (({arrayType}){colAccess}).IsNull(0) ? null : ({prop.Type.FullTypeName}?)(({arrayType}){colAccess}).{getMethod}(0);"); + else + Line($"var prop_{index} = (({arrayType}){colAccess}).{getMethod}(0).Value;"); + } + + private void EmitEnumDeserialize(PropertyModel prop, int index, string colAccess) + { + var enumType = prop.Type.FullTypeName; + if (prop.IsNullable) + { + Line($"var dictArr_{index} = (DictionaryArray){colAccess};"); + Line($"{enumType}? prop_{index} = dictArr_{index}.IsNull(0) ? null : System.Enum.Parse<{enumType}>(((StringArray)dictArr_{index}.Dictionary).GetString((int)((Int16Array)dictArr_{index}.Indices).GetValue(0).Value)!);"); + } + else + { + Line($"var dictArr_{index} = (DictionaryArray){colAccess};"); + Line($"var prop_{index} = System.Enum.Parse<{enumType}>(((StringArray)dictArr_{index}.Dictionary).GetString((int)((Int16Array)dictArr_{index}.Indices).GetValue(0).Value)!);"); + } + } + + private void EmitListDeserialize(PropertyModel prop, int index, string colAccess) + { + var elemType = prop.Type.ElementType!; + var listArr = $"listArr_{index}"; + Line($"var {listArr} = (ListArray){colAccess};"); + + if (prop.IsNullable) + { + var collTypeName = prop.Type.Kind == TypeKind2.Array + ? $"{elemType.FullTypeName}[]" + : $"System.Collections.Generic.List<{elemType.FullTypeName}>"; + Line($"{collTypeName}? prop_{index};"); + Line($"if ({listArr}.IsNull(0))"); + Line($" prop_{index} = null;"); + Line("else"); + Line("{"); + _indent++; + EmitListDeserializeInner(prop, elemType, index, listArr, isSet: false, assignTarget: $"prop_{index}"); + _indent--; + Line("}"); + } + else + { + EmitListDeserializeInner(prop, elemType, index, listArr, isSet: false, assignTarget: null); + } + } + + /// + /// Emits the inner list deserialization logic (non-null path). + /// If assignTarget is null, declares var prop_{index}; otherwise assigns to assignTarget. + /// + private void EmitListDeserializeInner(PropertyModel prop, TypeInfo elemType, int index, string listArr, bool isSet, string? assignTarget) + { + var declPrefix = assignTarget != null ? $"{assignTarget}" : $"var prop_{index}"; + if (NeedsManualListDeserialize(elemType)) + { + EmitManualListDeserialize(prop, elemType, index, listArr, isSet, assignTarget); + } + else + { + Line($"var {listArr}_values = {listArr}.GetSlicedValues(0);"); + var elemReader = GetElementReader(elemType, $"{listArr}_values"); + if (isSet) + Line($"{declPrefix} = new System.Collections.Generic.HashSet<{elemType.FullTypeName}>({elemReader});"); + else if (prop.Type.Kind == TypeKind2.Array) + Line($"{declPrefix} = {elemReader}.ToArray();"); + else + Line($"{declPrefix} = new System.Collections.Generic.List<{elemType.FullTypeName}>({elemReader});"); + } + } + + private void EmitSetDeserialize(PropertyModel prop, int index, string colAccess) + { + var elemType = prop.Type.ElementType!; + var listArr = $"listArr_{index}"; + Line($"var {listArr} = (ListArray){colAccess};"); + + if (prop.IsNullable) + { + var collTypeName = $"System.Collections.Generic.HashSet<{elemType.FullTypeName}>"; + Line($"{collTypeName}? prop_{index};"); + Line($"if ({listArr}.IsNull(0))"); + Line($" prop_{index} = null;"); + Line("else"); + Line("{"); + _indent++; + EmitListDeserializeInner(prop, elemType, index, listArr, isSet: true, assignTarget: $"prop_{index}"); + _indent--; + Line("}"); + } + else + { + EmitListDeserializeInner(prop, elemType, index, listArr, isSet: true, assignTarget: null); + } + } + + /// + /// Emits deserialization for single-row list/set with extension type elements (e.g. Guid) + /// that cannot use GetSlicedValues (which fails for ExtensionArray). + /// + private void EmitManualListDeserialize(PropertyModel prop, TypeInfo elemType, int index, string listArr, bool isSet, string? assignTarget = null) + { + // When assignTarget is set, assign to it; otherwise declare var prop_{index} + string PropDecl(string expr) => assignTarget != null ? $"{assignTarget} = {expr};" : $"var prop_{index} = {expr};"; + + // Read offsets and access the full values array directly + Line($"var {listArr}_start = {listArr}.ValueOffsets[0];"); + Line($"var {listArr}_len = {listArr}.GetValueLength(0);"); + switch (elemType.Kind) + { + case TypeKind2.Guid: + { + Line($"var {listArr}_allValues = (GuidArray){listArr}.Values;"); + var readExpr = $"Enumerable.Range({listArr}_start, {listArr}_len).Select(i => {listArr}_allValues.GetGuid(i)!.Value)"; + if (isSet) + Line(PropDecl($"new System.Collections.Generic.HashSet<{elemType.FullTypeName}>({readExpr})")); + else if (prop.Type.Kind == TypeKind2.Array) + Line(PropDecl($"{readExpr}.ToArray()")); + else + Line(PropDecl($"new System.Collections.Generic.List<{elemType.FullTypeName}>({readExpr})")); + break; + } + case TypeKind2.Enum: + { + // Enum list elements are dictionary-encoded: Dictionary(Int16, Utf8) + Line($"var {listArr}_dictArr = (DictionaryArray){listArr}.Values;"); + Line($"var {listArr}_dictValues = (StringArray){listArr}_dictArr.Dictionary;"); + var readExpr = $"Enumerable.Range({listArr}_start, {listArr}_len).Select(i => System.Enum.Parse<{elemType.FullTypeName}>({listArr}_dictValues.GetString(((Int16Array){listArr}_dictArr.Indices).GetValue(i)!.Value)!))"; + if (isSet) + Line(PropDecl($"new System.Collections.Generic.HashSet<{elemType.FullTypeName}>({readExpr})")); + else if (prop.Type.Kind == TypeKind2.Array) + Line(PropDecl($"{readExpr}.ToArray()")); + else + Line(PropDecl($"new System.Collections.Generic.List<{elemType.FullTypeName}>({readExpr})")); + break; + } + case TypeKind2.NestedRecord: + { + // Nested record list elements: read from StructArray via child's FromRecordBatch + var typeName = elemType.FullTypeName; + Line($"var {listArr}_struct = (StructArray){listArr}.Values;"); + Line($"var {listArr}_result = new System.Collections.Generic.List<{typeName}>({listArr}_len);"); + Line($"for (int i = {listArr}_start; i < {listArr}_start + {listArr}_len; i++)"); + Line("{"); + _indent++; + Line($"var childArrays_{index} = new IArrowArray[{listArr}_struct.Fields.Count];"); + Line($"for (int f = 0; f < {listArr}_struct.Fields.Count; f++) childArrays_{index}[f] = {listArr}_struct.Fields[f].Data.Slice(i, 1) is var sd_{index} ? ArrowArrayFactory.BuildArray(sd_{index}) : throw new System.InvalidOperationException();"); + Line($"var childBatch_{index} = new RecordBatch({typeName}.ArrowSchema, childArrays_{index}, 1);"); + Line($"{listArr}_result.Add({typeName}.FromRecordBatch(childBatch_{index}));"); + _indent--; + Line("}"); + if (isSet) + Line(PropDecl($"new System.Collections.Generic.HashSet<{typeName}>({listArr}_result)")); + else if (prop.Type.Kind == TypeKind2.Array) + Line(PropDecl($"{listArr}_result.ToArray()")); + else + Line(PropDecl($"{listArr}_result")); + break; + } + case TypeKind2.List: + case TypeKind2.Array: + { + // Nested list elements: read inner lists from ListArray values + var innerElemType = elemType.ElementType!; + var innerTypeName = elemType.FullTypeName; + Line($"var {listArr}_innerArr = (ListArray){listArr}.Values;"); + Line($"var {listArr}_result = new System.Collections.Generic.List<{innerTypeName}>({listArr}_len);"); + Line($"for (int i = {listArr}_start; i < {listArr}_start + {listArr}_len; i++)"); + Line("{"); + _indent++; + if (innerElemType.Kind == TypeKind2.Dictionary) + { + // Dict inner elements: read each dict from inner list's MapArray values + var keyType = innerElemType.KeyType!; + var valType = innerElemType.ValueType!; + var dictTypeName = $"System.Collections.Generic.Dictionary<{keyType.FullTypeName}, {valType.FullTypeName}>"; + Line($"var {listArr}_mapSlice = (MapArray){listArr}_innerArr.GetSlicedValues(i);"); + Line($"var {listArr}_innerResult = new System.Collections.Generic.List<{dictTypeName}>({listArr}_mapSlice.Length);"); + Line($"for (int j_{index} = 0; j_{index} < {listArr}_mapSlice.Length; j_{index}++)"); + Line("{"); + _indent++; + Line($"var kvs_{index} = (StructArray){listArr}_mapSlice.GetSlicedValues(j_{index});"); + Line($"var ks_{index} = kvs_{index}.Fields[0];"); + Line($"var vs_{index} = kvs_{index}.Fields[1];"); + Line($"var d_{index} = new {dictTypeName}();"); + Line($"for (int k_{index} = 0; k_{index} < kvs_{index}.Length; k_{index}++)"); + Line("{"); + _indent++; + var dKeyReader = GetScalarReader(keyType, $"ks_{index}", $"k_{index}"); + EmitDictValueRead(valType, $"d_{index}", dKeyReader, $"vs_{index}", $"k_{index}", index); + _indent--; + Line("}"); + Line($"{listArr}_innerResult.Add(d_{index});"); + _indent--; + Line("}"); + Line($"{listArr}_result.Add({listArr}_innerResult);"); + } + else + { + Line($"var {listArr}_innerValues = {listArr}_innerArr.GetSlicedValues(i);"); + var innerElemReader = GetElementReader(innerElemType, $"{listArr}_innerValues"); + if (elemType.Kind == TypeKind2.Array) + Line($"{listArr}_result.Add({innerElemReader}.ToArray());"); + else + Line($"{listArr}_result.Add(new {innerTypeName}({innerElemReader}));"); + } + _indent--; + Line("}"); + if (isSet) + Line(PropDecl($"new System.Collections.Generic.HashSet<{innerTypeName}>({listArr}_result)")); + else if (prop.Type.Kind == TypeKind2.Array) + Line(PropDecl($"{listArr}_result.ToArray()")); + else + Line(PropDecl($"{listArr}_result")); + break; + } + case TypeKind2.Dictionary: + { + // Dict elements inside a list: read each dict from MapArray values + var keyType = elemType.KeyType!; + var valType = elemType.ValueType!; + var dictTypeName = $"System.Collections.Generic.Dictionary<{keyType.FullTypeName}, {valType.FullTypeName}>"; + Line($"var mapArr_{index} = (MapArray){listArr}.Values;"); + Line($"var {listArr}_result = new System.Collections.Generic.List<{dictTypeName}>({listArr}_len);"); + Line($"for (int i = {listArr}_start; i < {listArr}_start + {listArr}_len; i++)"); + Line("{"); + _indent++; + Line($"var kvs_{index} = (StructArray)mapArr_{index}.GetSlicedValues(i);"); + Line($"var ks_{index} = kvs_{index}.Fields[0];"); + Line($"var vs_{index} = kvs_{index}.Fields[1];"); + Line($"var d_{index} = new {dictTypeName}();"); + Line($"for (int j_{index} = 0; j_{index} < kvs_{index}.Length; j_{index}++)"); + Line("{"); + _indent++; + var dKeyReader = GetScalarReader(keyType, $"ks_{index}", $"j_{index}"); + EmitDictValueRead(valType, $"d_{index}", dKeyReader, $"vs_{index}", $"j_{index}", index); + _indent--; + Line("}"); + Line($"{listArr}_result.Add(d_{index});"); + _indent--; + Line("}"); + if (isSet) + Line(PropDecl($"new System.Collections.Generic.HashSet<{dictTypeName}>({listArr}_result)")); + else if (prop.Type.Kind == TypeKind2.Array) + Line(PropDecl($"{listArr}_result.ToArray()")); + else + Line(PropDecl($"{listArr}_result")); + break; + } + } + } + + private string GetElementReader(TypeInfo elemType, string valuesVar) + { + var stringArrayType = GetArrayReadType(elemType); + return elemType.Kind switch + { + TypeKind2.String => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => (({stringArrayType}){valuesVar}).GetString(i)!)", + TypeKind2.Byte => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((UInt8Array){valuesVar}).GetValue(i).Value)", + TypeKind2.SByte => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((Int8Array){valuesVar}).GetValue(i).Value)", + TypeKind2.Int16 => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((Int16Array){valuesVar}).GetValue(i).Value)", + TypeKind2.UInt16 => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((UInt16Array){valuesVar}).GetValue(i).Value)", + TypeKind2.Int32 => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((Int32Array){valuesVar}).GetValue(i).Value)", + TypeKind2.UInt32 => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((UInt32Array){valuesVar}).GetValue(i).Value)", + TypeKind2.Int64 => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((Int64Array){valuesVar}).GetValue(i).Value)", + TypeKind2.UInt64 => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((UInt64Array){valuesVar}).GetValue(i).Value)", + TypeKind2.Float => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((FloatArray){valuesVar}).GetValue(i).Value)", + TypeKind2.Double => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((DoubleArray){valuesVar}).GetValue(i).Value)", + TypeKind2.Half => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((HalfFloatArray){valuesVar}).GetValue(i).Value)", + TypeKind2.Bool => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((BooleanArray){valuesVar}).GetValue(i).Value)", + TypeKind2.Decimal => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((Decimal128Array){valuesVar}).GetValue(i).Value)", + TypeKind2.DateTime => TypeInfoHasTimezone(elemType) + ? $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((TimestampArray){valuesVar}).GetTimestamp(i)!.Value.UtcDateTime)" + : $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((TimestampArray){valuesVar}).GetTimestamp(i)!.Value.DateTime)", + TypeKind2.DateTimeOffset => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((TimestampArray){valuesVar}).GetTimestamp(i)!.Value)", + TypeKind2.DateOnly => GetDateArrayReadType(elemType) == "Date64Array" + ? $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((Date64Array){valuesVar}).GetDateOnly(i).Value)" + : $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((Date32Array){valuesVar}).GetDateOnly(i).Value)", + TypeKind2.TimeOnly => GetTimeArrayReadType(elemType) == "Time32Array" + ? $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((Time32Array){valuesVar}).GetTime(i).Value)" + : $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((Time64Array){valuesVar}).GetTime(i).Value)", + TypeKind2.TimeSpan => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => Apache.Arrow.Serialization.ArrowArrayHelper.ReadDuration((DurationArray){valuesVar}, i))", + TypeKind2.Guid => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((GuidArray){valuesVar}).GetGuid(i)!.Value)", + TypeKind2.Binary => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((BinaryArray){valuesVar}).GetBytes(i).ToArray())", + TypeKind2.Enum => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => System.Enum.Parse<{elemType.FullTypeName}>(((StringArray){valuesVar}).GetString(i)!))", + _ => $"System.Array.Empty<{elemType.FullTypeName}>()", + }; + } + + private void EmitDictDeserialize(PropertyModel prop, int index, string colAccess) + { + var keyType = prop.Type.KeyType!; + var valueType = prop.Type.ValueType!; + var dictTypeName = $"System.Collections.Generic.Dictionary<{keyType.FullTypeName}, {valueType.FullTypeName}>"; + + Line($"var mapArr_{index} = (MapArray){colAccess};"); + if (prop.IsNullable) + { + Line($"{dictTypeName}? prop_{index};"); + Line($"if (mapArr_{index}.IsNull(0))"); + Line($" prop_{index} = null;"); + Line("else"); + Line("{"); + _indent++; + Line($"prop_{index} = new {dictTypeName}();"); + } + else + { + Line($"var prop_{index} = new {dictTypeName}();"); + Line($"if (!mapArr_{index}.IsNull(0))"); + Line("{"); + _indent++; + } + Line($"var kvStruct_{index} = (StructArray)mapArr_{index}.GetSlicedValues(0);"); + Line($"var keys_{index} = kvStruct_{index}.Fields[0];"); + Line($"var vals_{index} = kvStruct_{index}.Fields[1];"); + Line($"for (int i = 0; i < kvStruct_{index}.Length; i++)"); + Line("{"); + _indent++; + var keyReader = GetScalarReader(keyType, $"keys_{index}", "i"); + EmitDictValueRead(valueType, $"prop_{index}", keyReader, $"vals_{index}", "i", index); + _indent--; + Line("}"); + _indent--; + Line("}"); + } + + /// + /// Emits code to read a dict value and assign it to dictVar[keyExpr]. + /// Handles scalar values via GetScalarReader, and recurses for List/Dict values. + /// + private void EmitDictValueRead(TypeInfo valueType, string dictVar, string keyExpr, string valsArrayVar, string indexVar, int propIndex, int depth = 0) + { + var d = depth > 0 ? $"_{depth}" : ""; + switch (valueType.Kind) + { + case TypeKind2.List: + case TypeKind2.Array: + case TypeKind2.Set: + { + var elemType = valueType.ElementType!; + var listValsVar = $"lv_{propIndex}{d}"; + Line($"var {listValsVar} = ((ListArray){valsArrayVar}).GetSlicedValues({indexVar});"); + var elemReader = GetElementReader(elemType, listValsVar); + Line($"{dictVar}[{keyExpr}] = new System.Collections.Generic.List<{elemType.FullTypeName}>({elemReader});"); + break; + } + case TypeKind2.Dictionary: + { + var innerKeyType = valueType.KeyType!; + var innerValType = valueType.ValueType!; + var innerKvsVar = $"ikvs_{propIndex}{d}"; + var innerKeysVar = $"iks_{propIndex}{d}"; + var innerValsVar = $"ivs_{propIndex}{d}"; + var innerDictVar = $"id_{propIndex}{d}"; + var innerIdxVar = $"ii_{propIndex}{d}"; + Line($"var {innerKvsVar} = (StructArray)((MapArray){valsArrayVar}).GetSlicedValues({indexVar});"); + Line($"var {innerKeysVar} = {innerKvsVar}.Fields[0];"); + Line($"var {innerValsVar} = {innerKvsVar}.Fields[1];"); + Line($"var {innerDictVar} = new System.Collections.Generic.Dictionary<{innerKeyType.FullTypeName}, {innerValType.FullTypeName}>();"); + Line($"for (int {innerIdxVar} = 0; {innerIdxVar} < {innerKvsVar}.Length; {innerIdxVar}++)"); + Line("{"); + _indent++; + var innerKeyExpr = GetScalarReader(innerKeyType, innerKeysVar, innerIdxVar); + EmitDictValueRead(innerValType, innerDictVar, innerKeyExpr, innerValsVar, innerIdxVar, propIndex, depth + 1); + _indent--; + Line("}"); + Line($"{dictVar}[{keyExpr}] = {innerDictVar};"); + break; + } + case TypeKind2.Enum: + { + // Enum values in maps are dictionary-encoded: Dictionary(Int16, Utf8) + var enumType = valueType.FullTypeName; + var dvVar = $"dv_{propIndex}{d}"; + Line($"var {dvVar} = (DictionaryArray){valsArrayVar};"); + Line($"{dictVar}[{keyExpr}] = System.Enum.Parse<{enumType}>(((StringArray){dvVar}.Dictionary).GetString((int)((Int16Array){dvVar}.Indices).GetValue({indexVar}).Value)!);"); + break; + } + case TypeKind2.NestedRecord: + { + var typeName = valueType.FullTypeName; + var sVar = $"sv_{propIndex}{d}"; + Line($"var {sVar} = (StructArray){valsArrayVar};"); + Line($"var ca_{propIndex}{d} = new IArrowArray[{sVar}.Fields.Count];"); + Line($"for (int f = 0; f < {sVar}.Fields.Count; f++) ca_{propIndex}{d}[f] = ArrowArrayFactory.BuildArray({sVar}.Fields[f].Data.Slice({indexVar}, 1));"); + Line($"{dictVar}[{keyExpr}] = {typeName}.FromRecordBatch(new RecordBatch({typeName}.ArrowSchema, ca_{propIndex}{d}, 1));"); + break; + } + default: + { + var valueReader = GetScalarReader(valueType, valsArrayVar, indexVar); + Line($"{dictVar}[{keyExpr}] = {valueReader};"); + break; + } + } + } + + private string GetScalarReader(TypeInfo type, string arrayVar, string indexVar) + { + var stringArrayType = GetArrayReadType(type); + return type.Kind switch + { + TypeKind2.String => $"(({stringArrayType}){arrayVar}).GetString({indexVar})!", + TypeKind2.Byte => $"((UInt8Array){arrayVar}).GetValue({indexVar}).Value", + TypeKind2.SByte => $"((Int8Array){arrayVar}).GetValue({indexVar}).Value", + TypeKind2.Int16 => $"((Int16Array){arrayVar}).GetValue({indexVar}).Value", + TypeKind2.UInt16 => $"((UInt16Array){arrayVar}).GetValue({indexVar}).Value", + TypeKind2.Int32 => $"((Int32Array){arrayVar}).GetValue({indexVar}).Value", + TypeKind2.UInt32 => $"((UInt32Array){arrayVar}).GetValue({indexVar}).Value", + TypeKind2.Int64 => $"((Int64Array){arrayVar}).GetValue({indexVar}).Value", + TypeKind2.UInt64 => $"((UInt64Array){arrayVar}).GetValue({indexVar}).Value", + TypeKind2.Float => $"((FloatArray){arrayVar}).GetValue({indexVar}).Value", + TypeKind2.Double => $"((DoubleArray){arrayVar}).GetValue({indexVar}).Value", + TypeKind2.Half => $"((HalfFloatArray){arrayVar}).GetValue({indexVar}).Value", + TypeKind2.Bool => $"((BooleanArray){arrayVar}).GetValue({indexVar}).Value", + TypeKind2.Decimal => $"((Decimal128Array){arrayVar}).GetValue({indexVar}).Value", + TypeKind2.DateTime => TypeInfoHasTimezone(type) + ? $"((TimestampArray){arrayVar}).GetTimestamp({indexVar})!.Value.UtcDateTime" + : $"((TimestampArray){arrayVar}).GetTimestamp({indexVar})!.Value.DateTime", + TypeKind2.DateTimeOffset => $"((TimestampArray){arrayVar}).GetTimestamp({indexVar})!.Value", + TypeKind2.DateOnly => GetDateArrayReadType(type) == "Date64Array" + ? $"((Date64Array){arrayVar}).GetDateOnly({indexVar}).Value" + : $"((Date32Array){arrayVar}).GetDateOnly({indexVar}).Value", + TypeKind2.TimeOnly => GetTimeArrayReadType(type) == "Time32Array" + ? $"((Time32Array){arrayVar}).GetTime({indexVar}).Value" + : $"((Time64Array){arrayVar}).GetTime({indexVar}).Value", + TypeKind2.TimeSpan => $"Apache.Arrow.Serialization.ArrowArrayHelper.ReadDuration((DurationArray){arrayVar}, {indexVar})", + TypeKind2.Guid => $"((GuidArray){arrayVar}).GetGuid({indexVar})!.Value", + TypeKind2.Binary => $"((BinaryArray){arrayVar}).GetBytes({indexVar}).ToArray()", + TypeKind2.Enum => $"System.Enum.Parse<{type.FullTypeName}>(((StringArray){arrayVar}).GetString({indexVar})!)", + _ => $"default({type.FullTypeName})", + }; + } + + private void EmitNestedRecordDeserialize(PropertyModel prop, int index, string colAccess) + { + var typeName = prop.Type.FullTypeName; + + if (prop.IsNullable) + { + Line($"var structArr_{index} = (StructArray){colAccess};"); + Line($"{typeName}? prop_{index};"); + Line($"if (structArr_{index}.IsNull(0))"); + Line("{"); + _indent++; + Line($"prop_{index} = null;"); + _indent--; + Line("}"); + Line("else"); + Line("{"); + _indent++; + Line($"var nestedArrays_{index} = new IArrowArray[structArr_{index}.Fields.Count];"); + Line($"for (int i = 0; i < structArr_{index}.Fields.Count; i++) nestedArrays_{index}[i] = structArr_{index}.Fields[i];"); + Line($"var nestedBatch_{index} = new RecordBatch({typeName}.ArrowSchema, nestedArrays_{index}, 1);"); + Line($"prop_{index} = {typeName}.FromRecordBatch(nestedBatch_{index});"); + _indent--; + Line("}"); + } + else + { + Line($"var structArr_{index} = (StructArray){colAccess};"); + Line($"var nestedArrays_{index} = new IArrowArray[structArr_{index}.Fields.Count];"); + Line($"for (int i = 0; i < structArr_{index}.Fields.Count; i++) nestedArrays_{index}[i] = structArr_{index}.Fields[i];"); + Line($"var nestedBatch_{index} = new RecordBatch({typeName}.ArrowSchema, nestedArrays_{index}, 1);"); + Line($"var prop_{index} = {typeName}.FromRecordBatch(nestedBatch_{index});"); + } + } + + private void EmitMultiRowSerialize() + { + Line($"public static RecordBatch ToRecordBatch(IReadOnlyList<{_model.TypeName}> values)"); + Line("{"); + _indent++; + + Line("var count = values.Count;"); + Line(); + + // Create a builder for each column + for (int i = 0; i < _model.Properties.Count; i++) + { + var prop = _model.Properties[i]; + EmitMultiRowBuilderDecl(prop, i); + } + + Line(); + Line("for (int row = 0; row < count; row++)"); + Line("{"); + _indent++; + Line("var item = values[row];"); + + if (_model.HasSerializationCallback) + Line("((Apache.Arrow.Serialization.IArrowSerializationCallback)item).OnBeforeSerialize();"); + + for (int i = 0; i < _model.Properties.Count; i++) + { + var prop = _model.Properties[i]; + EmitMultiRowBuilderAppend(prop, i); + } + + _indent--; + Line("}"); + Line(); + + // Build arrays and return RecordBatch + for (int i = 0; i < _model.Properties.Count; i++) + { + var prop = _model.Properties[i]; + EmitMultiRowBuilderBuild(prop, i); + } + + Line(); + Line("return new RecordBatch(_arrowSchema, new IArrowArray[]"); + Line("{"); + _indent++; + for (int i = 0; i < _model.Properties.Count; i++) + { + var comma = i < _model.Properties.Count - 1 ? "," : ""; + Line($"arr_{i}{comma}"); + } + _indent--; + Line("}, count);"); + + _indent--; + Line("}"); + } + + private void EmitMultiRowBuilderDecl(PropertyModel prop, int index) + { + switch (prop.Type.Kind) + { + case TypeKind2.String: + { + var si = GetStringArrayInfo(prop); + Line($"var bld_{index} = new {si.BuilderType}();"); + break; + } + case TypeKind2.Bool: + { + var bl = GetBoolArrayInfo(prop); + Line($"var bld_{index} = new {bl.BuilderType}();"); + break; + } + case TypeKind2.Byte: + Line($"var bld_{index} = new UInt8Array.Builder();"); + break; + case TypeKind2.SByte: + Line($"var bld_{index} = new Int8Array.Builder();"); + break; + case TypeKind2.Int16: + Line($"var bld_{index} = new Int16Array.Builder();"); + break; + case TypeKind2.UInt16: + Line($"var bld_{index} = new UInt16Array.Builder();"); + break; + case TypeKind2.Int32: + Line($"var bld_{index} = new Int32Array.Builder();"); + break; + case TypeKind2.UInt32: + Line($"var bld_{index} = new UInt32Array.Builder();"); + break; + case TypeKind2.Int64: + Line($"var bld_{index} = new Int64Array.Builder();"); + break; + case TypeKind2.UInt64: + Line($"var bld_{index} = new UInt64Array.Builder();"); + break; + case TypeKind2.Float: + Line($"var bld_{index} = new FloatArray.Builder();"); + break; + case TypeKind2.Double: + Line($"var bld_{index} = new DoubleArray.Builder();"); + break; + case TypeKind2.Binary: + { + var bi = GetBinaryArrayInfo(prop); + Line($"var bld_{index} = new {bi.BuilderType}();"); + break; + } + case TypeKind2.DateTime: + case TypeKind2.DateTimeOffset: + Line($"var bld_{index} = new TimestampArray.Builder({GetTimestampTypeExpr(prop)});"); + break; + case TypeKind2.DateOnly: + { + var di = GetDateArrayInfo(prop); + Line($"var bld_{index} = new {di.BuilderType}();"); + break; + } + case TypeKind2.TimeOnly: + { + var ti = GetTimeArrayInfo(prop); + Line($"var bld_{index} = new {ti.BuilderType}({ti.TypeExpr});"); + break; + } + case TypeKind2.TimeSpan: + case TypeKind2.Decimal: + Line($"var bld_{index}_items = new List<{prop.Type.FullTypeName}{(prop.IsNullable ? "?" : "")}>(count);"); + break; + case TypeKind2.Guid: + Line($"var bld_{index}_items = new List<{prop.Type.FullTypeName}{(prop.IsNullable ? "?" : "")}>(count);"); + break; + case TypeKind2.Half: + Line($"var bld_{index} = new HalfFloatArray.Builder();"); + break; + case TypeKind2.Enum: + Line($"var bld_{index}_dict = new StringArray.Builder();"); + Line($"var bld_{index}_idx = new Int16Array.Builder();"); + Line($"var bld_{index}_map = new Dictionary();"); + break; + case TypeKind2.List: + case TypeKind2.Array: + case TypeKind2.Set: + { + var elemType = prop.Type.ElementType!; + if (NeedsManualListBuilder(elemType)) + { + // Collect raw items + offsets for manual ListArray construction + Line($"var bld_{index}_items = new List<{elemType.FullTypeName}>();"); + Line($"var bld_{index}_offsets = new ArrowBuffer.Builder();"); + if (prop.IsNullable) + Line($"var bld_{index}_nulls = new ArrowBuffer.BitmapBuilder();"); + } + else + { + var arrowElemType = GetArrowTypeExpression(elemType); + Line($"var bld_{index} = new ListArray.Builder({arrowElemType});"); + } + break; + } + case TypeKind2.Dictionary: + { + var valueType = prop.Type.ValueType!; + if (NeedsManualMapBuilder(valueType)) + { + var keyType = prop.Type.KeyType!; + var keyBuilderType = GetArrayBuilderType(keyType); + Line($"var bld_{index}_keys = new {keyBuilderType}();"); + switch (valueType.Kind) + { + case TypeKind2.Guid: + Line($"var bld_{index}_vals = new System.Collections.Generic.List();"); + break; + case TypeKind2.Enum: + Line($"var bld_{index}_valDict = new StringArray.Builder();"); + Line($"var bld_{index}_valIdx = new Int16Array.Builder();"); + Line($"var bld_{index}_valMap = new System.Collections.Generic.Dictionary();"); + break; + case TypeKind2.NestedRecord: + Line($"var bld_{index}_vals = new System.Collections.Generic.List<{valueType.FullTypeName}>();"); + break; + } + Line($"var bld_{index}_offsets = new ArrowBuffer.Builder();"); + if (prop.IsNullable) + Line($"var bld_{index}_nulls = new ArrowBuffer.BitmapBuilder();"); + } + else + { + var mapType = GetArrowTypeExpression(prop.Type); + Line($"var bld_{index} = new MapArray.Builder({mapType});"); + } + break; + } + case TypeKind2.NestedRecord: + { + var typeName = prop.Type.FullTypeName; + Line($"var bld_{index}_items = new List<{typeName}{(prop.IsNullable ? "?" : "")}>(count);"); + break; + } + case TypeKind2.Custom: + Line($"var bld_{index}_items = new List<{prop.Type.FullTypeName}>(count);"); + break; + default: + Line($"// TODO: Unsupported multi-row builder for {prop.Type.Kind}"); + break; + } + } + + private void EmitMultiRowBuilderAppend(PropertyModel prop, int index) + { + var access = $"item.{prop.PropertyName}"; + + switch (prop.Type.Kind) + { + case TypeKind2.String: + Line($"bld_{index}.Append({access});"); + break; + case TypeKind2.Bool: + if (prop.IsNullable) + { + Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(v_{index}); else bld_{index}.AppendNull();"); + } + else + Line($"bld_{index}.Append({access});"); + break; + case TypeKind2.Byte: + case TypeKind2.SByte: + case TypeKind2.Int16: + case TypeKind2.UInt16: + case TypeKind2.Int32: + case TypeKind2.UInt32: + case TypeKind2.Int64: + case TypeKind2.UInt64: + case TypeKind2.Float: + case TypeKind2.Double: + if (prop.IsNullable) + { + Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(v_{index}); else bld_{index}.AppendNull();"); + } + else + Line($"bld_{index}.Append({access});"); + break; + case TypeKind2.Binary: + Line($"bld_{index}.Append({BinaryToSpan(prop, access)});"); + break; + case TypeKind2.DateTime: + if (prop.IsNullable) + { + Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(Apache.Arrow.Serialization.ArrowArrayHelper.{TimestampConvertMethod(prop)}(v_{index})); else bld_{index}.AppendNull();"); + } + else + Line($"bld_{index}.Append(Apache.Arrow.Serialization.ArrowArrayHelper.{TimestampConvertMethod(prop)}({access}));"); + break; + case TypeKind2.DateTimeOffset: + if (prop.IsNullable) + { + Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(Apache.Arrow.Serialization.ArrowArrayHelper.{TimestampConvertMethod(prop)}(v_{index})); else bld_{index}.AppendNull();"); + } + else + Line($"bld_{index}.Append(Apache.Arrow.Serialization.ArrowArrayHelper.{TimestampConvertMethod(prop)}({access}));"); + break; + case TypeKind2.DateOnly: + if (prop.IsNullable) + { + Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(v_{index}); else bld_{index}.AppendNull();"); + } + else + Line($"bld_{index}.Append({access});"); + break; + case TypeKind2.TimeOnly: + if (prop.IsNullable) + { + Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(v_{index}); else bld_{index}.AppendNull();"); + } + else + Line($"bld_{index}.Append({access});"); + break; + case TypeKind2.TimeSpan: + case TypeKind2.Decimal: + Line($"bld_{index}_items.Add({access});"); + break; + case TypeKind2.Guid: + Line($"bld_{index}_items.Add({access});"); + break; + case TypeKind2.Half: + if (prop.IsNullable) + { + Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(v_{index}); else bld_{index}.AppendNull();"); + } + else + Line($"bld_{index}.Append({access});"); + break; + case TypeKind2.Enum: + if (prop.IsNullable) + { + Line($"if ({access} is {{ }} ev_{index})"); + Line("{"); + _indent++; + Line($"var es_{index} = ev_{index}.ToString();"); + Line($"if (!bld_{index}_map.TryGetValue(es_{index}, out var ei_{index})) {{ ei_{index} = (short)bld_{index}_map.Count; bld_{index}_map[es_{index}] = ei_{index}; }}"); + Line($"bld_{index}_idx.Append(ei_{index});"); + _indent--; + Line("}"); + Line($"else bld_{index}_idx.AppendNull();"); + } + else + { + Line($"var es_{index} = {access}.ToString();"); + Line($"if (!bld_{index}_map.TryGetValue(es_{index}, out var ei_{index})) {{ ei_{index} = (short)bld_{index}_map.Count; bld_{index}_map[es_{index}] = ei_{index}; }}"); + Line($"bld_{index}_idx.Append(ei_{index});"); + } + break; + case TypeKind2.List: + case TypeKind2.Array: + case TypeKind2.Set: + { + var elemType = prop.Type.ElementType!; + if (prop.IsNullable) + { + Line($"if ({access} != null)"); + Line("{"); + _indent++; + } + if (NeedsManualListBuilder(elemType)) + { + Line($"bld_{index}_offsets.Append(bld_{index}_items.Count);"); + Line($"bld_{index}_items.AddRange({access});"); + if (prop.IsNullable) Line($"bld_{index}_nulls.Append(true);"); + } + else + { + Line($"bld_{index}.Append();"); + EmitListValuePopulation(elemType, $"bld_{index}", access, index); + } + if (prop.IsNullable) + { + _indent--; + Line("}"); + Line("else"); + Line("{"); + _indent++; + if (NeedsManualListBuilder(elemType)) + { + Line($"bld_{index}_offsets.Append(bld_{index}_items.Count);"); + Line($"bld_{index}_nulls.Append(false);"); + } + else + { + Line($"bld_{index}.AppendNull();"); + } + _indent--; + Line("}"); + } + break; + } + case TypeKind2.Dictionary: + { + var keyType = prop.Type.KeyType!; + var valueType = prop.Type.ValueType!; + if (NeedsManualMapBuilder(valueType)) + { + if (prop.IsNullable) + { + Line($"if ({access} != null)"); + Line("{"); + _indent++; + } + Line($"bld_{index}_offsets.Append(bld_{index}_keys.Length);"); + Line($"foreach (var kv_{index} in {access})"); + Line("{"); + _indent++; + Line($"bld_{index}_keys.Append(kv_{index}.Key);"); + switch (valueType.Kind) + { + case TypeKind2.Guid: + Line($"bld_{index}_vals.Add(kv_{index}.Value);"); + break; + case TypeKind2.Enum: + { + Line($"var es_{index} = kv_{index}.Value.ToString();"); + Line($"if (!bld_{index}_valMap.TryGetValue(es_{index}, out var ei_{index})) {{ ei_{index} = (short)bld_{index}_valMap.Count; bld_{index}_valMap[es_{index}] = ei_{index}; }}"); + Line($"bld_{index}_valIdx.Append(ei_{index});"); + break; + } + case TypeKind2.NestedRecord: + Line($"bld_{index}_vals.Add(kv_{index}.Value);"); + break; + } + _indent--; + Line("}"); + if (prop.IsNullable) + { + Line($"bld_{index}_nulls.Append(true);"); + _indent--; + Line("}"); + Line("else"); + Line("{"); + _indent++; + Line($"bld_{index}_offsets.Append(bld_{index}_keys.Length);"); + Line($"bld_{index}_nulls.Append(false);"); + _indent--; + Line("}"); + } + } + else + { + if (prop.IsNullable) + { + Line($"if ({access} != null)"); + Line("{"); + _indent++; + } + Line($"bld_{index}.Append();"); + Line($"foreach (var kv_{index} in {access})"); + Line("{"); + _indent++; + EmitMapKeyAppend(keyType, $"bld_{index}", $"kv_{index}.Key", index); + EmitMapValueAppend(valueType, $"bld_{index}", $"kv_{index}.Value", index); + _indent--; + Line("}"); + if (prop.IsNullable) + { + _indent--; + Line("}"); + Line($"else bld_{index}.AppendNull();"); + } + } + break; + } + case TypeKind2.NestedRecord: + Line($"bld_{index}_items.Add({access});"); + break; + case TypeKind2.Custom: + Line($"bld_{index}_items.Add({access});"); + break; + default: + Line($"// TODO: Unsupported multi-row append for {prop.Type.Kind}"); + break; + } + } + + private void EmitMultiRowBuilderBuild(PropertyModel prop, int index) + { + switch (prop.Type.Kind) + { + case TypeKind2.Enum: + { + Line($"var dictValues_{index} = new StringArray.Builder();"); + Line($"var sortedEntries_{index} = bld_{index}_map.OrderBy(kv => kv.Value).Select(kv => kv.Key);"); + Line($"foreach (var e in sortedEntries_{index}) dictValues_{index}.Append(e);"); + Line($"var arr_{index} = new DictionaryArray(new DictionaryType(Int16Type.Default, StringType.Default, false), bld_{index}_idx.Build(), dictValues_{index}.Build());"); + break; + } + case TypeKind2.NestedRecord: + { + var typeName = prop.Type.FullTypeName; + if (prop.IsNullable) + { + // For nullable nested: substitute nulls with the first non-null item (value doesn't matter for null rows) + Line($"IArrowArray arr_{index};"); + Line("{"); + _indent++; + Line($"var firstNonNull_{index} = bld_{index}_items.FirstOrDefault(x => x != null);"); + Line($"var nullBitmap_{index} = new ArrowBuffer.BitmapBuilder();"); + Line($"int nullCount_{index} = 0;"); + Line($"if (firstNonNull_{index} != null)"); + Line("{"); + _indent++; + Line($"var resolved_{index} = new List<{typeName}>(count);"); + Line($"for (int r = 0; r < count; r++)"); + Line("{"); + _indent++; + Line($"if (bld_{index}_items[r] != null) {{ resolved_{index}.Add(bld_{index}_items[r]!); nullBitmap_{index}.Append(true); }}"); + Line($"else {{ resolved_{index}.Add(firstNonNull_{index}!); nullBitmap_{index}.Append(false); nullCount_{index}++; }}"); + _indent--; + Line("}"); + Line($"var nb_{index} = {typeName}.ToRecordBatch((IReadOnlyList<{typeName}>)resolved_{index});"); + Line($"var ca_{index} = new IArrowArray[nb_{index}.ColumnCount];"); + Line($"for (int f = 0; f < nb_{index}.ColumnCount; f++) ca_{index}[f] = nb_{index}.Column(f);"); + Line($"arr_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), count, ca_{index}, nullBitmap_{index}.Build(), nullCount_{index});"); + _indent--; + Line("}"); + Line("else"); + Line("{"); + _indent++; + Line($"for (int r = 0; r < count; r++) {{ nullBitmap_{index}.Append(false); nullCount_{index}++; }}"); + Line($"var nc_{index} = new IArrowArray[{typeName}.ArrowSchema.FieldsList.Count];"); + Line($"for (int f = 0; f < nc_{index}.Length; f++)"); + Line($" nc_{index}[f] = Apache.Arrow.Serialization.ArrowArrayHelper.BuildNullArray({typeName}.ArrowSchema.FieldsList[f].DataType, count);"); + Line($"arr_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), count, nc_{index}, nullBitmap_{index}.Build(), nullCount_{index});"); + _indent--; + Line("}"); + _indent--; + Line("}"); + } + else + { + // Non-nullable: directly use multi-row serializer + Line($"var nb_{index} = {typeName}.ToRecordBatch((IReadOnlyList<{typeName}>)bld_{index}_items);"); + Line($"var ca_{index} = new IArrowArray[nb_{index}.ColumnCount];"); + Line($"for (int f = 0; f < nb_{index}.ColumnCount; f++) ca_{index}[f] = nb_{index}.Column(f);"); + Line($"var arr_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), count, ca_{index}, ArrowBuffer.Empty);"); + } + break; + } + case TypeKind2.TimeOnly: + Line($"var arr_{index} = bld_{index}.Build();"); + break; + case TypeKind2.TimeSpan: + Line($"var arr_{index} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildDurationArray(bld_{index}_items);"); + break; + case TypeKind2.Decimal: + Line($"var arr_{index} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildDecimalArray(bld_{index}_items);"); + break; + case TypeKind2.Guid: + Line($"var arr_{index} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildGuidArray(bld_{index}_items);"); + break; + case TypeKind2.Custom: + Line($"var arr_{index} = new {prop.ConverterTypeName}().ToArray(bld_{index}_items);"); + break; + case TypeKind2.List: + case TypeKind2.Array: + case TypeKind2.Set: + { + var elemType = prop.Type.ElementType!; + if (NeedsManualListBuilder(elemType)) + { + var arrowElemType = GetArrowTypeExpression(elemType); + // Finalize the offsets with the total item count + Line($"bld_{index}_offsets.Append(bld_{index}_items.Count);"); + switch (elemType.Kind) + { + case TypeKind2.Guid: + { + Line($"var guidValues_{index} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildGuidArray(bld_{index}_items);"); + var guidNullBuf = prop.IsNullable ? $"bld_{index}_nulls.Build()" : "ArrowBuffer.Empty"; + Line($"var arr_{index} = new ListArray(new ListType(new Field(\"item\", {arrowElemType}, false)), count, bld_{index}_offsets.Build(), guidValues_{index}, {guidNullBuf});"); + break; + } + case TypeKind2.Enum: + { + // Dictionary-encode enum values: Dictionary(Int16, Utf8) + Line($"var enumDict_{index} = new StringArray.Builder();"); + Line($"var enumIdx_{index} = new Int16Array.Builder();"); + Line($"var enumMap_{index} = new System.Collections.Generic.Dictionary();"); + Line($"foreach (var e in bld_{index}_items) {{"); + Line($" var es = e.ToString();"); + Line($" if (!enumMap_{index}.TryGetValue(es, out var ei)) {{ ei = (short)enumMap_{index}.Count; enumMap_{index}[es] = ei; }}"); + Line($" enumIdx_{index}.Append(ei);"); + Line($"}}"); + Line($"foreach (var kv in enumMap_{index}.OrderBy(kv => kv.Value)) enumDict_{index}.Append(kv.Key);"); + Line($"var enumDictArr_{index} = new DictionaryArray(new DictionaryType(Int16Type.Default, StringType.Default, false), enumIdx_{index}.Build(), enumDict_{index}.Build());"); + var enumNullBuf = prop.IsNullable ? $"bld_{index}_nulls.Build()" : "ArrowBuffer.Empty"; + Line($"var arr_{index} = new ListArray(ArrowSchema.GetFieldByName(\"{prop.FieldName}\").DataType, count, bld_{index}_offsets.Build(), enumDictArr_{index}, {enumNullBuf});"); + break; + } + case TypeKind2.NestedRecord: + { + var typeName = elemType.FullTypeName; + Line($"IArrowArray structValues_{index};"); + Line($"if (bld_{index}_items.Count > 0)"); + Line("{"); + _indent++; + Line($"var nb_{index} = {typeName}.ToRecordBatch((IReadOnlyList<{typeName}>)bld_{index}_items);"); + Line($"var ca_{index} = new IArrowArray[nb_{index}.ColumnCount];"); + Line($"for (int f = 0; f < nb_{index}.ColumnCount; f++) ca_{index}[f] = nb_{index}.Column(f);"); + Line($"structValues_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), bld_{index}_items.Count, ca_{index}, ArrowBuffer.Empty);"); + _indent--; + Line("}"); + Line("else"); + Line("{"); + _indent++; + Line($"structValues_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), 0, System.Array.Empty(), ArrowBuffer.Empty);"); + _indent--; + Line("}"); + var nestedNullBuf = prop.IsNullable ? $"bld_{index}_nulls.Build()" : "ArrowBuffer.Empty"; + Line($"var arr_{index} = new ListArray(new ListType(new Field(\"item\", new StructType({typeName}.ArrowSchema.FieldsList), false)), count, bld_{index}_offsets.Build(), structValues_{index}, {nestedNullBuf});"); + break; + } + } + } + else + { + Line($"var arr_{index} = bld_{index}.Build();"); + } + break; + } + case TypeKind2.Dictionary: + { + var valueType = prop.Type.ValueType!; + if (NeedsManualMapBuilder(valueType)) + { + var mapType = GetArrowTypeExpression(prop.Type); + // Finalize offsets + Line($"bld_{index}_offsets.Append(bld_{index}_keys.Length);"); + Line($"var mkArr_{index} = bld_{index}_keys.Build();"); + + // Build value array + switch (valueType.Kind) + { + case TypeKind2.Guid: + Line($"var mvArr_{index} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildGuidArray(bld_{index}_vals);"); + break; + case TypeKind2.Enum: + { + Line($"foreach (var kv in bld_{index}_valMap.OrderBy(kv => kv.Value)) bld_{index}_valDict.Append(kv.Key);"); + Line($"var mvArr_{index} = new DictionaryArray(new DictionaryType(Int16Type.Default, StringType.Default, false), bld_{index}_valIdx.Build(), bld_{index}_valDict.Build());"); + break; + } + case TypeKind2.NestedRecord: + { + var typeName = valueType.FullTypeName; + Line($"IArrowArray mvArr_{index};"); + Line($"if (bld_{index}_vals.Count > 0)"); + Line("{"); + _indent++; + Line($"var nb_{index} = {typeName}.ToRecordBatch((IReadOnlyList<{typeName}>)bld_{index}_vals);"); + Line($"var ca_{index} = new IArrowArray[nb_{index}.ColumnCount];"); + Line($"for (int f = 0; f < nb_{index}.ColumnCount; f++) ca_{index}[f] = nb_{index}.Column(f);"); + Line($"mvArr_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), bld_{index}_vals.Count, ca_{index}, ArrowBuffer.Empty);"); + _indent--; + Line("}"); + Line($"else mvArr_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), 0, System.Array.Empty(), ArrowBuffer.Empty);"); + break; + } + } + + // Construct MapArray manually + var nullBuf = prop.IsNullable ? $"bld_{index}_nulls.Build()" : "ArrowBuffer.Empty"; + Line($"var mEntries_{index} = new StructArray(((MapType){mapType}).KeyValueType, mkArr_{index}.Length, new IArrowArray[] {{ mkArr_{index}, mvArr_{index} }}, ArrowBuffer.Empty);"); + Line($"var arr_{index} = new MapArray({mapType}, count, bld_{index}_offsets.Build(), mEntries_{index}, {nullBuf});"); + } + else + { + Line($"var arr_{index} = bld_{index}.Build();"); + } + break; + } + default: + Line($"var arr_{index} = bld_{index}.Build();"); + break; + } + } + + private void EmitMultiRowDeserialize() + { + var newKw = _model.HasArrowSerializableBase ? "new " : ""; + Line($"public static {newKw}IReadOnlyList<{_model.TypeName}> ListFromRecordBatch(RecordBatch batch)"); + Line("{"); + _indent++; + Line("var count = batch.Length;"); + Line($"var result = new List<{_model.TypeName}>(count);"); + Line(); + + // Get column references + for (int i = 0; i < _model.Properties.Count; i++) + { + var prop = _model.Properties[i]; + EmitMultiRowColumnRef(prop, i); + } + + Line(); + Line("for (int row = 0; row < count; row++)"); + Line("{"); + _indent++; + + // Read each property from the column at 'row' + for (int i = 0; i < _model.Properties.Count; i++) + { + var prop = _model.Properties[i]; + EmitMultiRowReadProperty(prop, i); + } + + // Construct the record + if (_model.HasSerializationCallback) + { + EmitObjectConstruction("var item = ", ";"); + Line("((Apache.Arrow.Serialization.IArrowSerializationCallback)item).OnAfterDeserialize();"); + Line("result.Add(item);"); + } + else + { + EmitObjectConstruction("result.Add(", ");"); + } + + _indent--; + Line("}"); + Line(); + Line("return result;"); + + _indent--; + Line("}"); + } + + private void EmitMultiRowColumnRef(PropertyModel prop, int index) + { + var colAccess = $"batch.Column(\"{prop.FieldName}\")"; + + switch (prop.Type.Kind) + { + case TypeKind2.String: + { + var si = GetStringArrayInfo(prop); + Line($"var col_{index} = ({si.ArrayType}){colAccess};"); + break; + } + case TypeKind2.Bool: + { + var bl = GetBoolArrayInfo(prop); + Line($"var col_{index} = ({bl.ArrayType}){colAccess};"); + break; + } + case TypeKind2.Byte: + Line($"var col_{index} = (UInt8Array){colAccess};"); + break; + case TypeKind2.SByte: + Line($"var col_{index} = (Int8Array){colAccess};"); + break; + case TypeKind2.Int16: + Line($"var col_{index} = (Int16Array){colAccess};"); + break; + case TypeKind2.UInt16: + Line($"var col_{index} = (UInt16Array){colAccess};"); + break; + case TypeKind2.Int32: + Line($"var col_{index} = (Int32Array){colAccess};"); + break; + case TypeKind2.UInt32: + Line($"var col_{index} = (UInt32Array){colAccess};"); + break; + case TypeKind2.Int64: + Line($"var col_{index} = (Int64Array){colAccess};"); + break; + case TypeKind2.UInt64: + Line($"var col_{index} = (UInt64Array){colAccess};"); + break; + case TypeKind2.Float: + Line($"var col_{index} = (FloatArray){colAccess};"); + break; + case TypeKind2.Double: + Line($"var col_{index} = (DoubleArray){colAccess};"); + break; + case TypeKind2.Binary: + { + var bi = GetBinaryArrayInfo(prop); + Line($"var col_{index} = ({bi.ArrayType}){colAccess};"); + break; + } + case TypeKind2.DateTime: + case TypeKind2.DateTimeOffset: + Line($"var col_{index} = (TimestampArray){colAccess};"); + break; + case TypeKind2.DateOnly: + { + var di = GetDateArrayInfo(prop); + Line($"var col_{index} = ({di.ArrayType}){colAccess};"); + break; + } + case TypeKind2.TimeOnly: + { + var ti = GetTimeArrayInfo(prop); + Line($"var col_{index} = ({ti.ArrayType}){colAccess};"); + break; + } + case TypeKind2.TimeSpan: + Line($"var col_{index} = (DurationArray){colAccess};"); + break; + case TypeKind2.Decimal: + Line($"var col_{index} = (Decimal128Array){colAccess};"); + break; + case TypeKind2.Guid: + Line($"var col_{index} = (GuidArray){colAccess};"); + break; + case TypeKind2.Half: + Line($"var col_{index} = (HalfFloatArray){colAccess};"); + break; + case TypeKind2.Enum: + Line($"var col_{index} = (DictionaryArray){colAccess};"); + break; + case TypeKind2.List: + case TypeKind2.Array: + case TypeKind2.Set: + Line($"var col_{index} = (ListArray){colAccess};"); + break; + case TypeKind2.Dictionary: + Line($"var col_{index} = (MapArray){colAccess};"); + break; + case TypeKind2.NestedRecord: + Line($"var col_{index} = (StructArray){colAccess};"); + break; + case TypeKind2.Custom: + Line($"var col_{index} = {colAccess};"); + break; + default: + Line($"var col_{index} = {colAccess};"); + break; + } + } + + private void EmitMultiRowReadProperty(PropertyModel prop, int index) + { + switch (prop.Type.Kind) + { + case TypeKind2.String: + if (prop.IsNullable) + Line($"var prop_{index} = col_{index}.IsNull(row) ? null : col_{index}.GetString(row);"); + else + Line($"var prop_{index} = col_{index}.GetString(row)!;"); + break; + case TypeKind2.Bool: + case TypeKind2.Byte: + case TypeKind2.SByte: + case TypeKind2.Int16: + case TypeKind2.UInt16: + case TypeKind2.Int32: + case TypeKind2.UInt32: + case TypeKind2.Int64: + case TypeKind2.UInt64: + case TypeKind2.Float: + case TypeKind2.Double: + if (prop.IsNullable) + Line($"var prop_{index} = col_{index}.IsNull(row) ? ({prop.Type.FullTypeName}?)null : col_{index}.GetValue(row).Value;"); + else + Line($"var prop_{index} = col_{index}.GetValue(row).Value;"); + break; + case TypeKind2.Binary: + if (IsReadOnlyMemoryByte(prop)) + Line($"ReadOnlyMemory prop_{index} = col_{index}.GetBytes(row).ToArray();"); + else + Line($"var prop_{index} = col_{index}.GetBytes(row).ToArray();"); + break; + case TypeKind2.DateTime: + { + var dtProp = TimestampHasTimezone(prop) ? "UtcDateTime" : "DateTime"; + if (prop.IsNullable) + Line($"var prop_{index} = col_{index}.IsNull(row) ? (System.DateTime?)null : col_{index}.GetTimestamp(row)!.Value.{dtProp};"); + else + Line($"var prop_{index} = col_{index}.GetTimestamp(row)!.Value.{dtProp};"); + break; + } + case TypeKind2.DateTimeOffset: + if (prop.IsNullable) + Line($"var prop_{index} = col_{index}.IsNull(row) ? (System.DateTimeOffset?)null : col_{index}.GetTimestamp(row)!.Value;"); + else + Line($"var prop_{index} = col_{index}.GetTimestamp(row)!.Value;"); + break; + case TypeKind2.DateOnly: + if (prop.IsNullable) + Line($"var prop_{index} = col_{index}.IsNull(row) ? (System.DateOnly?)null : col_{index}.GetDateOnly(row);"); + else + Line($"var prop_{index} = col_{index}.GetDateOnly(row)!.Value;"); + break; + case TypeKind2.TimeOnly: + if (prop.IsNullable) + Line($"var prop_{index} = col_{index}.GetTime(row);"); + else + Line($"var prop_{index} = col_{index}.GetTime(row)!.Value;"); + break; + case TypeKind2.TimeSpan: + if (prop.IsNullable) + Line($"var prop_{index} = col_{index}.IsNull(row) ? (System.TimeSpan?)null : Apache.Arrow.Serialization.ArrowArrayHelper.ReadDuration(col_{index}, row);"); + else + Line($"var prop_{index} = Apache.Arrow.Serialization.ArrowArrayHelper.ReadDuration(col_{index}, row);"); + break; + case TypeKind2.Decimal: + if (prop.IsNullable) + Line($"var prop_{index} = col_{index}.IsNull(row) ? (decimal?)null : ((Decimal128Array)col_{index}).GetValue(row).Value;"); + else + Line($"var prop_{index} = ((Decimal128Array)col_{index}).GetValue(row).Value;"); + break; + case TypeKind2.Guid: + if (prop.IsNullable) + Line($"var prop_{index} = col_{index}.GetGuid(row);"); + else + Line($"var prop_{index} = col_{index}.GetGuid(row)!.Value;"); + break; + case TypeKind2.Half: + if (prop.IsNullable) + Line($"var prop_{index} = col_{index}.IsNull(row) ? (System.Half?)null : col_{index}.GetValue(row).Value;"); + else + Line($"var prop_{index} = col_{index}.GetValue(row).Value;"); + break; + case TypeKind2.Enum: + { + var enumType = prop.Type.FullTypeName; + if (prop.IsNullable) + Line($"var prop_{index} = col_{index}.IsNull(row) ? ({enumType}?)null : System.Enum.Parse<{enumType}>(((StringArray)col_{index}.Dictionary).GetString((int)((Int16Array)col_{index}.Indices).GetValue(row).Value)!);"); + else + Line($"var prop_{index} = System.Enum.Parse<{enumType}>(((StringArray)col_{index}.Dictionary).GetString((int)((Int16Array)col_{index}.Indices).GetValue(row).Value)!);"); + break; + } + case TypeKind2.List: + case TypeKind2.Array: + case TypeKind2.Set: + { + var elemType = prop.Type.ElementType!; + // Nullable list: wrap in IsNull check + if (prop.IsNullable) + { + var collTypeName = prop.Type.Kind switch + { + TypeKind2.Array => $"{elemType.FullTypeName}[]", + TypeKind2.Set => $"System.Collections.Generic.HashSet<{elemType.FullTypeName}>", + _ => $"System.Collections.Generic.List<{elemType.FullTypeName}>" + }; + Line($"{collTypeName}? prop_{index};"); + Line($"if (col_{index}.IsNull(row))"); + Line($" prop_{index} = null;"); + Line("else"); + Line("{"); + _indent++; + } + if (NeedsManualListDeserialize(elemType)) + { + // Extension types can't use GetSlicedValues; use offsets + Values directly + Line($"var listStart_{index} = col_{index}.ValueOffsets[row];"); + Line($"var listLen_{index} = col_{index}.GetValueLength(row);"); + var multiDeclPrefix = prop.IsNullable ? $"prop_{index}" : $"var prop_{index}"; + switch (elemType.Kind) + { + case TypeKind2.Guid: + { + Line($"var guidArr_{index} = (GuidArray)col_{index}.Values;"); + var readExpr = $"Enumerable.Range(listStart_{index}, listLen_{index}).Select(i => guidArr_{index}.GetGuid(i)!.Value)"; + if (prop.Type.Kind == TypeKind2.Array) + Line($"{multiDeclPrefix} = {readExpr}.ToArray();"); + else if (prop.Type.Kind == TypeKind2.Set) + Line($"{multiDeclPrefix} = new System.Collections.Generic.HashSet<{elemType.FullTypeName}>({readExpr});"); + else + Line($"{multiDeclPrefix} = new System.Collections.Generic.List<{elemType.FullTypeName}>({readExpr});"); + break; + } + case TypeKind2.Enum: + { + // Dictionary-encoded enum values: Dictionary(Int16, Utf8) + Line($"var enumDictArr_{index} = (DictionaryArray)col_{index}.Values;"); + Line($"var enumDictValues_{index} = (StringArray)enumDictArr_{index}.Dictionary;"); + var readExpr2 = $"Enumerable.Range(listStart_{index}, listLen_{index}).Select(i => System.Enum.Parse<{elemType.FullTypeName}>(enumDictValues_{index}.GetString(((Int16Array)enumDictArr_{index}.Indices).GetValue(i)!.Value)!))"; + if (prop.Type.Kind == TypeKind2.Array) + Line($"{multiDeclPrefix} = {readExpr2}.ToArray();"); + else if (prop.Type.Kind == TypeKind2.Set) + Line($"{multiDeclPrefix} = new System.Collections.Generic.HashSet<{elemType.FullTypeName}>({readExpr2});"); + else + Line($"{multiDeclPrefix} = new System.Collections.Generic.List<{elemType.FullTypeName}>({readExpr2});"); + break; + } + case TypeKind2.NestedRecord: + { + var typeName = elemType.FullTypeName; + Line($"var structArr_{index} = (StructArray)col_{index}.Values;"); + Line($"{multiDeclPrefix} = new System.Collections.Generic.List<{typeName}>(listLen_{index});"); + Line($"for (int i = listStart_{index}; i < listStart_{index} + listLen_{index}; i++)"); + Line("{"); + _indent++; + Line($"var childArrays_{index} = new IArrowArray[structArr_{index}.Fields.Count];"); + Line($"for (int f = 0; f < structArr_{index}.Fields.Count; f++) childArrays_{index}[f] = ArrowArrayFactory.BuildArray(structArr_{index}.Fields[f].Data.Slice(i, 1));"); + Line($"prop_{index}.Add({typeName}.FromRecordBatch(new RecordBatch({typeName}.ArrowSchema, childArrays_{index}, 1)));"); + _indent--; + Line("}"); + break; + } + case TypeKind2.List: + case TypeKind2.Array: + { + // Nested list elements: read inner lists from ListArray values + var innerElemType = elemType.ElementType!; + var innerTypeName = elemType.FullTypeName; + Line($"var innerArr_{index} = (ListArray)col_{index}.Values;"); + Line($"{multiDeclPrefix} = new System.Collections.Generic.List<{innerTypeName}>(listLen_{index});"); + Line($"for (int i = listStart_{index}; i < listStart_{index} + listLen_{index}; i++)"); + Line("{"); + _indent++; + if (innerElemType.Kind == TypeKind2.Dictionary) + { + // Dict inner elements: read each dict from inner list's MapArray + var keyType2 = innerElemType.KeyType!; + var valType2 = innerElemType.ValueType!; + var dictTypeName2 = $"System.Collections.Generic.Dictionary<{keyType2.FullTypeName}, {valType2.FullTypeName}>"; + Line($"var mapSlice_{index} = (MapArray)innerArr_{index}.GetSlicedValues(i);"); + Line($"var innerResult_{index} = new System.Collections.Generic.List<{dictTypeName2}>(mapSlice_{index}.Length);"); + Line($"for (int j_{index} = 0; j_{index} < mapSlice_{index}.Length; j_{index}++)"); + Line("{"); + _indent++; + Line($"var kvs_{index} = (StructArray)mapSlice_{index}.GetSlicedValues(j_{index});"); + Line($"var ks_{index} = kvs_{index}.Fields[0];"); + Line($"var vs_{index} = kvs_{index}.Fields[1];"); + Line($"var d_{index} = new {dictTypeName2}();"); + Line($"for (int k_{index} = 0; k_{index} < kvs_{index}.Length; k_{index}++)"); + Line("{"); + _indent++; + var dKeyReader2 = GetScalarReader(keyType2, $"ks_{index}", $"k_{index}"); + EmitDictValueRead(valType2, $"d_{index}", dKeyReader2, $"vs_{index}", $"k_{index}", index); + _indent--; + Line("}"); + Line($"innerResult_{index}.Add(d_{index});"); + _indent--; + Line("}"); + Line($"prop_{index}.Add(innerResult_{index});"); + } + else + { + Line($"var innerValues_{index} = innerArr_{index}.GetSlicedValues(i);"); + var innerElemReader = GetElementReader(innerElemType, $"innerValues_{index}"); + if (elemType.Kind == TypeKind2.Array) + Line($"prop_{index}.Add({innerElemReader}.ToArray());"); + else + Line($"prop_{index}.Add(new {innerTypeName}({innerElemReader}));"); + } + _indent--; + Line("}"); + break; + } + case TypeKind2.Dictionary: + { + // Dict elements inside a list: read each dict from MapArray values + var keyType = elemType.KeyType!; + var valType = elemType.ValueType!; + var dictTypeName = $"System.Collections.Generic.Dictionary<{keyType.FullTypeName}, {valType.FullTypeName}>"; + Line($"var mapArr_{index} = (MapArray)col_{index}.Values;"); + Line($"{multiDeclPrefix} = new System.Collections.Generic.List<{dictTypeName}>(listLen_{index});"); + Line($"for (int i = listStart_{index}; i < listStart_{index} + listLen_{index}; i++)"); + Line("{"); + _indent++; + Line($"var kvs_{index} = (StructArray)mapArr_{index}.GetSlicedValues(i);"); + Line($"var ks_{index} = kvs_{index}.Fields[0];"); + Line($"var vs_{index} = kvs_{index}.Fields[1];"); + Line($"var d_{index} = new {dictTypeName}();"); + Line($"for (int j_{index} = 0; j_{index} < kvs_{index}.Length; j_{index}++)"); + Line("{"); + _indent++; + var dKeyReader = GetScalarReader(keyType, $"ks_{index}", $"j_{index}"); + EmitDictValueRead(valType, $"d_{index}", dKeyReader, $"vs_{index}", $"j_{index}", index); + _indent--; + Line("}"); + Line($"prop_{index}.Add(d_{index});"); + _indent--; + Line("}"); + break; + } + } + } + else + { + var nonManualDeclPrefix = prop.IsNullable ? $"prop_{index}" : $"var prop_{index}"; + Line($"var listValues_{index} = col_{index}.GetSlicedValues(row);"); + var elemReader = GetElementReader(elemType, $"listValues_{index}"); + if (prop.Type.Kind == TypeKind2.Array) + Line($"{nonManualDeclPrefix} = {elemReader}.ToArray();"); + else if (prop.Type.Kind == TypeKind2.Set) + Line($"{nonManualDeclPrefix} = new System.Collections.Generic.HashSet<{elemType.FullTypeName}>({elemReader});"); + else + Line($"{nonManualDeclPrefix} = new System.Collections.Generic.List<{elemType.FullTypeName}>({elemReader});"); + } + if (prop.IsNullable) + { + _indent--; + Line("}"); + } + break; + } + case TypeKind2.Dictionary: + { + var keyType = prop.Type.KeyType!; + var valueType = prop.Type.ValueType!; + var dictTypeName2 = $"System.Collections.Generic.Dictionary<{keyType.FullTypeName}, {valueType.FullTypeName}>"; + if (prop.IsNullable) + { + Line($"{dictTypeName2}? prop_{index};"); + Line($"if (col_{index}.IsNull(row))"); + Line($" prop_{index} = null;"); + Line("else"); + Line("{"); + _indent++; + Line($"prop_{index} = new {dictTypeName2}();"); + } + else + { + Line($"var prop_{index} = new {dictTypeName2}();"); + Line($"if (!col_{index}.IsNull(row))"); + Line("{"); + _indent++; + } + Line($"var kvStruct_{index} = (StructArray)col_{index}.GetSlicedValues(row);"); + Line($"var keys_{index} = kvStruct_{index}.Fields[0];"); + Line($"var vals_{index} = kvStruct_{index}.Fields[1];"); + Line($"for (int i = 0; i < kvStruct_{index}.Length; i++)"); + Line("{"); + _indent++; + var keyReader = GetScalarReader(keyType, $"keys_{index}", "i"); + EmitDictValueRead(valueType, $"prop_{index}", keyReader, $"vals_{index}", "i", index); + _indent--; + Line("}"); + _indent--; + Line("}"); + break; + } + case TypeKind2.NestedRecord: + { + var typeName = prop.Type.FullTypeName; + if (prop.IsNullable) + { + Line($"{typeName}? prop_{index};"); + Line($"if (col_{index}.IsNull(row))"); + Line("{"); + _indent++; + Line($"prop_{index} = null;"); + _indent--; + Line("}"); + Line("else"); + Line("{"); + _indent++; + EmitMultiRowNestedRead(typeName, index, declareVar: false); + _indent--; + Line("}"); + } + else + { + EmitMultiRowNestedRead(typeName, index, declareVar: true); + } + break; + } + case TypeKind2.Custom: + if (prop.IsNullable) + Line($"var prop_{index} = col_{index}.IsNull(row) ? ({prop.Type.FullTypeName}?)null : new {prop.ConverterTypeName}().FromArray(col_{index}, row);"); + else + Line($"var prop_{index} = new {prop.ConverterTypeName}().FromArray(col_{index}, row);"); + break; + default: + Line($"object? prop_{index} = null; // TODO: Unsupported type {prop.Type.Kind}"); + break; + } + } + + private void EmitMultiRowNestedRead(string typeName, int index, bool declareVar) + { + Line($"var na_{index} = new IArrowArray[col_{index}.Fields.Count];"); + Line($"for (int f = 0; f < col_{index}.Fields.Count; f++)"); + Line("{"); + _indent++; + Line($"na_{index}[f] = ArrowArrayFactory.BuildArray(col_{index}.Fields[f].Data.Slice(row, 1));"); + _indent--; + Line("}"); + var decl = declareVar ? "var " : ""; + Line($"{decl}prop_{index} = {typeName}.FromRecordBatch(new RecordBatch({typeName}.ArrowSchema, na_{index}, 1));"); + } + + // --- Arrow type expression helpers --- + + private string GetTypeKeyword() + { + if (_model.IsRecord && _model.IsValueType) return "record struct"; + if (_model.IsRecord) return "record"; + if (_model.IsValueType) return "struct"; + return "class"; + } + + private static string TimestampConvertMethod(PropertyModel prop) => + TimestampHasTimezone(prop) ? "ToUtcDateTimeOffset" : "ToWallClockDateTimeOffset"; + + private static bool IsReadOnlyMemoryByte(PropertyModel prop) => + prop.Type.FullTypeName.Contains("ReadOnlyMemory"); + + private static string BinaryToSpan(PropertyModel prop, string access) => + IsReadOnlyMemoryByte(prop) ? $"{access}.Span" : $"(ReadOnlySpan){access}"; + + /// + /// Like TimestampHasTimezone but works on TypeInfo directly (for element types in collections). + /// + private static bool TypeInfoHasTimezone(TypeInfo type) + { + var over = type.ArrowTypeOverride; + if (over == null) + return true; // default is UTC + var lower = over.Trim().ToLowerInvariant(); + if (!lower.StartsWith("timestamp")) + return true; + var start = lower.IndexOfAny(new[] { '[', '(' }); + if (start < 0) + return false; + var end = lower.IndexOfAny(new[] { ']', ')' }, start); + if (end < 0) + return false; + var inner = lower.Substring(start + 1, end - start - 1); + return inner.Contains(","); + } + + private static string GetDateArrayBuilderType(TypeInfo type) + { + var over = type.ArrowTypeOverride?.Trim().ToLowerInvariant(); + if (over != null && over.StartsWith("date64")) + return "Date64Array.Builder"; + return "Date32Array.Builder"; + } + + private static string GetTimeArrayBuilderType(TypeInfo type) + { + var over = type.ArrowTypeOverride?.Trim().ToLowerInvariant(); + if (over != null && over.StartsWith("time32")) + return "Time32Array.Builder"; + return "Time64Array.Builder"; + } + + private static string GetDateArrayReadType(TypeInfo type) + { + var over = type.ArrowTypeOverride?.Trim().ToLowerInvariant(); + if (over != null && over.StartsWith("date64")) + return "Date64Array"; + return "Date32Array"; + } + + private static string GetTimeArrayReadType(TypeInfo type) + { + var over = type.ArrowTypeOverride?.Trim().ToLowerInvariant(); + if (over != null && over.StartsWith("time32")) + return "Time32Array"; + return "Time64Array"; + } + + private string GetTimestampTypeExpr(PropertyModel prop) => GetArrowTypeExpression(prop.Type); + + /// + /// Returns true if the timestamp type has a timezone (default is UTC). + /// [ArrowType("timestamp[us]")] → no timezone. + /// No override or [ArrowType("timestamp[ns, UTC]")] → has timezone. + /// + private static bool TimestampHasTimezone(PropertyModel prop) + { + var over = prop.Type.ArrowTypeOverride; + if (over == null) + return true; // default is UTC + var lower = over.Trim().ToLowerInvariant(); + if (!lower.StartsWith("timestamp")) + return true; // not a timestamp override, use default + // Check if there's a comma (unit, tz) in the bracket params + var start = lower.IndexOfAny(new[] { '[', '(' }); + if (start < 0) + return false; // just "timestamp" with no params — no timezone + var end = lower.IndexOfAny(new[] { ']', ')' }, start); + if (end < 0) + return false; + var inner = lower.Substring(start + 1, end - start - 1); + return inner.Contains(","); + } + + /// + /// Returns ("Time64Array", "Time64Type", "new Time64Type(TimeUnit.Microsecond)") or + /// ("Time32Array", "Time32Type", "new Time32Type(TimeUnit.Millisecond)") based on override. + /// + private static (string ArrayType, string BuilderType, string TypeExpr) GetTimeArrayInfo(PropertyModel prop) + { + var over = prop.Type.ArrowTypeOverride?.Trim().ToLowerInvariant(); + if (over != null && over.StartsWith("time32")) + { + var unit = ParseUnitParam(over, "time32", "Millisecond"); + return ("Time32Array", "Time32Array.Builder", $"new Time32Type(TimeUnit.{unit})"); + } + // Default: time64 + if (over != null && over.StartsWith("time64")) + { + var unit = ParseUnitParam(over, "time64", "Microsecond"); + return ("Time64Array", "Time64Array.Builder", $"new Time64Type(TimeUnit.{unit})"); + } + return ("Time64Array", "Time64Array.Builder", "new Time64Type(TimeUnit.Microsecond)"); + } + + private static (string ArrayType, string BuilderType) GetDateArrayInfo(PropertyModel prop) + { + var over = prop.Type.ArrowTypeOverride?.Trim().ToLowerInvariant(); + if (over != null && over.StartsWith("date64")) + return ("Date64Array", "Date64Array.Builder"); + return ("Date32Array", "Date32Array.Builder"); + } + + private static bool IsStringView(PropertyModel prop) => + prop.Type.ArrowTypeOverride != null && + (prop.Type.ArrowTypeOverride.Trim().ToLowerInvariant() is "string_view" or "utf8_view" or "utf8view" or "stringview"); + + private static (string ArrayType, string BuilderType) GetStringArrayInfo(PropertyModel prop) => + IsStringView(prop) + ? ("StringViewArray", "StringViewArray.Builder") + : ("StringArray", "StringArray.Builder"); + + private static bool IsBool8(PropertyModel prop) => + prop.Type.ArrowTypeOverride != null && + (prop.Type.ArrowTypeOverride.Trim().ToLowerInvariant() is "bool8" or "arrow.bool8"); + + private static (string ArrayType, string BuilderType) GetBoolArrayInfo(PropertyModel prop) => + IsBool8(prop) + ? ("Bool8Array", "Bool8Array.Builder") + : ("BooleanArray", "BooleanArray.Builder"); + + private static bool IsBinaryView(PropertyModel prop) => + prop.Type.ArrowTypeOverride != null && + (prop.Type.ArrowTypeOverride.Trim().ToLowerInvariant() is "binary_view" or "binaryview"); + + private static (string ArrayType, string BuilderType) GetBinaryArrayInfo(PropertyModel prop) => + IsBinaryView(prop) + ? ("BinaryViewArray", "BinaryViewArray.Builder") + : ("BinaryArray", "BinaryArray.Builder"); + + private string GetArrowTypeExpression(TypeInfo type) + { + if (type.ArrowTypeOverride != null) + return ParseArrowTypeOverride(type.ArrowTypeOverride); + + return type.Kind switch + { + TypeKind2.String => "StringType.Default", + TypeKind2.Bool => "BooleanType.Default", + TypeKind2.Byte => "UInt8Type.Default", + TypeKind2.SByte => "Int8Type.Default", + TypeKind2.Int16 => "Int16Type.Default", + TypeKind2.UInt16 => "UInt16Type.Default", + TypeKind2.Int32 => "Int32Type.Default", + TypeKind2.UInt32 => "UInt32Type.Default", + TypeKind2.Int64 => "Int64Type.Default", + TypeKind2.UInt64 => "UInt64Type.Default", + TypeKind2.Float => "FloatType.Default", + TypeKind2.Double => "DoubleType.Default", + TypeKind2.Decimal => "new Decimal128Type(38, 18)", + TypeKind2.Binary => "BinaryType.Default", + TypeKind2.DateTime => "new TimestampType(TimeUnit.Microsecond, \"UTC\")", + TypeKind2.DateTimeOffset => "new TimestampType(TimeUnit.Microsecond, \"UTC\")", + TypeKind2.DateOnly => "Date32Type.Default", + TypeKind2.TimeOnly => "new Time64Type(TimeUnit.Microsecond)", + TypeKind2.TimeSpan => "DurationType.Microsecond", + TypeKind2.Guid => "GuidType.Default", + TypeKind2.Half => "HalfFloatType.Default", + TypeKind2.Enum => "new DictionaryType(Int16Type.Default, StringType.Default, false)", + TypeKind2.List or TypeKind2.Array or TypeKind2.Set => + $"new ListType(new Field(\"item\", {GetListElementTypeExpression(type.ElementType!)}, {(type.ElementType!.IsNullable ? "true" : "false")}))", + TypeKind2.Dictionary => + $"new MapType(new Field(\"key\", {GetArrowTypeExpression(type.KeyType!)}, false), new Field(\"value\", {GetArrowTypeExpression(type.ValueType!)}, true))", + TypeKind2.NestedRecord => + $"new StructType({type.FullTypeName}.ArrowSchema.FieldsList)", + _ => "StringType.Default /* unsupported */", + }; + } + + /// + /// Returns the Arrow type expression for list element types. + /// Enums inside lists use DictionaryType (matching Python's schema). + /// + private string GetListElementTypeExpression(TypeInfo elemType) + { + return GetArrowTypeExpression(elemType); + } + + private static string ParseArrowTypeOverride(string typeName) + { + var lower = typeName.ToLowerInvariant().Trim(); + var original = typeName.Trim(); + + // Parameterized types: timestamp[unit, tz], decimal128(precision, scale), + // time64[unit], time32[unit], duration[unit] + if (lower.StartsWith("timestamp")) + { + var (unit, tz) = ParseTimestampParams(original, "timestamp"); + return $"new TimestampType(TimeUnit.{unit}, \"{tz}\")"; + } + if (lower.StartsWith("decimal128") || lower.StartsWith("decimal")) + { + var (precision, scale) = ParseDecimalParams(lower); + return $"new Decimal128Type({precision}, {scale})"; + } + if (lower.StartsWith("time64")) + { + var unit = ParseUnitParam(lower, "time64", "Microsecond"); + return $"new Time64Type(TimeUnit.{unit})"; + } + if (lower.StartsWith("time32")) + { + var unit = ParseUnitParam(lower, "time32", "Millisecond"); + return $"new Time32Type(TimeUnit.{unit})"; + } + if (lower.StartsWith("duration")) + { + var unit = ParseUnitParam(lower, "duration", "Microsecond"); + return $"DurationType.{unit}"; + } + + return lower switch + { + "int8" => "Int8Type.Default", + "int16" => "Int16Type.Default", + "int32" => "Int32Type.Default", + "int64" => "Int64Type.Default", + "uint8" => "UInt8Type.Default", + "uint16" => "UInt16Type.Default", + "uint32" => "UInt32Type.Default", + "uint64" => "UInt64Type.Default", + "float16" or "halffloat" => "HalfFloatType.Default", + "float32" or "float" => "FloatType.Default", + "float64" or "double" => "DoubleType.Default", + "bool" or "boolean" => "BooleanType.Default", + "string" or "utf8" => "StringType.Default", + "binary" => "BinaryType.Default", + "date32" => "Date32Type.Default", + "date64" => "Date64Type.Default", + "string_view" or "utf8_view" or "utf8view" or "stringview" => "StringViewType.Default", + "binary_view" or "binaryview" => "BinaryViewType.Default", + "guid" => "GuidType.Default", + "bool8" or "arrow.bool8" => "Bool8Type.Default", + _ => $"StringType.Default /* unrecognized override: {typeName} */", + }; + } + + private static (string Unit, string Timezone) ParseTimestampParams(string input, string prefix) + { + var unit = "Microsecond"; + var tz = "UTC"; + var inner = ExtractBracketParams(input, prefix); + if (inner != null) + { + var parts = inner.Split(','); + if (parts.Length >= 1) unit = NormalizeTimeUnit(parts[0].Trim()); + if (parts.Length >= 2) tz = parts[1].Trim().Trim('"', '\''); + } + return (unit, tz); + } + + private static (int Precision, int Scale) ParseDecimalParams(string input) + { + var precision = 38; + var scale = 18; + // Match decimal128(p, s) or decimal(p, s) + var start = input.IndexOf('('); + var end = input.IndexOf(')'); + if (start >= 0 && end > start) + { + var parts = input.Substring(start + 1, end - start - 1).Split(','); + if (parts.Length >= 1 && int.TryParse(parts[0].Trim(), out var p)) precision = p; + if (parts.Length >= 2 && int.TryParse(parts[1].Trim(), out var s)) scale = s; + } + return (precision, scale); + } + + private static string ParseUnitParam(string input, string prefix, string defaultUnit) + { + var inner = ExtractBracketParams(input, prefix); + return inner != null ? NormalizeTimeUnit(inner.Trim()) : defaultUnit; + } + + private static string ExtractBracketParams(string input, string prefix) + { + // Supports both [] and () delimiters: timestamp[us, UTC] or timestamp(us, UTC) + var rest = input.Substring(prefix.Length).Trim(); + if (rest.Length >= 2) + { + if ((rest[0] == '[' && rest[rest.Length - 1] == ']') || + (rest[0] == '(' && rest[rest.Length - 1] == ')')) + return rest.Substring(1, rest.Length - 2); + } + return null!; + } + + private static string NormalizeTimeUnit(string unit) + { + switch (unit.ToLowerInvariant()) + { + case "s": case "second": case "seconds": return "Second"; + case "ms": case "millisecond": case "milliseconds": return "Millisecond"; + case "us": case "microsecond": case "microseconds": case "μs": return "Microsecond"; + case "ns": case "nanosecond": case "nanoseconds": return "Nanosecond"; + default: return "Microsecond"; + } + } + + private static string GetArrayBuilderType(TypeInfo type) + { + // Check for view type overrides + if (type.ArrowTypeOverride != null) + { + var ov = type.ArrowTypeOverride.ToLowerInvariant().Replace("_", ""); + if (ov is "stringview" or "utf8view") return "StringViewArray.Builder"; + if (ov is "binaryview") return "BinaryViewArray.Builder"; + } + return type.Kind switch + { + TypeKind2.String => "StringArray.Builder", + TypeKind2.Bool => "BooleanArray.Builder", + TypeKind2.Byte => "UInt8Array.Builder", + TypeKind2.SByte => "Int8Array.Builder", + TypeKind2.Int16 => "Int16Array.Builder", + TypeKind2.UInt16 => "UInt16Array.Builder", + TypeKind2.Int32 => "Int32Array.Builder", + TypeKind2.UInt32 => "UInt32Array.Builder", + TypeKind2.Int64 => "Int64Array.Builder", + TypeKind2.UInt64 => "UInt64Array.Builder", + TypeKind2.Float => "FloatArray.Builder", + TypeKind2.Double => "DoubleArray.Builder", + TypeKind2.Half => "HalfFloatArray.Builder", + TypeKind2.Decimal => "Decimal128Array.Builder", + TypeKind2.DateTime or TypeKind2.DateTimeOffset => "TimestampArray.Builder", + TypeKind2.DateOnly => GetDateArrayBuilderType(type), + TypeKind2.TimeOnly => GetTimeArrayBuilderType(type), + TypeKind2.TimeSpan => "DurationArray.Builder", + TypeKind2.Guid => "GuidArray.Builder", + TypeKind2.Binary => "BinaryArray.Builder", + TypeKind2.Enum => "StringArray.Builder", + _ => $"StringArray.Builder /* fallback for {type.Kind} */", + }; + } + + /// + /// Returns the Arrow array type name for reading, respecting view type overrides. + /// + private static string GetArrayReadType(TypeInfo type) + { + if (type.ArrowTypeOverride != null) + { + var ov = type.ArrowTypeOverride.ToLowerInvariant().Replace("_", ""); + if (ov is "stringview" or "utf8view") return "StringViewArray"; + if (ov is "binaryview") return "BinaryViewArray"; + } + return type.Kind switch + { + TypeKind2.String => "StringArray", + _ => null!, // Not used for non-string types + }; + } + + // --- Output helpers --- + + private void Line(string text = "") + { + if (string.IsNullOrEmpty(text)) + { + _sb.AppendLine(); + return; + } + _sb.Append(' ', _indent * 4); + _sb.AppendLine(text); + } +} + +} // namespace diff --git a/src/Apache.Arrow.Serialization.Generator/JsonSchemaEmitter.cs b/src/Apache.Arrow.Serialization.Generator/JsonSchemaEmitter.cs new file mode 100644 index 00000000..10381d4f --- /dev/null +++ b/src/Apache.Arrow.Serialization.Generator/JsonSchemaEmitter.cs @@ -0,0 +1,195 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Text; + +#nullable enable + +namespace Apache.Arrow.Serialization.Generator +{ + +/// +/// Emits a JSON schema descriptor as a static ArrowSchemaJson property. +/// The JSON describes the Arrow schema for cross-language codegen tools. +/// +internal static class JsonSchemaEmitter +{ + public static void Emit(StringBuilder sb, TypeModel model) + { + var json = BuildJson(model); + + sb.AppendLine("// "); + sb.AppendLine("#nullable enable"); + sb.AppendLine(); + + if (model.Namespace != null) + { + sb.AppendLine($"namespace {model.Namespace};"); + sb.AppendLine(); + } + + var typeKeyword = GetTypeKeyword(model); + sb.AppendLine($"partial {typeKeyword} {model.TypeName}"); + sb.AppendLine("{"); + sb.AppendLine($" public static string ArrowSchemaJson => @\"{json.Replace("\"", "\"\"")}\";"); + sb.AppendLine("}"); + } + + private static string BuildJson(TypeModel model) + { + var sb = new StringBuilder(); + sb.AppendLine("{"); + sb.AppendLine($" \"type\": \"{JsonEscape(model.FullTypeName)}\","); + sb.AppendLine($" \"namespace\": {(model.Namespace != null ? $"\"{JsonEscape(model.Namespace)}\"" : "null")},"); + sb.AppendLine($" \"typeName\": \"{JsonEscape(model.TypeName)}\","); + + // Metadata + sb.Append(" \"metadata\": {"); + if (model.Metadata.Count > 0) + { + sb.AppendLine(); + for (int i = 0; i < model.Metadata.Count; i++) + { + var kv = model.Metadata[i]; + var comma = i < model.Metadata.Count - 1 ? "," : ""; + sb.AppendLine($" \"{JsonEscape(kv.Key)}\": \"{JsonEscape(kv.Value)}\"{comma}"); + } + sb.AppendLine(" },"); + } + else + { + sb.AppendLine("},"); + } + + // Fields + sb.AppendLine(" \"fields\": ["); + for (int i = 0; i < model.Properties.Count; i++) + { + var prop = model.Properties[i]; + var comma = i < model.Properties.Count - 1 ? "," : ""; + EmitField(sb, prop); + // Remove trailing newline, add comma + sb.Length -= sb.ToString().EndsWith("\r\n") ? 2 : 1; + sb.AppendLine(comma); + } + sb.AppendLine(" ]"); + sb.Append("}"); + + return sb.ToString(); + } + + private static void EmitField(StringBuilder sb, PropertyModel prop) + { + sb.AppendLine(" {"); + sb.AppendLine($" \"name\": \"{JsonEscape(prop.FieldName)}\","); + sb.AppendLine($" \"propertyName\": \"{JsonEscape(prop.PropertyName)}\","); + sb.AppendLine($" \"arrowType\": \"{JsonEscape(GetArrowTypeName(prop.Type))}\","); + sb.AppendLine($" \"typeKind\": \"{prop.Type.Kind}\","); + sb.AppendLine($" \"nullable\": {(prop.IsNullable ? "true" : "false")},"); + sb.AppendLine($" \"csharpType\": \"{JsonEscape(prop.Type.FullTypeName)}\""); + + if (prop.Type.ElementType != null) + { + // Replace last newline to add comma + sb.Length -= sb.ToString().EndsWith("\r\n") ? 2 : 1; + sb.AppendLine(","); + sb.AppendLine($" \"elementType\": \"{JsonEscape(GetArrowTypeName(prop.Type.ElementType))}\""); + } + + if (prop.Type.KeyType != null) + { + sb.Length -= sb.ToString().EndsWith("\r\n") ? 2 : 1; + sb.AppendLine(","); + sb.AppendLine($" \"keyType\": \"{JsonEscape(GetArrowTypeName(prop.Type.KeyType))}\","); + sb.AppendLine($" \"valueType\": \"{JsonEscape(GetArrowTypeName(prop.Type.ValueType!))}\""); + } + + if (prop.ConverterTypeName != null) + { + sb.Length -= sb.ToString().EndsWith("\r\n") ? 2 : 1; + sb.AppendLine(","); + sb.AppendLine($" \"converter\": \"{JsonEscape(prop.ConverterTypeName)}\""); + } + + if (prop.Metadata.Count > 0) + { + sb.Length -= sb.ToString().EndsWith("\r\n") ? 2 : 1; + sb.AppendLine(","); + sb.Append(" \"metadata\": {"); + sb.AppendLine(); + for (int j = 0; j < prop.Metadata.Count; j++) + { + var kv = prop.Metadata[j]; + var mComma = j < prop.Metadata.Count - 1 ? "," : ""; + sb.AppendLine($" \"{JsonEscape(kv.Key)}\": \"{JsonEscape(kv.Value)}\"{mComma}"); + } + sb.AppendLine(" }"); + } + + sb.AppendLine(" }"); + } + + private static string GetArrowTypeName(TypeInfo type) + { + if (type.ArrowTypeOverride != null) + return type.ArrowTypeOverride; + + return type.Kind switch + { + TypeKind2.String => "utf8", + TypeKind2.Bool => "bool", + TypeKind2.Byte => "uint8", + TypeKind2.SByte => "int8", + TypeKind2.Int16 => "int16", + TypeKind2.UInt16 => "uint16", + TypeKind2.Int32 => "int32", + TypeKind2.UInt32 => "uint32", + TypeKind2.Int64 => "int64", + TypeKind2.UInt64 => "uint64", + TypeKind2.Float => "float32", + TypeKind2.Double => "float64", + TypeKind2.Decimal => "decimal128(38,18)", + TypeKind2.Binary => "binary", + TypeKind2.DateTime => "timestamp[us, UTC]", + TypeKind2.DateTimeOffset => "timestamp[us, UTC]", + TypeKind2.DateOnly => "date32", + TypeKind2.TimeOnly => "time64[us]", + TypeKind2.TimeSpan => "duration[us]", + TypeKind2.Guid => "fixed_size_binary(16)", + TypeKind2.Half => "float16", + TypeKind2.Enum => "dictionary", + TypeKind2.List => $"list<{GetArrowTypeName(type.ElementType!)}>", + TypeKind2.Array => $"list<{GetArrowTypeName(type.ElementType!)}>", + TypeKind2.Set => $"list<{GetArrowTypeName(type.ElementType!)}>", + TypeKind2.Dictionary => $"map<{GetArrowTypeName(type.KeyType!)}, {GetArrowTypeName(type.ValueType!)}>", + TypeKind2.NestedRecord => $"struct<{type.FullTypeName}>", + TypeKind2.Custom => "custom", + _ => "unknown", + }; + } + + private static string GetTypeKeyword(TypeModel model) + { + if (model.IsRecord && model.IsValueType) return "record struct"; + if (model.IsRecord) return "record"; + if (model.IsValueType) return "struct"; + return "class"; + } + + private static string JsonEscape(string s) => + s.Replace("\\", "\\\\").Replace("\"", "\\\"").Replace("\n", "\\n").Replace("\r", "\\r"); +} + +} // namespace diff --git a/src/Apache.Arrow.Serialization.Generator/Models.cs b/src/Apache.Arrow.Serialization.Generator/Models.cs new file mode 100644 index 00000000..27092ae9 --- /dev/null +++ b/src/Apache.Arrow.Serialization.Generator/Models.cs @@ -0,0 +1,151 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Collections.Generic; + +#nullable enable + +namespace Apache.Arrow.Serialization.Generator +{ + internal enum TypeKind2 + { + Unknown, + String, + Bool, + Byte, + SByte, + Int16, + UInt16, + Int32, + UInt32, + Int64, + UInt64, + Float, + Double, + Decimal, + Binary, + DateTime, + DateTimeOffset, + DateOnly, + TimeOnly, + TimeSpan, + Guid, + Half, + Enum, + List, + Array, + Dictionary, + Set, + NestedRecord, + Custom, + } + + internal class TypeInfo + { + public TypeKind2 Kind { get; set; } + public string FullTypeName { get; set; } = ""; + public bool IsNullable { get; set; } + public string? ArrowTypeOverride { get; set; } + public TypeInfo? ElementType { get; set; } + public TypeInfo? KeyType { get; set; } + public TypeInfo? ValueType { get; set; } + + public TypeInfo WithOverride(string arrowTypeOverride) + { + return new TypeInfo + { + Kind = Kind, + FullTypeName = FullTypeName, + IsNullable = IsNullable, + ArrowTypeOverride = arrowTypeOverride, + ElementType = ElementType, + KeyType = KeyType, + ValueType = ValueType, + }; + } + } + + internal class PropertyModel + { + public string PropertyName { get; set; } = ""; + public string FieldName { get; set; } = ""; + public int Order { get; set; } = int.MaxValue; + public int DeclOrder { get; set; } + public TypeInfo Type { get; set; } = new TypeInfo(); + public bool IsNullable { get; set; } + public bool HasDefaultValue { get; set; } + public List> Metadata { get; set; } = new List>(); + /// Fully qualified type name of the IArrowConverter implementation, if specified. + public string? ConverterTypeName { get; set; } + /// True if this member is a field (not a property). Fields require constructor-based deserialization. + public bool IsField { get; set; } + } + + internal class ConstructorParamModel + { + public string Name { get; set; } = ""; + public bool HasDefaultValue { get; set; } + } + + internal class DiagnosticInfo + { + public string Id { get; set; } = ""; + public string Message { get; set; } = ""; + public bool IsError { get; set; } + } + + internal class TypeModel + { + public string? Namespace { get; set; } + public string TypeName { get; set; } = ""; + public string FullTypeName { get; set; } = ""; + public List Properties { get; set; } = new List(); + public bool IsRecord { get; set; } + public bool IsValueType { get; set; } + public List> Metadata { get; set; } = new List>(); + public bool HasArrowSerializableBase { get; set; } + /// Constructor to use for deserialization. Null means use object initializer. + public List? ConstructorParams { get; set; } + /// True if the type implements IArrowSerializationCallback. + public bool HasSerializationCallback { get; set; } + public List Diagnostics { get; set; } = new List(); + } + + internal class DerivedTypeInfo + { + public string TypeDiscriminator { get; set; } = ""; + public string FullTypeName { get; set; } = ""; + public string TypeName { get; set; } = ""; + public List Properties { get; set; } = new List(); + public bool IsRecord { get; set; } + public bool IsValueType { get; set; } + } + + internal class PolymorphicModel + { + public string? Namespace { get; set; } + public string TypeName { get; set; } = ""; + public string FullTypeName { get; set; } = ""; + public string TypeDiscriminatorFieldName { get; set; } = "$type"; + public bool IsInterface { get; set; } + public bool IsRecord { get; set; } + public List DerivedTypes { get; set; } = new List(); + /// + /// Union of all derived type properties (all made nullable). Deduplicated by FieldName. + /// + public List UnionProperties { get; set; } = new List(); + public List> Metadata { get; set; } = new List>(); + } +} diff --git a/src/Apache.Arrow.Serialization.Generator/PolymorphicCodeEmitter.cs b/src/Apache.Arrow.Serialization.Generator/PolymorphicCodeEmitter.cs new file mode 100644 index 00000000..51d18a88 --- /dev/null +++ b/src/Apache.Arrow.Serialization.Generator/PolymorphicCodeEmitter.cs @@ -0,0 +1,816 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Text; + +#nullable enable + +namespace Apache.Arrow.Serialization.Generator +{ + +internal class PolymorphicCodeEmitter +{ + private readonly StringBuilder _sb; + private readonly PolymorphicModel _model; + private int _indent; + + public PolymorphicCodeEmitter(StringBuilder sb, PolymorphicModel model) + { + _sb = sb; + _model = model; + } + + public void Emit() + { + Line("// "); + Line("#nullable enable"); + Line("#pragma warning disable CS8629 // Nullable value type may be null"); + Line(); + Line("using System.Collections.Generic;"); + Line("using System.Linq;"); + Line("using Apache.Arrow;"); + Line("using Apache.Arrow.Arrays;"); + Line("using Apache.Arrow.Types;"); + Line("using Apache.Arrow.Serialization;"); + Line(); + + if (_model.Namespace != null) + { + Line($"namespace {_model.Namespace};"); + Line(); + } + + var typeKeyword = _model.IsInterface ? "interface" : (_model.IsRecord ? "record" : "class"); + Line($"partial {typeKeyword} {_model.TypeName} : IArrowSerializer<{_model.TypeName}>"); + Line("{"); + _indent++; + + EmitSchemaField(); + Line(); + EmitSerialize(); + Line(); + EmitDeserialize(); + Line(); + EmitMultiRowSerialize(); + Line(); + EmitMultiRowDeserialize(); + + _indent--; + Line("}"); + } + + private void EmitSchemaField() + { + Line($"private static readonly Schema _arrowSchema = new Schema.Builder()"); + _indent++; + + // Discriminator field + Line($".Field(new Field(\"{Escape(_model.TypeDiscriminatorFieldName)}\", StringType.Default, false))"); + + // Union properties — all nullable + foreach (var prop in _model.UnionProperties) + { + var arrowType = CodeEmitterHelpers.GetArrowTypeExpression(prop.Type); + Line($".Field(new Field(\"{Escape(prop.FieldName)}\", {arrowType}, true))"); + } + + _indent--; + Line($".Build();"); + Line(); + Line($"public static Schema ArrowSchema => _arrowSchema;"); + } + + private void EmitSerialize() + { + Line($"public static RecordBatch ToRecordBatch({_model.TypeName} value)"); + Line("{"); + _indent++; + + // Build discriminator column + Line("var discriminatorBuilder = new StringArray.Builder();"); + + // Build union property builders + for (int i = 0; i < _model.UnionProperties.Count; i++) + { + var prop = _model.UnionProperties[i]; + EmitBuilderDeclaration(prop, i); + } + + Line(); + // Switch on concrete type + Line("switch (value)"); + Line("{"); + _indent++; + + foreach (var dt in _model.DerivedTypes) + { + Line($"case {dt.FullTypeName} v_{dt.TypeName}:"); + Line("{"); + _indent++; + Line($"discriminatorBuilder.Append(\"{Escape(dt.TypeDiscriminator)}\");"); + + // For each union property, either append the value or null + for (int i = 0; i < _model.UnionProperties.Count; i++) + { + var unionProp = _model.UnionProperties[i]; + var derivedProp = FindProperty(dt, unionProp.FieldName); + if (derivedProp != null) + { + EmitAppendValue(derivedProp, i, $"v_{dt.TypeName}.{derivedProp.PropertyName}"); + } + else + { + EmitAppendNull(unionProp, i); + } + } + + Line("break;"); + _indent--; + Line("}"); + } + + Line($"default:"); + _indent++; + Line($"throw new System.ArgumentException($\"Unknown derived type: {{value.GetType().Name}}\");"); + _indent--; + + _indent--; + Line("}"); + + Line(); + // Build arrays and RecordBatch + Line("var columns = new IArrowArray[]"); + Line("{"); + _indent++; + Line("discriminatorBuilder.Build(),"); + for (int i = 0; i < _model.UnionProperties.Count; i++) + { + EmitBuildColumn(i); + } + _indent--; + Line("};"); + Line("return new RecordBatch(_arrowSchema, columns, 1);"); + + _indent--; + Line("}"); + } + + private void EmitDeserialize() + { + Line($"public static {_model.TypeName} FromRecordBatch(RecordBatch batch)"); + Line("{"); + _indent++; + + Line("var discriminator = ((StringArray)batch.Column(0)).GetString(0)!;"); + Line("switch (discriminator)"); + Line("{"); + _indent++; + + foreach (var dt in _model.DerivedTypes) + { + Line($"case \"{Escape(dt.TypeDiscriminator)}\":"); + Line("{"); + _indent++; + + // Read each property for this derived type from the batch columns + for (int i = 0; i < dt.Properties.Count; i++) + { + var prop = dt.Properties[i]; + var unionIndex = FindUnionPropertyIndex(prop.FieldName); + var colIndex = unionIndex + 1; // +1 for discriminator + EmitReadProperty(prop, i, colIndex); + } + + // Construct the derived type + Line($"return new {dt.FullTypeName}"); + Line("{"); + _indent++; + for (int i = 0; i < dt.Properties.Count; i++) + { + var prop = dt.Properties[i]; + Line($"{prop.PropertyName} = prop_{i},"); + } + _indent--; + Line("};"); + + _indent--; + Line("}"); + } + + Line("default:"); + _indent++; + Line("throw new System.ArgumentException($\"Unknown type discriminator: {discriminator}\");"); + _indent--; + + _indent--; + Line("}"); + + _indent--; + Line("}"); + } + + private void EmitMultiRowSerialize() + { + Line($"public static RecordBatch ToRecordBatch(IReadOnlyList<{_model.TypeName}> values)"); + Line("{"); + _indent++; + + Line("var discriminatorBuilder = new StringArray.Builder();"); + for (int i = 0; i < _model.UnionProperties.Count; i++) + { + var prop = _model.UnionProperties[i]; + EmitBuilderDeclaration(prop, i); + } + + Line(); + Line("foreach (var value in values)"); + Line("{"); + _indent++; + + Line("switch (value)"); + Line("{"); + _indent++; + + foreach (var dt in _model.DerivedTypes) + { + Line($"case {dt.FullTypeName} v_{dt.TypeName}:"); + Line("{"); + _indent++; + Line($"discriminatorBuilder.Append(\"{Escape(dt.TypeDiscriminator)}\");"); + + for (int i = 0; i < _model.UnionProperties.Count; i++) + { + var unionProp = _model.UnionProperties[i]; + var derivedProp = FindProperty(dt, unionProp.FieldName); + if (derivedProp != null) + { + EmitAppendValue(derivedProp, i, $"v_{dt.TypeName}.{derivedProp.PropertyName}"); + } + else + { + EmitAppendNull(unionProp, i); + } + } + + Line("break;"); + _indent--; + Line("}"); + } + + Line($"default:"); + _indent++; + Line($"throw new System.ArgumentException($\"Unknown derived type: {{value.GetType().Name}}\");"); + _indent--; + + _indent--; + Line("}"); + + _indent--; + Line("}"); + + Line(); + Line("var columns = new IArrowArray[]"); + Line("{"); + _indent++; + Line("discriminatorBuilder.Build(),"); + for (int i = 0; i < _model.UnionProperties.Count; i++) + { + EmitBuildColumn(i); + } + _indent--; + Line("};"); + Line("return new RecordBatch(_arrowSchema, columns, values.Count);"); + + _indent--; + Line("}"); + } + + private void EmitBuildColumn(int index) + { + var prop = _model.UnionProperties[index]; + if (prop.Type.Kind == TypeKind2.Enum) + { + // Build DictionaryArray from index + dict builders + Line($"new DictionaryArray(new DictionaryType(Int16Type.Default, StringType.Default, false), bld_{index}_idx.Build(), new StringArray.Builder().AppendRange(bld_{index}_dict).Build()),"); + } + else + { + Line($"bld_{index}.Build(),"); + } + } + + private void EmitMultiRowDeserialize() + { + Line($"public static IReadOnlyList<{_model.TypeName}> ListFromRecordBatch(RecordBatch batch)"); + Line("{"); + _indent++; + + Line("var discriminatorCol = (StringArray)batch.Column(0);"); + + // Cast all union property columns + for (int i = 0; i < _model.UnionProperties.Count; i++) + { + var prop = _model.UnionProperties[i]; + var colIndex = i + 1; + var castType = CodeEmitterHelpers.GetArrayCastType(prop); + Line($"var col_{i} = ({castType})batch.Column({colIndex});"); + } + + Line(); + Line($"var result = new List<{_model.TypeName}>(batch.Length);"); + Line("for (int row = 0; row < batch.Length; row++)"); + Line("{"); + _indent++; + + Line("var discriminator = discriminatorCol.GetString(row)!;"); + Line("switch (discriminator)"); + Line("{"); + _indent++; + + foreach (var dt in _model.DerivedTypes) + { + Line($"case \"{Escape(dt.TypeDiscriminator)}\":"); + Line("{"); + _indent++; + + for (int i = 0; i < dt.Properties.Count; i++) + { + var prop = dt.Properties[i]; + var unionIndex = FindUnionPropertyIndex(prop.FieldName); + EmitMultiRowReadProperty(prop, i, unionIndex); + } + + Line($"result.Add(new {dt.FullTypeName}"); + Line("{"); + _indent++; + for (int i = 0; i < dt.Properties.Count; i++) + { + var prop = dt.Properties[i]; + Line($"{prop.PropertyName} = prop_{i},"); + } + _indent--; + Line("});"); + Line("break;"); + + _indent--; + Line("}"); + } + + Line("default:"); + _indent++; + Line("throw new System.ArgumentException($\"Unknown type discriminator: {discriminator}\");"); + _indent--; + + _indent--; + Line("}"); + + _indent--; + Line("}"); + Line("return result;"); + + _indent--; + Line("}"); + } + + // --- Helpers --- + + private void EmitBuilderDeclaration(PropertyModel prop, int index) + { + if (prop.Type.Kind == TypeKind2.Enum) + { + // Enum uses Dictionary(Int16, Utf8) — built manually + Line($"var bld_{index}_idx = new Int16Array.Builder();"); + Line($"var bld_{index}_dict = new List();"); + } + else + { + var builderType = CodeEmitterHelpers.GetNullableBuilderDeclaration(prop); + Line($"var bld_{index} = {builderType};"); + } + } + + private void EmitAppendValue(PropertyModel prop, int index, string access) + { + // For non-nullable properties in the derived type, we still need to handle them + // as nullable in the union schema + switch (prop.Type.Kind) + { + case TypeKind2.String: + if (prop.IsNullable) + Line($"if ({access} != null) bld_{index}.Append({access}); else bld_{index}.AppendNull();"); + else + Line($"bld_{index}.Append({access});"); + break; + case TypeKind2.Bool: + case TypeKind2.Byte: + case TypeKind2.SByte: + case TypeKind2.Int16: + case TypeKind2.UInt16: + case TypeKind2.Int32: + case TypeKind2.UInt32: + case TypeKind2.Int64: + case TypeKind2.UInt64: + case TypeKind2.Float: + case TypeKind2.Double: + case TypeKind2.Half: + if (prop.IsNullable) + Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(v_{index}); else bld_{index}.AppendNull();"); + else + Line($"bld_{index}.Append({access});"); + break; + case TypeKind2.DateTime: + case TypeKind2.DateTimeOffset: + if (prop.IsNullable) + Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(Apache.Arrow.Serialization.ArrowArrayHelper.ToUtcDateTimeOffset(v_{index})); else bld_{index}.AppendNull();"); + else + Line($"bld_{index}.Append(Apache.Arrow.Serialization.ArrowArrayHelper.ToUtcDateTimeOffset({access}));"); + break; + case TypeKind2.DateOnly: + if (prop.IsNullable) + Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(v_{index}); else bld_{index}.AppendNull();"); + else + Line($"bld_{index}.Append({access});"); + break; + case TypeKind2.Decimal: + if (prop.IsNullable) + Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(v_{index}); else bld_{index}.AppendNull();"); + else + Line($"bld_{index}.Append({access});"); + break; + case TypeKind2.Enum: + if (prop.IsNullable) + Line($"if ({access} is {{ }} v_{index}) {{ bld_{index}_idx.Append((short)bld_{index}_dict.Count); bld_{index}_dict.Add(v_{index}.ToString()); }} else bld_{index}_idx.AppendNull();"); + else + { + Line($"bld_{index}_idx.Append((short)bld_{index}_dict.Count);"); + Line($"bld_{index}_dict.Add({access}.ToString());"); + } + break; + default: + // For complex types (Binary, Guid, TimeOnly, TimeSpan, nested, collections), + // fall back to AppendNull for now — these are less common in polymorphic scenarios + Line($"bld_{index}.AppendNull(); // TODO: complex type {prop.Type.Kind}"); + break; + } + } + + private void EmitAppendNull(PropertyModel prop, int index) + { + if (prop.Type.Kind == TypeKind2.Enum) + Line($"bld_{index}_idx.AppendNull();"); + else + Line($"bld_{index}.AppendNull();"); + } + + private void EmitReadProperty(PropertyModel prop, int propIndex, int colIndex) + { + var col = $"batch.Column({colIndex})"; + switch (prop.Type.Kind) + { + case TypeKind2.String: + if (prop.IsNullable) + Line($"var prop_{propIndex} = ((StringArray){col}).IsNull(0) ? null : ((StringArray){col}).GetString(0);"); + else + Line($"var prop_{propIndex} = ((StringArray){col}).GetString(0)!;"); + break; + case TypeKind2.Bool: + case TypeKind2.Byte: + case TypeKind2.SByte: + case TypeKind2.Int16: + case TypeKind2.UInt16: + case TypeKind2.Int32: + case TypeKind2.UInt32: + case TypeKind2.Int64: + case TypeKind2.UInt64: + case TypeKind2.Float: + case TypeKind2.Double: + case TypeKind2.Half: + { + var castType = CodeEmitterHelpers.GetArrayCastType(prop); + if (prop.IsNullable) + Line($"var prop_{propIndex} = (({castType}){col}).IsNull(0) ? ({prop.Type.FullTypeName}?)null : (({castType}){col}).GetValue(0).Value;"); + else + Line($"var prop_{propIndex} = (({castType}){col}).GetValue(0).Value;"); + break; + } + case TypeKind2.DateTime: + { + if (prop.IsNullable) + Line($"var prop_{propIndex} = ((TimestampArray){col}).IsNull(0) ? (System.DateTime?)null : ((TimestampArray){col}).GetTimestamp(0)!.Value.UtcDateTime;"); + else + Line($"var prop_{propIndex} = ((TimestampArray){col}).GetTimestamp(0)!.Value.UtcDateTime;"); + break; + } + case TypeKind2.DateTimeOffset: + { + if (prop.IsNullable) + Line($"var prop_{propIndex} = ((TimestampArray){col}).IsNull(0) ? (System.DateTimeOffset?)null : ((TimestampArray){col}).GetTimestamp(0)!.Value;"); + else + Line($"var prop_{propIndex} = ((TimestampArray){col}).GetTimestamp(0)!.Value;"); + break; + } + case TypeKind2.DateOnly: + { + var castType = CodeEmitterHelpers.GetArrayCastType(prop); + if (prop.IsNullable) + Line($"var prop_{propIndex} = (({castType}){col}).IsNull(0) ? (System.DateOnly?)null : (({castType}){col}).GetDateOnly(0);"); + else + Line($"var prop_{propIndex} = (({castType}){col}).GetDateOnly(0)!.Value;"); + break; + } + case TypeKind2.Decimal: + { + if (prop.IsNullable) + Line($"var prop_{propIndex} = ((Decimal128Array){col}).IsNull(0) ? (decimal?)null : ((Decimal128Array){col}).GetValue(0).Value;"); + else + Line($"var prop_{propIndex} = ((Decimal128Array){col}).GetValue(0)!.Value;"); + break; + } + case TypeKind2.Enum: + { + var enumType = prop.Type.FullTypeName; + if (prop.IsNullable) + { + Line($"{enumType}? prop_{propIndex} = null;"); + Line($"{{ var da = (DictionaryArray){col}; if (!da.IsNull(0)) {{ var idx = ((Int16Array)da.Indices).GetValue(0).Value; var name = ((StringArray)da.Dictionary).GetString(idx); prop_{propIndex} = System.Enum.Parse<{enumType}>(name!); }} }}"); + } + else + { + Line($"var da_{propIndex} = (DictionaryArray){col};"); + Line($"var idx_{propIndex} = ((Int16Array)da_{propIndex}.Indices).GetValue(0).Value;"); + Line($"var prop_{propIndex} = System.Enum.Parse<{enumType}>(((StringArray)da_{propIndex}.Dictionary).GetString(idx_{propIndex})!);"); + } + break; + } + default: + Line($"object? prop_{propIndex} = null; // TODO: unsupported type {prop.Type.Kind}"); + break; + } + } + + private void EmitMultiRowReadProperty(PropertyModel prop, int propIndex, int unionIndex) + { + var col = $"col_{unionIndex}"; + switch (prop.Type.Kind) + { + case TypeKind2.String: + if (prop.IsNullable) + Line($"var prop_{propIndex} = {col}.IsNull(row) ? null : {col}.GetString(row);"); + else + Line($"var prop_{propIndex} = {col}.GetString(row)!;"); + break; + case TypeKind2.Bool: + case TypeKind2.Byte: + case TypeKind2.SByte: + case TypeKind2.Int16: + case TypeKind2.UInt16: + case TypeKind2.Int32: + case TypeKind2.UInt32: + case TypeKind2.Int64: + case TypeKind2.UInt64: + case TypeKind2.Float: + case TypeKind2.Double: + case TypeKind2.Half: + if (prop.IsNullable) + Line($"var prop_{propIndex} = {col}.IsNull(row) ? ({prop.Type.FullTypeName}?)null : {col}.GetValue(row).Value;"); + else + Line($"var prop_{propIndex} = {col}.GetValue(row).Value;"); + break; + case TypeKind2.DateTime: + if (prop.IsNullable) + Line($"var prop_{propIndex} = {col}.IsNull(row) ? (System.DateTime?)null : {col}.GetTimestamp(row)!.Value.UtcDateTime;"); + else + Line($"var prop_{propIndex} = {col}.GetTimestamp(row)!.Value.UtcDateTime;"); + break; + case TypeKind2.DateTimeOffset: + if (prop.IsNullable) + Line($"var prop_{propIndex} = {col}.IsNull(row) ? (System.DateTimeOffset?)null : {col}.GetTimestamp(row)!.Value;"); + else + Line($"var prop_{propIndex} = {col}.GetTimestamp(row)!.Value;"); + break; + case TypeKind2.DateOnly: + if (prop.IsNullable) + Line($"var prop_{propIndex} = {col}.IsNull(row) ? (System.DateOnly?)null : {col}.GetDateOnly(row);"); + else + Line($"var prop_{propIndex} = {col}.GetDateOnly(row)!.Value;"); + break; + case TypeKind2.Decimal: + if (prop.IsNullable) + Line($"var prop_{propIndex} = {col}.IsNull(row) ? (decimal?)null : {col}.GetValue(row).Value;"); + else + Line($"var prop_{propIndex} = {col}.GetValue(row)!.Value;"); + break; + case TypeKind2.Enum: + { + var enumType = prop.Type.FullTypeName; + if (prop.IsNullable) + { + Line($"{enumType}? prop_{propIndex} = null;"); + Line($"if (!{col}.IsNull(row)) {{ var idx = ((Int16Array){col}.Indices).GetValue(row).Value; var name = ((StringArray){col}.Dictionary).GetString(idx); prop_{propIndex} = System.Enum.Parse<{enumType}>(name!); }}"); + } + else + { + Line($"var idx_{propIndex} = ((Int16Array){col}.Indices).GetValue(row).Value;"); + Line($"var prop_{propIndex} = System.Enum.Parse<{enumType}>(((StringArray){col}.Dictionary).GetString(idx_{propIndex})!);"); + } + break; + } + default: + Line($"object? prop_{propIndex} = null; // TODO: unsupported type {prop.Type.Kind}"); + break; + } + } + + private static PropertyModel? FindProperty(DerivedTypeInfo dt, string fieldName) + { + foreach (var prop in dt.Properties) + { + if (prop.FieldName == fieldName) + return prop; + } + return null; + } + + private int FindUnionPropertyIndex(string fieldName) + { + for (int i = 0; i < _model.UnionProperties.Count; i++) + { + if (_model.UnionProperties[i].FieldName == fieldName) + return i; + } + return -1; + } + + private static string Escape(string s) => s.Replace("\\", "\\\\").Replace("\"", "\\\""); + + private void Line(string text = "") + { + if (string.IsNullOrEmpty(text)) + { + _sb.AppendLine(); + return; + } + _sb.Append(' ', _indent * 4); + _sb.AppendLine(text); + } +} + +/// +/// Shared helpers used by both CodeEmitter and PolymorphicCodeEmitter. +/// +internal static class CodeEmitterHelpers +{ + /// + /// Returns the Arrow type expression for a TypeInfo (e.g., "Int32Type.Default"). + /// + public static string GetArrowTypeExpression(TypeInfo type) + { + if (type.ArrowTypeOverride != null) + return ParseArrowTypeOverride(type.ArrowTypeOverride); + + return type.Kind switch + { + TypeKind2.String => "StringType.Default", + TypeKind2.Bool => "BooleanType.Default", + TypeKind2.Byte => "UInt8Type.Default", + TypeKind2.SByte => "Int8Type.Default", + TypeKind2.Int16 => "Int16Type.Default", + TypeKind2.UInt16 => "UInt16Type.Default", + TypeKind2.Int32 => "Int32Type.Default", + TypeKind2.UInt32 => "UInt32Type.Default", + TypeKind2.Int64 => "Int64Type.Default", + TypeKind2.UInt64 => "UInt64Type.Default", + TypeKind2.Float => "FloatType.Default", + TypeKind2.Double => "DoubleType.Default", + TypeKind2.Decimal => "new Decimal128Type(38, 18)", + TypeKind2.Binary => "BinaryType.Default", + TypeKind2.DateTime => "new TimestampType(TimeUnit.Microsecond, \"UTC\")", + TypeKind2.DateTimeOffset => "new TimestampType(TimeUnit.Microsecond, \"UTC\")", + TypeKind2.DateOnly => "Date32Type.Default", + TypeKind2.TimeOnly => "new Time64Type(TimeUnit.Microsecond)", + TypeKind2.TimeSpan => "DurationType.Microsecond", + TypeKind2.Guid => "GuidType.Default", + TypeKind2.Half => "HalfFloatType.Default", + TypeKind2.Enum => "new DictionaryType(Int16Type.Default, StringType.Default, false)", + TypeKind2.List or TypeKind2.Array or TypeKind2.Set => + $"new ListType(new Field(\"item\", {GetArrowTypeExpression(type.ElementType!)}, {(type.ElementType!.IsNullable ? "true" : "false")}))", + TypeKind2.Dictionary => + $"new MapType(new Field(\"key\", {GetArrowTypeExpression(type.KeyType!)}, false), new Field(\"value\", {GetArrowTypeExpression(type.ValueType!)}, true))", + TypeKind2.NestedRecord => + $"new StructType({type.FullTypeName}.ArrowSchema.FieldsList)", + _ => "StringType.Default /* unsupported */", + }; + } + + private static string ParseArrowTypeOverride(string typeName) + { + var lower = typeName.ToLowerInvariant().Trim(); + return lower switch + { + "date32" => "Date32Type.Default", + "date64" => "Date64Type.Default", + _ => $"StringType.Default /* unrecognized: {typeName} */", + }; + } + + /// + /// Returns the Arrow array type for casting (e.g., "Int32Array", "StringArray"). + /// + public static string GetArrayCastType(PropertyModel prop) + { + return prop.Type.Kind switch + { + TypeKind2.String => "StringArray", + TypeKind2.Bool => "BooleanArray", + TypeKind2.Byte => "UInt8Array", + TypeKind2.SByte => "Int8Array", + TypeKind2.Int16 => "Int16Array", + TypeKind2.UInt16 => "UInt16Array", + TypeKind2.Int32 => "Int32Array", + TypeKind2.UInt32 => "UInt32Array", + TypeKind2.Int64 => "Int64Array", + TypeKind2.UInt64 => "UInt64Array", + TypeKind2.Float => "FloatArray", + TypeKind2.Double => "DoubleArray", + TypeKind2.Half => "HalfFloatArray", + TypeKind2.Decimal => "Decimal128Array", + TypeKind2.DateTime => "TimestampArray", + TypeKind2.DateTimeOffset => "TimestampArray", + TypeKind2.DateOnly => "Date32Array", + TypeKind2.TimeOnly => "Time64Array", + TypeKind2.TimeSpan => "DurationArray", + TypeKind2.Guid => "GuidArray", + TypeKind2.Binary => "BinaryArray", + TypeKind2.Enum => "DictionaryArray", + TypeKind2.List or TypeKind2.Array or TypeKind2.Set => "ListArray", + TypeKind2.Dictionary => "MapArray", + TypeKind2.NestedRecord => "StructArray", + _ => "IArrowArray", + }; + } + + /// + /// Returns a "new SomeBuilder()" expression for nullable builders. + /// + public static string GetNullableBuilderDeclaration(PropertyModel prop) + { + switch (prop.Type.Kind) + { + case TypeKind2.String: + return "new StringArray.Builder()"; + case TypeKind2.Bool: + return "new BooleanArray.Builder()"; + case TypeKind2.Byte: + return "new UInt8Array.Builder()"; + case TypeKind2.SByte: + return "new Int8Array.Builder()"; + case TypeKind2.Int16: + return "new Int16Array.Builder()"; + case TypeKind2.UInt16: + return "new UInt16Array.Builder()"; + case TypeKind2.Int32: + return "new Int32Array.Builder()"; + case TypeKind2.UInt32: + return "new UInt32Array.Builder()"; + case TypeKind2.Int64: + return "new Int64Array.Builder()"; + case TypeKind2.UInt64: + return "new UInt64Array.Builder()"; + case TypeKind2.Float: + return "new FloatArray.Builder()"; + case TypeKind2.Double: + return "new DoubleArray.Builder()"; + case TypeKind2.Half: + return "new HalfFloatArray.Builder()"; + case TypeKind2.Decimal: + return "new Decimal128Array.Builder(new Decimal128Type(38, 18))"; + case TypeKind2.DateTime: + case TypeKind2.DateTimeOffset: + return "new TimestampArray.Builder(new TimestampType(TimeUnit.Microsecond, \"UTC\"))"; + case TypeKind2.DateOnly: + return "new Date32Array.Builder()"; + case TypeKind2.TimeOnly: + return "new Time64Array.Builder(Time64Type.Default)"; + case TypeKind2.TimeSpan: + return "new DurationArray.Builder(DurationType.Microsecond)"; + default: + return $"new StringArray.Builder() /* fallback for {prop.Type.Kind} */"; + } + } +} + +} // namespace diff --git a/src/Apache.Arrow.Serialization/Apache.Arrow.Serialization.csproj b/src/Apache.Arrow.Serialization/Apache.Arrow.Serialization.csproj new file mode 100644 index 00000000..82893383 --- /dev/null +++ b/src/Apache.Arrow.Serialization/Apache.Arrow.Serialization.csproj @@ -0,0 +1,20 @@ + + + + net8.0;net10.0 + enable + enable + Source-generated Apache Arrow serialization for .NET. Provides [ArrowSerializable] attribute and IArrowSerializer<T> interface for compile-time Arrow schema derivation, serialization, and deserialization. + + + + + + + + + + + diff --git a/src/Apache.Arrow.Serialization/ArrowArrayHelper.cs b/src/Apache.Arrow.Serialization/ArrowArrayHelper.cs new file mode 100644 index 00000000..397e5170 --- /dev/null +++ b/src/Apache.Arrow.Serialization/ArrowArrayHelper.cs @@ -0,0 +1,409 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow; +using Apache.Arrow.Arrays; +using Apache.Arrow.Types; + +namespace Apache.Arrow.Serialization; + +/// +/// Utility methods for building Arrow arrays in generated code. +/// +public static class ArrowArrayHelper +{ + /// + /// Creates an array of the specified Arrow type with all null values. + /// Used by generated code for nullable nested record fields. + /// + public static IArrowArray BuildNullArray(IArrowType type, int length) + { + switch (type) + { + case BooleanType: + { + var b = new BooleanArray.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } + case Bool8Type: + { + var b = new Bool8Array.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } + case Int8Type: + { + var b = new Int8Array.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } + case UInt8Type: + { + var b = new UInt8Array.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } + case Int16Type: + { + var b = new Int16Array.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } + case UInt16Type: + { + var b = new UInt16Array.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } + case Int32Type: + { + var b = new Int32Array.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } + case UInt32Type: + { + var b = new UInt32Array.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } + case Int64Type: + { + var b = new Int64Array.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } + case UInt64Type: + { + var b = new UInt64Array.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } + case HalfFloatType: + { + var b = new HalfFloatArray.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } + case FloatType: + { + var b = new FloatArray.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } + case DoubleType: + { + var b = new DoubleArray.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } + case Decimal128Type dt: + { + var b = new Decimal128Array.Builder(dt); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } + case StringType: + { + var b = new StringArray.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } + case BinaryType: + { + var b = new BinaryArray.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } + case StringViewType: + { + var b = new StringViewArray.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } + case BinaryViewType: + { + var b = new BinaryViewArray.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } + case GuidType: + { + var b = new GuidArray.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } + case FixedSizeBinaryType fbt: + { + return BuildNullFixedSizeBinaryArray(fbt, length); + } + case Date32Type: + { + var b = new Date32Array.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } + case Date64Type: + { + var b = new Date64Array.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } + case TimestampType tsType: + { + var b = new TimestampArray.Builder(tsType); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } + case Time32Type t32: + { + var b = new Time32Array.Builder(t32); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } + case Time64Type t64: + { + var b = new Time64Array.Builder(t64); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } + case DurationType dur: + { + var b = new DurationArray.Builder(dur); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } + case DictionaryType dt: + { + var idx = new Int16Array.Builder(); + for (int i = 0; i < length; i++) idx.AppendNull(); + var dict = new StringArray.Builder().Build(); + return new DictionaryArray(dt, idx.Build(), dict); + } + case StructType st: + { + var children = new IArrowArray[st.Fields.Count]; + for (int i = 0; i < children.Length; i++) + children[i] = BuildNullArray(st.Fields[i].DataType, length); + var bitmapBuilder = new ArrowBuffer.BitmapBuilder(); + for (int i = 0; i < length; i++) bitmapBuilder.Append(false); + return new StructArray(st, length, children, bitmapBuilder.Build(), length); + } + case ListType lt: + { + var lb = new ListArray.Builder(lt.ValueDataType); + for (int i = 0; i < length; i++) lb.AppendNull(); + return lb.Build(); + } + case MapType mt: + { + var mb = new MapArray.Builder(mt); + for (int i = 0; i < length; i++) mb.AppendNull(); + return mb.Build(); + } + default: + return new NullArray(length); + } + } + + // --- FixedSizeBinary helpers (no concrete Builder class in Arrow C#) --- + + private static FixedSizeBinaryArray BuildNullFixedSizeBinaryArray(FixedSizeBinaryType type, int length) + { + var valueBytes = new byte[length * type.ByteWidth]; + var validityBuffer = new ArrowBuffer.BitmapBuilder(); + for (int i = 0; i < length; i++) + validityBuffer.Append(false); + var data = new ArrayData(type, length, length, 0, + new[] { validityBuffer.Build(), new ArrowBuffer(valueBytes) }); + return new FixedSizeBinaryArray(data); + } + + public static IArrowArray BuildGuidArray(Guid value) + { + var b = new GuidArray.Builder(); + b.Append(value); + return b.Build(); + } + + public static IArrowArray BuildGuidArray(Guid? value) + { + var b = new GuidArray.Builder(); + if (value is { } v) + b.Append(v); + else + b.AppendNull(); + return b.Build(); + } + + public static IArrowArray BuildGuidArray(IReadOnlyList items) + { + var b = new GuidArray.Builder(); + for (int i = 0; i < items.Count; i++) + { + if (items[i] is Guid v) + b.Append(v); + else + b.AppendNull(); + } + return b.Build(); + } + + // --- TimeOnly helpers (Time64) --- + + public static IArrowArray BuildTimeOnlyArray(TimeOnly value) + { + var b = new Time64Array.Builder(Time64Type.Default); + b.Append(value); + return b.Build(); + } + + public static IArrowArray BuildTimeOnlyArray(TimeOnly? value) + { + var b = new Time64Array.Builder(Time64Type.Default); + if (value is { } v) + b.Append(v); + else + b.AppendNull(); + return b.Build(); + } + + public static IArrowArray BuildTimeOnlyArray(IReadOnlyList items) + { + var b = new Time64Array.Builder(Time64Type.Default); + foreach (var item in items) + { + if (item is TimeOnly v) + b.Append(v); + else + b.AppendNull(); + } + return b.Build(); + } + + public static TimeOnly ReadTimeOnly(Time64Array array, int index) + { + return array.GetTime(index)!.Value; + } + + // --- TimeSpan helpers (Duration) --- + + public static IArrowArray BuildDurationArray(TimeSpan value) + { + var b = new DurationArray.Builder(DurationType.Microsecond); + b.Append(value); + return b.Build(); + } + + public static IArrowArray BuildDurationArray(TimeSpan? value) + { + var b = new DurationArray.Builder(DurationType.Microsecond); + if (value is { } v) + b.Append(v); + else + b.AppendNull(); + return b.Build(); + } + + public static IArrowArray BuildDurationArray(IReadOnlyList items) + { + var b = new DurationArray.Builder(DurationType.Microsecond); + foreach (var item in items) + { + if (item is TimeSpan v) + b.Append(v); + else + b.AppendNull(); + } + return b.Build(); + } + + public static TimeSpan ReadDuration(DurationArray array, int index) + { + return array.GetTimeSpan(index)!.Value; + } + + // --- Decimal helpers (Decimal128) --- + + public static IArrowArray BuildDecimalArray(decimal value) + { + var b = new Decimal128Array.Builder(new Decimal128Type(38, 18)); + b.Append(value); + return b.Build(); + } + + public static IArrowArray BuildDecimalArray(decimal? value) + { + var b = new Decimal128Array.Builder(new Decimal128Type(38, 18)); + if (value is { } v) + b.Append(v); + else + b.AppendNull(); + return b.Build(); + } + + public static IArrowArray BuildDecimalArray(IReadOnlyList items) + { + var b = new Decimal128Array.Builder(new Decimal128Type(38, 18)); + foreach (var item in items) + { + if (item is decimal v) + b.Append(v); + else + b.AppendNull(); + } + return b.Build(); + } + + // --- UTC normalization helpers --- + + /// + /// Converts a DateTime to a UTC DateTimeOffset. + /// Local/Unspecified kinds are converted via ToUniversalTime(); Utc is wrapped directly. + /// + public static DateTimeOffset ToUtcDateTimeOffset(DateTime value) + { + if (value.Kind == DateTimeKind.Utc) + return new DateTimeOffset(value, TimeSpan.Zero); + return new DateTimeOffset(value.ToUniversalTime(), TimeSpan.Zero); + } + + /// + /// Normalizes a DateTimeOffset to UTC. + /// + public static DateTimeOffset ToUtcDateTimeOffset(DateTimeOffset value) + { + return value.ToUniversalTime(); + } + + /// + /// Converts a DateTime to a wall-clock DateTimeOffset (no timezone conversion). + /// The raw ticks are preserved regardless of DateTimeKind. + /// + public static DateTimeOffset ToWallClockDateTimeOffset(DateTime value) + { + return new DateTimeOffset(value.Ticks, TimeSpan.Zero); + } + + /// + /// Converts a DateTimeOffset to a wall-clock DateTimeOffset (strips timezone offset). + /// The wall-clock time (DateTime) is preserved, offset is discarded. + /// + public static DateTimeOffset ToWallClockDateTimeOffset(DateTimeOffset value) + { + return new DateTimeOffset(value.DateTime.Ticks, TimeSpan.Zero); + } +} diff --git a/src/Apache.Arrow.Serialization/ArrowSerializerExtensions.cs b/src/Apache.Arrow.Serialization/ArrowSerializerExtensions.cs new file mode 100644 index 00000000..501e16ce --- /dev/null +++ b/src/Apache.Arrow.Serialization/ArrowSerializerExtensions.cs @@ -0,0 +1,137 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow; +using Apache.Arrow.Ipc; + +namespace Apache.Arrow.Serialization; + +/// +/// Extension methods for Arrow serialization/deserialization. +/// +public static class ArrowSerializerExtensions +{ + /// + /// Serialize an instance to Arrow IPC stream bytes. + /// + public static byte[] SerializeToBytes(this T value) where T : IArrowSerializer + { + var batch = T.ToRecordBatch(value); + return RecordBatchToBytes(batch); + } + + /// + /// Deserialize an instance from Arrow IPC stream bytes. + /// + public static T DeserializeFromBytes(byte[] data) where T : IArrowSerializer + { + var batch = BytesToRecordBatch(data); + return T.FromRecordBatch(batch); + } + + /// + /// Serialize an instance to an Arrow IPC stream. + /// + public static void SerializeToStream(this T value, Stream destination) where T : IArrowSerializer + { + var batch = T.ToRecordBatch(value); + WriteRecordBatch(batch, destination); + } + + /// + /// Deserialize an instance from an Arrow IPC stream. + /// + public static T DeserializeFromStream(Stream source) where T : IArrowSerializer + { + var batch = ReadRecordBatch(source); + return T.FromRecordBatch(batch); + } + + /// + /// Serialize multiple instances to a multi-row RecordBatch. + /// + public static RecordBatch ToRecordBatch(this IEnumerable items) where T : IArrowSerializer + { + var list = items as IReadOnlyList ?? items.ToList(); + return T.ToRecordBatch(list); + } + + /// + /// Deserialize all rows from a RecordBatch into a list of instances. + /// + public static IReadOnlyList ToList(this RecordBatch batch) where T : IArrowSerializer + { + return T.ListFromRecordBatch(batch); + } + + /// + /// Serialize multiple instances to Arrow IPC stream bytes. + /// + public static byte[] SerializeListToBytes(this IEnumerable items) where T : IArrowSerializer + { + var batch = items.ToRecordBatch(); + return RecordBatchToBytes(batch); + } + + /// + /// Deserialize multiple instances from Arrow IPC stream bytes. + /// + public static IReadOnlyList DeserializeListFromBytes(byte[] data) where T : IArrowSerializer + { + var batch = BytesToRecordBatch(data); + return T.ListFromRecordBatch(batch); + } + + /// + /// Serialize a RecordBatch to Arrow IPC stream bytes. + /// + public static byte[] RecordBatchToBytes(RecordBatch batch) + { + using var ms = new MemoryStream(); + WriteRecordBatch(batch, ms); + return ms.ToArray(); + } + + /// + /// Write a RecordBatch as an Arrow IPC stream (schema + batch + EOS). + /// + public static void WriteRecordBatch(RecordBatch batch, Stream destination) + { + var writer = new ArrowStreamWriter(destination, batch.Schema, leaveOpen: true); + writer.WriteRecordBatch(batch); + writer.WriteEnd(); + writer.Dispose(); + } + + /// + /// Deserialize a RecordBatch from Arrow IPC stream bytes. + /// + public static RecordBatch BytesToRecordBatch(byte[] data) + { + using var ms = new MemoryStream(data); + return ReadRecordBatch(ms); + } + + /// + /// Read a single RecordBatch from an Arrow IPC stream. + /// + public static RecordBatch ReadRecordBatch(Stream source) + { + using var reader = new ArrowStreamReader(source, leaveOpen: true); + var batch = reader.ReadNextRecordBatch() + ?? throw new InvalidOperationException("No RecordBatch found in Arrow IPC stream."); + return batch; + } +} diff --git a/src/Apache.Arrow.Serialization/Attributes.cs b/src/Apache.Arrow.Serialization/Attributes.cs new file mode 100644 index 00000000..6fbfe994 --- /dev/null +++ b/src/Apache.Arrow.Serialization/Attributes.cs @@ -0,0 +1,173 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace Apache.Arrow.Serialization; + +/// +/// Marks a type for Arrow serialization. The source generator will emit +/// schema derivation, serialization, and deserialization code. +/// Supports record, record struct, class, and struct types. +/// +[AttributeUsage(AttributeTargets.Class | AttributeTargets.Struct, Inherited = false)] +public sealed class ArrowSerializableAttribute : Attribute; + +/// +/// Overrides the field name and/or order for Arrow serialization. +/// +[AttributeUsage(AttributeTargets.Property | AttributeTargets.Field, Inherited = false)] +public sealed class ArrowFieldAttribute : Attribute +{ + /// + /// The Arrow field name. If null, the property name is used. + /// + public string? Name { get; } + + /// + /// The field order in the Arrow schema. Lower values come first. + /// If not set, properties are ordered by declaration order. + /// + public int Order { get; set; } = int.MaxValue; + + public ArrowFieldAttribute() { } + + public ArrowFieldAttribute(string name) + { + Name = name; + } +} + +/// +/// Overrides the inferred Arrow type for a property. +/// The type parameter must be a static property or field returning an Apache.Arrow IArrowType. +/// +[AttributeUsage(AttributeTargets.Property | AttributeTargets.Field | AttributeTargets.Parameter, Inherited = false)] +public sealed class ArrowTypeAttribute : Attribute +{ + /// + /// A well-known Arrow type name (e.g. "int32", "float32", "binary", "decimal128(38,18)"). + /// + public string? TypeName { get; } + + /// + /// A custom converter type implementing IArrowConverter<T> for the property type. + /// The converter must have a parameterless constructor. + /// + public Type? Converter { get; set; } + + /// + /// Overrides the Arrow type of the element in a List, Array, or HashSet property. + /// E.g. [ArrowType(ElementType = "string_view")] on a List<string>. + /// + public string? ElementType { get; set; } + + /// + /// Overrides the Arrow type of the key in a Dictionary property. + /// E.g. [ArrowType(KeyType = "string_view")] on a Dictionary<string, int>. + /// + public string? KeyType { get; set; } + + /// + /// Overrides the Arrow type of the value in a Dictionary property. + /// E.g. [ArrowType(ValueType = "timestamp[ns, UTC]")] on a Dictionary<string, DateTime>. + /// + public string? ValueType { get; set; } + + public ArrowTypeAttribute(string typeName) + { + TypeName = typeName; + } + + public ArrowTypeAttribute() + { + } +} + +/// +/// Adds key-value metadata to the Arrow schema (on class) or field (on property). +/// +[AttributeUsage(AttributeTargets.Class | AttributeTargets.Property | AttributeTargets.Field, AllowMultiple = true, Inherited = false)] +public sealed class ArrowMetadataAttribute : Attribute +{ + public string Key { get; } + public string Value { get; } + + public ArrowMetadataAttribute(string key, string value) + { + Key = key; + Value = value; + } +} + +/// +/// Excludes a property from Arrow serialization. The property must have a default value. +/// +[AttributeUsage(AttributeTargets.Property | AttributeTargets.Field, Inherited = false)] +public sealed class ArrowIgnoreAttribute : Attribute; + +/// +/// Marks a base type (abstract record or interface) for polymorphic Arrow serialization. +/// +[AttributeUsage(AttributeTargets.Class | AttributeTargets.Interface, Inherited = false)] +public sealed class ArrowPolymorphicAttribute : Attribute +{ + /// + /// The name of the type discriminator field in the Arrow schema. + /// Defaults to "$type". + /// + public string TypeDiscriminatorFieldName { get; set; } = "$type"; +} + +/// +/// Registers a derived type for polymorphic Arrow serialization on the base type. +/// +[AttributeUsage(AttributeTargets.Class | AttributeTargets.Interface, AllowMultiple = true, Inherited = false)] +public sealed class ArrowDerivedTypeAttribute : Attribute +{ + /// + /// The derived type to register. + /// + public Type DerivedType { get; } + + /// + /// The type discriminator value (string tag) for this derived type. + /// + public string TypeDiscriminator { get; } + + public ArrowDerivedTypeAttribute(Type derivedType, string typeDiscriminator) + { + DerivedType = derivedType; + TypeDiscriminator = typeDiscriminator; + } +} + +/// +/// Implement this interface to receive callbacks during Arrow serialization/deserialization. +/// is called before the object is serialized to a RecordBatch. +/// is called after the object is deserialized from a RecordBatch. +/// +public interface IArrowSerializationCallback +{ + /// + /// Called before the object is serialized to a RecordBatch. + /// Use this to compute derived fields, flush lazy state, or validate before serialization. + /// + void OnBeforeSerialize(); + + /// + /// Called after the object is deserialized from a RecordBatch. + /// Use this to rebuild computed/cached fields, initialize transient state, or validate invariants. + /// + void OnAfterDeserialize(); +} diff --git a/src/Apache.Arrow.Serialization/IArrowSerializer.cs b/src/Apache.Arrow.Serialization/IArrowSerializer.cs new file mode 100644 index 00000000..cb5a8f45 --- /dev/null +++ b/src/Apache.Arrow.Serialization/IArrowSerializer.cs @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow; +using Apache.Arrow.Types; + +namespace Apache.Arrow.Serialization; + +/// +/// Custom converter for Arrow serialization of a specific type. +/// Implement this interface to control how a type is stored in Arrow arrays. +/// +public interface IArrowConverter +{ + /// The Arrow type used to store values of type T. + IArrowType ArrowType { get; } + + /// Build a single-element Arrow array from a value. + IArrowArray ToArray(T value); + + /// Build a multi-element Arrow array from a list of values. + IArrowArray ToArray(IReadOnlyList values); + + /// Read a value from an Arrow array at the given index. + T FromArray(IArrowArray array, int index); +} + +/// +/// Interface implemented by source-generated code on [ArrowSerializable] records. +/// Provides static Arrow schema, serialization, and deserialization. +/// +public interface IArrowSerializer where T : IArrowSerializer +{ + /// + /// The Arrow schema for this type, derived from property types. + /// + static abstract Schema ArrowSchema { get; } + + /// + /// Serialize an instance to a single-row RecordBatch. + /// + static abstract RecordBatch ToRecordBatch(T value); + + /// + /// Deserialize an instance from a single-row RecordBatch. + /// + static abstract T FromRecordBatch(RecordBatch batch); + + /// + /// Serialize multiple instances to a multi-row RecordBatch. + /// + static abstract RecordBatch ToRecordBatch(IReadOnlyList values); + + /// + /// Deserialize all rows from a RecordBatch into a list of instances. + /// + static abstract IReadOnlyList ListFromRecordBatch(RecordBatch batch); +} diff --git a/src/Apache.Arrow.Serialization/README.md b/src/Apache.Arrow.Serialization/README.md new file mode 100644 index 00000000..5f21924d --- /dev/null +++ b/src/Apache.Arrow.Serialization/README.md @@ -0,0 +1,620 @@ +# Apache.Arrow.Serialization + +Source-generated [Apache Arrow](https://arrow.apache.org/) serialization for .NET. + +Mark any type with `[ArrowSerializable]` and a Roslyn source generator will emit compile-time Arrow schema derivation, serialization, and deserialization — zero reflection on the hot path, fully AOT-compatible. + +```csharp +[ArrowSerializable] +public partial record Person +{ + public string Name { get; init; } = ""; + public int Age { get; init; } +} + +// Single-row +var batch = Person.ToRecordBatch(new Person { Name = "Alice", Age = 30 }); +var alice = Person.FromRecordBatch(batch); + +// Multi-row +var people = new[] { alice, new Person { Name = "Bob", Age = 25 } }; +var table = Person.ToRecordBatch(people); +IReadOnlyList restored = Person.ListFromRecordBatch(table); + +// Arrow IPC bytes (cross-language compatible) +byte[] bytes = alice.SerializeToBytes(); +var roundTrip = ArrowSerializerExtensions.DeserializeFromBytes(bytes); +``` + +## Table of Contents + +- [Installation](#installation) +- [Quick Start](#quick-start) +- [Supported Types](#supported-types) + - [Type Declarations](#type-declarations) + - [Built-in Type Mappings](#built-in-type-mappings) + - [Collections](#collections) + - [Nullable Types](#nullable-types) +- [Attributes](#attributes) + - [ArrowSerializable](#arrowserializable) + - [ArrowField](#arrowfield) + - [ArrowType](#arrowtype) + - [ArrowIgnore](#arrowignore) + - [ArrowMetadata](#arrowmetadata) +- [Nested Types](#nested-types) +- [Readonly Fields and Constructors](#readonly-fields-and-constructors) +- [Enum Serialization](#enum-serialization) +- [Polymorphism](#polymorphism) +- [Custom Converters](#custom-converters) +- [Serialization Callbacks](#serialization-callbacks) +- [JSON Schema Emission](#json-schema-emission) +- [RecordBatchBuilder (Reflection-Based)](#recordbatchbuilder-reflection-based) +- [Extension Methods](#extension-methods) +- [Source Generator Diagnostics](#source-generator-diagnostics) +- [Cross-Language Compatibility](#cross-language-compatibility) + +## Installation + +``` +dotnet add package Apache.Arrow.Serialization +``` + +The NuGet package includes both the runtime library and the Roslyn source generator. Targets `net8.0` and `net10.0`. + +## Quick Start + +1. Add `[ArrowSerializable]` to your type +2. Make the type `partial` (required for source generation) +3. The generator emits `IArrowSerializer` — giving you `ArrowSchema`, `ToRecordBatch`, `FromRecordBatch`, and `ListFromRecordBatch` + +```csharp +using Apache.Arrow.Serialization; + +[ArrowSerializable] +public partial record SensorReading +{ + public string SensorId { get; init; } = ""; + public double Temperature { get; init; } + public DateTime Timestamp { get; init; } +} +``` + +The source generator produces a `partial` implementation with these static members: + +```csharp +partial record SensorReading : IArrowSerializer +{ + public static Schema ArrowSchema { get; } + public static RecordBatch ToRecordBatch(SensorReading value); + public static SensorReading FromRecordBatch(RecordBatch batch); + public static RecordBatch ToRecordBatch(IReadOnlyList values); + public static IReadOnlyList ListFromRecordBatch(RecordBatch batch); +} +``` + +## Supported Types + +### Type Declarations + +All four C# type kinds are supported: + +```csharp +[ArrowSerializable] +public partial record MyRecord { ... } + +[ArrowSerializable] +public partial record struct MyRecordStruct { ... } + +[ArrowSerializable] +public partial class MyClass { ... } + +[ArrowSerializable] +public partial struct MyStruct { ... } +``` + +Records use `{ get; init; }` properties. Classes and structs use `{ get; set; }`. + +### Built-in Type Mappings + +| C# Type | Arrow Type | Notes | +|---------|-----------|-------| +| `string` | `Utf8` | Override to `StringView` via `[ArrowType("string_view")]` | +| `bool` | `Boolean` | Override to `Bool8` via `[ArrowType("bool8")]` | +| `byte` | `UInt8` | | +| `sbyte` | `Int8` | | +| `short` | `Int16` | | +| `ushort` | `UInt16` | | +| `int` | `Int32` | | +| `uint` | `UInt32` | | +| `long` | `Int64` | | +| `ulong` | `UInt64` | | +| `float` | `Float32` | | +| `double` | `Float64` | | +| `Half` | `Float16` | | +| `decimal` | `Decimal128(38, 18)` | Configurable via `[ArrowType("decimal128(28, 10)")]` | +| `DateTime` | `Timestamp(us, UTC)` | Configurable resolution and timezone | +| `DateTimeOffset` | `Timestamp(us, UTC)` | Configurable resolution and timezone | +| `DateOnly` | `Date32` | Override to `Date64` via `[ArrowType("date64")]` | +| `TimeOnly` | `Time64(us)` | Override to `Time32` via `[ArrowType("time32[ms]")]` | +| `TimeSpan` | `Duration(us)` | | +| `Guid` | `FixedSizeBinary(16)` | UUID extension type | +| `byte[]` | `Binary` | Override to `BinaryView` via `[ArrowType("binary_view")]` | +| `ReadOnlyMemory` | `Binary` | Override to `BinaryView` via `[ArrowType("binary_view")]` | +| `enum` | `Dictionary(Int16, Utf8)` | Name-based encoding | + +### Collections + +| C# Type | Arrow Type | +|---------|-----------| +| `List`, `T[]`, `IList`, `IReadOnlyList` | `List(T)` | +| `IEnumerable`, `ICollection`, `IReadOnlyCollection` | `List(T)` | +| `HashSet`, `ISet`, `IReadOnlySet` | `List(T)` | +| `Dictionary`, `IDictionary`, `IReadOnlyDictionary` | `Map(K,V)` | + +Collections support nested types, enums, and nullable elements: + +```csharp +[ArrowSerializable] +public partial record DataSet +{ + public List Values { get; init; } = new(); + public Dictionary Metrics { get; init; } = new(); + public List OptionalItems { get; init; } = new(); + public HashSet UniqueColors { get; init; } = new(); +} +``` + +### Nullable Types + +Both nullable value types and nullable reference types are supported. Nullable properties produce nullable Arrow fields: + +```csharp +[ArrowSerializable] +public partial record WithNullables +{ + public int? MaybeCount { get; init; } + public string? MaybeName { get; init; } + public Inner? MaybeNested { get; init; } +} +``` + +## Attributes + +### ArrowSerializable + +Marks a type for source generation. Required on all types you want to serialize. + +```csharp +[ArrowSerializable] +public partial record MyType { ... } +``` + +### ArrowField + +Overrides the Arrow field name and/or controls field ordering: + +```csharp +[ArrowSerializable] +public partial record Measurement +{ + [ArrowField("sensor_id")] + public string SensorId { get; init; } = ""; + + [ArrowField("value", Order = 0)] + public double Value { get; init; } + + [ArrowField(Order = 1)] + public DateTime Timestamp { get; init; } +} +``` + +- `Name` — the Arrow field name (defaults to C# property name) +- `Order` — controls field position in the schema (lower values first; default is declaration order) + +### ArrowType + +Overrides the inferred Arrow type for a property: + +```csharp +[ArrowSerializable] +public partial record Precise +{ + [ArrowType("decimal128(28, 10)")] + public decimal Value { get; init; } + + [ArrowType("timestamp[ns, UTC]")] + public DateTime Created { get; init; } + + [ArrowType("timestamp[us]")] + public DateTime LocalTime { get; init; } // wall-clock (no timezone) + + [ArrowType("date64")] + public DateOnly Birthday { get; init; } + + [ArrowType("time32[ms]")] + public TimeOnly Alarm { get; init; } + + [ArrowType("string_view")] + public string Name { get; init; } = ""; + + [ArrowType("binary_view")] + public byte[] Payload { get; init; } = []; + + [ArrowType("bool8")] + public bool Flag { get; init; } +} +``` + +For collections, override element, key, or value types: + +```csharp +[ArrowSerializable] +public partial record WithOverrides +{ + [ArrowType(ElementType = "string_view")] + public List Tags { get; init; } = new(); + + [ArrowType(KeyType = "string_view", ValueType = "timestamp[ns, UTC]")] + public Dictionary Events { get; init; } = new(); +} +``` + +#### Timestamp Semantics + +| ArrowType | Semantics | Serialize | Deserialize | +|-----------|-----------|-----------|-------------| +| `timestamp[us, UTC]` (default) | Instant | UTC-normalized | `.UtcDateTime` | +| `timestamp[us]` | Wall-clock | Raw ticks preserved | `.DateTime` | + +### ArrowIgnore + +Excludes a property from serialization. The property must have a default value: + +```csharp +[ArrowSerializable] +public partial record WithCache +{ + public int Value { get; init; } + + [ArrowIgnore] + public int CachedDouble { get; set; } +} +``` + +### ArrowMetadata + +Adds key-value metadata to the Arrow schema (on class) or field (on property). Multiple entries are supported: + +```csharp +[ArrowSerializable] +[ArrowMetadata("version", "2")] +[ArrowMetadata("source", "sensor-array")] +public partial record Annotated +{ + [ArrowMetadata("unit", "celsius")] + public double Temperature { get; init; } + + [ArrowMetadata("unit", "hPa")] + public double Pressure { get; init; } +} +``` + +Metadata is stored in the Arrow schema and is accessible from any Arrow implementation. + +## Nested Types + +Any `[ArrowSerializable]` type can be used as a property of another, mapping to Arrow `Struct`: + +```csharp +[ArrowSerializable] +public partial record Address +{ + public string Street { get; init; } = ""; + public string City { get; init; } = ""; +} + +[ArrowSerializable] +public partial record Customer +{ + public string Name { get; init; } = ""; + public Address Address { get; init; } = new(); + public Address? BillingAddress { get; init; } // nullable nested +} +``` + +Nesting works to any depth. Nested types in collections are also supported (`List
`, `Dictionary`). + +## Readonly Fields and Constructors + +Structs with `readonly` fields are supported. The generator resolves a constructor by matching parameter names to field names (case-insensitive): + +```csharp +[ArrowSerializable] +public partial struct Vector3 +{ + public readonly float X; + public readonly float Y; + public readonly float Z; + + public Vector3(float x, float y, float z) + { + X = x; + Y = y; + Z = z; + } +} +``` + +Fields and properties can be mixed. Fields support all the same attributes as properties (`[ArrowField]`, `[ArrowType]`, `[ArrowIgnore]`, `[ArrowMetadata]`). + +If readonly fields exist without a matching constructor, the generator reports diagnostic [`ARROW002`](#source-generator-diagnostics). + +## Enum Serialization + +Enums are serialized as Arrow `Dictionary(Int16, Utf8)` using their string name: + +```csharp +public enum Priority { Low, Medium, High } + +[ArrowSerializable] +public partial record Task +{ + public string Title { get; init; } = ""; + public Priority Priority { get; init; } +} +``` + +This produces compact dictionary-encoded arrays and is compatible with Python's string-based enum encoding. + +## Polymorphism + +For type hierarchies, use `[ArrowPolymorphic]` on the base type and `[ArrowDerivedType]` to register subtypes: + +```csharp +[ArrowPolymorphic] +[ArrowDerivedType(typeof(Circle), "circle")] +[ArrowDerivedType(typeof(Rectangle), "rectangle")] +public abstract partial record Shape; + +[ArrowSerializable] +public partial record Circle : Shape +{ + public double Radius { get; init; } +} + +[ArrowSerializable] +public partial record Rectangle : Shape +{ + public double Width { get; init; } + public double Height { get; init; } +} +``` + +Serialize and deserialize through the base type: + +```csharp +var shapes = new Shape[] { new Circle { Radius = 5 }, new Rectangle { Width = 3, Height = 4 } }; +var batch = Shape.ToRecordBatch(shapes); +IReadOnlyList restored = Shape.ListFromRecordBatch(batch); +``` + +**Wire format:** Flat schema with a string discriminator column (`$type` by default) plus the union of all derived type fields (all made nullable). Customize the discriminator name: + +```csharp +[ArrowPolymorphic(TypeDiscriminatorFieldName = "kind")] +``` + +## Custom Converters + +For types that don't have a built-in mapping, implement `IArrowConverter`: + +```csharp +public struct Point2D +{ + public double X { get; set; } + public double Y { get; set; } +} + +public class Point2DConverter : IArrowConverter +{ + public IArrowType ArrowType => StringType.Default; + + public IArrowArray ToArray(Point2D value) => + new StringArray.Builder().Append($"{value.X},{value.Y}").Build(); + + public IArrowArray ToArray(IReadOnlyList values) + { + var b = new StringArray.Builder(); + foreach (var v in values) b.Append($"{v.X},{v.Y}"); + return b.Build(); + } + + public Point2D FromArray(IArrowArray array, int index) + { + var parts = ((StringArray)array).GetString(index)!.Split(','); + return new Point2D + { + X = double.Parse(parts[0]), + Y = double.Parse(parts[1]), + }; + } +} +``` + +Apply via `[ArrowType(Converter = ...)]`: + +```csharp +[ArrowSerializable] +public partial record WithLocation +{ + public string Name { get; init; } = ""; + + [ArrowType(Converter = typeof(Point2DConverter))] + public Point2D Location { get; init; } +} +``` + +The converter class must implement `IArrowConverter` and have a parameterless constructor. + +## Serialization Callbacks + +Types implementing `IArrowSerializationCallback` receive calls during serialization and deserialization: + +```csharp +[ArrowSerializable] +public partial class Computed : IArrowSerializationCallback +{ + public int Value { get; set; } + + [ArrowIgnore] + public int DoubleValue { get; set; } + + public void OnBeforeSerialize() + { + // Called before serialization — validate, flush lazy state, etc. + } + + public void OnAfterDeserialize() + { + // Called after deserialization — rebuild computed/cached fields + DoubleValue = Value * 2; + } +} +``` + +- `OnBeforeSerialize()` is called once per instance before it is written to a RecordBatch +- `OnAfterDeserialize()` is called once per instance after it is read from a RecordBatch + +Callbacks fire for both single-row and multi-row serialization. + +## JSON Schema Emission + +Opt in to generate a `static string ArrowSchemaJson` property for cross-language tooling: + +```xml + + true + +``` + +This produces a JSON descriptor of the Arrow schema: + +```csharp +string json = SensorReading.ArrowSchemaJson; +``` + +```json +{ + "type": "SensorReading", + "namespace": "MyApp", + "typeName": "SensorReading", + "metadata": {}, + "fields": [ + { + "name": "SensorId", + "propertyName": "SensorId", + "arrowType": "utf8", + "typeKind": "String", + "nullable": false, + "csharpType": "string" + }, + { + "name": "Temperature", + "propertyName": "Temperature", + "arrowType": "float64", + "typeKind": "Double", + "nullable": false, + "csharpType": "double" + } + ] +} +``` + +The JSON schema is off by default — it's intended for external tooling (code generators, schema registries), not runtime use. + +## RecordBatchBuilder (Reflection-Based) + +For quick prototyping or anonymous types, `RecordBatchBuilder` provides reflection-based serialization without any attributes: + +```csharp +var data = new[] +{ + new { Name = "Alice", Score = 95.5 }, + new { Name = "Bob", Score = 87.0 }, +}; + +RecordBatch batch = RecordBatchBuilder.FromObjects(data); +``` + +```csharp +// Single object +RecordBatch single = RecordBatchBuilder.FromObject(new { X = 1, Y = 2 }); +``` + +This is **serialize-only** — anonymous types can't be deserialized. Annotated with `[RequiresUnreferencedCode]` (not AOT-safe). For production use, prefer `[ArrowSerializable]` with the source generator. + +When `RecordBatchBuilder` encounters a nested `[ArrowSerializable]` type, it delegates to the generated serializer for correct attribute handling and performance. + +## Extension Methods + +`ArrowSerializerExtensions` provides convenience methods for any `IArrowSerializer` type: + +| Method | Description | +|--------|-------------| +| `value.SerializeToBytes()` | Serialize to Arrow IPC stream bytes | +| `DeserializeFromBytes(bytes)` | Deserialize from Arrow IPC stream bytes | +| `value.SerializeToStream(stream)` | Serialize to a `Stream` | +| `DeserializeFromStream(stream)` | Deserialize from a `Stream` | +| `items.ToRecordBatch()` | `IEnumerable` to multi-row RecordBatch | +| `batch.ToList()` | RecordBatch to `IReadOnlyList` | +| `items.SerializeListToBytes()` | Serialize list to Arrow IPC bytes | +| `DeserializeListFromBytes(bytes)` | Deserialize list from Arrow IPC bytes | + +```csharp +// Round-trip through bytes +byte[] bytes = person.SerializeToBytes(); +var restored = ArrowSerializerExtensions.DeserializeFromBytes(bytes); + +// Multi-row convenience +var people = new[] { alice, bob }; +byte[] listBytes = people.SerializeListToBytes(); +IReadOnlyList all = ArrowSerializerExtensions.DeserializeListFromBytes(listBytes); +``` + +## Source Generator Diagnostics + +The source generator reports compile-time diagnostics for common mistakes: + +| ID | Severity | Description | +|----|----------|-------------| +| `ARROW001` | Error | Type with `[ArrowSerializable]` must be declared `partial` | +| `ARROW002` | Error | Readonly fields found but no constructor with matching parameters | +| `ARROW003` | Warning | Unsupported property type (e.g. `Queue`, `Stack`) — property will be skipped | +| `ARROW004` | Error | Duplicate Arrow field names (e.g. two properties with `[ArrowField("same")]`) | +| `ARROW005` | Warning | Get-only property without `init` accessor — property will be skipped | +| `ARROW006` | Warning | Arrow attribute on private or static member — attribute is ignored | + +## Cross-Language Compatibility + +Serialized data uses the standard [Arrow IPC streaming format](https://arrow.apache.org/docs/format/Columnar.html#serialization-and-interprocess-communication-ipc) and is compatible with any Arrow implementation (Python, Java, Rust, C++, Go, etc.). + +```python +# Python — read data serialized by C# +import pyarrow as pa + +with open("data.arrow", "rb") as f: + reader = pa.ipc.open_stream(f) + table = reader.read_all() + +print(table.to_pandas()) +``` + +Type mappings are designed to match Python `pyarrow` conventions: +- Enums as `Dictionary(Int16, Utf8)` +- Timestamps as `Timestamp(us, UTC)` by default +- Nested records as `Struct` +- Lists, maps, and sets follow standard Arrow nested types + +## License + +Apache License, Version 2.0 diff --git a/src/Apache.Arrow.Serialization/RecordBatchBuilder.cs b/src/Apache.Arrow.Serialization/RecordBatchBuilder.cs new file mode 100644 index 00000000..f5825b34 --- /dev/null +++ b/src/Apache.Arrow.Serialization/RecordBatchBuilder.cs @@ -0,0 +1,711 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Diagnostics.CodeAnalysis; +using System.Reflection; +using Apache.Arrow; +using Apache.Arrow.Arrays; +using Apache.Arrow.Types; + +namespace Apache.Arrow.Serialization; + +/// +/// Reflection-based serializer for converting arbitrary .NET objects (including anonymous types) +/// to Arrow RecordBatches. Analogous to System.Text.Json's reflection-based path — +/// works without attributes or source generation but is not AOT-safe. +/// +public static class RecordBatchBuilder +{ + /// + /// Convert a collection of objects to a RecordBatch. Schema is inferred from the + /// public readable properties of . + /// Works with anonymous types, records, classes, and structs. + /// + [RequiresUnreferencedCode("Uses reflection to inspect properties. Use [ArrowSerializable] for AOT-safe serialization.")] + public static RecordBatch FromObjects(IEnumerable items) + { + var list = items as IReadOnlyList ?? items.ToList(); + if (list.Count == 0) + throw new ArgumentException("Cannot infer schema from empty collection.", nameof(items)); + + var properties = typeof(T).GetProperties(BindingFlags.Public | BindingFlags.Instance) + .Where(p => p.CanRead) + .ToArray(); + + var fields = new List(); + var builders = new List(); + + foreach (var prop in properties) + { + var propType = prop.PropertyType; + var (arrowType, nullable) = InferArrowType(propType); + fields.Add(new Field(prop.Name, arrowType, nullable)); + builders.Add(CreateColumnBuilder(propType, arrowType)); + } + + var schema = new Schema.Builder(); + foreach (var f in fields) schema.Field(f); + + // Populate builders + for (int row = 0; row < list.Count; row++) + { + var item = list[row]!; + for (int col = 0; col < properties.Length; col++) + { + var value = properties[col].GetValue(item); + builders[col].Append(value); + } + } + + var arrays = builders.Select(b => b.Build()).ToArray(); + return new RecordBatch(schema.Build(), arrays, list.Count); + } + + /// + /// Convert a single object to a single-row RecordBatch. + /// + [RequiresUnreferencedCode("Uses reflection to inspect properties. Use [ArrowSerializable] for AOT-safe serialization.")] + public static RecordBatch FromObject(T item) + => FromObjects(new[] { item }); + + private static (IArrowType Type, bool Nullable) InferArrowType(Type clrType) + { + var underlying = Nullable.GetUnderlyingType(clrType); + if (underlying is not null) + { + var (inner, _) = InferArrowType(underlying); + return (inner, true); + } + + if (clrType == typeof(string)) return (StringType.Default, true); + if (clrType == typeof(bool)) return (BooleanType.Default, false); + if (clrType == typeof(sbyte)) return (Int8Type.Default, false); + if (clrType == typeof(byte)) return (UInt8Type.Default, false); + if (clrType == typeof(short)) return (Int16Type.Default, false); + if (clrType == typeof(ushort)) return (UInt16Type.Default, false); + if (clrType == typeof(int)) return (Int32Type.Default, false); + if (clrType == typeof(uint)) return (UInt32Type.Default, false); + if (clrType == typeof(long)) return (Int64Type.Default, false); + if (clrType == typeof(ulong)) return (UInt64Type.Default, false); + if (clrType == typeof(Half)) return (HalfFloatType.Default, false); + if (clrType == typeof(float)) return (FloatType.Default, false); + if (clrType == typeof(double)) return (DoubleType.Default, false); + if (clrType == typeof(decimal)) return (new Decimal128Type(38, 18), false); + if (clrType == typeof(DateTime)) return (new TimestampType(TimeUnit.Microsecond, "UTC"), false); + if (clrType == typeof(DateTimeOffset)) return (new TimestampType(TimeUnit.Microsecond, "UTC"), false); + if (clrType == typeof(DateOnly)) return (Date32Type.Default, false); + if (clrType == typeof(TimeOnly)) return (new Time64Type(TimeUnit.Microsecond), false); + if (clrType == typeof(TimeSpan)) return (DurationType.Microsecond, false); + if (clrType == typeof(Guid)) return (new GuidType(), false); + if (clrType == typeof(byte[])) return (BinaryType.Default, true); + if (clrType == typeof(ReadOnlyMemory)) return (BinaryType.Default, false); + + if (clrType.IsEnum) + return (new DictionaryType(Int16Type.Default, StringType.Default, false), false); + + // T[] arrays (not byte[] which is handled above) + if (clrType.IsArray) + { + var elemType = clrType.GetElementType()!; + var (elemArrow, elemNullable) = InferArrowType(elemType); + return (new ListType(new Field("item", elemArrow, elemNullable)), true); + } + + if (clrType.IsGenericType) + { + var genDef = clrType.GetGenericTypeDefinition(); + if (genDef == typeof(List<>) || genDef == typeof(HashSet<>)) + { + var elemType = clrType.GetGenericArguments()[0]; + var (elemArrow, elemNullable) = InferArrowType(elemType); + return (new ListType(new Field("item", elemArrow, elemNullable)), true); + } + if (genDef == typeof(Dictionary<,>)) + { + var args = clrType.GetGenericArguments(); + var (keyArrow, _) = InferArrowType(args[0]); + var (valArrow, valNullable) = InferArrowType(args[1]); + return (new MapType(new Field("key", keyArrow, false), new Field("value", valArrow, valNullable)), true); + } + } + + // Check for [ArrowSerializable] types with source-generated IArrowSerializer + var genSchema = GetGeneratedArrowSchema(clrType); + if (genSchema is not null) + { + var structFields = new List(genSchema.FieldsList); + return (new StructType(structFields), true); + } + + // Nested object type (anonymous, record, class, struct with readable properties) + if (clrType.IsClass || clrType.IsValueType) + { + var nestedProps = clrType.GetProperties(BindingFlags.Public | BindingFlags.Instance) + .Where(p => p.CanRead) + .ToArray(); + if (nestedProps.Length > 0) + { + var nestedFields = nestedProps.Select(p => + { + var (ft, fn) = InferArrowType(p.PropertyType); + return new Field(p.Name, ft, fn); + }).ToList(); + return (new StructType(nestedFields), true); + } + } + + throw new NotSupportedException($"Cannot infer Arrow type for {clrType.FullName}"); + } + + /// + /// Check if a type implements IArrowSerializer<T> (i.e. has [ArrowSerializable] source-generated code) + /// and return its static ArrowSchema if so. + /// + private static Schema? GetGeneratedArrowSchema(Type clrType) + { + var iface = clrType.GetInterfaces() + .FirstOrDefault(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IArrowSerializer<>)); + if (iface is null) return null; + + var schemaProp = clrType.GetProperty("ArrowSchema", BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy); + return schemaProp?.GetValue(null) as Schema; + } + + /// + /// Try to get the static ToRecordBatch(IReadOnlyList<T>) method from a source-generated type. + /// + private static MethodInfo? GetGeneratedToRecordBatchList(Type clrType) + { + var listType = typeof(IReadOnlyList<>).MakeGenericType(clrType); + return clrType.GetMethod("ToRecordBatch", BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy, [listType]); + } + + private static IColumnBuilder CreateColumnBuilder(Type clrType, IArrowType arrowType) + { + var underlying = Nullable.GetUnderlyingType(clrType); + if (underlying is not null) + return CreateColumnBuilder(underlying, arrowType); // inner builders all handle null + + if (clrType == typeof(string)) return new StringColumnBuilder(); + if (clrType == typeof(bool)) return new BoolColumnBuilder(); + if (clrType == typeof(sbyte)) return new TypedColumnBuilder(new(), (b, v) => b.Append(v), b => b.AppendNull(), b => b.Build()); + if (clrType == typeof(byte)) return new TypedColumnBuilder(new(), (b, v) => b.Append(v), b => b.AppendNull(), b => b.Build()); + if (clrType == typeof(short)) return new TypedColumnBuilder(new(), (b, v) => b.Append(v), b => b.AppendNull(), b => b.Build()); + if (clrType == typeof(ushort)) return new TypedColumnBuilder(new(), (b, v) => b.Append(v), b => b.AppendNull(), b => b.Build()); + if (clrType == typeof(int)) return new TypedColumnBuilder(new(), (b, v) => b.Append(v), b => b.AppendNull(), b => b.Build()); + if (clrType == typeof(uint)) return new TypedColumnBuilder(new(), (b, v) => b.Append(v), b => b.AppendNull(), b => b.Build()); + if (clrType == typeof(long)) return new TypedColumnBuilder(new(), (b, v) => b.Append(v), b => b.AppendNull(), b => b.Build()); + if (clrType == typeof(ulong)) return new TypedColumnBuilder(new(), (b, v) => b.Append(v), b => b.AppendNull(), b => b.Build()); + if (clrType == typeof(Half)) return new TypedColumnBuilder(new(), (b, v) => b.Append(v), b => b.AppendNull(), b => b.Build()); + if (clrType == typeof(float)) return new TypedColumnBuilder(new(), (b, v) => b.Append(v), b => b.AppendNull(), b => b.Build()); + if (clrType == typeof(double)) return new TypedColumnBuilder(new(), (b, v) => b.Append(v), b => b.AppendNull(), b => b.Build()); + if (clrType == typeof(decimal)) return new DecimalColumnBuilder(); + if (clrType == typeof(DateTime)) return new DateTimeColumnBuilder(); + if (clrType == typeof(DateTimeOffset)) return new DateTimeOffsetColumnBuilder(); + if (clrType == typeof(DateOnly)) return new TypedColumnBuilder(new(), (b, v) => b.Append(v), b => b.AppendNull(), b => b.Build()); + if (clrType == typeof(TimeOnly)) return new TimeOnlyColumnBuilder(); + if (clrType == typeof(TimeSpan)) return new TimeSpanColumnBuilder(); + if (clrType == typeof(Guid)) return new GuidColumnBuilder(); + if (clrType == typeof(byte[])) return new BinaryColumnBuilder(); + if (clrType == typeof(ReadOnlyMemory)) return new ReadOnlyMemoryByteColumnBuilder(); + if (clrType.IsEnum) return new EnumColumnBuilder(); + + // List, T[], HashSet → ListArray + if (arrowType is ListType listType) + { + var elemClrType = clrType.IsArray + ? clrType.GetElementType()! + : clrType.GetGenericArguments()[0]; + var elemBuilder = CreateColumnBuilder(elemClrType, listType.ValueDataType); + return new ListColumnBuilder(listType, elemClrType, elemBuilder); + } + + // Dictionary → MapArray + if (arrowType is MapType mapType) + { + var args = clrType.GetGenericArguments(); + var keyBuilder = CreateColumnBuilder(args[0], mapType.KeyField.DataType); + var valBuilder = CreateColumnBuilder(args[1], mapType.ValueField.DataType); + return new MapColumnBuilder(mapType, args[0], args[1], keyBuilder, valBuilder); + } + + // Nested object → StructArray + if (arrowType is StructType structType) + { + // If the type has source-generated IArrowSerializer, delegate to it + var toRecordBatchList = GetGeneratedToRecordBatchList(clrType); + if (toRecordBatchList is not null) + return new SourceGenStructColumnBuilder(clrType, structType, toRecordBatchList); + + // Otherwise, fall back to reflection-based struct builder + var nestedProps = clrType.GetProperties(BindingFlags.Public | BindingFlags.Instance) + .Where(p => p.CanRead) + .ToArray(); + var childBuilders = new List(); + for (int i = 0; i < nestedProps.Length; i++) + { + var childArrowType = structType.Fields[i].DataType; + childBuilders.Add(CreateColumnBuilder(nestedProps[i].PropertyType, childArrowType)); + } + return new StructColumnBuilder(structType, nestedProps, childBuilders); + } + + throw new NotSupportedException($"Column builder not available for {clrType.FullName}"); + } + + // --- Column builder interface and implementations --- + + private interface IColumnBuilder + { + void Append(object? value); + IArrowArray Build(); + } + + private sealed class StringColumnBuilder : IColumnBuilder + { + private readonly StringArray.Builder _b = new(); + public void Append(object? value) { if (value is null) _b.AppendNull(); else _b.Append((string)value); } + public IArrowArray Build() => _b.Build(); + } + + private sealed class BoolColumnBuilder : IColumnBuilder + { + private readonly BooleanArray.Builder _b = new(); + public void Append(object? value) { if (value is null) _b.AppendNull(); else _b.Append((bool)value); } + public IArrowArray Build() => _b.Build(); + } + + private sealed class TypedColumnBuilder : IColumnBuilder + where T : struct + where TBuilder : class + { + private readonly TBuilder _builder; + private readonly Action _append; + private readonly Action _appendNull; + private readonly Func _build; + + public TypedColumnBuilder(TBuilder builder, Action append, + Action appendNull, Func build) + { + _builder = builder; + _append = append; + _appendNull = appendNull; + _build = build; + } + + public void Append(object? value) + { + if (value is null) _appendNull(_builder); + else _append(_builder, (T)value); + } + + public IArrowArray Build() => _build(_builder); + } + + private sealed class DecimalColumnBuilder : IColumnBuilder + { + private readonly List<(decimal Value, bool IsNull)> _values = new(); + public void Append(object? value) + { + if (value is null) _values.Add((0, true)); + else _values.Add(((decimal)value, false)); + } + public IArrowArray Build() + { + var b = new Decimal128Array.Builder(new Decimal128Type(38, 18)); + foreach (var (v, isNull) in _values) + if (isNull) b.AppendNull(); else b.Append(v); + return b.Build(); + } + } + + private sealed class DateTimeColumnBuilder : IColumnBuilder + { + private readonly TimestampArray.Builder _b = new(new TimestampType(TimeUnit.Microsecond, "UTC")); + public void Append(object? value) + { + if (value is null) _b.AppendNull(); + else _b.Append(new DateTimeOffset((DateTime)value, TimeSpan.Zero)); + } + public IArrowArray Build() => _b.Build(); + } + + private sealed class DateTimeOffsetColumnBuilder : IColumnBuilder + { + private readonly TimestampArray.Builder _b = new(new TimestampType(TimeUnit.Microsecond, "UTC")); + public void Append(object? value) + { + if (value is null) _b.AppendNull(); + else _b.Append((DateTimeOffset)value); + } + public IArrowArray Build() => _b.Build(); + } + + private sealed class TimeOnlyColumnBuilder : IColumnBuilder + { + private readonly List<(TimeOnly Value, bool IsNull)> _values = new(); + public void Append(object? value) + { + if (value is null) _values.Add((default, true)); + else _values.Add(((TimeOnly)value, false)); + } + public IArrowArray Build() + { + var b = new Time64Array.Builder(new Time64Type(TimeUnit.Microsecond)); + foreach (var (v, isNull) in _values) + if (isNull) b.AppendNull(); else b.Append(v); + return b.Build(); + } + } + + private sealed class TimeSpanColumnBuilder : IColumnBuilder + { + private readonly List<(TimeSpan Value, bool IsNull)> _values = new(); + public void Append(object? value) + { + if (value is null) _values.Add((default, true)); + else _values.Add(((TimeSpan)value, false)); + } + public IArrowArray Build() + { + var b = new DurationArray.Builder(DurationType.Microsecond); + foreach (var (v, isNull) in _values) + if (isNull) b.AppendNull(); else b.Append(v); + return b.Build(); + } + } + + private sealed class GuidColumnBuilder : IColumnBuilder + { + private readonly GuidArray.Builder _b = new(); + public void Append(object? value) + { + if (value is null) _b.AppendNull(); + else _b.Append((Guid)value); + } + public IArrowArray Build() => _b.Build(); + } + + private sealed class BinaryColumnBuilder : IColumnBuilder + { + private readonly BinaryArray.Builder _b = new(); + public void Append(object? value) + { + if (value is null) _b.AppendNull(); + else _b.Append((ReadOnlySpan)(byte[])value); + } + public IArrowArray Build() => _b.Build(); + } + + private sealed class EnumColumnBuilder : IColumnBuilder + { + private readonly Dictionary _dict = new(); + private readonly List _values = new(); + + public void Append(object? value) + { + if (value is null) { _values.Add(null); return; } + var name = value.ToString()!; + if (!_dict.ContainsKey(name)) + _dict[name] = (short)_dict.Count; + _values.Add(name); + } + + public IArrowArray Build() + { + var dictNames = _dict.OrderBy(kv => kv.Value).Select(kv => kv.Key).ToArray(); + var dictBuilder = new StringArray.Builder(); + foreach (var n in dictNames) dictBuilder.Append(n); + var dictArray = dictBuilder.Build(); + + var idxBuilder = new Int16Array.Builder(); + foreach (var v in _values) + { + if (v is null) idxBuilder.AppendNull(); + else idxBuilder.Append(_dict[v]); + } + + return new DictionaryArray( + new DictionaryType(Int16Type.Default, StringType.Default, false), + idxBuilder.Build(), dictArray); + } + } + + private sealed class ReadOnlyMemoryByteColumnBuilder : IColumnBuilder + { + private readonly BinaryArray.Builder _b = new(); + public void Append(object? value) + { + if (value is null) _b.AppendNull(); + else _b.Append(((ReadOnlyMemory)value).Span); + } + public IArrowArray Build() => _b.Build(); + } + + private sealed class ListColumnBuilder : IColumnBuilder + { + private readonly ListType _listType; + private readonly IColumnBuilder _elemBuilder; + private readonly List _offsets = new() { 0 }; + private readonly List _validity = new(); + private int _totalElements; + + public ListColumnBuilder(ListType listType, Type elemClrType, IColumnBuilder elemBuilder) + { + _listType = listType; + _elemBuilder = elemBuilder; + } + + public void Append(object? value) + { + if (value is null) + { + _validity.Add(false); + _offsets.Add(_totalElements); + return; + } + + _validity.Add(true); + var enumerable = (System.Collections.IEnumerable)value; + foreach (var item in enumerable) + { + _elemBuilder.Append(item); + _totalElements++; + } + _offsets.Add(_totalElements); + } + + public IArrowArray Build() + { + var valueArray = _elemBuilder.Build(); + int length = _validity.Count; + int nullCount = _validity.Count(v => !v); + + var offsetBuffer = new ArrowBuffer( + _offsets.SelectMany(BitConverter.GetBytes).ToArray()); + + ArrowBuffer nullBitmap; + if (nullCount == 0) + { + nullBitmap = ArrowBuffer.Empty; + } + else + { + var bitmapBytes = new byte[(length + 7) / 8]; + for (int i = 0; i < length; i++) + if (_validity[i]) + bitmapBytes[i / 8] |= (byte)(1 << (i % 8)); + nullBitmap = new ArrowBuffer(bitmapBytes); + } + + var data = new ArrayData(_listType, length, nullCount, + 0, [nullBitmap, offsetBuffer], [valueArray.Data]); + return new ListArray(data); + } + } + + private sealed class MapColumnBuilder : IColumnBuilder + { + private readonly MapType _mapType; + private readonly IColumnBuilder _keyBuilder; + private readonly IColumnBuilder _valBuilder; + private readonly List _offsets = new() { 0 }; + private readonly List _validity = new(); + private int _totalEntries; + + public MapColumnBuilder(MapType mapType, Type keyClrType, Type valClrType, + IColumnBuilder keyBuilder, IColumnBuilder valBuilder) + { + _mapType = mapType; + _keyBuilder = keyBuilder; + _valBuilder = valBuilder; + } + + public void Append(object? value) + { + if (value is null) + { + _validity.Add(false); + _offsets.Add(_totalEntries); + return; + } + + _validity.Add(true); + var dict = (System.Collections.IDictionary)value; + foreach (System.Collections.DictionaryEntry entry in dict) + { + _keyBuilder.Append(entry.Key); + _valBuilder.Append(entry.Value); + _totalEntries++; + } + _offsets.Add(_totalEntries); + } + + public IArrowArray Build() + { + var keyArray = _keyBuilder.Build(); + var valArray = _valBuilder.Build(); + int length = _validity.Count; + int nullCount = _validity.Count(v => !v); + + var offsetBuffer = new ArrowBuffer( + _offsets.SelectMany(BitConverter.GetBytes).ToArray()); + + ArrowBuffer nullBitmap; + if (nullCount == 0) + { + nullBitmap = ArrowBuffer.Empty; + } + else + { + var bitmapBytes = new byte[(length + 7) / 8]; + for (int i = 0; i < length; i++) + if (_validity[i]) + bitmapBytes[i / 8] |= (byte)(1 << (i % 8)); + nullBitmap = new ArrowBuffer(bitmapBytes); + } + + // MapArray's child is a StructArray of (key, value) entries + var entryType = new StructType(new List { _mapType.KeyField, _mapType.ValueField }); + var entryArray = new StructArray(entryType, _totalEntries, + new IArrowArray[] { keyArray, valArray }, ArrowBuffer.Empty, 0); + + var data = new ArrayData(_mapType, length, nullCount, + 0, [nullBitmap, offsetBuffer], [entryArray.Data]); + return new MapArray(data); + } + } + + /// + /// Column builder that delegates to source-generated ToRecordBatch(IReadOnlyList<T>) + /// for [ArrowSerializable] types, then wraps the RecordBatch columns into a StructArray. + /// + private sealed class SourceGenStructColumnBuilder : IColumnBuilder + { + private readonly Type _clrType; + private readonly StructType _structType; + private readonly MethodInfo _toRecordBatchList; + private readonly List _items = new(); + + public SourceGenStructColumnBuilder(Type clrType, StructType structType, MethodInfo toRecordBatchList) + { + _clrType = clrType; + _structType = structType; + _toRecordBatchList = toRecordBatchList; + } + + public void Append(object? value) => _items.Add(value); + + public IArrowArray Build() + { + int length = _items.Count; + int nullCount = _items.Count(v => v is null); + + // Build a typed list for the source-generated method + var listType = typeof(List<>).MakeGenericType(_clrType); + var typedList = (System.Collections.IList)Activator.CreateInstance(listType, length)!; + + // For null slots, we need a stand-in value (first non-null item) + object? standIn = _items.FirstOrDefault(v => v is not null); + foreach (var item in _items) + typedList.Add(item ?? standIn!); + + // Call the generated ToRecordBatch(IReadOnlyList) + var batch = (RecordBatch)_toRecordBatchList.Invoke(null, [typedList])!; + + // Extract columns as child arrays for the StructArray + var childArrays = new IArrowArray[batch.ColumnCount]; + for (int i = 0; i < batch.ColumnCount; i++) + childArrays[i] = batch.Column(i); + + // Build null bitmap + ArrowBuffer nullBitmap; + if (nullCount == 0) + { + nullBitmap = ArrowBuffer.Empty; + } + else + { + var bitmapBytes = new byte[(length + 7) / 8]; + for (int i = 0; i < length; i++) + { + if (_items[i] is not null) + bitmapBytes[i / 8] |= (byte)(1 << (i % 8)); + } + nullBitmap = new ArrowBuffer(bitmapBytes); + } + + return new StructArray(_structType, length, childArrays, nullBitmap, nullCount); + } + } + + private sealed class StructColumnBuilder : IColumnBuilder + { + private readonly StructType _structType; + private readonly PropertyInfo[] _properties; + private readonly List _childBuilders; + private readonly List _validity = new(); + + public StructColumnBuilder(StructType structType, PropertyInfo[] properties, List childBuilders) + { + _structType = structType; + _properties = properties; + _childBuilders = childBuilders; + } + + public void Append(object? value) + { + if (value is null) + { + _validity.Add(false); + // Append nulls/defaults to all children to keep lengths aligned + for (int i = 0; i < _childBuilders.Count; i++) + _childBuilders[i].Append(null); + } + else + { + _validity.Add(true); + for (int i = 0; i < _properties.Length; i++) + _childBuilders[i].Append(_properties[i].GetValue(value)); + } + } + + public IArrowArray Build() + { + var childArrays = _childBuilders.Select(b => b.Build()).ToArray(); + int length = _validity.Count; + int nullCount = _validity.Count(v => !v); + + // Build null bitmap + ArrowBuffer nullBitmap; + if (nullCount == 0) + { + nullBitmap = ArrowBuffer.Empty; + } + else + { + var bitmapBytes = new byte[(length + 7) / 8]; + for (int i = 0; i < length; i++) + { + if (_validity[i]) + bitmapBytes[i / 8] |= (byte)(1 << (i % 8)); + } + nullBitmap = new ArrowBuffer(bitmapBytes); + } + + return new StructArray(_structType, length, childArrays, nullBitmap, nullCount); + } + } +} diff --git a/test/Apache.Arrow.Serialization.Tests/Apache.Arrow.Serialization.Tests.csproj b/test/Apache.Arrow.Serialization.Tests/Apache.Arrow.Serialization.Tests.csproj new file mode 100644 index 00000000..16166422 --- /dev/null +++ b/test/Apache.Arrow.Serialization.Tests/Apache.Arrow.Serialization.Tests.csproj @@ -0,0 +1,26 @@ + + + + net10.0 + enable + enable + false + true + true + true + + + + + + + + + + + + + + + diff --git a/test/Apache.Arrow.Serialization.Tests/DiagnosticTests.cs b/test/Apache.Arrow.Serialization.Tests/DiagnosticTests.cs new file mode 100644 index 00000000..1b414b05 --- /dev/null +++ b/test/Apache.Arrow.Serialization.Tests/DiagnosticTests.cs @@ -0,0 +1,255 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.Collections.Immutable; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Apache.Arrow.Serialization.Generator; +using Xunit; + +namespace Apache.Arrow.Serialization.Tests; + +public class DiagnosticTests +{ + private static ImmutableArray GetGeneratorDiagnostics(string source) + { + var syntaxTree = CSharpSyntaxTree.ParseText(source); + + // Reference the runtime assembly for attributes + var runtimeAssembly = typeof(ArrowSerializableAttribute).Assembly.Location; + var references = new List + { + MetadataReference.CreateFromFile(typeof(object).Assembly.Location), + MetadataReference.CreateFromFile(runtimeAssembly), + }; + + // Add System.Runtime for netcoreapp + var runtimeDir = Path.GetDirectoryName(typeof(object).Assembly.Location)!; + references.Add(MetadataReference.CreateFromFile(Path.Combine(runtimeDir, "System.Runtime.dll"))); + + var compilation = CSharpCompilation.Create("TestAssembly", + [syntaxTree], + references, + new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary)); + + var generator = new ArrowSerializerGenerator(); + GeneratorDriver driver = CSharpGeneratorDriver.Create(generator); + driver = driver.RunGeneratorsAndUpdateCompilation(compilation, out _, out var diagnostics); + + return diagnostics; + } + + private static bool HasDiagnostic(ImmutableArray diagnostics, string id) + { + foreach (var d in diagnostics) + if (d.Id == id) return true; + return false; + } + + [Fact] + public void ARROW001_NonPartialType() + { + var source = """ + using Apache.Arrow.Serialization; + + [ArrowSerializable] + public struct NotPartial + { + public int X { get; set; } + } + """; + + var diagnostics = GetGeneratorDiagnostics(source); + Assert.True(HasDiagnostic(diagnostics, "ARROW001"), + "Expected ARROW001 for non-partial type"); + } + + [Fact] + public void ARROW002_ReadonlyFieldNoConstructor() + { + var source = """ + using Apache.Arrow.Serialization; + + [ArrowSerializable] + public partial struct BadStruct + { + public readonly int X; + public readonly int Y; + } + """; + + var diagnostics = GetGeneratorDiagnostics(source); + Assert.True(HasDiagnostic(diagnostics, "ARROW002"), + "Expected ARROW002 for readonly fields without matching constructor"); + } + + [Fact] + public void ARROW002_NotFired_WhenConstructorExists() + { + var source = """ + using Apache.Arrow.Serialization; + + [ArrowSerializable] + public partial struct GoodStruct + { + public readonly int X; + public readonly int Y; + + public GoodStruct(int x, int y) { X = x; Y = y; } + } + """; + + var diagnostics = GetGeneratorDiagnostics(source); + Assert.False(HasDiagnostic(diagnostics, "ARROW002"), + "ARROW002 should not fire when a matching constructor exists"); + } + + [Fact] + public void ARROW003_UnsupportedType() + { + var source = """ + using System.Collections.Generic; + using Apache.Arrow.Serialization; + + [ArrowSerializable] + public partial record WithQueue + { + public Queue Items { get; init; } = new(); + } + """; + + var diagnostics = GetGeneratorDiagnostics(source); + Assert.True(HasDiagnostic(diagnostics, "ARROW003"), + "Expected ARROW003 for unsupported Queue type"); + } + + [Fact] + public void ARROW004_DuplicateFieldNames() + { + var source = """ + using Apache.Arrow.Serialization; + + [ArrowSerializable] + public partial record DuplicateFields + { + [ArrowField("same")] + public int A { get; init; } + [ArrowField("same")] + public int B { get; init; } + } + """; + + var diagnostics = GetGeneratorDiagnostics(source); + Assert.True(HasDiagnostic(diagnostics, "ARROW004"), + "Expected ARROW004 for duplicate Arrow field names"); + } + + [Fact] + public void ARROW005_GetOnlyProperty() + { + var source = """ + using Apache.Arrow.Serialization; + + [ArrowSerializable] + public partial class GetOnlyProp + { + public int X { get; } + public string Name { get; init; } = ""; + } + """; + + var diagnostics = GetGeneratorDiagnostics(source); + Assert.True(HasDiagnostic(diagnostics, "ARROW005"), + "Expected ARROW005 for get-only property without init"); + } + + [Fact] + public void NoDiagnostics_ForValidType() + { + var source = """ + using Apache.Arrow.Serialization; + + [ArrowSerializable] + public partial record ValidType + { + public int X { get; init; } + public string Name { get; init; } = ""; + } + """; + + var diagnostics = GetGeneratorDiagnostics(source); + var arrowDiags = diagnostics.Where(d => d.Id.StartsWith("ARROW")).ToList(); + Assert.Empty(arrowDiags); + } + + [Fact] + public void ARROW006_ArrowAttributeOnPrivateProperty() + { + var source = """ + using Apache.Arrow.Serialization; + + [ArrowSerializable] + public partial record WithPrivate + { + public int X { get; init; } + [ArrowField("secret")] + private int Y { get; init; } + } + """; + + var diagnostics = GetGeneratorDiagnostics(source); + Assert.True(HasDiagnostic(diagnostics, "ARROW006"), + "Expected ARROW006 for Arrow attribute on private property"); + } + + [Fact] + public void ARROW006_ArrowAttributeOnStaticProperty() + { + var source = """ + using Apache.Arrow.Serialization; + + [ArrowSerializable] + public partial record WithStatic + { + public int X { get; init; } + [ArrowField("shared")] + public static int Y { get; set; } + } + """; + + var diagnostics = GetGeneratorDiagnostics(source); + Assert.True(HasDiagnostic(diagnostics, "ARROW006"), + "Expected ARROW006 for Arrow attribute on static property"); + } + + [Fact] + public void ARROW006_NotFired_ForPrivateWithoutAttributes() + { + var source = """ + using Apache.Arrow.Serialization; + + [ArrowSerializable] + public partial record WithPrivateNoAttr + { + public int X { get; init; } + private int Y { get; init; } + } + """; + + var diagnostics = GetGeneratorDiagnostics(source); + Assert.False(HasDiagnostic(diagnostics, "ARROW006"), + "ARROW006 should not fire for private member without Arrow attributes"); + } +} diff --git a/test/Apache.Arrow.Serialization.Tests/RecordBatchBuilderTests.cs b/test/Apache.Arrow.Serialization.Tests/RecordBatchBuilderTests.cs new file mode 100644 index 00000000..5ef14003 --- /dev/null +++ b/test/Apache.Arrow.Serialization.Tests/RecordBatchBuilderTests.cs @@ -0,0 +1,400 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow; +using Apache.Arrow.Types; +using Apache.Arrow.Serialization; +using Xunit; + +namespace Apache.Arrow.Serialization.Tests; + +public class RecordBatchBuilderTests +{ + [Fact] + public void AnonymousType_BasicColumns() + { + var tabelle = new[] + { + new { Id = 1, Name = "Alice", Alter = 30, Stadt = "Berlin" }, + new { Id = 2, Name = "Bob", Alter = 25, Stadt = "Hamburg" }, + new { Id = 3, Name = "Charlie", Alter = 35, Stadt = "München" }, + }; + + var batch = RecordBatchBuilder.FromObjects(tabelle); + + Assert.Equal(3, batch.Length); + Assert.Equal(4, batch.Schema.FieldsList.Count); + + // Check schema + Assert.Equal("Id", batch.Schema.FieldsList[0].Name); + Assert.IsType(batch.Schema.FieldsList[0].DataType); + Assert.Equal("Name", batch.Schema.FieldsList[1].Name); + Assert.IsType(batch.Schema.FieldsList[1].DataType); + + // Check values + var idCol = (Int32Array)batch.Column(0); + Assert.Equal(1, idCol.GetValue(0)); + Assert.Equal(2, idCol.GetValue(1)); + Assert.Equal(3, idCol.GetValue(2)); + + var nameCol = (StringArray)batch.Column(1); + Assert.Equal("Alice", nameCol.GetString(0)); + Assert.Equal("Bob", nameCol.GetString(1)); + Assert.Equal("Charlie", nameCol.GetString(2)); + + var stadtCol = (StringArray)batch.Column(3); + Assert.Equal("Berlin", stadtCol.GetString(0)); + Assert.Equal("München", stadtCol.GetString(2)); + } + + [Fact] + public void AnonymousType_NumericTypes() + { + var data = new[] + { + new { I = 1, L = 100L, F = 1.5f, D = 2.5, B = true, By = (byte)7 }, + }; + + var batch = RecordBatchBuilder.FromObjects(data); + Assert.Equal(1, batch.Length); + Assert.Equal(6, batch.Schema.FieldsList.Count); + + Assert.Equal(1, ((Int32Array)batch.Column(0)).GetValue(0)); + Assert.Equal(100L, ((Int64Array)batch.Column(1)).GetValue(0)); + Assert.Equal(1.5f, ((FloatArray)batch.Column(2)).GetValue(0)); + Assert.Equal(2.5, ((DoubleArray)batch.Column(3)).GetValue(0)); + Assert.True(((BooleanArray)batch.Column(4)).GetValue(0)); + Assert.Equal((byte)7, ((UInt8Array)batch.Column(5)).GetValue(0)); + } + + [Fact] + public void NullableProperties() + { + var data = new[] + { + new { Value = 1, Extra = (int?)42 }, + new { Value = 2, Extra = (int?)null }, + }; + + var batch = RecordBatchBuilder.FromObjects(data); + Assert.Equal(2, batch.Length); + + var extraCol = (Int32Array)batch.Column(1); + Assert.Equal(42, extraCol.GetValue(0)); + Assert.True(extraCol.IsNull(1)); + } + + [Fact] + public void NullableStrings() + { + var data = new[] + { + new { Name = "Alice", Tag = (string?)"admin" }, + new { Name = "Bob", Tag = (string?)null }, + }; + + var batch = RecordBatchBuilder.FromObjects(data); + var tagCol = (StringArray)batch.Column(1); + Assert.Equal("admin", tagCol.GetString(0)); + Assert.True(tagCol.IsNull(1)); + } + + [Fact] + public void SingleObject() + { + var obj = new { X = 10, Y = 20.5 }; + var batch = RecordBatchBuilder.FromObject(obj); + + Assert.Equal(1, batch.Length); + Assert.Equal(10, ((Int32Array)batch.Column(0)).GetValue(0)); + Assert.Equal(20.5, ((DoubleArray)batch.Column(1)).GetValue(0)); + } + + [Fact] + public void NamedRecord() + { + var data = new List + { + new(1.0, 2.0, "A"), + new(3.0, 4.0, "B"), + }; + + var batch = RecordBatchBuilder.FromObjects(data); + Assert.Equal(2, batch.Length); + Assert.Equal(1.0, ((DoubleArray)batch.Column(0)).GetValue(0)); + Assert.Equal("B", ((StringArray)batch.Column(2)).GetString(1)); + } + + [Fact] + public void EnumProperty() + { + var data = new[] + { + new { Color = Color.Red }, + new { Color = Color.Blue }, + new { Color = Color.Red }, + }; + + var batch = RecordBatchBuilder.FromObjects(data); + Assert.Equal(3, batch.Length); + var col = (DictionaryArray)batch.Column(0); + var dict = (StringArray)col.Dictionary; + var indices = (Int16Array)col.Indices; + // Red=0, Blue=1 + Assert.Equal((short)0, indices.GetValue(0)); + Assert.Equal((short)1, indices.GetValue(1)); + Assert.Equal((short)0, indices.GetValue(2)); + Assert.Equal("Red", dict.GetString(0)); + Assert.Equal("Blue", dict.GetString(1)); + } + + [Fact] + public void DateTimeAndGuid() + { + var dt = new DateTime(2024, 1, 15, 12, 0, 0, DateTimeKind.Utc); + var guid = Guid.NewGuid(); + var data = new[] { new { When = dt, Id = guid } }; + + var batch = RecordBatchBuilder.FromObjects(data); + Assert.Equal(1, batch.Length); + Assert.IsType(batch.Schema.FieldsList[0].DataType); + Assert.IsType(batch.Schema.FieldsList[1].DataType); + } + + [Fact] + public void EmptyCollection_Throws() + { + Assert.Throws(() => + RecordBatchBuilder.FromObjects(System.Array.Empty())); + } + + [Fact] + public void NestedAnonymousType() + { + var data = new[] + { + new { Name = "Alice", Address = new { City = "Berlin", Zip = 10115 } }, + new { Name = "Bob", Address = new { City = "Hamburg", Zip = 20095 } }, + }; + + var batch = RecordBatchBuilder.FromObjects(data); + Assert.Equal(2, batch.Length); + Assert.Equal(2, batch.Schema.FieldsList.Count); + Assert.Equal("Name", batch.Schema.FieldsList[0].Name); + Assert.Equal("Address", batch.Schema.FieldsList[1].Name); + Assert.IsType(batch.Schema.FieldsList[1].DataType); + + var structCol = (StructArray)batch.Column(1); + var cityCol = (StringArray)structCol.Fields[0]; + var zipCol = (Int32Array)structCol.Fields[1]; + + Assert.Equal("Berlin", cityCol.GetString(0)); + Assert.Equal("Hamburg", cityCol.GetString(1)); + Assert.Equal(10115, zipCol.GetValue(0)); + Assert.Equal(20095, zipCol.GetValue(1)); + } + + [Fact] + public void DeeplyNestedAnonymousType() + { + var data = new[] + { + new { A = new { B = new { C = 42 } } }, + }; + + var batch = RecordBatchBuilder.FromObjects(data); + Assert.Equal(1, batch.Length); + + var aCol = (StructArray)batch.Column(0); + var bCol = (StructArray)aCol.Fields[0]; + var cCol = (Int32Array)bCol.Fields[0]; + Assert.Equal(42, cCol.GetValue(0)); + } + + [Fact] + public void NestedNamedRecord() + { + var data = new[] + { + new { Point = new PointRecord(1.0, 2.0, "A"), Tag = "first" }, + new { Point = new PointRecord(3.0, 4.0, "B"), Tag = "second" }, + }; + + var batch = RecordBatchBuilder.FromObjects(data); + Assert.Equal(2, batch.Length); + + var pointCol = (StructArray)batch.Column(0); + var xCol = (DoubleArray)pointCol.Fields[0]; + var yCol = (DoubleArray)pointCol.Fields[1]; + Assert.Equal(1.0, xCol.GetValue(0)); + Assert.Equal(4.0, yCol.GetValue(1)); + } + + [Fact] + public void NestedArrowSerializable_UsesSourceGeneratedSchema() + { + // Inner is [ArrowSerializable] and implements IArrowSerializer + var data = new[] + { + new { Tag = "first", Inner = new Inner { X = 10, Label = "A" } }, + new { Tag = "second", Inner = new Inner { X = 20, Label = "B" } }, + }; + + var batch = RecordBatchBuilder.FromObjects(data); + Assert.Equal(2, batch.Length); + Assert.Equal(2, batch.Schema.FieldsList.Count); + Assert.Equal("Tag", batch.Schema.FieldsList[0].Name); + Assert.Equal("Inner", batch.Schema.FieldsList[1].Name); + Assert.IsType(batch.Schema.FieldsList[1].DataType); + + // Verify the struct schema matches the source-generated one + var structType = (StructType)batch.Schema.FieldsList[1].DataType; + Assert.Equal(2, structType.Fields.Count); + Assert.Equal("X", structType.Fields[0].Name); + Assert.Equal("Label", structType.Fields[1].Name); + + // Verify values + var tagCol = (StringArray)batch.Column(0); + Assert.Equal("first", tagCol.GetString(0)); + + var innerCol = (StructArray)batch.Column(1); + var xCol = (Int32Array)innerCol.Fields[0]; + var labelCol = (StringArray)innerCol.Fields[1]; + Assert.Equal(10, xCol.GetValue(0)); + Assert.Equal(20, xCol.GetValue(1)); + Assert.Equal("A", labelCol.GetString(0)); + Assert.Equal("B", labelCol.GetString(1)); + } + + [Fact] + public void ListOfInts() + { + var data = new[] + { + new { Name = "Alice", Scores = new List { 90, 85, 92 } }, + new { Name = "Bob", Scores = new List { 78, 88 } }, + }; + + var batch = RecordBatchBuilder.FromObjects(data); + Assert.Equal(2, batch.Length); + Assert.IsType(batch.Schema.FieldsList[1].DataType); + + var listCol = (ListArray)batch.Column(1); + Assert.Equal(3, listCol.GetValueLength(0)); + Assert.Equal(2, listCol.GetValueLength(1)); + + var values = (Int32Array)listCol.Values; + Assert.Equal(90, values.GetValue(0)); + Assert.Equal(85, values.GetValue(1)); + Assert.Equal(92, values.GetValue(2)); + Assert.Equal(78, values.GetValue(3)); + Assert.Equal(88, values.GetValue(4)); + } + + [Fact] + public void ListOfStrings() + { + var data = new[] + { + new { Tags = new List { "admin", "user" } }, + new { Tags = new List { "guest" } }, + }; + + var batch = RecordBatchBuilder.FromObjects(data); + var listCol = (ListArray)batch.Column(0); + var values = (StringArray)listCol.Values; + Assert.Equal("admin", values.GetString(0)); + Assert.Equal("user", values.GetString(1)); + Assert.Equal("guest", values.GetString(2)); + } + + [Fact] + public void NullableList() + { + var data = new[] + { + new { Items = (List?)new List { 1, 2 } }, + new { Items = (List?)null }, + }; + + var batch = RecordBatchBuilder.FromObjects(data); + var listCol = (ListArray)batch.Column(0); + Assert.False(listCol.IsNull(0)); + Assert.True(listCol.IsNull(1)); + } + + [Fact] + public void ArrayProperty() + { + var data = new[] + { + new { Values = new[] { 1.0, 2.0, 3.0 } }, + new { Values = new[] { 4.0 } }, + }; + + var batch = RecordBatchBuilder.FromObjects(data); + Assert.IsType(batch.Schema.FieldsList[0].DataType); + + var listCol = (ListArray)batch.Column(0); + var values = (DoubleArray)listCol.Values; + Assert.Equal(1.0, values.GetValue(0)); + Assert.Equal(4.0, values.GetValue(3)); + } + + [Fact] + public void DictionaryProperty() + { + var data = new[] + { + new { Props = new Dictionary { ["a"] = 1, ["b"] = 2 } }, + new { Props = new Dictionary { ["c"] = 3 } }, + }; + + var batch = RecordBatchBuilder.FromObjects(data); + Assert.IsType(batch.Schema.FieldsList[0].DataType); + + var mapCol = (MapArray)batch.Column(0); + Assert.Equal(2, mapCol.GetValueLength(0)); + Assert.Equal(1, mapCol.GetValueLength(1)); + + var keys = (StringArray)mapCol.Keys; + var vals = (Int32Array)mapCol.Values; + Assert.Equal("a", keys.GetString(0)); + Assert.Equal(1, vals.GetValue(0)); + Assert.Equal("c", keys.GetString(2)); + Assert.Equal(3, vals.GetValue(2)); + } + + [Fact] + public void ReadOnlyMemoryByte() + { + var bytes1 = new ReadOnlyMemory(new byte[] { 1, 2, 3 }); + var bytes2 = new ReadOnlyMemory(new byte[] { 4, 5 }); + var data = new[] + { + new { Data = bytes1 }, + new { Data = bytes2 }, + }; + + var batch = RecordBatchBuilder.FromObjects(data); + Assert.IsType(batch.Schema.FieldsList[0].DataType); + + var binCol = (BinaryArray)batch.Column(0); + Assert.Equal(new byte[] { 1, 2, 3 }, binCol.GetBytes(0).ToArray()); + Assert.Equal(new byte[] { 4, 5 }, binCol.GetBytes(1).ToArray()); + } + + public record PointRecord(double X, double Y, string Label); +} diff --git a/test/Apache.Arrow.Serialization.Tests/SerializationTests.cs b/test/Apache.Arrow.Serialization.Tests/SerializationTests.cs new file mode 100644 index 00000000..c36c7184 --- /dev/null +++ b/test/Apache.Arrow.Serialization.Tests/SerializationTests.cs @@ -0,0 +1,2277 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow; +using Apache.Arrow.Types; +using Apache.Arrow.Serialization; +using Xunit; + +namespace Apache.Arrow.Serialization.Tests; + +public class SerializationTests +{ + [Fact] + public void Primitives_RoundTrip() + { + var obj = new WithPrimitives + { + Name = "hello", + Count = 42, + BigCount = 123456789L, + Score = 3.14f, + Precise = 2.718281828, + Flag = true, + Small = 7, + }; + + var batch = WithPrimitives.ToRecordBatch(obj); + Assert.Equal(1, batch.Length); + + var restored = WithPrimitives.FromRecordBatch(batch); + Assert.Equal(obj.Name, restored.Name); + Assert.Equal(obj.Count, restored.Count); + Assert.Equal(obj.BigCount, restored.BigCount); + Assert.Equal(obj.Score, restored.Score); + Assert.Equal(obj.Precise, restored.Precise); + Assert.Equal(obj.Flag, restored.Flag); + Assert.Equal(obj.Small, restored.Small); + } + + [Fact] + public void String_And_Int_RoundTrip() + { + var obj = new Inner { X = 42, Label = "hello" }; + var batch = Inner.ToRecordBatch(obj); + var restored = Inner.FromRecordBatch(batch); + Assert.Equal(obj, restored); + } + + [Fact] + public void Bool_RoundTrip() + { + foreach (var val in new[] { true, false }) + { + var obj = new WithBool { Flag = val }; + var restored = WithBool.FromRecordBatch(WithBool.ToRecordBatch(obj)); + Assert.Equal(val, restored.Flag); + } + } + + [Fact] + public void Bytes_RoundTrip() + { + var obj = new WithBytes { Data = new byte[] { 0x00, 0x01, 0xFF } }; + var restored = WithBytes.FromRecordBatch(WithBytes.ToRecordBatch(obj)); + Assert.Equal(obj.Data, restored.Data); + } + + [Fact] + public void ReadOnlyMemoryBytes_RoundTrip() + { + var obj = new WithReadOnlyMemoryBytes { Data = new byte[] { 0x00, 0x01, 0xFF } }; + var batch = WithReadOnlyMemoryBytes.ToRecordBatch(obj); + var restored = WithReadOnlyMemoryBytes.FromRecordBatch(batch); + Assert.Equal(obj.Data.ToArray(), restored.Data.ToArray()); + } + + [Fact] + public void ReadOnlyMemoryBytes_MultiRow_RoundTrip() + { + var items = new List + { + new() { Data = new byte[] { 1, 2, 3 } }, + new() { Data = new byte[] { 4, 5 } }, + }; + var batch = WithReadOnlyMemoryBytes.ToRecordBatch(items); + var restored = WithReadOnlyMemoryBytes.ListFromRecordBatch(batch); + Assert.Equal(items[0].Data.ToArray(), restored[0].Data.ToArray()); + Assert.Equal(items[1].Data.ToArray(), restored[1].Data.ToArray()); + } + + [Fact] + public void Enum_RoundTrip() + { + var obj = new WithEnum { Color = Color.Red }; + var restored = WithEnum.FromRecordBatch(WithEnum.ToRecordBatch(obj)); + Assert.Equal(Color.Red, restored.Color); + } + + [Fact] + public void NullableEnum_None_RoundTrip() + { + var obj = new WithNullableEnum { Color = null }; + var restored = WithNullableEnum.FromRecordBatch(WithNullableEnum.ToRecordBatch(obj)); + Assert.Null(restored.Color); + } + + [Fact] + public void NullableEnum_Present_RoundTrip() + { + var obj = new WithNullableEnum { Color = Color.Green }; + var restored = WithNullableEnum.FromRecordBatch(WithNullableEnum.ToRecordBatch(obj)); + Assert.Equal(Color.Green, restored.Color); + } + + [Fact] + public void NestedRecord_RoundTrip() + { + var obj = new Outer { Inner = new Inner { X = 42, Label = "hi" }, Name = "test" }; + var restored = Outer.FromRecordBatch(Outer.ToRecordBatch(obj)); + Assert.Equal(obj, restored); + } + + [Fact] + public void DeeplyNested_RoundTrip() + { + var obj = new Level3 + { + Level2 = new Level2 + { + Inner = new Inner { X = 1, Label = "deep" }, + Tag = "t", + }, + Score = 3.14, + }; + var restored = Level3.FromRecordBatch(Level3.ToRecordBatch(obj)); + Assert.Equal(obj, restored); + } + + [Fact] + public void OptionalString_None_RoundTrip() + { + var obj = new WithOptional { Value = "hello", Extra = null }; + var restored = WithOptional.FromRecordBatch(WithOptional.ToRecordBatch(obj)); + Assert.Null(restored.Extra); + } + + [Fact] + public void OptionalString_Present_RoundTrip() + { + var obj = new WithOptional { Value = "hello", Extra = "world" }; + var restored = WithOptional.FromRecordBatch(WithOptional.ToRecordBatch(obj)); + Assert.Equal("world", restored.Extra); + } + + [Fact] + public void OptionalInt_None_RoundTrip() + { + var obj = new WithOptionalInt { Value = 5, Extra = null }; + var restored = WithOptionalInt.FromRecordBatch(WithOptionalInt.ToRecordBatch(obj)); + Assert.Null(restored.Extra); + } + + [Fact] + public void OptionalInt_Present_RoundTrip() + { + var obj = new WithOptionalInt { Value = 5, Extra = 42 }; + var restored = WithOptionalInt.FromRecordBatch(WithOptionalInt.ToRecordBatch(obj)); + Assert.Equal(42, restored.Extra); + } + + [Fact] + public void List_RoundTrip() + { + var obj = new WithList { Items = new List { 1, 2, 3 } }; + var restored = WithList.FromRecordBatch(WithList.ToRecordBatch(obj)); + Assert.Equal(obj.Items, restored.Items); + } + + [Fact] + public void List_Empty_RoundTrip() + { + var obj = new WithList { Items = new List() }; + var restored = WithList.FromRecordBatch(WithList.ToRecordBatch(obj)); + Assert.Empty(restored.Items); + } + + [Fact] + public void StringList_RoundTrip() + { + var obj = new WithStringList { Tags = new List { "a", "b", "c" } }; + var restored = WithStringList.FromRecordBatch(WithStringList.ToRecordBatch(obj)); + Assert.Equal(obj.Tags, restored.Tags); + } + + [Fact] + public void Dict_RoundTrip() + { + var obj = new WithDict { Mapping = new Dictionary { ["a"] = 1, ["b"] = 2 } }; + var restored = WithDict.FromRecordBatch(WithDict.ToRecordBatch(obj)); + Assert.Equal(obj.Mapping, restored.Mapping); + } + + [Fact] + public void Dict_Empty_RoundTrip() + { + var obj = new WithDict { Mapping = new Dictionary() }; + var restored = WithDict.FromRecordBatch(WithDict.ToRecordBatch(obj)); + Assert.Empty(restored.Mapping); + } + + [Fact] + public void Set_RoundTrip() + { + var obj = new WithSet { Tags = new HashSet { 10, 20 } }; + var restored = WithSet.FromRecordBatch(WithSet.ToRecordBatch(obj)); + Assert.Equal(obj.Tags, restored.Tags); + } + + [Fact] + public void OptionalNested_None_RoundTrip() + { + var obj = new WithOptionalNested { Inner = null }; + var restored = WithOptionalNested.FromRecordBatch(WithOptionalNested.ToRecordBatch(obj)); + Assert.Null(restored.Inner); + } + + [Fact] + public void OptionalNested_Present_RoundTrip() + { + var obj = new WithOptionalNested { Inner = new Inner { X = 7, Label = "nested" } }; + var restored = WithOptionalNested.FromRecordBatch(WithOptionalNested.ToRecordBatch(obj)); + Assert.Equal(obj, restored); + } +} + +public class MultiRowTests +{ + [Fact] + public void Primitives_MultiRow_RoundTrip() + { + var items = new List + { + new() { Name = "a", Count = 1, BigCount = 10L, Score = 1.1f, Precise = 1.11, Flag = true, Small = 1 }, + new() { Name = "b", Count = 2, BigCount = 20L, Score = 2.2f, Precise = 2.22, Flag = false, Small = 2 }, + new() { Name = "c", Count = 3, BigCount = 30L, Score = 3.3f, Precise = 3.33, Flag = true, Small = 3 }, + }; + + var batch = WithPrimitives.ToRecordBatch(items); + Assert.Equal(3, batch.Length); + + var restored = WithPrimitives.ListFromRecordBatch(batch); + Assert.Equal(3, restored.Count); + for (int i = 0; i < 3; i++) + { + Assert.Equal(items[i].Name, restored[i].Name); + Assert.Equal(items[i].Count, restored[i].Count); + Assert.Equal(items[i].BigCount, restored[i].BigCount); + Assert.Equal(items[i].Score, restored[i].Score); + Assert.Equal(items[i].Precise, restored[i].Precise); + Assert.Equal(items[i].Flag, restored[i].Flag); + Assert.Equal(items[i].Small, restored[i].Small); + } + } + + [Fact] + public void Inner_MultiRow_RoundTrip() + { + var items = new List + { + new() { X = 1, Label = "one" }, + new() { X = 2, Label = "two" }, + new() { X = 3, Label = "three" }, + }; + + var batch = Inner.ToRecordBatch(items); + Assert.Equal(3, batch.Length); + + var restored = Inner.ListFromRecordBatch(batch); + Assert.Equal(items, restored); + } + + [Fact] + public void Enum_MultiRow_RoundTrip() + { + var items = new List + { + new() { Color = Color.Red }, + new() { Color = Color.Green }, + new() { Color = Color.Blue }, + new() { Color = Color.Red }, + }; + + var batch = WithEnum.ToRecordBatch(items); + Assert.Equal(4, batch.Length); + + var restored = WithEnum.ListFromRecordBatch(batch); + for (int i = 0; i < items.Count; i++) + Assert.Equal(items[i].Color, restored[i].Color); + } + + [Fact] + public void NullableEnum_MultiRow_RoundTrip() + { + var items = new List + { + new() { Color = Color.Red }, + new() { Color = null }, + new() { Color = Color.Blue }, + }; + + var batch = WithNullableEnum.ToRecordBatch(items); + var restored = WithNullableEnum.ListFromRecordBatch(batch); + Assert.Equal(Color.Red, restored[0].Color); + Assert.Null(restored[1].Color); + Assert.Equal(Color.Blue, restored[2].Color); + } + + [Fact] + public void NestedRecord_MultiRow_RoundTrip() + { + var items = new List + { + new() { Inner = new Inner { X = 1, Label = "a" }, Name = "first" }, + new() { Inner = new Inner { X = 2, Label = "b" }, Name = "second" }, + }; + + var batch = Outer.ToRecordBatch(items); + Assert.Equal(2, batch.Length); + + var restored = Outer.ListFromRecordBatch(batch); + Assert.Equal(items, restored); + } + + [Fact] + public void OptionalNested_MultiRow_RoundTrip() + { + var items = new List + { + new() { Inner = new Inner { X = 1, Label = "a" } }, + new() { Inner = null }, + new() { Inner = new Inner { X = 3, Label = "c" } }, + }; + + var batch = WithOptionalNested.ToRecordBatch(items); + var restored = WithOptionalNested.ListFromRecordBatch(batch); + Assert.Equal(items[0], restored[0]); + Assert.Null(restored[1].Inner); + Assert.Equal(items[2], restored[2]); + } + + [Fact] + public void List_MultiRow_RoundTrip() + { + var items = new List + { + new() { Items = new List { 1, 2, 3 } }, + new() { Items = new List { 4, 5 } }, + new() { Items = new List() }, + }; + + var batch = WithList.ToRecordBatch(items); + var restored = WithList.ListFromRecordBatch(batch); + Assert.Equal(items[0].Items, restored[0].Items); + Assert.Equal(items[1].Items, restored[1].Items); + Assert.Empty(restored[2].Items); + } + + [Fact] + public void Dict_MultiRow_RoundTrip() + { + var items = new List + { + new() { Mapping = new Dictionary { ["a"] = 1, ["b"] = 2 } }, + new() { Mapping = new Dictionary { ["c"] = 3 } }, + }; + + var batch = WithDict.ToRecordBatch(items); + var restored = WithDict.ListFromRecordBatch(batch); + Assert.Equal(items[0].Mapping, restored[0].Mapping); + Assert.Equal(items[1].Mapping, restored[1].Mapping); + } + + [Fact] + public void EnumList_SingleRow_RoundTrip() + { + var original = new WithColorEnumList { Colors = [Color.Red, Color.Green, Color.Blue] }; + var batch = WithColorEnumList.ToRecordBatch(original); + var restored = WithColorEnumList.FromRecordBatch(batch); + Assert.Equal(original.Colors, restored.Colors); + } + + [Fact] + public void EnumList_MultiRow_RoundTrip() + { + var items = new List + { + new() { Colors = [Color.Red, Color.Green] }, + new() { Colors = [Color.Blue] }, + new() { Colors = [] }, + }; + var batch = WithColorEnumList.ToRecordBatch(items); + var restored = WithColorEnumList.ListFromRecordBatch(batch); + Assert.Equal(items[0].Colors, restored[0].Colors); + Assert.Equal(items[1].Colors, restored[1].Colors); + Assert.Empty(restored[2].Colors); + } + + [Fact] + public void NestedList_SingleRow_RoundTrip() + { + var original = new WithNestedList + { + Entries = [new Inner { X = 1, Label = "a" }, new Inner { X = 2, Label = "b" }], + }; + var batch = WithNestedList.ToRecordBatch(original); + var restored = WithNestedList.FromRecordBatch(batch); + Assert.Equal(2, restored.Entries.Count); + Assert.Equal(1, restored.Entries[0].X); + Assert.Equal("a", restored.Entries[0].Label); + Assert.Equal(2, restored.Entries[1].X); + Assert.Equal("b", restored.Entries[1].Label); + } + + [Fact] + public void NestedList_MultiRow_RoundTrip() + { + var items = new List + { + new() { Entries = [new Inner { X = 1, Label = "a" }, new Inner { X = 2, Label = "b" }] }, + new() { Entries = [new Inner { X = 3, Label = "c" }] }, + new() { Entries = [] }, + }; + var batch = WithNestedList.ToRecordBatch(items); + var restored = WithNestedList.ListFromRecordBatch(batch); + Assert.Equal(2, restored[0].Entries.Count); + Assert.Equal(1, restored[0].Entries[0].X); + Assert.Single(restored[1].Entries); + Assert.Equal(3, restored[1].Entries[0].X); + Assert.Empty(restored[2].Entries); + } + + [Fact] + public void EnumList_IPC_RoundTrip() + { + var original = new WithColorEnumList { Colors = [Color.Red, Color.Blue] }; + var bytes = original.SerializeToBytes(); + var restored = ArrowSerializerExtensions.DeserializeFromBytes(bytes); + Assert.Equal(original.Colors, restored.Colors); + } + + [Fact] + public void NestedList_IPC_RoundTrip() + { + var original = new WithNestedList + { + Entries = [new Inner { X = 42, Label = "hello" }], + }; + var bytes = original.SerializeToBytes(); + var restored = ArrowSerializerExtensions.DeserializeFromBytes(bytes); + Assert.Single(restored.Entries); + Assert.Equal(42, restored.Entries[0].X); + Assert.Equal("hello", restored.Entries[0].Label); + } + + [Fact] + public void Empty_MultiRow_RoundTrip() + { + var items = new List(); + var batch = Inner.ToRecordBatch(items); + Assert.Equal(0, batch.Length); + + var restored = Inner.ListFromRecordBatch(batch); + Assert.Empty(restored); + } + + [Fact] + public void ExtensionMethod_ToRecordBatch_RoundTrip() + { + var items = new[] { new Inner { X = 1, Label = "one" }, new Inner { X = 2, Label = "two" } }; + var batch = items.ToRecordBatch(); + Assert.Equal(2, batch.Length); + + var restored = batch.ToList(); + Assert.Equal(items, restored); + } + + [Fact] + public void IpcMultiRow_RoundTrip() + { + var items = new List + { + new() { X = 1, Label = "one" }, + new() { X = 2, Label = "two" }, + }; + + var bytes = items.SerializeListToBytes(); + var restored = ArrowSerializerExtensions.DeserializeListFromBytes(bytes); + Assert.Equal(items, restored); + } + + [Fact] + public void OptionalInt_MultiRow_RoundTrip() + { + var items = new List + { + new() { Value = 1, Extra = 10 }, + new() { Value = 2, Extra = null }, + new() { Value = 3, Extra = 30 }, + }; + + var batch = WithOptionalInt.ToRecordBatch(items); + var restored = WithOptionalInt.ListFromRecordBatch(batch); + Assert.Equal(10, restored[0].Extra); + Assert.Null(restored[1].Extra); + Assert.Equal(30, restored[2].Extra); + } +} + +public class DateTimeTypeTests +{ + [Fact] + public void DateTime_RoundTrip() + { + var obj = new WithDateTimeTypes + { + Timestamp = new DateTime(2025, 6, 15, 12, 30, 0, DateTimeKind.Utc), + TimestampOffset = new DateTimeOffset(2025, 6, 15, 12, 30, 0, TimeSpan.Zero), + Date = new DateOnly(2025, 6, 15), + Time = new TimeOnly(12, 30, 45), + Duration = TimeSpan.FromHours(2.5), + }; + + var batch = WithDateTimeTypes.ToRecordBatch(obj); + var restored = WithDateTimeTypes.FromRecordBatch(batch); + Assert.Equal(obj.Timestamp, restored.Timestamp); + Assert.Equal(obj.TimestampOffset, restored.TimestampOffset); + Assert.Equal(obj.Date, restored.Date); + Assert.Equal(obj.Time, restored.Time); + Assert.Equal(obj.Duration, restored.Duration); + } + + [Fact] + public void DateTime_LocalNormalizesToUtc() + { + // A local DateTime should be converted to UTC on serialize, and come back as UTC + var localDt = new DateTime(2025, 6, 15, 14, 30, 0, DateTimeKind.Local); + var expectedUtc = localDt.ToUniversalTime(); + + var obj = new WithDateTimeTypes + { + Timestamp = localDt, + TimestampOffset = new DateTimeOffset(2025, 6, 15, 14, 30, 0, TimeSpan.FromHours(2)), + Date = new DateOnly(2025, 6, 15), + Time = new TimeOnly(14, 30, 0), + Duration = TimeSpan.Zero, + }; + + var batch = WithDateTimeTypes.ToRecordBatch(obj); + var restored = WithDateTimeTypes.FromRecordBatch(batch); + + // DateTime comes back as UTC-normalized value + Assert.Equal(expectedUtc, restored.Timestamp); + Assert.Equal(DateTimeKind.Utc, restored.Timestamp.Kind); + + // DateTimeOffset with +02:00 offset normalizes to UTC + Assert.Equal(new DateTimeOffset(2025, 6, 15, 12, 30, 0, TimeSpan.Zero), restored.TimestampOffset); + } + + [Fact] + public void DateTime_MultiRow_RoundTrip() + { + var items = new List + { + new() { Timestamp = new DateTime(2025, 1, 1, 0, 0, 0, DateTimeKind.Utc), TimestampOffset = DateTimeOffset.UtcNow, + Date = new DateOnly(2025, 1, 1), Time = new TimeOnly(8, 0), Duration = TimeSpan.FromMinutes(30) }, + new() { Timestamp = new DateTime(2025, 12, 31, 23, 59, 59, DateTimeKind.Utc), TimestampOffset = DateTimeOffset.UtcNow, + Date = new DateOnly(2025, 12, 31), Time = new TimeOnly(23, 59, 59), Duration = TimeSpan.FromDays(1) }, + }; + + var batch = WithDateTimeTypes.ToRecordBatch(items); + var restored = WithDateTimeTypes.ListFromRecordBatch(batch); + Assert.Equal(items[0].Date, restored[0].Date); + Assert.Equal(items[0].Time, restored[0].Time); + Assert.Equal(items[0].Duration, restored[0].Duration); + Assert.Equal(items[1].Date, restored[1].Date); + Assert.Equal(items[1].Time, restored[1].Time); + Assert.Equal(items[1].Duration, restored[1].Duration); + } + + [Fact] + public void DecimalAndGuid_RoundTrip() + { + var obj = new WithDecimalAndGuid + { + Amount = 123456.789m, + Id = Guid.Parse("12345678-1234-1234-1234-123456789abc"), + }; + + var batch = WithDecimalAndGuid.ToRecordBatch(obj); + var restored = WithDecimalAndGuid.FromRecordBatch(batch); + Assert.Equal(obj.Amount, restored.Amount); + Assert.Equal(obj.Id, restored.Id); + } + + [Fact] + public void DecimalAndGuid_MultiRow_RoundTrip() + { + var items = new List + { + new() { Amount = 100.50m, Id = Guid.NewGuid() }, + new() { Amount = 0.001m, Id = Guid.NewGuid() }, + new() { Amount = -999.99m, Id = Guid.Empty }, + }; + + var batch = WithDecimalAndGuid.ToRecordBatch(items); + var restored = WithDecimalAndGuid.ListFromRecordBatch(batch); + for (int i = 0; i < items.Count; i++) + { + Assert.Equal(items[i].Amount, restored[i].Amount); + Assert.Equal(items[i].Id, restored[i].Id); + } + } + + [Fact] + public void Half_RoundTrip() + { + var obj = new WithHalf { Value = (Half)3.14 }; + var batch = WithHalf.ToRecordBatch(obj); + var restored = WithHalf.FromRecordBatch(batch); + Assert.Equal(obj.Value, restored.Value); + } + + [Fact] + public void NullableDateTimeTypes_RoundTrip_WithValues() + { + var obj = new WithNullableDateTimeTypes + { + Timestamp = new DateTime(2025, 6, 15, 12, 0, 0, DateTimeKind.Utc), + Date = new DateOnly(2025, 6, 15), + Time = new TimeOnly(12, 30), + Duration = TimeSpan.FromSeconds(90), + Amount = 42.5m, + Id = Guid.NewGuid(), + HalfVal = (Half)1.5, + }; + + var batch = WithNullableDateTimeTypes.ToRecordBatch(obj); + var restored = WithNullableDateTimeTypes.FromRecordBatch(batch); + Assert.Equal(obj.Timestamp, restored.Timestamp); + Assert.Equal(obj.Date, restored.Date); + Assert.Equal(obj.Time, restored.Time); + Assert.Equal(obj.Duration, restored.Duration); + Assert.Equal(obj.Amount, restored.Amount); + Assert.Equal(obj.Id, restored.Id); + Assert.Equal(obj.HalfVal, restored.HalfVal); + } + + [Fact] + public void NullableDateTimeTypes_RoundTrip_WithNulls() + { + var obj = new WithNullableDateTimeTypes(); + + var batch = WithNullableDateTimeTypes.ToRecordBatch(obj); + var restored = WithNullableDateTimeTypes.FromRecordBatch(batch); + Assert.Null(restored.Timestamp); + Assert.Null(restored.Date); + Assert.Null(restored.Time); + Assert.Null(restored.Duration); + Assert.Null(restored.Amount); + Assert.Null(restored.Id); + Assert.Null(restored.HalfVal); + } + + [Fact] + public void NullableDateTimeTypes_MultiRow_RoundTrip() + { + var items = new List + { + new() { Timestamp = new DateTime(2025, 1, 1, 0, 0, 0, DateTimeKind.Utc), Date = new DateOnly(2025, 1, 1), + Time = new TimeOnly(8, 0), Duration = TimeSpan.FromMinutes(30), + Amount = 10.5m, Id = Guid.NewGuid(), HalfVal = (Half)1.0 }, + new(), // all nulls + new() { Timestamp = new DateTime(2025, 12, 31, 0, 0, 0, DateTimeKind.Utc), Date = new DateOnly(2025, 12, 31), + Time = new TimeOnly(23, 59), Duration = TimeSpan.FromHours(1), + Amount = 99.99m, Id = Guid.NewGuid(), HalfVal = (Half)2.0 }, + }; + + var batch = WithNullableDateTimeTypes.ToRecordBatch(items); + var restored = WithNullableDateTimeTypes.ListFromRecordBatch(batch); + Assert.Equal(items[0].Date, restored[0].Date); + Assert.Equal(items[0].Time, restored[0].Time); + Assert.Equal(items[0].Duration, restored[0].Duration); + Assert.Equal(items[0].Amount, restored[0].Amount); + Assert.Equal(items[0].Id, restored[0].Id); + Assert.Null(restored[1].Timestamp); + Assert.Null(restored[1].Date); + Assert.Null(restored[1].Time); + Assert.Null(restored[1].Duration); + Assert.Null(restored[1].Amount); + Assert.Null(restored[1].Id); + Assert.Equal(items[2].Date, restored[2].Date); + } +} + +public class ArrowTypeOverrideTests +{ + [Fact] + public void TimestampOverride_UsesNanoseconds() + { + var schema = WithTimestampOverride.ArrowSchema; + var tsType = Assert.IsType(schema.FieldsList[0].DataType); + Assert.Equal(TimeUnit.Nanosecond, tsType.Unit); + Assert.Equal("UTC", tsType.Timezone); + } + + [Fact] + public void DecimalOverride_UsesCustomPrecisionScale() + { + var schema = WithDecimalOverride.ArrowSchema; + var decType = Assert.IsType(schema.FieldsList[0].DataType); + Assert.Equal(28, decType.Precision); + Assert.Equal(10, decType.Scale); + } + + [Fact] + public void TimestampOverride_DateTimeMilliseconds() + { + var schema = WithTimestampOverride.ArrowSchema; + var tsType = Assert.IsType(schema.FieldsList[1].DataType); + Assert.Equal(TimeUnit.Millisecond, tsType.Unit); + Assert.Equal("UTC", tsType.Timezone); + + var obj = new WithTimestampOverride + { + Value = new DateTimeOffset(2025, 6, 15, 12, 0, 0, TimeSpan.Zero), + DateTimeValue = new DateTime(2025, 3, 1, 8, 30, 0, DateTimeKind.Utc), + }; + var batch = WithTimestampOverride.ToRecordBatch(obj); + var restored = WithTimestampOverride.FromRecordBatch(batch); + Assert.Equal(obj.Value, restored.Value); + Assert.Equal(obj.DateTimeValue, restored.DateTimeValue); + } + + [Fact] + public void TimestampOverride_RoundTrip() + { + var obj = new WithTimestampOverride { Value = new DateTimeOffset(2025, 6, 15, 12, 0, 0, TimeSpan.Zero) }; + var batch = WithTimestampOverride.ToRecordBatch(obj); + var restored = WithTimestampOverride.FromRecordBatch(batch); + Assert.Equal(obj.Value, restored.Value); + } + + [Fact] + public void DecimalOverride_RoundTrip() + { + var obj = new WithDecimalOverride { Value = 12345.6789m }; + var batch = WithDecimalOverride.ToRecordBatch(obj); + var restored = WithDecimalOverride.FromRecordBatch(batch); + Assert.Equal(obj.Value, restored.Value); + } + + [Fact] + public void TimeOverride_Time32MillisAndTime64Nanos() + { + var schema = WithTimeOverride.ArrowSchema; + var t32 = Assert.IsType(schema.FieldsList[0].DataType); + Assert.Equal(TimeUnit.Millisecond, t32.Unit); + var t64 = Assert.IsType(schema.FieldsList[1].DataType); + Assert.Equal(TimeUnit.Nanosecond, t64.Unit); + + var obj = new WithTimeOverride + { + Millis = new TimeOnly(14, 30, 45), + Nanos = new TimeOnly(8, 15, 30, 500), + }; + var batch = WithTimeOverride.ToRecordBatch(obj); + var restored = WithTimeOverride.FromRecordBatch(batch); + // time32[ms] has millisecond precision — truncates sub-ms + Assert.Equal(obj.Millis, restored.Millis); + Assert.Equal(obj.Nanos, restored.Nanos); + } + + [Fact] + public void TimeOverride_MultiRow_RoundTrip() + { + var items = new List + { + new() { Millis = new TimeOnly(0, 0, 0), Nanos = new TimeOnly(12, 0, 0) }, + new() { Millis = new TimeOnly(23, 59, 59), Nanos = new TimeOnly(6, 30, 15) }, + }; + var batch = WithTimeOverride.ToRecordBatch(items); + var restored = WithTimeOverride.ListFromRecordBatch(batch); + Assert.Equal(items[0].Millis, restored[0].Millis); + Assert.Equal(items[0].Nanos, restored[0].Nanos); + Assert.Equal(items[1].Millis, restored[1].Millis); + Assert.Equal(items[1].Nanos, restored[1].Nanos); + } + + [Fact] + public void DateOverride_Schema_Date64AndDate32() + { + var schema = WithDateOverride.ArrowSchema; + Assert.IsType(schema.FieldsList[0].DataType); // default + Assert.IsType(schema.FieldsList[1].DataType); // date64 override + Assert.IsType(schema.FieldsList[2].DataType); // explicit date32 + } + + [Fact] + public void DateOverride_SingleRow_RoundTrip() + { + var obj = new WithDateOverride + { + Default = new DateOnly(2025, 3, 15), + Date64Value = new DateOnly(2024, 12, 25), + Date32Value = new DateOnly(2000, 1, 1), + }; + var batch = WithDateOverride.ToRecordBatch(obj); + var restored = WithDateOverride.FromRecordBatch(batch); + Assert.Equal(obj.Default, restored.Default); + Assert.Equal(obj.Date64Value, restored.Date64Value); + Assert.Equal(obj.Date32Value, restored.Date32Value); + } + + [Fact] + public void DateOverride_MultiRow_RoundTrip() + { + var items = new List + { + new() { Default = new DateOnly(2020, 1, 1), Date64Value = new DateOnly(2020, 6, 15), Date32Value = new DateOnly(1999, 12, 31) }, + new() { Default = new DateOnly(2025, 12, 31), Date64Value = new DateOnly(2025, 7, 4), Date32Value = new DateOnly(2030, 3, 1) }, + }; + var batch = WithDateOverride.ToRecordBatch(items); + var restored = WithDateOverride.ListFromRecordBatch(batch); + Assert.Equal(2, restored.Count); + Assert.Equal(items[0].Default, restored[0].Default); + Assert.Equal(items[0].Date64Value, restored[0].Date64Value); + Assert.Equal(items[0].Date32Value, restored[0].Date32Value); + Assert.Equal(items[1].Default, restored[1].Default); + Assert.Equal(items[1].Date64Value, restored[1].Date64Value); + Assert.Equal(items[1].Date32Value, restored[1].Date32Value); + } + + [Fact] + public void ViewTypes_Schema_UsesViewTypes() + { + var schema = WithViewTypes.ArrowSchema; + Assert.IsType(schema.FieldsList[0].DataType); + Assert.IsType(schema.FieldsList[1].DataType); + Assert.IsType(schema.FieldsList[2].DataType); + Assert.IsType(schema.FieldsList[3].DataType); + } + + [Fact] + public void ViewTypes_SingleRow_RoundTrip() + { + var obj = new WithViewTypes + { + ViewString = "hello view", + ViewBinary = new byte[] { 1, 2, 3 }, + NormalString = "hello normal", + NormalBinary = new byte[] { 4, 5, 6 }, + }; + var batch = WithViewTypes.ToRecordBatch(obj); + var restored = WithViewTypes.FromRecordBatch(batch); + Assert.Equal(obj.ViewString, restored.ViewString); + Assert.Equal(obj.ViewBinary, restored.ViewBinary); + Assert.Equal(obj.NormalString, restored.NormalString); + Assert.Equal(obj.NormalBinary, restored.NormalBinary); + } + + [Fact] + public void ViewTypes_MultiRow_RoundTrip() + { + var items = new List + { + new() { ViewString = "a", ViewBinary = new byte[] { 10 }, NormalString = "x", NormalBinary = new byte[] { 20 } }, + new() { ViewString = "b", ViewBinary = new byte[] { 30, 40 }, NormalString = "y", NormalBinary = new byte[] { 50 } }, + }; + var batch = WithViewTypes.ToRecordBatch(items); + var restored = WithViewTypes.ListFromRecordBatch(batch); + Assert.Equal(2, restored.Count); + Assert.Equal("a", restored[0].ViewString); + Assert.Equal(new byte[] { 10 }, restored[0].ViewBinary); + Assert.Equal("b", restored[1].ViewString); + Assert.Equal(new byte[] { 30, 40 }, restored[1].ViewBinary); + } + + // --- Collection element type override tests --- + + [Fact] + public void ListStringView_Schema_UsesStringViewElementType() + { + var schema = WithListStringView.ArrowSchema; + // Items: List(StringView) + var itemsField = schema.FieldsList[0]; + var listType = Assert.IsType(itemsField.DataType); + Assert.IsType(listType.ValueDataType); + // NormalItems: List(Utf8) + var normalField = schema.FieldsList[1]; + var normalListType = Assert.IsType(normalField.DataType); + Assert.IsType(normalListType.ValueDataType); + } + + [Fact] + public void ListStringView_SingleRow_RoundTrip() + { + var obj = new WithListStringView + { + Items = new List { "hello", "world" }, + NormalItems = new List { "foo", "bar" }, + }; + var batch = WithListStringView.ToRecordBatch(obj); + var restored = WithListStringView.FromRecordBatch(batch); + Assert.Equal(obj.Items, restored.Items); + Assert.Equal(obj.NormalItems, restored.NormalItems); + } + + [Fact] + public void ListStringView_MultiRow_RoundTrip() + { + var items = new List + { + new() { Items = new List { "a", "b" }, NormalItems = new List { "x" } }, + new() { Items = new List { "c" }, NormalItems = new List { "y", "z" } }, + }; + var batch = WithListStringView.ToRecordBatch(items); + var restored = WithListStringView.ListFromRecordBatch(batch); + Assert.Equal(2, restored.Count); + Assert.Equal(new[] { "a", "b" }, restored[0].Items); + Assert.Equal(new[] { "c" }, restored[1].Items); + } + + [Fact] + public void DictStringViewKey_Schema_UsesStringViewKeyType() + { + var schema = WithDictStringViewKey.ArrowSchema; + var mapType = Assert.IsType(schema.FieldsList[0].DataType); + Assert.IsType(mapType.KeyField.DataType); + } + + [Fact] + public void DictStringViewKey_SingleRow_RoundTrip() + { + var obj = new WithDictStringViewKey + { + Mapping = new Dictionary { ["alpha"] = 1, ["beta"] = 2 }, + }; + var batch = WithDictStringViewKey.ToRecordBatch(obj); + var restored = WithDictStringViewKey.FromRecordBatch(batch); + Assert.Equal(obj.Mapping, restored.Mapping); + } + + [Fact] + public void DictValueOverride_Schema_UsesStringViewValueType() + { + var schema = WithDictValueOverride.ArrowSchema; + var mapType = Assert.IsType(schema.FieldsList[0].DataType); + Assert.IsType(mapType.KeyField.DataType); // key stays normal + Assert.IsType(mapType.ValueField.DataType); // value overridden + } + + [Fact] + public void DictValueOverride_SingleRow_RoundTrip() + { + var obj = new WithDictValueOverride + { + Labels = new Dictionary { ["key1"] = "val1", ["key2"] = "val2" }, + }; + var batch = WithDictValueOverride.ToRecordBatch(obj); + var restored = WithDictValueOverride.FromRecordBatch(batch); + Assert.Equal(obj.Labels, restored.Labels); + } + + [Fact] + public void Bool8_Schema_UsesBool8Type() + { + var schema = WithBool8.ArrowSchema; + Assert.IsType(schema.FieldsList[0].DataType); // Bool8Value + Assert.IsType(schema.FieldsList[1].DataType); // NormalBool + Assert.IsType(schema.FieldsList[2].DataType); // NullableBool8 + } + + [Fact] + public void Bool8_SingleRow_RoundTrip() + { + var original = new WithBool8 { Bool8Value = true, NormalBool = false, NullableBool8 = true }; + var batch = WithBool8.ToRecordBatch(original); + var restored = WithBool8.FromRecordBatch(batch); + Assert.Equal(original, restored); + } + + [Fact] + public void Bool8_MultiRow_RoundTrip() + { + var items = new List + { + new() { Bool8Value = true, NormalBool = false, NullableBool8 = null }, + new() { Bool8Value = false, NormalBool = true, NullableBool8 = true }, + }; + var batch = WithBool8.ToRecordBatch(items); + var restored = WithBool8.ListFromRecordBatch(batch); + Assert.Equal(2, restored.Count); + Assert.True(restored[0].Bool8Value); + Assert.False(restored[0].NormalBool); + Assert.Null(restored[0].NullableBool8); + Assert.False(restored[1].Bool8Value); + Assert.True(restored[1].NormalBool); + Assert.True(restored[1].NullableBool8); + } + + [Fact] + public void CustomConverter_Schema_UsesConverterType() + { + var schema = WithCustomConverter.ArrowSchema; + Assert.IsType(schema.FieldsList[1].DataType); // Origin — converter uses StringType + } + + [Fact] + public void CustomConverter_SingleRow_RoundTrip() + { + var original = new WithCustomConverter + { + Name = "route1", + Origin = new Point2D { X = 1.5, Y = 2.5 }, + Destination = new Point2D { X = 3.0, Y = 4.0 }, + }; + var batch = WithCustomConverter.ToRecordBatch(original); + var restored = WithCustomConverter.FromRecordBatch(batch); + Assert.Equal("route1", restored.Name); + Assert.Equal(1.5, restored.Origin.X); + Assert.Equal(2.5, restored.Origin.Y); + Assert.Equal(3.0, restored.Destination.X); + Assert.Equal(4.0, restored.Destination.Y); + } + + [Fact] + public void CustomConverter_MultiRow_RoundTrip() + { + var items = new List + { + new() { Name = "a", Origin = new Point2D { X = 1, Y = 2 }, Destination = new Point2D { X = 3, Y = 4 } }, + new() { Name = "b", Origin = new Point2D { X = 5, Y = 6 }, Destination = new Point2D { X = 7, Y = 8 } }, + }; + var batch = WithCustomConverter.ToRecordBatch(items); + var restored = WithCustomConverter.ListFromRecordBatch(batch); + Assert.Equal(2, restored.Count); + Assert.Equal(1, restored[0].Origin.X); + Assert.Equal(6, restored[1].Origin.Y); + Assert.Equal(7, restored[1].Destination.X); + } +} + +public class SchemaTests +{ + [Fact] + public void Inner_Schema_HasCorrectFields() + { + var schema = Inner.ArrowSchema; + Assert.Equal(2, schema.FieldsList.Count); + Assert.Equal("X", schema.FieldsList[0].Name); + Assert.Equal(Int32Type.Default, schema.FieldsList[0].DataType); + Assert.Equal("Label", schema.FieldsList[1].Name); + Assert.Equal(StringType.Default, schema.FieldsList[1].DataType); + } + + [Fact] + public void OptionalField_IsNullable() + { + var schema = WithOptional.ArrowSchema; + Assert.False(schema.FieldsList[0].IsNullable); // Value + Assert.True(schema.FieldsList[1].IsNullable); // Extra + } + + [Fact] + public void Enum_IsDictionaryEncoded() + { + var schema = WithEnum.ArrowSchema; + Assert.IsType(schema.FieldsList[0].DataType); + } + + [Fact] + public void NestedRecord_IsStruct() + { + var schema = Outer.ArrowSchema; + Assert.IsType(schema.FieldsList[0].DataType); + } + + [Fact] + public void List_IsListType() + { + var schema = WithList.ArrowSchema; + Assert.IsType(schema.FieldsList[0].DataType); + } + + [Fact] + public void Dict_IsMapType() + { + var schema = WithDict.ArrowSchema; + Assert.IsType(schema.FieldsList[0].DataType); + } + + [Fact] + public void Metadata_SchemaHasTypeMetadata() + { + var schema = WithMetadata.ArrowSchema; + Assert.True(schema.HasMetadata); + Assert.Equal("vgi-rpc-cs", schema.Metadata["source"]); + Assert.Equal("1.0", schema.Metadata["version"]); + } + + [Fact] + public void Metadata_FieldHasMetadata() + { + var schema = WithMetadata.ArrowSchema; + var distanceField = schema.FieldsList[0]; + Assert.Equal("Distance", distanceField.Name); + Assert.True(distanceField.HasMetadata); + Assert.Equal("meters", distanceField.Metadata["unit"]); + Assert.Equal("Distance traveled", distanceField.Metadata["description"]); + } + + [Fact] + public void Metadata_FieldWithoutMetadata_HasNoMetadata() + { + var schema = WithMetadata.ArrowSchema; + var labelField = schema.FieldsList[1]; + Assert.Equal("Label", labelField.Name); + Assert.False(labelField.HasMetadata); + } + + [Fact] + public void Metadata_PreservedThroughIpc() + { + var obj = new WithMetadata { Distance = 42.5, Label = "test" }; + var bytes = obj.SerializeToBytes(); + var restored = ArrowSerializerExtensions.DeserializeFromBytes(bytes); + Assert.Equal(obj.Distance, restored.Distance); + Assert.Equal(obj.Label, restored.Label); + } + + [Fact] + public void Transient_ExcludedFromSchema() + { + var schema = WithTransient.ArrowSchema; + Assert.Equal(2, schema.FieldsList.Count); + Assert.Equal("Id", schema.FieldsList[0].Name); + Assert.Equal("Name", schema.FieldsList[1].Name); + } + + [Fact] + public void Transient_RoundTrip_PreservesDefault() + { + var obj = new WithTransient { Id = 1, Computed = "custom", Name = "test" }; + var batch = WithTransient.ToRecordBatch(obj); + var restored = WithTransient.FromRecordBatch(batch); + Assert.Equal(1, restored.Id); + Assert.Equal("test", restored.Name); + Assert.Equal("default", restored.Computed); // transient field gets default, not "custom" + } + + [Fact] + public void Transient_MultiRow_RoundTrip() + { + var items = new List + { + new() { Id = 1, Computed = "a", Name = "first" }, + new() { Id = 2, Computed = "b", Name = "second" }, + }; + var batch = WithTransient.ToRecordBatch(items); + var restored = WithTransient.ListFromRecordBatch(batch); + Assert.Equal(2, restored.Count); + Assert.Equal(1, restored[0].Id); + Assert.Equal("first", restored[0].Name); + Assert.Equal("default", restored[0].Computed); + Assert.Equal(2, restored[1].Id); + Assert.Equal("second", restored[1].Name); + Assert.Equal("default", restored[1].Computed); + } +} + +public class TimezoneAwarenessTests +{ + [Fact] + public void WallClock_DateTime_PreservesRawValue() + { + // A local DateTime with no timezone override should preserve the raw value (wall clock) + var local = new DateTime(2024, 6, 15, 14, 30, 0, DateTimeKind.Local); + var original = new WithWallClockTimestamp + { + WallDateTime = local, + WallDateTimeOffset = new DateTimeOffset(2024, 6, 15, 14, 30, 0, TimeSpan.FromHours(5)), + UtcDateTime = local, + UtcDateTimeOffset = new DateTimeOffset(2024, 6, 15, 14, 30, 0, TimeSpan.FromHours(5)), + }; + + var batch = WithWallClockTimestamp.ToRecordBatch(original); + var restored = WithWallClockTimestamp.FromRecordBatch(batch); + + // Wall clock: raw ticks preserved, no UTC conversion + Assert.Equal(local.Ticks, restored.WallDateTime.Ticks); + // Wall clock DateTimeOffset: wall-clock time preserved (14:30), offset stripped + Assert.Equal(new DateTime(2024, 6, 15, 14, 30, 0).Ticks, restored.WallDateTimeOffset.DateTime.Ticks); + + // UTC: should be converted to UTC + Assert.Equal(DateTimeKind.Utc, restored.UtcDateTime.Kind); + Assert.Equal(local.ToUniversalTime(), restored.UtcDateTime); + // UTC DateTimeOffset: converted to UTC (14:30 +05:00 = 09:30 UTC) + Assert.Equal(TimeSpan.Zero, restored.UtcDateTimeOffset.Offset); + Assert.Equal(new DateTimeOffset(2024, 6, 15, 9, 30, 0, TimeSpan.Zero), restored.UtcDateTimeOffset); + } + + [Fact] + public void WallClock_MultiRow_PreservesRawValues() + { + var items = new List + { + new() + { + WallDateTime = new DateTime(2024, 1, 1, 10, 0, 0, DateTimeKind.Local), + WallDateTimeOffset = new DateTimeOffset(2024, 1, 1, 10, 0, 0, TimeSpan.FromHours(3)), + UtcDateTime = new DateTime(2024, 1, 1, 10, 0, 0, DateTimeKind.Local), + UtcDateTimeOffset = new DateTimeOffset(2024, 1, 1, 10, 0, 0, TimeSpan.FromHours(3)), + }, + new() + { + WallDateTime = new DateTime(2024, 6, 15, 20, 0, 0, DateTimeKind.Unspecified), + WallDateTimeOffset = new DateTimeOffset(2024, 6, 15, 20, 0, 0, TimeSpan.FromHours(-5)), + UtcDateTime = new DateTime(2024, 6, 15, 20, 0, 0, DateTimeKind.Utc), + UtcDateTimeOffset = new DateTimeOffset(2024, 6, 15, 20, 0, 0, TimeSpan.FromHours(-5)), + }, + }; + + var batch = WithWallClockTimestamp.ToRecordBatch(items); + var restored = WithWallClockTimestamp.ListFromRecordBatch(batch); + + Assert.Equal(2, restored.Count); + + // Row 0: wall clock preserves raw ticks + Assert.Equal(new DateTime(2024, 1, 1, 10, 0, 0).Ticks, restored[0].WallDateTime.Ticks); + Assert.Equal(new DateTime(2024, 1, 1, 10, 0, 0).Ticks, restored[0].WallDateTimeOffset.DateTime.Ticks); + + // Row 1: wall clock preserves raw ticks + Assert.Equal(new DateTime(2024, 6, 15, 20, 0, 0).Ticks, restored[1].WallDateTime.Ticks); + Assert.Equal(new DateTime(2024, 6, 15, 20, 0, 0).Ticks, restored[1].WallDateTimeOffset.DateTime.Ticks); + + // Row 1 UTC: DateTime 20:00 UTC stays 20:00 UTC, DateTimeOffset 20:00 -05:00 = 01:00+1 UTC + Assert.Equal(DateTimeKind.Utc, restored[1].UtcDateTime.Kind); + Assert.Equal(new DateTime(2024, 6, 15, 20, 0, 0, DateTimeKind.Utc), restored[1].UtcDateTime); + Assert.Equal(new DateTimeOffset(2024, 6, 16, 1, 0, 0, TimeSpan.Zero), restored[1].UtcDateTimeOffset); + } +} + +public class TypeVariantTests +{ + [Fact] + public void RecordStruct_RoundTrip() + { + var obj = new PointRecordStruct { X = 1.5, Y = 2.5 }; + var batch = PointRecordStruct.ToRecordBatch(obj); + var restored = PointRecordStruct.FromRecordBatch(batch); + Assert.Equal(obj.X, restored.X); + Assert.Equal(obj.Y, restored.Y); + } + + [Fact] + public void RecordStruct_MultiRow_RoundTrip() + { + var items = new List + { + new() { X = 1.0, Y = 2.0 }, + new() { X = 3.0, Y = 4.0 }, + }; + var batch = PointRecordStruct.ToRecordBatch(items); + var restored = PointRecordStruct.ListFromRecordBatch(batch); + Assert.Equal(1.0, restored[0].X); + Assert.Equal(4.0, restored[1].Y); + } + + [Fact] + public void Class_RoundTrip() + { + var obj = new SimpleClass { Id = 42, Name = "hello" }; + var batch = SimpleClass.ToRecordBatch(obj); + var restored = SimpleClass.FromRecordBatch(batch); + Assert.Equal(obj.Id, restored.Id); + Assert.Equal(obj.Name, restored.Name); + } + + [Fact] + public void Class_MultiRow_RoundTrip() + { + var items = new List + { + new() { Id = 1, Name = "a" }, + new() { Id = 2, Name = "b" }, + }; + var batch = SimpleClass.ToRecordBatch(items); + var restored = SimpleClass.ListFromRecordBatch(batch); + Assert.Equal(1, restored[0].Id); + Assert.Equal("b", restored[1].Name); + } + + [Fact] + public void Struct_RoundTrip() + { + var obj = new SimpleStruct { Value = 7, Label = "test" }; + var batch = SimpleStruct.ToRecordBatch(obj); + var restored = SimpleStruct.FromRecordBatch(batch); + Assert.Equal(obj.Value, restored.Value); + Assert.Equal(obj.Label, restored.Label); + } + + [Fact] + public void Struct_MultiRow_RoundTrip() + { + var items = new List + { + new() { Value = 1, Label = "x" }, + new() { Value = 2, Label = "y" }, + }; + var batch = SimpleStruct.ToRecordBatch(items); + var restored = SimpleStruct.ListFromRecordBatch(batch); + Assert.Equal(1, restored[0].Value); + Assert.Equal("y", restored[1].Label); + } + + [Fact] + public void FieldStruct_RoundTrip() + { + var obj = new PointWithFields(3, 7); + var batch = PointWithFields.ToRecordBatch(obj); + var restored = PointWithFields.FromRecordBatch(batch); + Assert.Equal(3, restored.X); + Assert.Equal(7, restored.Y); + } + + [Fact] + public void FieldStruct_MultiRow_RoundTrip() + { + var items = new List { new(1, 2), new(3, 4) }; + var batch = PointWithFields.ToRecordBatch(items); + var restored = PointWithFields.ListFromRecordBatch(batch); + Assert.Equal(1, restored[0].X); + Assert.Equal(4, restored[1].Y); + } + + [Fact] + public void FieldStruct_MultiCtor_RoundTrip() + { + var obj = new PointMultiCtor(5, 10); + var batch = PointMultiCtor.ToRecordBatch(obj); + var restored = PointMultiCtor.FromRecordBatch(batch); + Assert.Equal(5, restored.X); + Assert.Equal(10, restored.Y); + } + + [Fact] + public void FieldStruct_MixedFieldsAndProps_RoundTrip() + { + var obj = new MixedFieldsAndProps(42, "hello"); + var batch = MixedFieldsAndProps.ToRecordBatch(obj); + var restored = MixedFieldsAndProps.FromRecordBatch(batch); + Assert.Equal(42, restored.Id); + Assert.Equal("hello", restored.Name); + } + + [Fact] + public void FieldStruct_ArrowFieldAttribute_RoundTrip() + { + var obj = new FieldWithArrowField(1.5, 2.5); + var batch = FieldWithArrowField.ToRecordBatch(obj); + // Verify schema uses overridden field names + Assert.Equal("x_coord", batch.Schema.FieldsList[0].Name); + Assert.Equal("y_coord", batch.Schema.FieldsList[1].Name); + var restored = FieldWithArrowField.FromRecordBatch(batch); + Assert.Equal(1.5, restored.X); + Assert.Equal(2.5, restored.Y); + } + + [Fact] + public void FieldStruct_IpcRoundTrip() + { + var obj = new PointWithFields(99, -1); + var bytes = obj.SerializeToBytes(); + var restored = ArrowSerializerExtensions.DeserializeFromBytes(bytes); + Assert.Equal(99, restored.X); + Assert.Equal(-1, restored.Y); + } + + [Fact] + public void Callback_OnBeforeSerialize_Called() + { + var obj = new WithCallback { Value = 5 }; + Assert.False(obj.BeforeSerializeCalled); + var batch = WithCallback.ToRecordBatch(obj); + Assert.True(obj.BeforeSerializeCalled); + } + + [Fact] + public void Callback_OnAfterDeserialize_Called() + { + var obj = new WithCallback { Value = 7 }; + var batch = WithCallback.ToRecordBatch(obj); + var restored = WithCallback.FromRecordBatch(batch); + Assert.True(restored.AfterDeserializeCalled); + Assert.Equal(14, restored.ComputedDouble); + } + + [Fact] + public void Callback_MultiRow_OnBeforeSerialize_Called() + { + var items = new List + { + new() { Value = 1 }, + new() { Value = 2 }, + }; + Assert.False(items[0].BeforeSerializeCalled); + Assert.False(items[1].BeforeSerializeCalled); + WithCallback.ToRecordBatch(items); + Assert.True(items[0].BeforeSerializeCalled); + Assert.True(items[1].BeforeSerializeCalled); + } + + [Fact] + public void Callback_MultiRow_OnAfterDeserialize_Called() + { + var items = new List + { + new() { Value = 3 }, + new() { Value = 5 }, + }; + var batch = WithCallback.ToRecordBatch(items); + var restored = WithCallback.ListFromRecordBatch(batch); + Assert.Equal(2, restored.Count); + Assert.True(restored[0].AfterDeserializeCalled); + Assert.Equal(6, restored[0].ComputedDouble); + Assert.True(restored[1].AfterDeserializeCalled); + Assert.Equal(10, restored[1].ComputedDouble); + } +} + +public class IpcTests +{ + [Fact] + public void SerializeToBytes_DeserializeFromBytes_RoundTrip() + { + var obj = new Inner { X = 42, Label = "hello" }; + var bytes = obj.SerializeToBytes(); + Assert.True(bytes.Length > 0); + var restored = ArrowSerializerExtensions.DeserializeFromBytes(bytes); + Assert.Equal(obj, restored); + } + + [Fact] + public void SerializeToStream_DeserializeFromStream_RoundTrip() + { + var obj = new Inner { X = 99, Label = "stream" }; + using var ms = new MemoryStream(); + obj.SerializeToStream(ms); + ms.Position = 0; + var restored = ArrowSerializerExtensions.DeserializeFromStream(ms); + Assert.Equal(obj, restored); + } +} + +public class PolymorphicTests +{ + [Fact] + public void Schema_HasDiscriminatorAndUnionFields() + { + var schema = Shape.ArrowSchema; + Assert.Equal("$type", schema.FieldsList[0].Name); + Assert.Equal(StringType.Default, schema.FieldsList[0].DataType); + // Union fields: Radius, Width, Height — all nullable + Assert.Equal(3, schema.FieldsList.Count - 1); + Assert.True(schema.FieldsList[1].IsNullable); + Assert.True(schema.FieldsList[2].IsNullable); + Assert.True(schema.FieldsList[3].IsNullable); + } + + [Fact] + public void CustomDiscriminatorFieldName() + { + var schema = Message.ArrowSchema; + Assert.Equal("kind", schema.FieldsList[0].Name); + } + + [Fact] + public void Circle_SingleRow_RoundTrip() + { + Shape original = new Circle { Radius = 5.0 }; + var batch = Shape.ToRecordBatch(original); + var restored = Shape.FromRecordBatch(batch); + var circle = Assert.IsType(restored); + Assert.Equal(5.0, circle.Radius); + } + + [Fact] + public void Rectangle_SingleRow_RoundTrip() + { + Shape original = new Rectangle { Width = 3.0, Height = 4.0 }; + var batch = Shape.ToRecordBatch(original); + var restored = Shape.FromRecordBatch(batch); + var rect = Assert.IsType(restored); + Assert.Equal(3.0, rect.Width); + Assert.Equal(4.0, rect.Height); + } + + [Fact] + public void MultiRow_MixedTypes_RoundTrip() + { + var items = new List + { + new Circle { Radius = 1.0 }, + new Rectangle { Width = 2.0, Height = 3.0 }, + new Circle { Radius = 4.5 }, + }; + var batch = Shape.ToRecordBatch(items); + Assert.Equal(3, batch.Length); + + var restored = Shape.ListFromRecordBatch(batch); + Assert.Equal(3, restored.Count); + + var c1 = Assert.IsType(restored[0]); + Assert.Equal(1.0, c1.Radius); + + var r1 = Assert.IsType(restored[1]); + Assert.Equal(2.0, r1.Width); + Assert.Equal(3.0, r1.Height); + + var c2 = Assert.IsType(restored[2]); + Assert.Equal(4.5, c2.Radius); + } + + [Fact] + public void Message_TextMessage_RoundTrip() + { + Message original = new TextMessage { Content = "hello" }; + var batch = Message.ToRecordBatch(original); + var restored = Message.FromRecordBatch(batch); + var text = Assert.IsType(restored); + Assert.Equal("hello", text.Content); + } + + [Fact] + public void Message_ImageMessage_RoundTrip() + { + Message original = new ImageMessage { Url = "https://example.com/img.png", Width = 800, Height = 600 }; + var batch = Message.ToRecordBatch(original); + var restored = Message.FromRecordBatch(batch); + var img = Assert.IsType(restored); + Assert.Equal("https://example.com/img.png", img.Url); + Assert.Equal(800, img.Width); + Assert.Equal(600, img.Height); + } + + [Fact] + public void Message_MultiRow_RoundTrip() + { + var items = new List + { + new TextMessage { Content = "hi" }, + new ImageMessage { Url = "pic.jpg", Width = 100, Height = 200 }, + new TextMessage { Content = "bye" }, + }; + var batch = Message.ToRecordBatch(items); + var restored = Message.ListFromRecordBatch(batch); + Assert.Equal(3, restored.Count); + + Assert.IsType(restored[0]); + Assert.IsType(restored[1]); + Assert.IsType(restored[2]); + Assert.Equal("hi", ((TextMessage)restored[0]).Content); + Assert.Equal("pic.jpg", ((ImageMessage)restored[1]).Url); + Assert.Equal("bye", ((TextMessage)restored[2]).Content); + } + + // --- Collection tests for newly supported element types --- + + [Fact] + public void ByteList_RoundTrip() + { + var obj = new WithByteList { Values = new List { 0, 127, 255 } }; + var restored = WithByteList.FromRecordBatch(WithByteList.ToRecordBatch(obj)); + Assert.Equal(obj.Values, restored.Values); + } + + [Fact] + public void ByteList_MultiRow_RoundTrip() + { + var items = new List + { + new() { Values = new List { 1, 2, 3 } }, + new() { Values = new List { 4, 5 } }, + new() { Values = new List() }, + }; + var batch = WithByteList.ToRecordBatch(items); + var restored = WithByteList.ListFromRecordBatch(batch); + Assert.Equal(items[0].Values, restored[0].Values); + Assert.Equal(items[1].Values, restored[1].Values); + Assert.Empty(restored[2].Values); + } + + [Fact] + public void DateTimeList_RoundTrip() + { + var now = new DateTime(2025, 6, 15, 10, 30, 0, DateTimeKind.Utc); + var obj = new WithDateTimeList { Timestamps = new List { now, now.AddHours(1) } }; + var restored = WithDateTimeList.FromRecordBatch(WithDateTimeList.ToRecordBatch(obj)); + Assert.Equal(obj.Timestamps.Count, restored.Timestamps.Count); + Assert.Equal(obj.Timestamps[0], restored.Timestamps[0]); + Assert.Equal(obj.Timestamps[1], restored.Timestamps[1]); + } + + [Fact] + public void DateTimeList_MultiRow_RoundTrip() + { + var dt1 = new DateTime(2025, 1, 1, 0, 0, 0, DateTimeKind.Utc); + var dt2 = new DateTime(2025, 6, 15, 12, 0, 0, DateTimeKind.Utc); + var items = new List + { + new() { Timestamps = new List { dt1 } }, + new() { Timestamps = new List { dt1, dt2 } }, + }; + var batch = WithDateTimeList.ToRecordBatch(items); + var restored = WithDateTimeList.ListFromRecordBatch(batch); + Assert.Equal(items[0].Timestamps, restored[0].Timestamps); + Assert.Equal(items[1].Timestamps, restored[1].Timestamps); + } + + [Fact] + public void DecimalList_RoundTrip() + { + var obj = new WithDecimalList { Amounts = new List { 1.23m, 456.789m, 0m } }; + var restored = WithDecimalList.FromRecordBatch(WithDecimalList.ToRecordBatch(obj)); + Assert.Equal(obj.Amounts, restored.Amounts); + } + + [Fact] + public void DecimalList_MultiRow_RoundTrip() + { + var items = new List + { + new() { Amounts = new List { 1.1m, 2.2m } }, + new() { Amounts = new List { 3.3m } }, + }; + var batch = WithDecimalList.ToRecordBatch(items); + var restored = WithDecimalList.ListFromRecordBatch(batch); + Assert.Equal(items[0].Amounts, restored[0].Amounts); + Assert.Equal(items[1].Amounts, restored[1].Amounts); + } + + [Fact] + public void GuidList_RoundTrip() + { + var g1 = Guid.NewGuid(); + var g2 = Guid.NewGuid(); + var obj = new WithGuidList { Ids = new List { g1, g2 } }; + var restored = WithGuidList.FromRecordBatch(WithGuidList.ToRecordBatch(obj)); + Assert.Equal(obj.Ids, restored.Ids); + } + + [Fact] + public void GuidList_MultiRow_RoundTrip() + { + var items = new List + { + new() { Ids = new List { Guid.NewGuid(), Guid.NewGuid() } }, + new() { Ids = new List { Guid.NewGuid() } }, + }; + var batch = WithGuidList.ToRecordBatch(items); + var restored = WithGuidList.ListFromRecordBatch(batch); + Assert.Equal(items[0].Ids, restored[0].Ids); + Assert.Equal(items[1].Ids, restored[1].Ids); + } + + [Fact] + public void BinaryList_RoundTrip() + { + var obj = new WithBinaryList { Blobs = new List { new byte[] { 1, 2 }, new byte[] { 3, 4, 5 } } }; + var restored = WithBinaryList.FromRecordBatch(WithBinaryList.ToRecordBatch(obj)); + Assert.Equal(obj.Blobs.Count, restored.Blobs.Count); + Assert.Equal(obj.Blobs[0], restored.Blobs[0]); + Assert.Equal(obj.Blobs[1], restored.Blobs[1]); + } + + [Fact] + public void BinaryList_MultiRow_RoundTrip() + { + var items = new List + { + new() { Blobs = new List { new byte[] { 0xFF } } }, + new() { Blobs = new List { new byte[] { 0x00, 0x01 }, new byte[] { 0x02 } } }, + }; + var batch = WithBinaryList.ToRecordBatch(items); + var restored = WithBinaryList.ListFromRecordBatch(batch); + Assert.Equal(items[0].Blobs[0], restored[0].Blobs[0]); + Assert.Equal(items[1].Blobs[0], restored[1].Blobs[0]); + Assert.Equal(items[1].Blobs[1], restored[1].Blobs[1]); + } + + [Fact] + public void IntKeyDict_RoundTrip() + { + var obj = new WithIntKeyDict { Lookup = new Dictionary { [1] = "one", [2] = "two" } }; + var restored = WithIntKeyDict.FromRecordBatch(WithIntKeyDict.ToRecordBatch(obj)); + Assert.Equal(obj.Lookup, restored.Lookup); + } + + [Fact] + public void IntKeyDict_MultiRow_RoundTrip() + { + var items = new List + { + new() { Lookup = new Dictionary { [10] = "ten" } }, + new() { Lookup = new Dictionary { [20] = "twenty", [30] = "thirty" } }, + }; + var batch = WithIntKeyDict.ToRecordBatch(items); + var restored = WithIntKeyDict.ListFromRecordBatch(batch); + Assert.Equal(items[0].Lookup, restored[0].Lookup); + Assert.Equal(items[1].Lookup, restored[1].Lookup); + } + + [Fact] + public void DateTimeOffsetList_RoundTrip() + { + var dto1 = new DateTimeOffset(2025, 6, 15, 10, 30, 0, TimeSpan.Zero); + var dto2 = new DateTimeOffset(2025, 12, 25, 0, 0, 0, TimeSpan.Zero); + var obj = new WithDateTimeOffsetList { Timestamps = new List { dto1, dto2 } }; + var restored = WithDateTimeOffsetList.FromRecordBatch(WithDateTimeOffsetList.ToRecordBatch(obj)); + Assert.Equal(obj.Timestamps, restored.Timestamps); + } + + [Fact] + public void DateOnlyList_RoundTrip() + { + var obj = new WithDateOnlyList { Dates = new List { new DateOnly(2025, 1, 1), new DateOnly(2025, 12, 31) } }; + var restored = WithDateOnlyList.FromRecordBatch(WithDateOnlyList.ToRecordBatch(obj)); + Assert.Equal(obj.Dates, restored.Dates); + } + + [Fact] + public void TimeOnlyList_RoundTrip() + { + var obj = new WithTimeOnlyList { Times = new List { new TimeOnly(10, 30, 0), new TimeOnly(23, 59, 59) } }; + var restored = WithTimeOnlyList.FromRecordBatch(WithTimeOnlyList.ToRecordBatch(obj)); + Assert.Equal(obj.Times, restored.Times); + } + + [Fact] + public void TimeSpanList_RoundTrip() + { + var obj = new WithTimeSpanList { Durations = new List { TimeSpan.FromHours(1), TimeSpan.FromMinutes(30) } }; + var restored = WithTimeSpanList.FromRecordBatch(WithTimeSpanList.ToRecordBatch(obj)); + Assert.Equal(obj.Durations, restored.Durations); + } + + [Fact] + public void HalfList_RoundTrip() + { + var obj = new WithHalfList { Values = new List { (Half)1.5f, (Half)2.25f } }; + var restored = WithHalfList.FromRecordBatch(WithHalfList.ToRecordBatch(obj)); + Assert.Equal(obj.Values, restored.Values); + } + + [Fact] + public void AllPrimitiveLists_RoundTrip() + { + var obj = new WithAllPrimitiveLists + { + SBytes = new List { -128, 0, 127 }, + Shorts = new List { -32768, 0, 32767 }, + UShorts = new List { 0, 100, 65535 }, + UInts = new List { 0, 42, uint.MaxValue }, + ULongs = new List { 0, 99, ulong.MaxValue }, + }; + var restored = WithAllPrimitiveLists.FromRecordBatch(WithAllPrimitiveLists.ToRecordBatch(obj)); + Assert.Equal(obj.SBytes, restored.SBytes); + Assert.Equal(obj.Shorts, restored.Shorts); + Assert.Equal(obj.UShorts, restored.UShorts); + Assert.Equal(obj.UInts, restored.UInts); + Assert.Equal(obj.ULongs, restored.ULongs); + } + + [Fact] + public void AllPrimitiveLists_MultiRow_RoundTrip() + { + var items = new List + { + new() { SBytes = new List { 1 }, Shorts = new List { 2 }, UShorts = new List { 3 }, UInts = new List { 4 }, ULongs = new List { 5 } }, + new() { SBytes = new List { -1 }, Shorts = new List { -2 }, UShorts = new List { 0 }, UInts = new List { 0 }, ULongs = new List { 0 } }, + }; + var batch = WithAllPrimitiveLists.ToRecordBatch(items); + var restored = WithAllPrimitiveLists.ListFromRecordBatch(batch); + Assert.Equal(items[0].SBytes, restored[0].SBytes); + Assert.Equal(items[1].Shorts, restored[1].Shorts); + Assert.Equal(items[0].UShorts, restored[0].UShorts); + Assert.Equal(items[1].UInts, restored[1].UInts); + Assert.Equal(items[0].ULongs, restored[0].ULongs); + } + + // --- Deeply nested collection tests --- + + [Fact] + public void ListOfDicts_SingleRow_RoundTrip() + { + var original = new WithListOfDicts + { + Items = + [ + new Dictionary { ["a"] = 1, ["b"] = 2 }, + new Dictionary { ["c"] = 3 }, + ], + }; + var batch = WithListOfDicts.ToRecordBatch(original); + var restored = WithListOfDicts.FromRecordBatch(batch); + Assert.Equal(2, restored.Items.Count); + Assert.Equal(1, restored.Items[0]["a"]); + Assert.Equal(2, restored.Items[0]["b"]); + Assert.Equal(3, restored.Items[1]["c"]); + } + + [Fact] + public void ListOfDicts_MultiRow_RoundTrip() + { + var items = new List + { + new() { Items = [new Dictionary { ["x"] = 10 }] }, + new() { Items = [new Dictionary { ["y"] = 20 }, new Dictionary { ["z"] = 30 }] }, + new() { Items = [] }, + }; + var batch = WithListOfDicts.ToRecordBatch(items); + var restored = WithListOfDicts.ListFromRecordBatch(batch); + Assert.Single(restored[0].Items); + Assert.Equal(10, restored[0].Items[0]["x"]); + Assert.Equal(2, restored[1].Items.Count); + Assert.Equal(20, restored[1].Items[0]["y"]); + Assert.Equal(30, restored[1].Items[1]["z"]); + Assert.Empty(restored[2].Items); + } + + [Fact] + public void ListOfDicts_IPC_RoundTrip() + { + var original = new WithListOfDicts + { + Items = [new Dictionary { ["key"] = 42 }], + }; + var bytes = original.SerializeToBytes(); + var restored = ArrowSerializerExtensions.DeserializeFromBytes(bytes); + Assert.Single(restored.Items); + Assert.Equal(42, restored.Items[0]["key"]); + } + + [Fact] + public void DictOfLists_SingleRow_RoundTrip() + { + var original = new WithDictOfLists + { + Groups = new Dictionary> + { + ["evens"] = [2, 4, 6], + ["odds"] = [1, 3, 5], + }, + }; + var batch = WithDictOfLists.ToRecordBatch(original); + var restored = WithDictOfLists.FromRecordBatch(batch); + Assert.Equal(2, restored.Groups.Count); + Assert.Equal([2, 4, 6], restored.Groups["evens"]); + Assert.Equal([1, 3, 5], restored.Groups["odds"]); + } + + [Fact] + public void DictOfLists_MultiRow_RoundTrip() + { + var items = new List + { + new() { Groups = new() { ["a"] = [1, 2] } }, + new() { Groups = new() { ["b"] = [3], ["c"] = [4, 5, 6] } }, + }; + var batch = WithDictOfLists.ToRecordBatch(items); + var restored = WithDictOfLists.ListFromRecordBatch(batch); + Assert.Equal([1, 2], restored[0].Groups["a"]); + Assert.Equal([3], restored[1].Groups["b"]); + Assert.Equal([4, 5, 6], restored[1].Groups["c"]); + } + + [Fact] + public void DictOfLists_IPC_RoundTrip() + { + var original = new WithDictOfLists + { + Groups = new() { ["nums"] = [7, 8, 9] }, + }; + var bytes = original.SerializeToBytes(); + var restored = ArrowSerializerExtensions.DeserializeFromBytes(bytes); + Assert.Equal([7, 8, 9], restored.Groups["nums"]); + } + + [Fact] + public void NestedDicts_SingleRow_RoundTrip() + { + var original = new WithNestedDicts + { + Nested = new() + { + ["group1"] = new() { ["a"] = 1, ["b"] = 2 }, + ["group2"] = new() { ["c"] = 3 }, + }, + }; + var batch = WithNestedDicts.ToRecordBatch(original); + var restored = WithNestedDicts.FromRecordBatch(batch); + Assert.Equal(2, restored.Nested.Count); + Assert.Equal(1, restored.Nested["group1"]["a"]); + Assert.Equal(2, restored.Nested["group1"]["b"]); + Assert.Equal(3, restored.Nested["group2"]["c"]); + } + + [Fact] + public void NestedDicts_MultiRow_RoundTrip() + { + var items = new List + { + new() { Nested = new() { ["x"] = new() { ["k"] = 10 } } }, + new() { Nested = new() { ["y"] = new() { ["m"] = 20, ["n"] = 30 } } }, + }; + var batch = WithNestedDicts.ToRecordBatch(items); + var restored = WithNestedDicts.ListFromRecordBatch(batch); + Assert.Equal(10, restored[0].Nested["x"]["k"]); + Assert.Equal(20, restored[1].Nested["y"]["m"]); + Assert.Equal(30, restored[1].Nested["y"]["n"]); + } + + [Fact] + public void NestedDicts_IPC_RoundTrip() + { + var original = new WithNestedDicts + { + Nested = new() { ["g"] = new() { ["v"] = 99 } }, + }; + var bytes = original.SerializeToBytes(); + var restored = ArrowSerializerExtensions.DeserializeFromBytes(bytes); + Assert.Equal(99, restored.Nested["g"]["v"]); + } + + [Fact] + public void ListOfListOfDicts_SingleRow_RoundTrip() + { + var original = new WithListOfListOfDicts + { + Matrix = + [ + [new Dictionary { ["a"] = 1 }, new Dictionary { ["b"] = 2 }], + [new Dictionary { ["c"] = 3 }], + ], + }; + var batch = WithListOfListOfDicts.ToRecordBatch(original); + var restored = WithListOfListOfDicts.FromRecordBatch(batch); + Assert.Equal(2, restored.Matrix.Count); + Assert.Equal(2, restored.Matrix[0].Count); + Assert.Equal(1, restored.Matrix[0][0]["a"]); + Assert.Equal(2, restored.Matrix[0][1]["b"]); + Assert.Single(restored.Matrix[1]); + Assert.Equal(3, restored.Matrix[1][0]["c"]); + } + + [Fact] + public void ListOfListOfDicts_MultiRow_RoundTrip() + { + var items = new List + { + new() { Matrix = [[new Dictionary { ["k"] = 1 }]] }, + new() { Matrix = [[new Dictionary { ["m"] = 2 }], [new Dictionary { ["n"] = 3 }]] }, + }; + var batch = WithListOfListOfDicts.ToRecordBatch(items); + var restored = WithListOfListOfDicts.ListFromRecordBatch(batch); + Assert.Equal(1, restored[0].Matrix[0][0]["k"]); + Assert.Equal(2, restored[1].Matrix[0][0]["m"]); + Assert.Equal(3, restored[1].Matrix[1][0]["n"]); + } + + [Fact] + public void DictOfDictOfLists_SingleRow_RoundTrip() + { + var original = new WithDictOfDictOfLists + { + Deep = new() + { + ["outer"] = new() + { + ["inner"] = [10, 20, 30], + }, + }, + }; + var batch = WithDictOfDictOfLists.ToRecordBatch(original); + var restored = WithDictOfDictOfLists.FromRecordBatch(batch); + Assert.Equal([10, 20, 30], restored.Deep["outer"]["inner"]); + } + + [Fact] + public void DictOfDictOfLists_MultiRow_RoundTrip() + { + var items = new List + { + new() { Deep = new() { ["a"] = new() { ["x"] = [1, 2] } } }, + new() { Deep = new() { ["b"] = new() { ["y"] = [3], ["z"] = [4, 5] } } }, + }; + var batch = WithDictOfDictOfLists.ToRecordBatch(items); + var restored = WithDictOfDictOfLists.ListFromRecordBatch(batch); + Assert.Equal([1, 2], restored[0].Deep["a"]["x"]); + Assert.Equal([3], restored[1].Deep["b"]["y"]); + Assert.Equal([4, 5], restored[1].Deep["b"]["z"]); + } + + // --- Nullable Collection Tests --- + + [Fact] + public void NullableList_SingleRow_NonNull() + { + var record = new WithNullableList { Values = [1, 2, 3] }; + var batch = WithNullableList.ToRecordBatch(record); + var restored = WithNullableList.FromRecordBatch(batch); + Assert.Equal([1, 2, 3], restored.Values); + } + + [Fact] + public void NullableList_SingleRow_Null() + { + var record = new WithNullableList { Values = null }; + var batch = WithNullableList.ToRecordBatch(record); + var restored = WithNullableList.FromRecordBatch(batch); + Assert.Null(restored.Values); + } + + [Fact] + public void NullableList_MultiRow() + { + var items = new List + { + new() { Values = [1, 2] }, + new() { Values = null }, + new() { Values = [3] }, + }; + var batch = WithNullableList.ToRecordBatch(items); + var restored = WithNullableList.ListFromRecordBatch(batch); + Assert.Equal(3, restored.Count); + Assert.Equal([1, 2], restored[0].Values); + Assert.Null(restored[1].Values); + Assert.Equal([3], restored[2].Values); + } + + [Fact] + public void NullableDict_SingleRow_NonNull() + { + var record = new WithNullableDict { Mapping = new() { ["a"] = 1 } }; + var batch = WithNullableDict.ToRecordBatch(record); + var restored = WithNullableDict.FromRecordBatch(batch); + Assert.NotNull(restored.Mapping); + Assert.Equal(1, restored.Mapping!["a"]); + } + + [Fact] + public void NullableDict_SingleRow_Null() + { + var record = new WithNullableDict { Mapping = null }; + var batch = WithNullableDict.ToRecordBatch(record); + var restored = WithNullableDict.FromRecordBatch(batch); + Assert.Null(restored.Mapping); + } + + [Fact] + public void NullableDict_MultiRow() + { + var items = new List + { + new() { Mapping = new() { ["a"] = 1 } }, + new() { Mapping = null }, + new() { Mapping = new() { ["b"] = 2 } }, + }; + var batch = WithNullableDict.ToRecordBatch(items); + var restored = WithNullableDict.ListFromRecordBatch(batch); + Assert.Equal(3, restored.Count); + Assert.Equal(1, restored[0].Mapping!["a"]); + Assert.Null(restored[1].Mapping); + Assert.Equal(2, restored[2].Mapping!["b"]); + } + + [Fact] + public void NullableListOfDicts_SingleRow_NonNull() + { + var record = new WithNullableListOfDicts { Items = [new() { ["x"] = 1L }] }; + var batch = WithNullableListOfDicts.ToRecordBatch(record); + var restored = WithNullableListOfDicts.FromRecordBatch(batch); + Assert.NotNull(restored.Items); + Assert.Equal(1L, restored.Items![0]["x"]); + } + + [Fact] + public void NullableListOfDicts_SingleRow_Null() + { + var record = new WithNullableListOfDicts { Items = null }; + var batch = WithNullableListOfDicts.ToRecordBatch(record); + var restored = WithNullableListOfDicts.FromRecordBatch(batch); + Assert.Null(restored.Items); + } + + [Fact] + public void NullableListOfDicts_MultiRow() + { + var items = new List + { + new() { Items = [new() { ["x"] = 1L }] }, + new() { Items = null }, + new() { Items = [new() { ["y"] = 2L }, new() { ["z"] = 3L }] }, + }; + var batch = WithNullableListOfDicts.ToRecordBatch(items); + var restored = WithNullableListOfDicts.ListFromRecordBatch(batch); + Assert.Equal(3, restored.Count); + Assert.Equal(1L, restored[0].Items![0]["x"]); + Assert.Null(restored[1].Items); + Assert.Equal(2L, restored[2].Items![0]["y"]); + } + + [Fact] + public void NullableDictOfLists_SingleRow_NonNull() + { + var record = new WithNullableDictOfLists { Groups = new() { ["a"] = [1L, 2L] } }; + var batch = WithNullableDictOfLists.ToRecordBatch(record); + var restored = WithNullableDictOfLists.FromRecordBatch(batch); + Assert.NotNull(restored.Groups); + Assert.Equal([1L, 2L], restored.Groups!["a"]); + } + + [Fact] + public void NullableDictOfLists_SingleRow_Null() + { + var record = new WithNullableDictOfLists { Groups = null }; + var batch = WithNullableDictOfLists.ToRecordBatch(record); + var restored = WithNullableDictOfLists.FromRecordBatch(batch); + Assert.Null(restored.Groups); + } + + [Fact] + public void NullableDictOfLists_MultiRow() + { + var items = new List + { + new() { Groups = new() { ["a"] = [1L] } }, + new() { Groups = null }, + new() { Groups = new() { ["b"] = [2L, 3L] } }, + }; + var batch = WithNullableDictOfLists.ToRecordBatch(items); + var restored = WithNullableDictOfLists.ListFromRecordBatch(batch); + Assert.Equal(3, restored.Count); + Assert.Equal([1L], restored[0].Groups!["a"]); + Assert.Null(restored[1].Groups); + Assert.Equal([2L, 3L], restored[2].Groups!["b"]); + } + + // --- Dict with Guid values --- + + [Fact] + public void WithDictOfGuids_SingleRow_RoundTrips() + { + var g1 = Guid.NewGuid(); + var g2 = Guid.NewGuid(); + var record = new WithDictOfGuids { Items = new() { ["a"] = g1, ["b"] = g2 } }; + var batch = WithDictOfGuids.ToRecordBatch(record); + var restored = WithDictOfGuids.FromRecordBatch(batch); + Assert.Equal(g1, restored.Items["a"]); + Assert.Equal(g2, restored.Items["b"]); + } + + [Fact] + public void WithDictOfGuids_MultiRow_RoundTrips() + { + var g1 = Guid.NewGuid(); + var g2 = Guid.NewGuid(); + var items = new List + { + new() { Items = new() { ["x"] = g1 } }, + new() { Items = new() { ["y"] = g2 } }, + }; + var batch = WithDictOfGuids.ToRecordBatch(items); + var restored = WithDictOfGuids.ListFromRecordBatch(batch); + Assert.Equal(2, restored.Count); + Assert.Equal(g1, restored[0].Items["x"]); + Assert.Equal(g2, restored[1].Items["y"]); + } + + // --- Dict with Enum values --- + + [Fact] + public void WithDictOfEnums_SingleRow_RoundTrips() + { + var record = new WithDictOfEnums { Items = new() { ["a"] = Color.Red, ["b"] = Color.Blue } }; + var batch = WithDictOfEnums.ToRecordBatch(record); + var restored = WithDictOfEnums.FromRecordBatch(batch); + Assert.Equal(Color.Red, restored.Items["a"]); + Assert.Equal(Color.Blue, restored.Items["b"]); + } + + [Fact] + public void WithDictOfEnums_MultiRow_RoundTrips() + { + var items = new List + { + new() { Items = new() { ["x"] = Color.Green } }, + new() { Items = new() { ["y"] = Color.Red, ["z"] = Color.Blue } }, + }; + var batch = WithDictOfEnums.ToRecordBatch(items); + var restored = WithDictOfEnums.ListFromRecordBatch(batch); + Assert.Equal(2, restored.Count); + Assert.Equal(Color.Green, restored[0].Items["x"]); + Assert.Equal(Color.Red, restored[1].Items["y"]); + Assert.Equal(Color.Blue, restored[1].Items["z"]); + } + + // --- Dict with NestedRecord values --- + + [Fact] + public void WithDictOfNested_SingleRow_RoundTrips() + { + var record = new WithDictOfNested + { + Items = new() + { + ["a"] = new Inner { X = 1, Label = "one" }, + ["b"] = new Inner { X = 2, Label = "two" }, + } + }; + var batch = WithDictOfNested.ToRecordBatch(record); + var restored = WithDictOfNested.FromRecordBatch(batch); + Assert.Equal(1, restored.Items["a"].X); + Assert.Equal("one", restored.Items["a"].Label); + Assert.Equal(2, restored.Items["b"].X); + Assert.Equal("two", restored.Items["b"].Label); + } + + [Fact] + public void WithDictOfNested_MultiRow_RoundTrips() + { + var items = new List + { + new() { Items = new() { ["x"] = new Inner { X = 10, Label = "ten" } } }, + new() { Items = new() { ["y"] = new Inner { X = 20, Label = "twenty" }, ["z"] = new Inner { X = 30, Label = "thirty" } } }, + }; + var batch = WithDictOfNested.ToRecordBatch(items); + var restored = WithDictOfNested.ListFromRecordBatch(batch); + Assert.Equal(2, restored.Count); + Assert.Equal(10, restored[0].Items["x"].X); + Assert.Equal("ten", restored[0].Items["x"].Label); + Assert.Equal(20, restored[1].Items["y"].X); + Assert.Equal(30, restored[1].Items["z"].X); + } + + [Fact] + public void WithDictOfNested_EmptyDict_RoundTrips() + { + var record = new WithDictOfNested { Items = new() }; + var batch = WithDictOfNested.ToRecordBatch(record); + var restored = WithDictOfNested.FromRecordBatch(batch); + Assert.Empty(restored.Items); + } +} diff --git a/test/Apache.Arrow.Serialization.Tests/TestTypes.cs b/test/Apache.Arrow.Serialization.Tests/TestTypes.cs new file mode 100644 index 00000000..d750681f --- /dev/null +++ b/test/Apache.Arrow.Serialization.Tests/TestTypes.cs @@ -0,0 +1,673 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Apache.Arrow.Serialization; + +namespace Apache.Arrow.Serialization.Tests; + +public enum Color +{ + Red, + Green, + Blue, +} + +[ArrowSerializable] +public partial record Inner +{ + public int X { get; init; } + public string Label { get; init; } = ""; +} + +[ArrowSerializable] +public partial record Outer +{ + public Inner Inner { get; init; } = new(); + public string Name { get; init; } = ""; +} + +[ArrowSerializable] +public partial record WithPrimitives +{ + public string Name { get; init; } = ""; + public int Count { get; init; } + public long BigCount { get; init; } + public float Score { get; init; } + public double Precise { get; init; } + public bool Flag { get; init; } + public byte Small { get; init; } +} + +[ArrowSerializable] +public partial record WithOptional +{ + public string Value { get; init; } = ""; + public string? Extra { get; init; } +} + +[ArrowSerializable] +public partial record WithOptionalInt +{ + public int Value { get; init; } + public int? Extra { get; init; } +} + +[ArrowSerializable] +public partial record WithEnum +{ + public Color Color { get; init; } +} + +[ArrowSerializable] +public partial record WithList +{ + public List Items { get; init; } = new(); +} + +[ArrowSerializable] +public partial record WithStringList +{ + public List Tags { get; init; } = new(); +} + +[ArrowSerializable] +public partial record WithDict +{ + public Dictionary Mapping { get; init; } = new(); +} + +[ArrowSerializable] +public partial record WithSet +{ + public HashSet Tags { get; init; } = new(); +} + +[ArrowSerializable] +public partial record WithBytes +{ + public byte[] Data { get; init; } = System.Array.Empty(); +} + +[ArrowSerializable] +public partial record WithReadOnlyMemoryBytes +{ + public ReadOnlyMemory Data { get; init; } +} + +[ArrowSerializable] +public partial record WithBool +{ + public bool Flag { get; init; } +} + +[ArrowSerializable] +public partial record WithOptionalNested +{ + public Inner? Inner { get; init; } +} + +[ArrowSerializable] +public partial record WithNullableEnum +{ + public Color? Color { get; init; } +} + +[ArrowSerializable] +public partial record Level2 +{ + public Inner Inner { get; init; } = new(); + public string Tag { get; init; } = ""; +} + +[ArrowSerializable] +public partial record Level3 +{ + public Level2 Level2 { get; init; } = new(); + public double Score { get; init; } +} + +[ArrowSerializable] +public partial record WithEnumList +{ + public List Colors { get; init; } = new(); +} + +[ArrowSerializable] +public partial record WithColorEnumList +{ + public List Colors { get; init; } = new(); +} + +[ArrowSerializable] +public partial record WithNestedList +{ + public List Entries { get; init; } = new(); +} + +[ArrowSerializable] +public partial record WithArrowTypeOverride +{ + [ArrowType("int64")] + public int Value { get; init; } +} + +[ArrowSerializable] +public partial record WithDateTimeTypes +{ + public DateTime Timestamp { get; init; } + public DateTimeOffset TimestampOffset { get; init; } + public DateOnly Date { get; init; } + public TimeOnly Time { get; init; } + public TimeSpan Duration { get; init; } +} + +[ArrowSerializable] +public partial record WithDecimalAndGuid +{ + public decimal Amount { get; init; } + public Guid Id { get; init; } +} + +[ArrowSerializable] +public partial record WithHalf +{ + public Half Value { get; init; } +} + +[ArrowSerializable] +public partial record WithNullableDateTimeTypes +{ + public DateTime? Timestamp { get; init; } + public DateOnly? Date { get; init; } + public TimeOnly? Time { get; init; } + public TimeSpan? Duration { get; init; } + public decimal? Amount { get; init; } + public Guid? Id { get; init; } + public Half? HalfVal { get; init; } +} + +[ArrowSerializable] +public partial record WithTimestampOverride +{ + [ArrowType("timestamp[ns, UTC]")] + public DateTimeOffset Value { get; init; } + + [ArrowType("timestamp[ms, UTC]")] + public DateTime DateTimeValue { get; init; } +} + +[ArrowSerializable] +public partial record WithTimeOverride +{ + [ArrowType("time32[ms]")] + public TimeOnly Millis { get; init; } + + [ArrowType("time64[ns]")] + public TimeOnly Nanos { get; init; } +} + +[ArrowSerializable] +public partial record WithDecimalOverride +{ + [ArrowType("decimal128(28, 10)")] + public decimal Value { get; init; } +} + +[ArrowSerializable] +[ArrowMetadata("source", "vgi-rpc-cs")] +[ArrowMetadata("version", "1.0")] +public partial record WithMetadata +{ + [ArrowMetadata("unit", "meters")] + [ArrowMetadata("description", "Distance traveled")] + public double Distance { get; init; } + + public string Label { get; init; } = ""; +} + +[ArrowSerializable] +public partial record WithTransient +{ + public int Id { get; init; } + + [ArrowIgnore] + public string Computed { get; init; } = "default"; + + public string Name { get; init; } = ""; +} + +[ArrowSerializable] +public partial record struct PointRecordStruct +{ + public double X { get; init; } + public double Y { get; init; } +} + +[ArrowSerializable] +public partial class SimpleClass +{ + public int Id { get; init; } + public string Name { get; init; } = ""; +} + +[ArrowSerializable] +public partial struct SimpleStruct +{ + public int Value { get; set; } + public string Label { get; set; } +} + +[ArrowSerializable] +public partial record WithWallClockTimestamp +{ + [ArrowType("timestamp[us]")] + public DateTime WallDateTime { get; init; } + + [ArrowType("timestamp[us]")] + public DateTimeOffset WallDateTimeOffset { get; init; } + + [ArrowType("timestamp[us, UTC]")] + public DateTime UtcDateTime { get; init; } + + [ArrowType("timestamp[us, UTC]")] + public DateTimeOffset UtcDateTimeOffset { get; init; } +} + +[ArrowSerializable] +public partial record WithDateOverride +{ + public DateOnly Default { get; init; } + + [ArrowType("date64")] + public DateOnly Date64Value { get; init; } + + [ArrowType("date32")] + public DateOnly Date32Value { get; init; } +} + +[ArrowSerializable] +public partial record WithBool8 +{ + [ArrowType("bool8")] + public bool Bool8Value { get; init; } + + public bool NormalBool { get; init; } + + [ArrowType("bool8")] + public bool? NullableBool8 { get; init; } +} + +[ArrowSerializable] +public partial record WithViewTypes +{ + [ArrowType("string_view")] + public string ViewString { get; init; } = ""; + + [ArrowType("binary_view")] + public byte[] ViewBinary { get; init; } = System.Array.Empty(); + + public string NormalString { get; init; } = ""; + public byte[] NormalBinary { get; init; } = System.Array.Empty(); +} + +// --- Collection element type override test types --- + +[ArrowSerializable] +public partial record WithListStringView +{ + [ArrowType(ElementType = "string_view")] + public List Items { get; init; } = new(); + + public List NormalItems { get; init; } = new(); +} + +[ArrowSerializable] +public partial record WithDictStringViewKey +{ + [ArrowType(KeyType = "string_view")] + public Dictionary Mapping { get; init; } = new(); +} + +[ArrowSerializable] +public partial record WithDictValueOverride +{ + [ArrowType(ValueType = "string_view")] + public Dictionary Labels { get; init; } = new(); +} + +// --- Custom converter test types --- + +public struct Point2D +{ + public double X { get; set; } + public double Y { get; set; } +} + +public class Point2DArrowConverter : IArrowConverter +{ + public Apache.Arrow.Types.IArrowType ArrowType => Apache.Arrow.Types.StringType.Default; + + public Apache.Arrow.IArrowArray ToArray(Point2D value) + { + return new Apache.Arrow.StringArray.Builder().Append(string.Create(System.Globalization.CultureInfo.InvariantCulture, $"{value.X},{value.Y}")).Build(); + } + + public Apache.Arrow.IArrowArray ToArray(IReadOnlyList values) + { + var b = new Apache.Arrow.StringArray.Builder(); + foreach (var v in values) b.Append(string.Create(System.Globalization.CultureInfo.InvariantCulture, $"{v.X},{v.Y}")); + return b.Build(); + } + + public Point2D FromArray(Apache.Arrow.IArrowArray array, int index) + { + var str = ((Apache.Arrow.StringArray)array).GetString(index)!; + var parts = str.Split(','); + return new Point2D + { + X = double.Parse(parts[0], System.Globalization.CultureInfo.InvariantCulture), + Y = double.Parse(parts[1], System.Globalization.CultureInfo.InvariantCulture), + }; + } +} + +[ArrowSerializable] +public partial record WithCustomConverter +{ + public string Name { get; init; } = ""; + + [ArrowType(Converter = typeof(Point2DArrowConverter))] + public Point2D Origin { get; init; } + + [ArrowType(Converter = typeof(Point2DArrowConverter))] + public Point2D Destination { get; init; } +} + +// --- Polymorphic test types --- + +[ArrowPolymorphic] +[ArrowDerivedType(typeof(Circle), "circle")] +[ArrowDerivedType(typeof(Rectangle), "rectangle")] +public abstract partial record Shape; + +[ArrowSerializable] +public partial record Circle : Shape +{ + public double Radius { get; init; } +} + +[ArrowSerializable] +public partial record Rectangle : Shape +{ + public double Width { get; init; } + public double Height { get; init; } +} + +[ArrowPolymorphic(TypeDiscriminatorFieldName = "kind")] +[ArrowDerivedType(typeof(TextMessage), "text")] +[ArrowDerivedType(typeof(ImageMessage), "image")] +public abstract partial record Message; + +[ArrowSerializable] +public partial record TextMessage : Message +{ + public string Content { get; init; } = ""; +} + +[ArrowSerializable] +public partial record ImageMessage : Message +{ + public string Url { get; init; } = ""; + public int Width { get; init; } + public int Height { get; init; } +} + +// --- Collection types with various element types --- + +[ArrowSerializable] +public partial record WithByteList +{ + public List Values { get; init; } = new(); +} + +[ArrowSerializable] +public partial record WithDateTimeList +{ + public List Timestamps { get; init; } = new(); +} + +[ArrowSerializable] +public partial record WithDecimalList +{ + public List Amounts { get; init; } = new(); +} + +[ArrowSerializable] +public partial record WithGuidList +{ + public List Ids { get; init; } = new(); +} + +[ArrowSerializable] +public partial record WithBinaryList +{ + public List Blobs { get; init; } = new(); +} + +[ArrowSerializable] +public partial record WithIntKeyDict +{ + public Dictionary Lookup { get; init; } = new(); +} + +[ArrowSerializable] +public partial record WithDateTimeOffsetList +{ + public List Timestamps { get; init; } = new(); +} + +[ArrowSerializable] +public partial record WithDateOnlyList +{ + public List Dates { get; init; } = new(); +} + +[ArrowSerializable] +public partial record WithTimeOnlyList +{ + public List Times { get; init; } = new(); +} + +[ArrowSerializable] +public partial record WithTimeSpanList +{ + public List Durations { get; init; } = new(); +} + +[ArrowSerializable] +public partial record WithHalfList +{ + public List Values { get; init; } = new(); +} + +[ArrowSerializable] +public partial record WithAllPrimitiveLists +{ + public List SBytes { get; init; } = new(); + public List Shorts { get; init; } = new(); + public List UShorts { get; init; } = new(); + public List UInts { get; init; } = new(); + public List ULongs { get; init; } = new(); +} + +// --- Deeply nested collection types --- + +[ArrowSerializable] +public partial record WithListOfDicts +{ + public List> Items { get; init; } = new(); +} + +[ArrowSerializable] +public partial record WithDictOfLists +{ + public Dictionary> Groups { get; init; } = new(); +} + +[ArrowSerializable] +public partial record WithNestedDicts +{ + public Dictionary> Nested { get; init; } = new(); +} + +[ArrowSerializable] +public partial record WithListOfListOfDicts +{ + public List>> Matrix { get; init; } = new(); +} + +[ArrowSerializable] +public partial record WithDictOfDictOfLists +{ + public Dictionary>> Deep { get; init; } = new(); +} + +// --- Nullable collection types --- + +[ArrowSerializable] +public partial record WithNullableList +{ + public List? Values { get; init; } +} + +[ArrowSerializable] +public partial record WithNullableDict +{ + public Dictionary? Mapping { get; init; } +} + +[ArrowSerializable] +public partial record WithNullableListOfDicts +{ + public List>? Items { get; init; } +} + +[ArrowSerializable] +public partial record WithNullableDictOfLists +{ + public Dictionary>? Groups { get; init; } +} + +// --- Dict/List with NestedRecord/Enum/Guid leaves --- + +[ArrowSerializable] +public partial record WithDictOfGuids +{ + public Dictionary Items { get; init; } = new(); +} + +[ArrowSerializable] +public partial record WithDictOfEnums +{ + public Dictionary Items { get; init; } = new(); +} + +[ArrowSerializable] +public partial record WithDictOfNested +{ + public Dictionary Items { get; init; } = new(); +} + +// --- Field + constructor types --- + +[ArrowSerializable] +public partial struct PointWithFields +{ + public readonly int X; + public readonly int Y; + + public PointWithFields(int x, int y) + { + X = x; + Y = y; + } +} + +[ArrowSerializable] +public partial struct PointMultiCtor +{ + public readonly int X; + public readonly int Y; + + public PointMultiCtor(int x) + { + X = x; + Y = -1; + } + + public PointMultiCtor(int x, int y) + { + X = x; + Y = y; + } +} + +[ArrowSerializable] +public partial struct MixedFieldsAndProps +{ + public readonly int Id; + public string Name { get; set; } + + public MixedFieldsAndProps(int id, string name) + { + Id = id; + Name = name; + } +} + +[ArrowSerializable] +public partial struct FieldWithArrowField +{ + [ArrowField("x_coord")] + public readonly double X; + [ArrowField("y_coord")] + public readonly double Y; + + public FieldWithArrowField(double x, double y) + { + X = x; + Y = y; + } +} + +[ArrowSerializable] +public partial class WithCallback : IArrowSerializationCallback +{ + public int Value { get; set; } + [ArrowIgnore] public bool BeforeSerializeCalled { get; set; } + [ArrowIgnore] public bool AfterDeserializeCalled { get; set; } + [ArrowIgnore] public int ComputedDouble { get; set; } + + public void OnBeforeSerialize() + { + BeforeSerializeCalled = true; + } + + public void OnAfterDeserialize() + { + AfterDeserializeCalled = true; + ComputedDouble = Value * 2; + } +} From ddc0fe0c09ab76e8f87c57681f29c8e6836a3e09 Mon Sep 17 00:00:00 2001 From: Christoph Mettler Date: Wed, 11 Mar 2026 13:31:27 +0100 Subject: [PATCH 2/9] Change test project target from net10.0 to net8.0 Align with upstream CI which uses .NET 8.0 SDK. All 197 tests pass. Co-Authored-By: Claude Opus 4.6 --- .../Apache.Arrow.Serialization.Tests.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Apache.Arrow.Serialization.Tests/Apache.Arrow.Serialization.Tests.csproj b/test/Apache.Arrow.Serialization.Tests/Apache.Arrow.Serialization.Tests.csproj index 16166422..943f2302 100644 --- a/test/Apache.Arrow.Serialization.Tests/Apache.Arrow.Serialization.Tests.csproj +++ b/test/Apache.Arrow.Serialization.Tests/Apache.Arrow.Serialization.Tests.csproj @@ -1,7 +1,7 @@ - net10.0 + net8.0 enable enable false From c5135356738d7ad9e533c325000ba9b20fecacf0 Mon Sep 17 00:00:00 2001 From: Christoph Mettler Date: Wed, 11 Mar 2026 13:43:08 +0100 Subject: [PATCH 3/9] Fix solution folder nesting and remove net10.0 target Remove src/test solution folders from sln so serialization projects appear at root level like all other projects. Change serialization library target from net8.0;net10.0 to net8.0 for CI compatibility. Co-Authored-By: Claude Opus 4.6 --- Apache.Arrow.sln | 9 --------- .../Apache.Arrow.Serialization.csproj | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/Apache.Arrow.sln b/Apache.Arrow.sln index 310625a4..693efd50 100644 --- a/Apache.Arrow.sln +++ b/Apache.Arrow.sln @@ -38,14 +38,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Apache.Arrow.Operations", " EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Apache.Arrow.Operations.Tests", "test\Apache.Arrow.Operations.Tests\Apache.Arrow.Operations.Tests.csproj", "{73EBE132-AE05-4C32-9525-515F4768156B}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Apache.Arrow.Serialization", "src\Apache.Arrow.Serialization\Apache.Arrow.Serialization.csproj", "{E0C418BE-DD55-4FB1-973E-272B142BAA9E}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Apache.Arrow.Serialization.Generator", "src\Apache.Arrow.Serialization.Generator\Apache.Arrow.Serialization.Generator.csproj", "{FD8B13D7-16F4-4DBF-BB25-13EA5131EE03}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{0C88DD14-F956-CE84-757C-A364CCF449FC}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Apache.Arrow.Serialization.Tests", "test\Apache.Arrow.Serialization.Tests\Apache.Arrow.Serialization.Tests.csproj", "{3726633C-7093-40A1-8ABB-13A5CD64033A}" EndProject Global @@ -314,11 +310,6 @@ Global GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection - GlobalSection(NestedProjects) = preSolution - {E0C418BE-DD55-4FB1-973E-272B142BAA9E} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} - {FD8B13D7-16F4-4DBF-BB25-13EA5131EE03} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} - {3726633C-7093-40A1-8ABB-13A5CD64033A} = {0C88DD14-F956-CE84-757C-A364CCF449FC} - EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {FD0BB617-6031-4844-B99D-B331E335B572} EndGlobalSection diff --git a/src/Apache.Arrow.Serialization/Apache.Arrow.Serialization.csproj b/src/Apache.Arrow.Serialization/Apache.Arrow.Serialization.csproj index 82893383..031efaf9 100644 --- a/src/Apache.Arrow.Serialization/Apache.Arrow.Serialization.csproj +++ b/src/Apache.Arrow.Serialization/Apache.Arrow.Serialization.csproj @@ -1,7 +1,7 @@ - net8.0;net10.0 + net8.0 enable enable Source-generated Apache Arrow serialization for .NET. Provides [ArrowSerializable] attribute and IArrowSerializer<T> interface for compile-time Arrow schema derivation, serialization, and deserialization. From 4f1fcd7e61f9fac3798810c20b7f1edbb020ae9e Mon Sep 17 00:00:00 2001 From: Christoph Mettler Date: Wed, 11 Mar 2026 21:02:41 +0100 Subject: [PATCH 4/9] added license to readme and downgraded Microsoft.CodeAnalysis.CSharp --- Directory.Packages.props | 2 +- src/Apache.Arrow.Serialization/README.md | 19 ++++++++++++++- .../Apache.Arrow.Tests/Apache.Arrow.Tests.sln | 24 +++++++++++++++++++ 3 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 test/Apache.Arrow.Tests/Apache.Arrow.Tests.sln diff --git a/Directory.Packages.props b/Directory.Packages.props index a4a12e79..2127fa77 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -33,7 +33,7 @@ - + diff --git a/src/Apache.Arrow.Serialization/README.md b/src/Apache.Arrow.Serialization/README.md index 5f21924d..3a0eb041 100644 --- a/src/Apache.Arrow.Serialization/README.md +++ b/src/Apache.Arrow.Serialization/README.md @@ -1,3 +1,20 @@ + + # Apache.Arrow.Serialization Source-generated [Apache Arrow](https://arrow.apache.org/) serialization for .NET. @@ -59,7 +76,7 @@ var roundTrip = ArrowSerializerExtensions.DeserializeFromBytes(bytes); dotnet add package Apache.Arrow.Serialization ``` -The NuGet package includes both the runtime library and the Roslyn source generator. Targets `net8.0` and `net10.0`. +The NuGet package includes both the runtime library and the Roslyn source generator. Targets `net8.0`. ## Quick Start diff --git a/test/Apache.Arrow.Tests/Apache.Arrow.Tests.sln b/test/Apache.Arrow.Tests/Apache.Arrow.Tests.sln new file mode 100644 index 00000000..799b96cf --- /dev/null +++ b/test/Apache.Arrow.Tests/Apache.Arrow.Tests.sln @@ -0,0 +1,24 @@ +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.5.2.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Apache.Arrow.Tests", "Apache.Arrow.Tests.csproj", "{3BAAF56E-2C12-8C44-31E2-A16C966B1D20}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {3BAAF56E-2C12-8C44-31E2-A16C966B1D20}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3BAAF56E-2C12-8C44-31E2-A16C966B1D20}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3BAAF56E-2C12-8C44-31E2-A16C966B1D20}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3BAAF56E-2C12-8C44-31E2-A16C966B1D20}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {4E19A52B-785E-4817-8013-698DA567C663} + EndGlobalSection +EndGlobal From 60e173540b0386ff2c248f2cd9ee0d5359120f91 Mon Sep 17 00:00:00 2001 From: Christoph Mettler Date: Wed, 11 Mar 2026 21:08:05 +0100 Subject: [PATCH 5/9] removed Apache.Arrow.Tests.sln --- .../Apache.Arrow.Tests/Apache.Arrow.Tests.sln | 24 ------------------- 1 file changed, 24 deletions(-) delete mode 100644 test/Apache.Arrow.Tests/Apache.Arrow.Tests.sln diff --git a/test/Apache.Arrow.Tests/Apache.Arrow.Tests.sln b/test/Apache.Arrow.Tests/Apache.Arrow.Tests.sln deleted file mode 100644 index 799b96cf..00000000 --- a/test/Apache.Arrow.Tests/Apache.Arrow.Tests.sln +++ /dev/null @@ -1,24 +0,0 @@ -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.5.2.0 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Apache.Arrow.Tests", "Apache.Arrow.Tests.csproj", "{3BAAF56E-2C12-8C44-31E2-A16C966B1D20}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Release|Any CPU = Release|Any CPU - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {3BAAF56E-2C12-8C44-31E2-A16C966B1D20}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {3BAAF56E-2C12-8C44-31E2-A16C966B1D20}.Debug|Any CPU.Build.0 = Debug|Any CPU - {3BAAF56E-2C12-8C44-31E2-A16C966B1D20}.Release|Any CPU.ActiveCfg = Release|Any CPU - {3BAAF56E-2C12-8C44-31E2-A16C966B1D20}.Release|Any CPU.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {4E19A52B-785E-4817-8013-698DA567C663} - EndGlobalSection -EndGlobal From 922cb4ae6264a0407f81cef5e21d49c3f9381b6d Mon Sep 17 00:00:00 2001 From: Christoph Mettler Date: Tue, 7 Jul 2026 14:40:32 +0200 Subject: [PATCH 6/9] Address PR review feedback - Rename TypeKind2 enum to ArrowTypeKind (review suggestion) - RecordBatchBuilder: allow empty collections (schema is inferred from the type, not the data) and add fields directly to Schema.Builder - RecordBatchBuilder: convert non-UTC DateTime kinds via ToUtcDateTimeOffset instead of misinterpreting local time as UTC - RecordBatchBuilder: handle all-null struct columns by creating a default stand-in instance instead of NullReferenceException - PolymorphicCodeEmitter: deduplicate enum dictionary entries so repeated values share one index - PolymorphicCodeEmitter: throw NotSupportedException for unsupported complex property types instead of silently serializing null - Pack the source generator into the Apache.Arrow.Serialization nupkg (analyzers/dotnet/cs) and add the package README - docs.sh: build the serialization library before docfx metadata - verify_rc.sh: verify the Apache.Arrow.Serialization package - Directory.Packages.props: restore alphabetical package ordering - README: document CLR decimal vs Arrow Decimal128 mismatch - Run dotnet format over the serialization projects Co-Authored-By: Claude Fable 5 --- Directory.Packages.props | 2 +- ci/scripts/docs.sh | 4 + dev/release/verify_rc.sh | 6 + .../ArrowSerializerGenerator.cs | 1350 ++-- .../CodeEmitter.cs | 5570 ++++++++--------- .../JsonSchemaEmitter.cs | 292 +- .../Models.cs | 4 +- .../PolymorphicCodeEmitter.cs | 1328 ++-- .../Apache.Arrow.Serialization.csproj | 14 + .../ArrowArrayHelper.cs | 313 +- src/Apache.Arrow.Serialization/README.md | 2 + .../RecordBatchBuilder.cs | 26 +- .../DiagnosticTests.cs | 2 +- .../RecordBatchBuilderTests.cs | 14 +- .../SerializationTests.cs | 2 +- 15 files changed, 4492 insertions(+), 4437 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 2127fa77..07a50b19 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -32,9 +32,9 @@ + - diff --git a/ci/scripts/docs.sh b/ci/scripts/docs.sh index 9b837fce..61a36395 100755 --- a/ci/scripts/docs.sh +++ b/ci/scripts/docs.sh @@ -21,6 +21,10 @@ set -eux source_dir=${1} +# Build the serialization library first so its source generator output is +# available when docfx extracts metadata. +dotnet build "${source_dir}/src/Apache.Arrow.Serialization/Apache.Arrow.Serialization.csproj" + pushd "${source_dir}/docs" dotnet tool install -g docfx diff --git a/dev/release/verify_rc.sh b/dev/release/verify_rc.sh index e2c22941..8f592b2f 100755 --- a/dev/release/verify_rc.sh +++ b/dev/release/verify_rc.sh @@ -186,6 +186,12 @@ test_binary_distribution() { reference_package "Apache.Arrow.Flight.AspNetCore" "Apache.Arrow.Flight.TestWeb" reference_package "Apache.Arrow.Operations" "Apache.Arrow.Operations.Tests" "Apache.Arrow.Scalars.Tests" reference_package "Apache.Arrow.Scalars" "Apache.Arrow.Scalars.Tests" "Apache.Arrow.Tests" "Apache.Arrow.Operations.Tests" + reference_package "Apache.Arrow.Serialization" "Apache.Arrow.Serialization.Tests" + # The source generator ships inside the Apache.Arrow.Serialization package + # as a Roslyn analyzer; drop the direct project reference so the packaged + # generator is exercised instead. + dotnet remove "test/Apache.Arrow.Serialization.Tests" \ + reference "src/Apache.Arrow.Serialization.Generator/Apache.Arrow.Serialization.Generator.csproj" # Move src directory to ensure we are only testing against built packages mv src src.backup diff --git a/src/Apache.Arrow.Serialization.Generator/ArrowSerializerGenerator.cs b/src/Apache.Arrow.Serialization.Generator/ArrowSerializerGenerator.cs index 27f04265..95cf42c7 100644 --- a/src/Apache.Arrow.Serialization.Generator/ArrowSerializerGenerator.cs +++ b/src/Apache.Arrow.Serialization.Generator/ArrowSerializerGenerator.cs @@ -26,830 +26,830 @@ namespace Apache.Arrow.Serialization.Generator { -[Generator(LanguageNames.CSharp)] -public class ArrowSerializerGenerator : IIncrementalGenerator -{ - private static readonly DiagnosticDescriptor NonPartialType = new DiagnosticDescriptor( - "ARROW001", "Type must be partial", - "[ArrowSerializable] type '{0}' must be declared as partial", - "ArrowSerialization", DiagnosticSeverity.Error, true); - - private static readonly DiagnosticDescriptor NoMatchingConstructor = new DiagnosticDescriptor( - "ARROW002", "No matching constructor for readonly fields", - "[ArrowSerializable] type '{0}' has readonly fields but no public constructor with matching parameters", - "ArrowSerialization", DiagnosticSeverity.Error, true); - - private static readonly DiagnosticDescriptor UnsupportedMemberType = new DiagnosticDescriptor( - "ARROW003", "Unsupported member type", - "Member '{0}' on type '{1}' has unsupported type '{2}'", - "ArrowSerialization", DiagnosticSeverity.Error, true); - - private static readonly DiagnosticDescriptor DuplicateFieldName = new DiagnosticDescriptor( - "ARROW004", "Duplicate Arrow field name", - "Members '{0}' and '{1}' on type '{2}' both map to Arrow field name '{3}'", - "ArrowSerialization", DiagnosticSeverity.Error, true); - - private static readonly DiagnosticDescriptor NonSettableProperty = new DiagnosticDescriptor( - "ARROW005", "Property is not settable", - "Property '{0}' on type '{1}' has no set or init accessor and cannot be deserialized", - "ArrowSerialization", DiagnosticSeverity.Error, true); - - private static readonly DiagnosticDescriptor ArrowAttributeOnIgnoredMember = new DiagnosticDescriptor( - "ARROW006", "Arrow attribute on non-serialized member", - "Member '{0}' on type '{1}' has Arrow attributes but is not serialized because it is {2}", - "ArrowSerialization", DiagnosticSeverity.Warning, true); - - public void Initialize(IncrementalGeneratorInitializationContext context) + [Generator(LanguageNames.CSharp)] + public class ArrowSerializerGenerator : IIncrementalGenerator { - var emitSchemaJson = context.AnalyzerConfigOptionsProvider.Select(static (options, _) => - { - options.GlobalOptions.TryGetValue("build_property.ArrowSerializerEmitSchemaJson", out var value); - return string.Equals(value, "true", StringComparison.OrdinalIgnoreCase); - }); - - var types = context.SyntaxProvider.ForAttributeWithMetadataName( - "Apache.Arrow.Serialization.ArrowSerializableAttribute", - predicate: static (node, _) => node is RecordDeclarationSyntax || node is ClassDeclarationSyntax || node is StructDeclarationSyntax, - transform: static (ctx, _) => GetTypeModel(ctx)) - .Where(static m => m != null) - .Select(static (m, _) => m!); - - var typesWithConfig = types.Combine(emitSchemaJson); - context.RegisterSourceOutput(typesWithConfig, static (spc, pair) => Execute(spc, pair.Left, pair.Right)); - - var polyTypes = context.SyntaxProvider.ForAttributeWithMetadataName( - "Apache.Arrow.Serialization.ArrowPolymorphicAttribute", - predicate: static (node, _) => node is RecordDeclarationSyntax || node is ClassDeclarationSyntax || node is StructDeclarationSyntax || node is InterfaceDeclarationSyntax, - transform: static (ctx, _) => GetPolymorphicModel(ctx)) - .Where(static m => m != null) - .Select(static (m, _) => m!); - - context.RegisterSourceOutput(polyTypes, static (spc, model) => ExecutePolymorphic(spc, model)); - } + private static readonly DiagnosticDescriptor NonPartialType = new DiagnosticDescriptor( + "ARROW001", "Type must be partial", + "[ArrowSerializable] type '{0}' must be declared as partial", + "ArrowSerialization", DiagnosticSeverity.Error, true); + + private static readonly DiagnosticDescriptor NoMatchingConstructor = new DiagnosticDescriptor( + "ARROW002", "No matching constructor for readonly fields", + "[ArrowSerializable] type '{0}' has readonly fields but no public constructor with matching parameters", + "ArrowSerialization", DiagnosticSeverity.Error, true); + + private static readonly DiagnosticDescriptor UnsupportedMemberType = new DiagnosticDescriptor( + "ARROW003", "Unsupported member type", + "Member '{0}' on type '{1}' has unsupported type '{2}'", + "ArrowSerialization", DiagnosticSeverity.Error, true); + + private static readonly DiagnosticDescriptor DuplicateFieldName = new DiagnosticDescriptor( + "ARROW004", "Duplicate Arrow field name", + "Members '{0}' and '{1}' on type '{2}' both map to Arrow field name '{3}'", + "ArrowSerialization", DiagnosticSeverity.Error, true); + + private static readonly DiagnosticDescriptor NonSettableProperty = new DiagnosticDescriptor( + "ARROW005", "Property is not settable", + "Property '{0}' on type '{1}' has no set or init accessor and cannot be deserialized", + "ArrowSerialization", DiagnosticSeverity.Error, true); + + private static readonly DiagnosticDescriptor ArrowAttributeOnIgnoredMember = new DiagnosticDescriptor( + "ARROW006", "Arrow attribute on non-serialized member", + "Member '{0}' on type '{1}' has Arrow attributes but is not serialized because it is {2}", + "ArrowSerialization", DiagnosticSeverity.Warning, true); + + public void Initialize(IncrementalGeneratorInitializationContext context) + { + var emitSchemaJson = context.AnalyzerConfigOptionsProvider.Select(static (options, _) => + { + options.GlobalOptions.TryGetValue("build_property.ArrowSerializerEmitSchemaJson", out var value); + return string.Equals(value, "true", StringComparison.OrdinalIgnoreCase); + }); - private static TypeModel? GetTypeModel(GeneratorAttributeSyntaxContext ctx) - { - if (!(ctx.TargetSymbol is INamedTypeSymbol typeSymbol)) - return null; + var types = context.SyntaxProvider.ForAttributeWithMetadataName( + "Apache.Arrow.Serialization.ArrowSerializableAttribute", + predicate: static (node, _) => node is RecordDeclarationSyntax || node is ClassDeclarationSyntax || node is StructDeclarationSyntax, + transform: static (ctx, _) => GetTypeModel(ctx)) + .Where(static m => m != null) + .Select(static (m, _) => m!); - var properties = new List(); - var ignoredMemberWarnings = new List(); - int declOrder = 0; - foreach (var member in typeSymbol.GetMembers()) - { - // Support both properties and fields - ITypeSymbol memberType; - string memberName; - bool isField = false; - if (member is IPropertySymbol prop) - { - string? skipReason = null; - if (prop.IsStatic) skipReason = "static"; - else if (prop.IsIndexer) skipReason = "an indexer"; - else if (prop.DeclaredAccessibility != Accessibility.Public) skipReason = "not public"; - - if (skipReason != null) - { - if (HasArrowAttributes(prop)) - ignoredMemberWarnings.Add(new DiagnosticInfo { Id = "ARROW006", Message = $"{prop.Name}\t{typeSymbol.Name}\t{skipReason}", IsError = false }); - continue; - } - memberType = prop.Type; - memberName = prop.Name; - } - else if (member is IFieldSymbol field) - { - if (field.IsImplicitlyDeclared) - continue; + var typesWithConfig = types.Combine(emitSchemaJson); + context.RegisterSourceOutput(typesWithConfig, static (spc, pair) => Execute(spc, pair.Left, pair.Right)); - string? skipReason = null; - if (field.IsStatic || field.IsConst) skipReason = "static or const"; - else if (field.DeclaredAccessibility != Accessibility.Public) skipReason = "not public"; + var polyTypes = context.SyntaxProvider.ForAttributeWithMetadataName( + "Apache.Arrow.Serialization.ArrowPolymorphicAttribute", + predicate: static (node, _) => node is RecordDeclarationSyntax || node is ClassDeclarationSyntax || node is StructDeclarationSyntax || node is InterfaceDeclarationSyntax, + transform: static (ctx, _) => GetPolymorphicModel(ctx)) + .Where(static m => m != null) + .Select(static (m, _) => m!); - if (skipReason != null) - { - if (HasArrowAttributes(field)) - ignoredMemberWarnings.Add(new DiagnosticInfo { Id = "ARROW006", Message = $"{field.Name}\t{typeSymbol.Name}\t{skipReason}", IsError = false }); - continue; - } - memberType = field.Type; - memberName = field.Name; - isField = true; - } - else - { - continue; - } + context.RegisterSourceOutput(polyTypes, static (spc, model) => ExecutePolymorphic(spc, model)); + } - // Check for [ArrowIgnore] - bool isTransient = false; - string? arrowTypeName = null; - string? converterTypeName = null; - string? elementTypeOverride = null; - string? keyTypeOverride = null; - string? valueTypeOverride = null; - string? fieldName = null; - int order = int.MaxValue; - var propMetadata = new List>(); + private static TypeModel? GetTypeModel(GeneratorAttributeSyntaxContext ctx) + { + if (!(ctx.TargetSymbol is INamedTypeSymbol typeSymbol)) + return null; - foreach (var attr in member.GetAttributes()) + var properties = new List(); + var ignoredMemberWarnings = new List(); + int declOrder = 0; + foreach (var member in typeSymbol.GetMembers()) { - var attrName = attr.AttributeClass?.ToDisplayString(); - if (attrName == "Apache.Arrow.Serialization.ArrowIgnoreAttribute") - { - isTransient = true; - } - else if (attrName == "Apache.Arrow.Serialization.ArrowTypeAttribute") + // Support both properties and fields + ITypeSymbol memberType; + string memberName; + bool isField = false; + if (member is IPropertySymbol prop) { - if (attr.ConstructorArguments.Length > 0) - arrowTypeName = attr.ConstructorArguments[0].Value as string; - foreach (var named in attr.NamedArguments) + string? skipReason = null; + if (prop.IsStatic) skipReason = "static"; + else if (prop.IsIndexer) skipReason = "an indexer"; + else if (prop.DeclaredAccessibility != Accessibility.Public) skipReason = "not public"; + + if (skipReason != null) { - if (named.Key == "Converter" && named.Value.Value is INamedTypeSymbol converterSym) - converterTypeName = converterSym.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat); - else if (named.Key == "ElementType" && named.Value.Value is string et) - elementTypeOverride = et; - else if (named.Key == "KeyType" && named.Value.Value is string kt) - keyTypeOverride = kt; - else if (named.Key == "ValueType" && named.Value.Value is string vt) - valueTypeOverride = vt; + if (HasArrowAttributes(prop)) + ignoredMemberWarnings.Add(new DiagnosticInfo { Id = "ARROW006", Message = $"{prop.Name}\t{typeSymbol.Name}\t{skipReason}", IsError = false }); + continue; } + memberType = prop.Type; + memberName = prop.Name; } - else if (attrName == "Apache.Arrow.Serialization.ArrowFieldAttribute") + else if (member is IFieldSymbol field) { - if (attr.ConstructorArguments.Length > 0) - fieldName = attr.ConstructorArguments[0].Value as string; - foreach (var named in attr.NamedArguments) + if (field.IsImplicitlyDeclared) + continue; + + string? skipReason = null; + if (field.IsStatic || field.IsConst) skipReason = "static or const"; + else if (field.DeclaredAccessibility != Accessibility.Public) skipReason = "not public"; + + if (skipReason != null) { - if (named.Key == "Order" && named.Value.Value is int o) - order = o; + if (HasArrowAttributes(field)) + ignoredMemberWarnings.Add(new DiagnosticInfo { Id = "ARROW006", Message = $"{field.Name}\t{typeSymbol.Name}\t{skipReason}", IsError = false }); + continue; } + memberType = field.Type; + memberName = field.Name; + isField = true; } - else if (attrName == "Apache.Arrow.Serialization.ArrowMetadataAttribute") + else { - if (attr.ConstructorArguments.Length >= 2) + continue; + } + + // Check for [ArrowIgnore] + bool isTransient = false; + string? arrowTypeName = null; + string? converterTypeName = null; + string? elementTypeOverride = null; + string? keyTypeOverride = null; + string? valueTypeOverride = null; + string? fieldName = null; + int order = int.MaxValue; + var propMetadata = new List>(); + + foreach (var attr in member.GetAttributes()) + { + var attrName = attr.AttributeClass?.ToDisplayString(); + if (attrName == "Apache.Arrow.Serialization.ArrowIgnoreAttribute") + { + isTransient = true; + } + else if (attrName == "Apache.Arrow.Serialization.ArrowTypeAttribute") + { + if (attr.ConstructorArguments.Length > 0) + arrowTypeName = attr.ConstructorArguments[0].Value as string; + foreach (var named in attr.NamedArguments) + { + if (named.Key == "Converter" && named.Value.Value is INamedTypeSymbol converterSym) + converterTypeName = converterSym.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat); + else if (named.Key == "ElementType" && named.Value.Value is string et) + elementTypeOverride = et; + else if (named.Key == "KeyType" && named.Value.Value is string kt) + keyTypeOverride = kt; + else if (named.Key == "ValueType" && named.Value.Value is string vt) + valueTypeOverride = vt; + } + } + else if (attrName == "Apache.Arrow.Serialization.ArrowFieldAttribute") + { + if (attr.ConstructorArguments.Length > 0) + fieldName = attr.ConstructorArguments[0].Value as string; + foreach (var named in attr.NamedArguments) + { + if (named.Key == "Order" && named.Value.Value is int o) + order = o; + } + } + else if (attrName == "Apache.Arrow.Serialization.ArrowMetadataAttribute") { - var key = attr.ConstructorArguments[0].Value as string; - var val = attr.ConstructorArguments[1].Value as string; - if (key != null && val != null) - propMetadata.Add(new KeyValuePair(key, val)); + if (attr.ConstructorArguments.Length >= 2) + { + var key = attr.ConstructorArguments[0].Value as string; + var val = attr.ConstructorArguments[1].Value as string; + if (key != null && val != null) + propMetadata.Add(new KeyValuePair(key, val)); + } } } - } - if (isTransient) - continue; + if (isTransient) + continue; - bool isNullable = memberType.NullableAnnotation == NullableAnnotation.Annotated; - if (isNullable && memberType is INamedTypeSymbol nullableType && nullableType.OriginalDefinition.SpecialType == SpecialType.System_Nullable_T) - { - memberType = nullableType.TypeArguments[0]; + bool isNullable = memberType.NullableAnnotation == NullableAnnotation.Annotated; + if (isNullable && memberType is INamedTypeSymbol nullableType && nullableType.OriginalDefinition.SpecialType == SpecialType.System_Nullable_T) + { + memberType = nullableType.TypeArguments[0]; + } + + var typeInfo = AnalyzeType(memberType, isNullable); + if (converterTypeName != null) + typeInfo = new TypeInfo { Kind = ArrowTypeKind.Custom, FullTypeName = typeInfo.FullTypeName, IsNullable = isNullable }; + else if (arrowTypeName != null) + typeInfo = typeInfo.WithOverride(arrowTypeName); + + // Apply element/key/value type overrides for collections + if (elementTypeOverride != null && typeInfo.ElementType != null) + typeInfo.ElementType = typeInfo.ElementType.WithOverride(elementTypeOverride); + if (keyTypeOverride != null && typeInfo.KeyType != null) + typeInfo.KeyType = typeInfo.KeyType.WithOverride(keyTypeOverride); + if (valueTypeOverride != null && typeInfo.ValueType != null) + typeInfo.ValueType = typeInfo.ValueType.WithOverride(valueTypeOverride); + + properties.Add(new PropertyModel + { + PropertyName = memberName, + FieldName = fieldName ?? memberName, + Order = order, + DeclOrder = declOrder++, + Type = typeInfo, + IsNullable = isNullable, + HasDefaultValue = isField ? false : HasDefaultValue((IPropertySymbol)member, typeSymbol), + Metadata = propMetadata, + ConverterTypeName = converterTypeName, + IsField = isField, + }); } - var typeInfo = AnalyzeType(memberType, isNullable); - if (converterTypeName != null) - typeInfo = new TypeInfo { Kind = TypeKind2.Custom, FullTypeName = typeInfo.FullTypeName, IsNullable = isNullable }; - else if (arrowTypeName != null) - typeInfo = typeInfo.WithOverride(arrowTypeName); - - // Apply element/key/value type overrides for collections - if (elementTypeOverride != null && typeInfo.ElementType != null) - typeInfo.ElementType = typeInfo.ElementType.WithOverride(elementTypeOverride); - if (keyTypeOverride != null && typeInfo.KeyType != null) - typeInfo.KeyType = typeInfo.KeyType.WithOverride(keyTypeOverride); - if (valueTypeOverride != null && typeInfo.ValueType != null) - typeInfo.ValueType = typeInfo.ValueType.WithOverride(valueTypeOverride); - - properties.Add(new PropertyModel - { - PropertyName = memberName, - FieldName = fieldName ?? memberName, - Order = order, - DeclOrder = declOrder++, - Type = typeInfo, - IsNullable = isNullable, - HasDefaultValue = isField ? false : HasDefaultValue((IPropertySymbol)member, typeSymbol), - Metadata = propMetadata, - ConverterTypeName = converterTypeName, - IsField = isField, + properties.Sort((a, b) => + { + int cmp = a.Order.CompareTo(b.Order); + return cmp != 0 ? cmp : a.DeclOrder.CompareTo(b.DeclOrder); }); - } - properties.Sort((a, b) => - { - int cmp = a.Order.CompareTo(b.Order); - return cmp != 0 ? cmp : a.DeclOrder.CompareTo(b.DeclOrder); - }); + var typeMetadata = new List>(); + foreach (var attr in typeSymbol.GetAttributes()) + { + if (attr.AttributeClass?.ToDisplayString() == "Apache.Arrow.Serialization.ArrowMetadataAttribute" + && attr.ConstructorArguments.Length >= 2) + { + var key = attr.ConstructorArguments[0].Value as string; + var val = attr.ConstructorArguments[1].Value as string; + if (key != null && val != null) + typeMetadata.Add(new KeyValuePair(key, val)); + } + } - var typeMetadata = new List>(); - foreach (var attr in typeSymbol.GetAttributes()) - { - if (attr.AttributeClass?.ToDisplayString() == "Apache.Arrow.Serialization.ArrowMetadataAttribute" - && attr.ConstructorArguments.Length >= 2) + // Determine if constructor-based deserialization is needed + // This is required when any member is a readonly field (can't use object initializer) + bool needsConstructor = false; + foreach (var p in properties) { - var key = attr.ConstructorArguments[0].Value as string; - var val = attr.ConstructorArguments[1].Value as string; - if (key != null && val != null) - typeMetadata.Add(new KeyValuePair(key, val)); + if (p.IsField) { needsConstructor = true; break; } } - } - // Determine if constructor-based deserialization is needed - // This is required when any member is a readonly field (can't use object initializer) - bool needsConstructor = false; - foreach (var p in properties) - { - if (p.IsField) { needsConstructor = true; break; } - } + List? ctorParams = null; + if (needsConstructor) + { + ctorParams = ResolveConstructor(typeSymbol, properties); + } - List? ctorParams = null; - if (needsConstructor) - { - ctorParams = ResolveConstructor(typeSymbol, properties); - } + // Validate and collect diagnostics + var diagnostics = new List(ignoredMemberWarnings); + var typeName = typeSymbol.Name; - // Validate and collect diagnostics - var diagnostics = new List(ignoredMemberWarnings); - var typeName = typeSymbol.Name; + // ARROW001: non-partial type + if (ctx.TargetNode is TypeDeclarationSyntax tds && !tds.Modifiers.Any(Microsoft.CodeAnalysis.CSharp.SyntaxKind.PartialKeyword)) + { + diagnostics.Add(new DiagnosticInfo { Id = "ARROW001", Message = typeName, IsError = true }); + } - // ARROW001: non-partial type - if (ctx.TargetNode is TypeDeclarationSyntax tds && !tds.Modifiers.Any(Microsoft.CodeAnalysis.CSharp.SyntaxKind.PartialKeyword)) - { - diagnostics.Add(new DiagnosticInfo { Id = "ARROW001", Message = typeName, IsError = true }); - } + // ARROW002: readonly fields with no matching constructor + if (needsConstructor && ctorParams == null) + { + diagnostics.Add(new DiagnosticInfo { Id = "ARROW002", Message = typeName, IsError = true }); + } - // ARROW002: readonly fields with no matching constructor - if (needsConstructor && ctorParams == null) - { - diagnostics.Add(new DiagnosticInfo { Id = "ARROW002", Message = typeName, IsError = true }); - } + // ARROW003: unsupported member types + foreach (var p in properties) + { + if (p.Type.Kind == ArrowTypeKind.Unknown) + diagnostics.Add(new DiagnosticInfo { Id = "ARROW003", Message = $"{p.PropertyName}\t{typeName}\t{p.Type.FullTypeName}", IsError = true }); + } - // ARROW003: unsupported member types - foreach (var p in properties) - { - if (p.Type.Kind == TypeKind2.Unknown) - diagnostics.Add(new DiagnosticInfo { Id = "ARROW003", Message = $"{p.PropertyName}\t{typeName}\t{p.Type.FullTypeName}", IsError = true }); - } + // ARROW004: duplicate Arrow field names + var seenFields = new Dictionary(); + foreach (var p in properties) + { + if (seenFields.TryGetValue(p.FieldName, out var existingMember)) + diagnostics.Add(new DiagnosticInfo { Id = "ARROW004", Message = $"{existingMember}\t{p.PropertyName}\t{typeName}\t{p.FieldName}", IsError = true }); + else + seenFields[p.FieldName] = p.PropertyName; + } - // ARROW004: duplicate Arrow field names - var seenFields = new Dictionary(); - foreach (var p in properties) - { - if (seenFields.TryGetValue(p.FieldName, out var existingMember)) - diagnostics.Add(new DiagnosticInfo { Id = "ARROW004", Message = $"{existingMember}\t{p.PropertyName}\t{typeName}\t{p.FieldName}", IsError = true }); - else - seenFields[p.FieldName] = p.PropertyName; + // ARROW005: non-settable properties (no set/init, not in constructor) + if (ctorParams == null) + { + foreach (var member in typeSymbol.GetMembers()) + { + if (!(member is IPropertySymbol ps) || ps.IsStatic || ps.IsIndexer) + continue; + if (ps.DeclaredAccessibility != Accessibility.Public) + continue; + if (ps.GetAttributes().Any(a => a.AttributeClass?.ToDisplayString() == "Apache.Arrow.Serialization.ArrowIgnoreAttribute")) + continue; + if (ps.SetMethod == null) + diagnostics.Add(new DiagnosticInfo { Id = "ARROW005", Message = $"{ps.Name}\t{typeName}", IsError = true }); + } + } + + return new TypeModel + { + Namespace = typeSymbol.ContainingNamespace.IsGlobalNamespace ? null : typeSymbol.ContainingNamespace.ToDisplayString(), + TypeName = typeSymbol.Name, + FullTypeName = typeSymbol.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat), + Properties = properties, + IsRecord = typeSymbol.IsRecord, + IsValueType = typeSymbol.IsValueType, + Metadata = typeMetadata, + HasArrowSerializableBase = HasArrowSerializableBaseType(typeSymbol), + ConstructorParams = ctorParams, + HasSerializationCallback = ImplementsInterface(typeSymbol, "Apache.Arrow.Serialization.IArrowSerializationCallback"), + Diagnostics = diagnostics, + }; } - // ARROW005: non-settable properties (no set/init, not in constructor) - if (ctorParams == null) + private static bool HasArrowSerializableBaseType(INamedTypeSymbol typeSymbol) { - foreach (var member in typeSymbol.GetMembers()) + var baseType = typeSymbol.BaseType; + while (baseType != null && baseType.SpecialType != SpecialType.System_Object) { - if (!(member is IPropertySymbol ps) || ps.IsStatic || ps.IsIndexer) - continue; - if (ps.DeclaredAccessibility != Accessibility.Public) - continue; - if (ps.GetAttributes().Any(a => a.AttributeClass?.ToDisplayString() == "Apache.Arrow.Serialization.ArrowIgnoreAttribute")) - continue; - if (ps.SetMethod == null) - diagnostics.Add(new DiagnosticInfo { Id = "ARROW005", Message = $"{ps.Name}\t{typeName}", IsError = true }); + foreach (var attr in baseType.GetAttributes()) + { + var attrName = attr.AttributeClass?.ToDisplayString(); + if (attrName == "Apache.Arrow.Serialization.ArrowSerializableAttribute" || + attrName == "Apache.Arrow.Serialization.ArrowPolymorphicAttribute") + return true; + } + baseType = baseType.BaseType; } + return false; } - return new TypeModel - { - Namespace = typeSymbol.ContainingNamespace.IsGlobalNamespace ? null : typeSymbol.ContainingNamespace.ToDisplayString(), - TypeName = typeSymbol.Name, - FullTypeName = typeSymbol.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat), - Properties = properties, - IsRecord = typeSymbol.IsRecord, - IsValueType = typeSymbol.IsValueType, - Metadata = typeMetadata, - HasArrowSerializableBase = HasArrowSerializableBaseType(typeSymbol), - ConstructorParams = ctorParams, - HasSerializationCallback = ImplementsInterface(typeSymbol, "Apache.Arrow.Serialization.IArrowSerializationCallback"), - Diagnostics = diagnostics, - }; - } - - private static bool HasArrowSerializableBaseType(INamedTypeSymbol typeSymbol) - { - var baseType = typeSymbol.BaseType; - while (baseType != null && baseType.SpecialType != SpecialType.System_Object) + private static bool ImplementsInterface(INamedTypeSymbol typeSymbol, string interfaceFullName) { - foreach (var attr in baseType.GetAttributes()) + foreach (var iface in typeSymbol.AllInterfaces) { - var attrName = attr.AttributeClass?.ToDisplayString(); - if (attrName == "Apache.Arrow.Serialization.ArrowSerializableAttribute" || - attrName == "Apache.Arrow.Serialization.ArrowPolymorphicAttribute") + if (iface.ToDisplayString() == interfaceFullName) return true; } - baseType = baseType.BaseType; + return false; } - return false; - } - private static bool ImplementsInterface(INamedTypeSymbol typeSymbol, string interfaceFullName) - { - foreach (var iface in typeSymbol.AllInterfaces) + private static bool HasArrowAttributes(ISymbol member) { - if (iface.ToDisplayString() == interfaceFullName) - return true; + foreach (var attr in member.GetAttributes()) + { + var name = attr.AttributeClass?.ToDisplayString(); + if (name != null && name.StartsWith("Apache.Arrow.Serialization.Arrow")) + return true; + } + return false; } - return false; - } - private static bool HasArrowAttributes(ISymbol member) - { - foreach (var attr in member.GetAttributes()) + private static bool HasDefaultValue(IPropertySymbol prop, INamedTypeSymbol containingType) { - var name = attr.AttributeClass?.ToDisplayString(); - if (name != null && name.StartsWith("Apache.Arrow.Serialization.Arrow")) + // Check if the property has a default in the constructor or an initializer + // For simplicity, we check if it's nullable or if there are constructors with optional params + if (prop.NullableAnnotation == NullableAnnotation.Annotated) return true; - } - return false; - } - - private static bool HasDefaultValue(IPropertySymbol prop, INamedTypeSymbol containingType) - { - // Check if the property has a default in the constructor or an initializer - // For simplicity, we check if it's nullable or if there are constructors with optional params - if (prop.NullableAnnotation == NullableAnnotation.Annotated) - return true; - foreach (var ctor in containingType.Constructors) - { - if (ctor.IsImplicitlyDeclared) continue; - foreach (var param in ctor.Parameters) + foreach (var ctor in containingType.Constructors) { - if (param.Name.Equals(prop.Name, StringComparison.OrdinalIgnoreCase) && param.HasExplicitDefaultValue) - return true; + if (ctor.IsImplicitlyDeclared) continue; + foreach (var param in ctor.Parameters) + { + if (param.Name.Equals(prop.Name, StringComparison.OrdinalIgnoreCase) && param.HasExplicitDefaultValue) + return true; + } } + + return false; } - return false; - } + /// + /// Find the best constructor for deserialization. Picks the constructor whose parameters + /// best match the serialized members (by name, case-insensitive). Prefers the constructor + /// with the most matching parameters. + /// + private static List? ResolveConstructor(INamedTypeSymbol typeSymbol, List properties) + { + var memberNames = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var p in properties) + memberNames.Add(p.PropertyName); - /// - /// Find the best constructor for deserialization. Picks the constructor whose parameters - /// best match the serialized members (by name, case-insensitive). Prefers the constructor - /// with the most matching parameters. - /// - private static List? ResolveConstructor(INamedTypeSymbol typeSymbol, List properties) - { - var memberNames = new HashSet(StringComparer.OrdinalIgnoreCase); - foreach (var p in properties) - memberNames.Add(p.PropertyName); + IMethodSymbol? bestCtor = null; + int bestMatch = -1; - IMethodSymbol? bestCtor = null; - int bestMatch = -1; + foreach (var ctor in typeSymbol.Constructors) + { + if (ctor.IsStatic || ctor.IsImplicitlyDeclared) + continue; + if (ctor.DeclaredAccessibility != Accessibility.Public) + continue; - foreach (var ctor in typeSymbol.Constructors) - { - if (ctor.IsStatic || ctor.IsImplicitlyDeclared) - continue; - if (ctor.DeclaredAccessibility != Accessibility.Public) - continue; - - // Count how many required params match members - int matchCount = 0; - bool allRequiredMatch = true; - foreach (var param in ctor.Parameters) - { - if (memberNames.Contains(param.Name)) - matchCount++; - else if (!param.HasExplicitDefaultValue) + // Count how many required params match members + int matchCount = 0; + bool allRequiredMatch = true; + foreach (var param in ctor.Parameters) { - allRequiredMatch = false; - break; + if (memberNames.Contains(param.Name)) + matchCount++; + else if (!param.HasExplicitDefaultValue) + { + allRequiredMatch = false; + break; + } } - } - if (allRequiredMatch && matchCount > bestMatch) - { - bestMatch = matchCount; - bestCtor = ctor; + if (allRequiredMatch && matchCount > bestMatch) + { + bestMatch = matchCount; + bestCtor = ctor; + } } - } - if (bestCtor == null) - return null; + if (bestCtor == null) + return null; - var result = new List(); - foreach (var param in bestCtor.Parameters) - { - result.Add(new ConstructorParamModel + var result = new List(); + foreach (var param in bestCtor.Parameters) { - Name = param.Name, - HasDefaultValue = param.HasExplicitDefaultValue, - }); - } - return result; - } - - private static TypeInfo AnalyzeType(ITypeSymbol type, bool isNullable) - { - // Unwrap Nullable for value types - if (type is INamedTypeSymbol nt && nt.OriginalDefinition.SpecialType == SpecialType.System_Nullable_T) - { - type = nt.TypeArguments[0]; - isNullable = true; + result.Add(new ConstructorParamModel + { + Name = param.Name, + HasDefaultValue = param.HasExplicitDefaultValue, + }); + } + return result; } - var fullName = type.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat); - - // Check for enum - if (type.TypeKind == TypeKind.Enum) + private static TypeInfo AnalyzeType(ITypeSymbol type, bool isNullable) { - return new TypeInfo + // Unwrap Nullable for value types + if (type is INamedTypeSymbol nt && nt.OriginalDefinition.SpecialType == SpecialType.System_Nullable_T) { - Kind = TypeKind2.Enum, - FullTypeName = fullName, - IsNullable = isNullable, - }; - } + type = nt.TypeArguments[0]; + isNullable = true; + } - // Check for nested ArrowSerializable - foreach (var attr in type.GetAttributes()) - { - if (attr.AttributeClass?.ToDisplayString() == "Apache.Arrow.Serialization.ArrowSerializableAttribute") + var fullName = type.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat); + + // Check for enum + if (type.TypeKind == TypeKind.Enum) { return new TypeInfo { - Kind = TypeKind2.NestedRecord, + Kind = ArrowTypeKind.Enum, FullTypeName = fullName, IsNullable = isNullable, }; } - } - - // Check for collections - if (type is INamedTypeSymbol namedType) - { - var origDef = namedType.OriginalDefinition.ToDisplayString(); - // List, IList, IReadOnlyList - if (origDef == "System.Collections.Generic.List" - || origDef == "System.Collections.Generic.IList" - || origDef == "System.Collections.Generic.IReadOnlyList" - || origDef == "System.Collections.Generic.IEnumerable" - || origDef == "System.Collections.Generic.ICollection" - || origDef == "System.Collections.Generic.IReadOnlyCollection") + // Check for nested ArrowSerializable + foreach (var attr in type.GetAttributes()) { - var elemType = AnalyzeType(namedType.TypeArguments[0], false); - return new TypeInfo + if (attr.AttributeClass?.ToDisplayString() == "Apache.Arrow.Serialization.ArrowSerializableAttribute") { - Kind = TypeKind2.List, - FullTypeName = fullName, - IsNullable = isNullable, - ElementType = elemType, - }; + return new TypeInfo + { + Kind = ArrowTypeKind.NestedRecord, + FullTypeName = fullName, + IsNullable = isNullable, + }; + } } - // Dictionary, IReadOnlyDictionary - if (origDef == "System.Collections.Generic.Dictionary" - || origDef == "System.Collections.Generic.IDictionary" - || origDef == "System.Collections.Generic.IReadOnlyDictionary") + // Check for collections + if (type is INamedTypeSymbol namedType) { - var keyType = AnalyzeType(namedType.TypeArguments[0], false); - var valueType = AnalyzeType(namedType.TypeArguments[1], false); - return new TypeInfo + var origDef = namedType.OriginalDefinition.ToDisplayString(); + + // List, IList, IReadOnlyList + if (origDef == "System.Collections.Generic.List" + || origDef == "System.Collections.Generic.IList" + || origDef == "System.Collections.Generic.IReadOnlyList" + || origDef == "System.Collections.Generic.IEnumerable" + || origDef == "System.Collections.Generic.ICollection" + || origDef == "System.Collections.Generic.IReadOnlyCollection") { - Kind = TypeKind2.Dictionary, - FullTypeName = fullName, - IsNullable = isNullable, - KeyType = keyType, - ValueType = valueType, - }; + var elemType = AnalyzeType(namedType.TypeArguments[0], false); + return new TypeInfo + { + Kind = ArrowTypeKind.List, + FullTypeName = fullName, + IsNullable = isNullable, + ElementType = elemType, + }; + } + + // Dictionary, IReadOnlyDictionary + if (origDef == "System.Collections.Generic.Dictionary" + || origDef == "System.Collections.Generic.IDictionary" + || origDef == "System.Collections.Generic.IReadOnlyDictionary") + { + var keyType = AnalyzeType(namedType.TypeArguments[0], false); + var valueType = AnalyzeType(namedType.TypeArguments[1], false); + return new TypeInfo + { + Kind = ArrowTypeKind.Dictionary, + FullTypeName = fullName, + IsNullable = isNullable, + KeyType = keyType, + ValueType = valueType, + }; + } + + // HashSet, ISet, IReadOnlySet + if (origDef == "System.Collections.Generic.HashSet" + || origDef == "System.Collections.Generic.ISet" + || origDef == "System.Collections.Generic.IReadOnlySet" + || origDef == "System.Collections.Frozen.FrozenSet") + { + var elemType = AnalyzeType(namedType.TypeArguments[0], false); + return new TypeInfo + { + Kind = ArrowTypeKind.Set, + FullTypeName = fullName, + IsNullable = isNullable, + ElementType = elemType, + }; + } } - // HashSet, ISet, IReadOnlySet - if (origDef == "System.Collections.Generic.HashSet" - || origDef == "System.Collections.Generic.ISet" - || origDef == "System.Collections.Generic.IReadOnlySet" - || origDef == "System.Collections.Frozen.FrozenSet") + // Array T[] — but byte[] is Binary, not Array + if (type is IArrayTypeSymbol arrayType) { - var elemType = AnalyzeType(namedType.TypeArguments[0], false); + if (arrayType.ElementType.SpecialType == SpecialType.System_Byte) + { + return new TypeInfo + { + Kind = ArrowTypeKind.Binary, + FullTypeName = fullName, + IsNullable = isNullable, + }; + } + + var elemType = AnalyzeType(arrayType.ElementType, false); return new TypeInfo { - Kind = TypeKind2.Set, + Kind = ArrowTypeKind.Array, FullTypeName = fullName, IsNullable = isNullable, ElementType = elemType, }; } - } - // Array T[] — but byte[] is Binary, not Array - if (type is IArrayTypeSymbol arrayType) - { - if (arrayType.ElementType.SpecialType == SpecialType.System_Byte) + // Primitive types + var kind = fullName switch { - return new TypeInfo - { - Kind = TypeKind2.Binary, - FullTypeName = fullName, - IsNullable = isNullable, - }; - } + "string" => ArrowTypeKind.String, + "bool" => ArrowTypeKind.Bool, + "byte" => ArrowTypeKind.Byte, + "sbyte" => ArrowTypeKind.SByte, + "short" => ArrowTypeKind.Int16, + "ushort" => ArrowTypeKind.UInt16, + "int" => ArrowTypeKind.Int32, + "uint" => ArrowTypeKind.UInt32, + "long" => ArrowTypeKind.Int64, + "ulong" => ArrowTypeKind.UInt64, + "float" => ArrowTypeKind.Float, + "double" => ArrowTypeKind.Double, + "decimal" => ArrowTypeKind.Decimal, + "byte[]" => ArrowTypeKind.Binary, + "System.ReadOnlyMemory" => ArrowTypeKind.Binary, + "global::System.ReadOnlyMemory" => ArrowTypeKind.Binary, + "System.DateTime" => ArrowTypeKind.DateTime, + "global::System.DateTime" => ArrowTypeKind.DateTime, + "System.DateTimeOffset" => ArrowTypeKind.DateTimeOffset, + "global::System.DateTimeOffset" => ArrowTypeKind.DateTimeOffset, + "System.DateOnly" => ArrowTypeKind.DateOnly, + "global::System.DateOnly" => ArrowTypeKind.DateOnly, + "System.TimeOnly" => ArrowTypeKind.TimeOnly, + "global::System.TimeOnly" => ArrowTypeKind.TimeOnly, + "System.TimeSpan" => ArrowTypeKind.TimeSpan, + "global::System.TimeSpan" => ArrowTypeKind.TimeSpan, + "System.Guid" => ArrowTypeKind.Guid, + "global::System.Guid" => ArrowTypeKind.Guid, + "System.Half" => ArrowTypeKind.Half, + "global::System.Half" => ArrowTypeKind.Half, + _ => ArrowTypeKind.Unknown, + }; - var elemType = AnalyzeType(arrayType.ElementType, false); return new TypeInfo { - Kind = TypeKind2.Array, + Kind = kind, FullTypeName = fullName, IsNullable = isNullable, - ElementType = elemType, }; } - // Primitive types - var kind = fullName switch - { - "string" => TypeKind2.String, - "bool" => TypeKind2.Bool, - "byte" => TypeKind2.Byte, - "sbyte" => TypeKind2.SByte, - "short" => TypeKind2.Int16, - "ushort" => TypeKind2.UInt16, - "int" => TypeKind2.Int32, - "uint" => TypeKind2.UInt32, - "long" => TypeKind2.Int64, - "ulong" => TypeKind2.UInt64, - "float" => TypeKind2.Float, - "double" => TypeKind2.Double, - "decimal" => TypeKind2.Decimal, - "byte[]" => TypeKind2.Binary, - "System.ReadOnlyMemory" => TypeKind2.Binary, - "global::System.ReadOnlyMemory" => TypeKind2.Binary, - "System.DateTime" => TypeKind2.DateTime, - "global::System.DateTime" => TypeKind2.DateTime, - "System.DateTimeOffset" => TypeKind2.DateTimeOffset, - "global::System.DateTimeOffset" => TypeKind2.DateTimeOffset, - "System.DateOnly" => TypeKind2.DateOnly, - "global::System.DateOnly" => TypeKind2.DateOnly, - "System.TimeOnly" => TypeKind2.TimeOnly, - "global::System.TimeOnly" => TypeKind2.TimeOnly, - "System.TimeSpan" => TypeKind2.TimeSpan, - "global::System.TimeSpan" => TypeKind2.TimeSpan, - "System.Guid" => TypeKind2.Guid, - "global::System.Guid" => TypeKind2.Guid, - "System.Half" => TypeKind2.Half, - "global::System.Half" => TypeKind2.Half, - _ => TypeKind2.Unknown, - }; - - return new TypeInfo - { - Kind = kind, - FullTypeName = fullName, - IsNullable = isNullable, - }; - } - - private static void Execute(SourceProductionContext spc, TypeModel model, bool emitSchemaJson) - { - // Report diagnostics - foreach (var diag in model.Diagnostics) + private static void Execute(SourceProductionContext spc, TypeModel model, bool emitSchemaJson) { - var parts = diag.Message.Split('\t'); - var descriptor = diag.Id switch - { - "ARROW001" => NonPartialType, - "ARROW002" => NoMatchingConstructor, - "ARROW003" => UnsupportedMemberType, - "ARROW004" => DuplicateFieldName, - "ARROW005" => NonSettableProperty, - "ARROW006" => ArrowAttributeOnIgnoredMember, - _ => null, - }; - if (descriptor != null) - spc.ReportDiagnostic(Diagnostic.Create(descriptor, Location.None, parts)); - } + // Report diagnostics + foreach (var diag in model.Diagnostics) + { + var parts = diag.Message.Split('\t'); + var descriptor = diag.Id switch + { + "ARROW001" => NonPartialType, + "ARROW002" => NoMatchingConstructor, + "ARROW003" => UnsupportedMemberType, + "ARROW004" => DuplicateFieldName, + "ARROW005" => NonSettableProperty, + "ARROW006" => ArrowAttributeOnIgnoredMember, + _ => null, + }; + if (descriptor != null) + spc.ReportDiagnostic(Diagnostic.Create(descriptor, Location.None, parts)); + } - // Don't emit code if there are errors - bool hasErrors = false; - foreach (var d in model.Diagnostics) - { - if (d.IsError) { hasErrors = true; break; } - } - if (hasErrors) return; + // Don't emit code if there are errors + bool hasErrors = false; + foreach (var d in model.Diagnostics) + { + if (d.IsError) { hasErrors = true; break; } + } + if (hasErrors) return; - var sb = new StringBuilder(); - var emitter = new CodeEmitter(sb, model); - emitter.Emit(); - spc.AddSource($"{model.TypeName}.ArrowSerializer.g.cs", sb.ToString()); + var sb = new StringBuilder(); + var emitter = new CodeEmitter(sb, model); + emitter.Emit(); + spc.AddSource($"{model.TypeName}.ArrowSerializer.g.cs", sb.ToString()); - if (emitSchemaJson) - { - var jsonSb = new StringBuilder(); - JsonSchemaEmitter.Emit(jsonSb, model); - spc.AddSource($"{model.TypeName}.ArrowSchemaJson.g.cs", jsonSb.ToString()); + if (emitSchemaJson) + { + var jsonSb = new StringBuilder(); + JsonSchemaEmitter.Emit(jsonSb, model); + spc.AddSource($"{model.TypeName}.ArrowSchemaJson.g.cs", jsonSb.ToString()); + } } - } - - private static PolymorphicModel? GetPolymorphicModel(GeneratorAttributeSyntaxContext ctx) - { - if (!(ctx.TargetSymbol is INamedTypeSymbol typeSymbol)) - return null; - // Read [ArrowPolymorphic] attribute - string discriminatorFieldName = "$type"; - foreach (var attr in typeSymbol.GetAttributes()) + private static PolymorphicModel? GetPolymorphicModel(GeneratorAttributeSyntaxContext ctx) { - if (attr.AttributeClass?.ToDisplayString() == "Apache.Arrow.Serialization.ArrowPolymorphicAttribute") + if (!(ctx.TargetSymbol is INamedTypeSymbol typeSymbol)) + return null; + + // Read [ArrowPolymorphic] attribute + string discriminatorFieldName = "$type"; + foreach (var attr in typeSymbol.GetAttributes()) { - foreach (var named in attr.NamedArguments) + if (attr.AttributeClass?.ToDisplayString() == "Apache.Arrow.Serialization.ArrowPolymorphicAttribute") { - if (named.Key == "TypeDiscriminatorFieldName" && named.Value.Value is string s) - discriminatorFieldName = s; + foreach (var named in attr.NamedArguments) + { + if (named.Key == "TypeDiscriminatorFieldName" && named.Value.Value is string s) + discriminatorFieldName = s; + } } } - } - // Collect [ArrowDerivedType] attributes - var derivedTypes = new List(); - foreach (var attr in typeSymbol.GetAttributes()) - { - if (attr.AttributeClass?.ToDisplayString() != "Apache.Arrow.Serialization.ArrowDerivedTypeAttribute") - continue; - if (attr.ConstructorArguments.Length < 2) - continue; - - var derivedTypeSymbol = attr.ConstructorArguments[0].Value as INamedTypeSymbol; - var discriminator = attr.ConstructorArguments[1].Value as string; - if (derivedTypeSymbol == null || discriminator == null) - continue; - - var props = CollectProperties(derivedTypeSymbol); - derivedTypes.Add(new DerivedTypeInfo - { - TypeDiscriminator = discriminator, - FullTypeName = derivedTypeSymbol.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat), - TypeName = derivedTypeSymbol.Name, - Properties = props, - IsRecord = derivedTypeSymbol.IsRecord, - IsValueType = derivedTypeSymbol.IsValueType, - }); - } + // Collect [ArrowDerivedType] attributes + var derivedTypes = new List(); + foreach (var attr in typeSymbol.GetAttributes()) + { + if (attr.AttributeClass?.ToDisplayString() != "Apache.Arrow.Serialization.ArrowDerivedTypeAttribute") + continue; + if (attr.ConstructorArguments.Length < 2) + continue; - // Build union of all properties (deduplicated by FieldName, all nullable) - var unionProps = new List(); - var seenFields = new HashSet(); - foreach (var dt in derivedTypes) - { - foreach (var prop in dt.Properties) + var derivedTypeSymbol = attr.ConstructorArguments[0].Value as INamedTypeSymbol; + var discriminator = attr.ConstructorArguments[1].Value as string; + if (derivedTypeSymbol == null || discriminator == null) + continue; + + var props = CollectProperties(derivedTypeSymbol); + derivedTypes.Add(new DerivedTypeInfo + { + TypeDiscriminator = discriminator, + FullTypeName = derivedTypeSymbol.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat), + TypeName = derivedTypeSymbol.Name, + Properties = props, + IsRecord = derivedTypeSymbol.IsRecord, + IsValueType = derivedTypeSymbol.IsValueType, + }); + } + + // Build union of all properties (deduplicated by FieldName, all nullable) + var unionProps = new List(); + var seenFields = new HashSet(); + foreach (var dt in derivedTypes) { - if (seenFields.Add(prop.FieldName)) + foreach (var prop in dt.Properties) { - // Make a nullable copy for the union schema - unionProps.Add(new PropertyModel + if (seenFields.Add(prop.FieldName)) { - PropertyName = prop.PropertyName, - FieldName = prop.FieldName, - Order = prop.Order, - DeclOrder = prop.DeclOrder, - Type = prop.Type, - IsNullable = true, // always nullable in union schema - HasDefaultValue = true, - Metadata = prop.Metadata, - }); + // Make a nullable copy for the union schema + unionProps.Add(new PropertyModel + { + PropertyName = prop.PropertyName, + FieldName = prop.FieldName, + Order = prop.Order, + DeclOrder = prop.DeclOrder, + Type = prop.Type, + IsNullable = true, // always nullable in union schema + HasDefaultValue = true, + Metadata = prop.Metadata, + }); + } } } - } - // Collect metadata - var typeMetadata = new List>(); - foreach (var attr in typeSymbol.GetAttributes()) - { - if (attr.AttributeClass?.ToDisplayString() == "Apache.Arrow.Serialization.ArrowMetadataAttribute" - && attr.ConstructorArguments.Length >= 2) + // Collect metadata + var typeMetadata = new List>(); + foreach (var attr in typeSymbol.GetAttributes()) { - var key = attr.ConstructorArguments[0].Value as string; - var val = attr.ConstructorArguments[1].Value as string; - if (key != null && val != null) - typeMetadata.Add(new KeyValuePair(key, val)); + if (attr.AttributeClass?.ToDisplayString() == "Apache.Arrow.Serialization.ArrowMetadataAttribute" + && attr.ConstructorArguments.Length >= 2) + { + var key = attr.ConstructorArguments[0].Value as string; + var val = attr.ConstructorArguments[1].Value as string; + if (key != null && val != null) + typeMetadata.Add(new KeyValuePair(key, val)); + } } + + return new PolymorphicModel + { + Namespace = typeSymbol.ContainingNamespace.IsGlobalNamespace ? null : typeSymbol.ContainingNamespace.ToDisplayString(), + TypeName = typeSymbol.Name, + FullTypeName = typeSymbol.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat), + TypeDiscriminatorFieldName = discriminatorFieldName, + IsInterface = typeSymbol.TypeKind == TypeKind.Interface, + IsRecord = typeSymbol.IsRecord, + DerivedTypes = derivedTypes, + UnionProperties = unionProps, + Metadata = typeMetadata, + }; } - return new PolymorphicModel + /// + /// Collects properties from a type symbol (same logic as GetTypeModel but returns just the properties). + /// + private static List CollectProperties(INamedTypeSymbol typeSymbol) { - Namespace = typeSymbol.ContainingNamespace.IsGlobalNamespace ? null : typeSymbol.ContainingNamespace.ToDisplayString(), - TypeName = typeSymbol.Name, - FullTypeName = typeSymbol.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat), - TypeDiscriminatorFieldName = discriminatorFieldName, - IsInterface = typeSymbol.TypeKind == TypeKind.Interface, - IsRecord = typeSymbol.IsRecord, - DerivedTypes = derivedTypes, - UnionProperties = unionProps, - Metadata = typeMetadata, - }; - } + var properties = new List(); + int declOrder = 0; + foreach (var member in typeSymbol.GetMembers()) + { + if (!(member is IPropertySymbol prop)) + continue; + if (prop.IsStatic || prop.IsIndexer) + continue; + if (prop.DeclaredAccessibility != Accessibility.Public) + continue; - /// - /// Collects properties from a type symbol (same logic as GetTypeModel but returns just the properties). - /// - private static List CollectProperties(INamedTypeSymbol typeSymbol) - { - var properties = new List(); - int declOrder = 0; - foreach (var member in typeSymbol.GetMembers()) - { - if (!(member is IPropertySymbol prop)) - continue; - if (prop.IsStatic || prop.IsIndexer) - continue; - if (prop.DeclaredAccessibility != Accessibility.Public) - continue; - - bool isTransient = false; - string? arrowTypeName = null; - string? converterTypeName2 = null; - string? fieldName = null; - int order = int.MaxValue; - var propMetadata = new List>(); - - foreach (var attr in prop.GetAttributes()) - { - var attrName = attr.AttributeClass?.ToDisplayString(); - if (attrName == "Apache.Arrow.Serialization.ArrowIgnoreAttribute") - isTransient = true; - else if (attrName == "Apache.Arrow.Serialization.ArrowTypeAttribute") + bool isTransient = false; + string? arrowTypeName = null; + string? converterTypeName2 = null; + string? fieldName = null; + int order = int.MaxValue; + var propMetadata = new List>(); + + foreach (var attr in prop.GetAttributes()) { - if (attr.ConstructorArguments.Length > 0) - arrowTypeName = attr.ConstructorArguments[0].Value as string; - foreach (var named in attr.NamedArguments) + var attrName = attr.AttributeClass?.ToDisplayString(); + if (attrName == "Apache.Arrow.Serialization.ArrowIgnoreAttribute") + isTransient = true; + else if (attrName == "Apache.Arrow.Serialization.ArrowTypeAttribute") { - if (named.Key == "Converter" && named.Value.Value is INamedTypeSymbol converterSym) - converterTypeName2 = converterSym.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat); + if (attr.ConstructorArguments.Length > 0) + arrowTypeName = attr.ConstructorArguments[0].Value as string; + foreach (var named in attr.NamedArguments) + { + if (named.Key == "Converter" && named.Value.Value is INamedTypeSymbol converterSym) + converterTypeName2 = converterSym.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat); + } } - } - else if (attrName == "Apache.Arrow.Serialization.ArrowFieldAttribute") - { - if (attr.ConstructorArguments.Length > 0) - fieldName = attr.ConstructorArguments[0].Value as string; - foreach (var named in attr.NamedArguments) + else if (attrName == "Apache.Arrow.Serialization.ArrowFieldAttribute") { - if (named.Key == "Order" && named.Value.Value is int o) - order = o; + if (attr.ConstructorArguments.Length > 0) + fieldName = attr.ConstructorArguments[0].Value as string; + foreach (var named in attr.NamedArguments) + { + if (named.Key == "Order" && named.Value.Value is int o) + order = o; + } } - } - else if (attrName == "Apache.Arrow.Serialization.ArrowMetadataAttribute") - { - if (attr.ConstructorArguments.Length >= 2) + else if (attrName == "Apache.Arrow.Serialization.ArrowMetadataAttribute") { - var key = attr.ConstructorArguments[0].Value as string; - var val = attr.ConstructorArguments[1].Value as string; - if (key != null && val != null) - propMetadata.Add(new KeyValuePair(key, val)); + if (attr.ConstructorArguments.Length >= 2) + { + var key = attr.ConstructorArguments[0].Value as string; + var val = attr.ConstructorArguments[1].Value as string; + if (key != null && val != null) + propMetadata.Add(new KeyValuePair(key, val)); + } } } - } - if (isTransient) - continue; + if (isTransient) + continue; + + var propType = prop.Type; + bool isNullable = propType.NullableAnnotation == NullableAnnotation.Annotated; + if (isNullable && propType is INamedTypeSymbol nullableType && nullableType.OriginalDefinition.SpecialType == SpecialType.System_Nullable_T) + propType = nullableType.TypeArguments[0]; - var propType = prop.Type; - bool isNullable = propType.NullableAnnotation == NullableAnnotation.Annotated; - if (isNullable && propType is INamedTypeSymbol nullableType && nullableType.OriginalDefinition.SpecialType == SpecialType.System_Nullable_T) - propType = nullableType.TypeArguments[0]; + var typeInfo = AnalyzeType(propType, isNullable); + if (converterTypeName2 != null) + typeInfo = new TypeInfo { Kind = ArrowTypeKind.Custom, FullTypeName = typeInfo.FullTypeName, IsNullable = isNullable }; + else if (arrowTypeName != null) + typeInfo = typeInfo.WithOverride(arrowTypeName); - var typeInfo = AnalyzeType(propType, isNullable); - if (converterTypeName2 != null) - typeInfo = new TypeInfo { Kind = TypeKind2.Custom, FullTypeName = typeInfo.FullTypeName, IsNullable = isNullable }; - else if (arrowTypeName != null) - typeInfo = typeInfo.WithOverride(arrowTypeName); + properties.Add(new PropertyModel + { + PropertyName = prop.Name, + FieldName = fieldName ?? prop.Name, + Order = order, + DeclOrder = declOrder++, + Type = typeInfo, + IsNullable = isNullable, + HasDefaultValue = HasDefaultValue(prop, typeSymbol), + Metadata = propMetadata, + ConverterTypeName = converterTypeName2, + }); + } - properties.Add(new PropertyModel + properties.Sort((a, b) => { - PropertyName = prop.Name, - FieldName = fieldName ?? prop.Name, - Order = order, - DeclOrder = declOrder++, - Type = typeInfo, - IsNullable = isNullable, - HasDefaultValue = HasDefaultValue(prop, typeSymbol), - Metadata = propMetadata, - ConverterTypeName = converterTypeName2, + int cmp = a.Order.CompareTo(b.Order); + return cmp != 0 ? cmp : a.DeclOrder.CompareTo(b.DeclOrder); }); + + return properties; } - properties.Sort((a, b) => + private static void ExecutePolymorphic(SourceProductionContext spc, PolymorphicModel model) { - int cmp = a.Order.CompareTo(b.Order); - return cmp != 0 ? cmp : a.DeclOrder.CompareTo(b.DeclOrder); - }); - - return properties; - } - - private static void ExecutePolymorphic(SourceProductionContext spc, PolymorphicModel model) - { - var sb = new StringBuilder(); - var emitter = new PolymorphicCodeEmitter(sb, model); - emitter.Emit(); - spc.AddSource($"{model.TypeName}.ArrowPolymorphic.g.cs", sb.ToString()); + var sb = new StringBuilder(); + var emitter = new PolymorphicCodeEmitter(sb, model); + emitter.Emit(); + spc.AddSource($"{model.TypeName}.ArrowPolymorphic.g.cs", sb.ToString()); + } } -} } // namespace diff --git a/src/Apache.Arrow.Serialization.Generator/CodeEmitter.cs b/src/Apache.Arrow.Serialization.Generator/CodeEmitter.cs index e515a58f..85e23376 100644 --- a/src/Apache.Arrow.Serialization.Generator/CodeEmitter.cs +++ b/src/Apache.Arrow.Serialization.Generator/CodeEmitter.cs @@ -22,3094 +22,3094 @@ namespace Apache.Arrow.Serialization.Generator { -internal class CodeEmitter -{ - private readonly StringBuilder _sb; - private readonly TypeModel _model; - private int _indent; - - public CodeEmitter(StringBuilder sb, TypeModel model) + internal class CodeEmitter { - _sb = sb; - _model = model; - } + private readonly StringBuilder _sb; + private readonly TypeModel _model; + private int _indent; - public void Emit() - { - Line("// "); - Line("#nullable enable"); - Line("#pragma warning disable CS8629 // Nullable value type may be null"); - Line(); - Line("using System.Collections.Generic;"); - Line("using System.Linq;"); - Line("using Apache.Arrow;"); - Line("using Apache.Arrow.Arrays;"); - Line("using Apache.Arrow.Types;"); - Line("using Apache.Arrow.Serialization;"); - Line(); - - if (_model.Namespace != null) + public CodeEmitter(StringBuilder sb, TypeModel model) { - Line($"namespace {_model.Namespace};"); - Line(); + _sb = sb; + _model = model; } - var typeKeyword = GetTypeKeyword(); - Line($"partial {typeKeyword} {_model.TypeName} : IArrowSerializer<{_model.TypeName}>"); - Line("{"); - _indent++; - - EmitSchemaField(); - Line(); - EmitSerialize(); - Line(); - EmitDeserialize(); - Line(); - EmitMultiRowSerialize(); - Line(); - EmitMultiRowDeserialize(); - - _indent--; - Line("}"); - } - - private void EmitSchemaField() - { - Line($"private static readonly Schema _arrowSchema = new Schema.Builder()"); - _indent++; - foreach (var prop in _model.Properties) + public void Emit() { - var arrowType = prop.ConverterTypeName != null - ? $"new {prop.ConverterTypeName}().ArrowType" - : GetArrowTypeExpression(prop.Type); - var nullable = prop.IsNullable ? "true" : "false"; - if (prop.Metadata.Count > 0) - { - var metadataEntries = string.Join(", ", - prop.Metadata.ConvertAll(kv => - $"new System.Collections.Generic.KeyValuePair(\"{Escape(kv.Key)}\", \"{Escape(kv.Value)}\")")); - Line($".Field(new Field(\"{prop.FieldName}\", {arrowType}, {nullable}, new[] {{ {metadataEntries} }}))"); - } - else + Line("// "); + Line("#nullable enable"); + Line("#pragma warning disable CS8629 // Nullable value type may be null"); + Line(); + Line("using System.Collections.Generic;"); + Line("using System.Linq;"); + Line("using Apache.Arrow;"); + Line("using Apache.Arrow.Arrays;"); + Line("using Apache.Arrow.Types;"); + Line("using Apache.Arrow.Serialization;"); + Line(); + + if (_model.Namespace != null) { - Line($".Field(new Field(\"{prop.FieldName}\", {arrowType}, {nullable}))"); + Line($"namespace {_model.Namespace};"); + Line(); } - } - foreach (var kv in _model.Metadata) - { - Line($".Metadata(\"{Escape(kv.Key)}\", \"{Escape(kv.Value)}\")"); - } - Line(".Build();"); - _indent--; - Line(); - var newKw = _model.HasArrowSerializableBase ? "new " : ""; - Line($"public static {newKw}Schema ArrowSchema => _arrowSchema;"); - } - private static string Escape(string s) => s.Replace("\\", "\\\\").Replace("\"", "\\\""); - - private void EmitSerialize() - { - Line($"public static RecordBatch ToRecordBatch({_model.TypeName} value)"); - Line("{"); - _indent++; + var typeKeyword = GetTypeKeyword(); + Line($"partial {typeKeyword} {_model.TypeName} : IArrowSerializer<{_model.TypeName}>"); + Line("{"); + _indent++; - if (_model.HasSerializationCallback) - { - Line("((Apache.Arrow.Serialization.IArrowSerializationCallback)value).OnBeforeSerialize();"); + EmitSchemaField(); Line(); - } + EmitSerialize(); + Line(); + EmitDeserialize(); + Line(); + EmitMultiRowSerialize(); + Line(); + EmitMultiRowDeserialize(); - // Build each column array - for (int i = 0; i < _model.Properties.Count; i++) - { - var prop = _model.Properties[i]; - EmitArrayBuilder(prop, i); + _indent--; + Line("}"); } - // Build RecordBatch - Line(); - Line("return new RecordBatch(_arrowSchema, new IArrowArray[]"); - Line("{"); - _indent++; - for (int i = 0; i < _model.Properties.Count; i++) + private void EmitSchemaField() { - var comma = i < _model.Properties.Count - 1 ? "," : ""; - Line($"arr_{i}{comma}"); + Line($"private static readonly Schema _arrowSchema = new Schema.Builder()"); + _indent++; + foreach (var prop in _model.Properties) + { + var arrowType = prop.ConverterTypeName != null + ? $"new {prop.ConverterTypeName}().ArrowType" + : GetArrowTypeExpression(prop.Type); + var nullable = prop.IsNullable ? "true" : "false"; + if (prop.Metadata.Count > 0) + { + var metadataEntries = string.Join(", ", + prop.Metadata.ConvertAll(kv => + $"new System.Collections.Generic.KeyValuePair(\"{Escape(kv.Key)}\", \"{Escape(kv.Value)}\")")); + Line($".Field(new Field(\"{prop.FieldName}\", {arrowType}, {nullable}, new[] {{ {metadataEntries} }}))"); + } + else + { + Line($".Field(new Field(\"{prop.FieldName}\", {arrowType}, {nullable}))"); + } + } + foreach (var kv in _model.Metadata) + { + Line($".Metadata(\"{Escape(kv.Key)}\", \"{Escape(kv.Value)}\")"); + } + Line(".Build();"); + _indent--; + Line(); + var newKw = _model.HasArrowSerializableBase ? "new " : ""; + Line($"public static {newKw}Schema ArrowSchema => _arrowSchema;"); } - _indent--; - Line("}, 1);"); - - _indent--; - Line("}"); - } - private void EmitArrayBuilder(PropertyModel prop, int index) - { - var varName = $"arr_{index}"; - var access = $"value.{prop.PropertyName}"; - - if (prop.IsNullable) - { - EmitNullableArrayBuilder(prop, index, varName, access); - return; - } + private static string Escape(string s) => s.Replace("\\", "\\\\").Replace("\"", "\\\""); - switch (prop.Type.Kind) + private void EmitSerialize() { - case TypeKind2.String: - { - var si = GetStringArrayInfo(prop); - Line($"var {varName} = new {si.BuilderType}().Append({access}).Build();"); - break; - } - case TypeKind2.Bool: - { - var bl = GetBoolArrayInfo(prop); - Line($"var {varName} = new {bl.BuilderType}().Append({access}).Build();"); - break; - } - case TypeKind2.Byte: - Line($"var {varName} = new UInt8Array.Builder().Append({access}).Build();"); - break; - case TypeKind2.SByte: - Line($"var {varName} = new Int8Array.Builder().Append({access}).Build();"); - break; - case TypeKind2.Int16: - Line($"var {varName} = new Int16Array.Builder().Append({access}).Build();"); - break; - case TypeKind2.UInt16: - Line($"var {varName} = new UInt16Array.Builder().Append({access}).Build();"); - break; - case TypeKind2.Int32: - Line($"var {varName} = new Int32Array.Builder().Append({access}).Build();"); - break; - case TypeKind2.UInt32: - Line($"var {varName} = new UInt32Array.Builder().Append({access}).Build();"); - break; - case TypeKind2.Int64: - Line($"var {varName} = new Int64Array.Builder().Append({access}).Build();"); - break; - case TypeKind2.UInt64: - Line($"var {varName} = new UInt64Array.Builder().Append({access}).Build();"); - break; - case TypeKind2.Float: - Line($"var {varName} = new FloatArray.Builder().Append({access}).Build();"); - break; - case TypeKind2.Double: - Line($"var {varName} = new DoubleArray.Builder().Append({access}).Build();"); - break; - case TypeKind2.Binary: - { - var bi = GetBinaryArrayInfo(prop); - Line($"var {varName} = new {bi.BuilderType}().Append({BinaryToSpan(prop, access)}).Build();"); - break; - } - case TypeKind2.DateTime: - Line($"var {varName} = new TimestampArray.Builder({GetTimestampTypeExpr(prop)}).Append(Apache.Arrow.Serialization.ArrowArrayHelper.{TimestampConvertMethod(prop)}({access})).Build();"); - break; - case TypeKind2.DateTimeOffset: - Line($"var {varName} = new TimestampArray.Builder({GetTimestampTypeExpr(prop)}).Append(Apache.Arrow.Serialization.ArrowArrayHelper.{TimestampConvertMethod(prop)}({access})).Build();"); - break; - case TypeKind2.DateOnly: - { - var di = GetDateArrayInfo(prop); - Line($"var {varName} = new {di.BuilderType}().Append({access}).Build();"); - break; - } - case TypeKind2.TimeOnly: - { - var ti = GetTimeArrayInfo(prop); - Line($"var {varName} = new {ti.BuilderType}({ti.TypeExpr}).Append({access}).Build();"); - break; - } - case TypeKind2.TimeSpan: - Line($"var {varName} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildDurationArray({access});"); - break; - case TypeKind2.Decimal: - Line($"var {varName} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildDecimalArray({access});"); - break; - case TypeKind2.Guid: - Line($"var {varName} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildGuidArray({access});"); - break; - case TypeKind2.Half: - Line($"var {varName} = new HalfFloatArray.Builder().Append({access}).Build();"); - break; - case TypeKind2.Enum: - EmitEnumArrayBuilder(prop, index, varName, access); - break; - case TypeKind2.List: - case TypeKind2.Array: - EmitListArrayBuilder(prop, index, varName, access); - break; - case TypeKind2.Set: - EmitSetArrayBuilder(prop, index, varName, access); - break; - case TypeKind2.Dictionary: - EmitMapArrayBuilder(prop, index, varName, access); - break; - case TypeKind2.NestedRecord: - EmitNestedRecordSerializer(prop, index, varName, access); - break; - case TypeKind2.Custom: - Line($"var {varName} = new {prop.ConverterTypeName}().ToArray({access});"); - break; - default: - Line($"// TODO: Unsupported type {prop.Type.Kind} for {prop.PropertyName}"); - Line($"IArrowArray {varName} = new NullArray(1);"); - break; - } - } + Line($"public static RecordBatch ToRecordBatch({_model.TypeName} value)"); + Line("{"); + _indent++; - private void EmitNullableArrayBuilder(PropertyModel prop, int index, string varName, string access) - { - switch (prop.Type.Kind) - { - case TypeKind2.String: - { - var si = GetStringArrayInfo(prop); - Line($"var {varName} = new {si.BuilderType}().Append({access}).Build();"); - break; - } - case TypeKind2.Bool: - { - var bl = GetBoolArrayInfo(prop); - Line($"var b_{index} = new {bl.BuilderType}();"); - Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(v_{index}); else b_{index}.AppendNull();"); - Line($"var {varName} = b_{index}.Build();"); - break; - } - case TypeKind2.Int32: - Line($"var b_{index} = new Int32Array.Builder();"); - Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(v_{index}); else b_{index}.AppendNull();"); - Line($"var {varName} = b_{index}.Build();"); - break; - case TypeKind2.Int64: - Line($"var b_{index} = new Int64Array.Builder();"); - Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(v_{index}); else b_{index}.AppendNull();"); - Line($"var {varName} = b_{index}.Build();"); - break; - case TypeKind2.Float: - Line($"var b_{index} = new FloatArray.Builder();"); - Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(v_{index}); else b_{index}.AppendNull();"); - Line($"var {varName} = b_{index}.Build();"); - break; - case TypeKind2.Double: - Line($"var b_{index} = new DoubleArray.Builder();"); - Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(v_{index}); else b_{index}.AppendNull();"); - Line($"var {varName} = b_{index}.Build();"); - break; - case TypeKind2.DateTime: - Line($"var b_{index} = new TimestampArray.Builder({GetTimestampTypeExpr(prop)});"); - Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(Apache.Arrow.Serialization.ArrowArrayHelper.{TimestampConvertMethod(prop)}(v_{index})); else b_{index}.AppendNull();"); - Line($"var {varName} = b_{index}.Build();"); - break; - case TypeKind2.DateTimeOffset: - Line($"var b_{index} = new TimestampArray.Builder({GetTimestampTypeExpr(prop)});"); - Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(Apache.Arrow.Serialization.ArrowArrayHelper.{TimestampConvertMethod(prop)}(v_{index})); else b_{index}.AppendNull();"); - Line($"var {varName} = b_{index}.Build();"); - break; - case TypeKind2.DateOnly: - { - var di = GetDateArrayInfo(prop); - Line($"var b_{index} = new {di.BuilderType}();"); - Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(v_{index}); else b_{index}.AppendNull();"); - Line($"var {varName} = b_{index}.Build();"); - break; - } - case TypeKind2.TimeOnly: - { - var ti = GetTimeArrayInfo(prop); - Line($"var b_{index} = new {ti.BuilderType}({ti.TypeExpr});"); - Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(v_{index}); else b_{index}.AppendNull();"); - Line($"var {varName} = b_{index}.Build();"); - break; - } - case TypeKind2.TimeSpan: - Line($"var {varName} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildDurationArray({access});"); - break; - case TypeKind2.Decimal: - Line($"var {varName} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildDecimalArray({access});"); - break; - case TypeKind2.Guid: - Line($"var {varName} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildGuidArray({access});"); - break; - case TypeKind2.Half: - Line($"var b_{index} = new HalfFloatArray.Builder();"); - Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(v_{index}); else b_{index}.AppendNull();"); - Line($"var {varName} = b_{index}.Build();"); - break; - case TypeKind2.Enum: - EmitEnumArrayBuilder(prop, index, varName, access); - break; - case TypeKind2.NestedRecord: - EmitNestedRecordSerializer(prop, index, varName, access); - break; - case TypeKind2.Custom: - Line($"IArrowArray {varName};"); - Line($"if ({access} is {{ }} cv_{index})"); - Line($" {varName} = new {prop.ConverterTypeName}().ToArray(cv_{index});"); - Line($"else"); - Line($" {varName} = new NullArray(1);"); - break; - case TypeKind2.List: - case TypeKind2.Array: - case TypeKind2.Set: + if (_model.HasSerializationCallback) { - Line($"IArrowArray {varName};"); - Line($"if ({access} != null)"); - Line("{"); - _indent++; - var tempVar = $"_nonnull_{index}"; - if (prop.Type.Kind == TypeKind2.Set) - EmitSetArrayBuilder(prop, index, tempVar, access); - else - EmitListArrayBuilder(prop, index, tempVar, access); - Line($"{varName} = {tempVar};"); - _indent--; - Line("}"); - Line("else"); - Line("{"); - _indent++; - Line($"{varName} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildNullArray(ArrowSchema.GetFieldByName(\"{prop.FieldName}\").DataType, 1);"); - _indent--; - Line("}"); - break; + Line("((Apache.Arrow.Serialization.IArrowSerializationCallback)value).OnBeforeSerialize();"); + Line(); } - case TypeKind2.Dictionary: + + // Build each column array + for (int i = 0; i < _model.Properties.Count; i++) { - Line($"IArrowArray {varName};"); - Line($"if ({access} != null)"); - Line("{"); - _indent++; - var tempVar = $"_nonnull_{index}"; - EmitMapArrayBuilder(prop, index, tempVar, access); - Line($"{varName} = {tempVar};"); - _indent--; - Line("}"); - Line("else"); - Line("{"); - _indent++; - Line($"{varName} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildNullArray(ArrowSchema.GetFieldByName(\"{prop.FieldName}\").DataType, 1);"); - _indent--; - Line("}"); - break; + var prop = _model.Properties[i]; + EmitArrayBuilder(prop, i); } - default: - // Fallback: use the non-nullable path (many builders accept null for ref types) - EmitArrayBuilder(new PropertyModel - { - PropertyName = prop.PropertyName, - FieldName = prop.FieldName, - Type = prop.Type, - IsNullable = false, - }, index); - break; - } - } - private void EmitEnumArrayBuilder(PropertyModel prop, int index, string varName, string access) - { - // Dictionary-encoded string array: indices + dictionary - // For a single value, build a StringArray for the dictionary and Int16Array for indices - if (prop.IsNullable) - { - Line($"StringArray dict_{index};"); - Line($"Int16Array idx_{index};"); - Line($"if ({access} is {{ }} ev_{index})"); + // Build RecordBatch + Line(); + Line("return new RecordBatch(_arrowSchema, new IArrowArray[]"); Line("{"); _indent++; - Line($"dict_{index} = new StringArray.Builder().Append(ev_{index}.ToString()).Build();"); - Line($"idx_{index} = new Int16Array.Builder().Append(0).Build();"); + for (int i = 0; i < _model.Properties.Count; i++) + { + var comma = i < _model.Properties.Count - 1 ? "," : ""; + Line($"arr_{i}{comma}"); + } _indent--; - Line("}"); - Line("else"); - Line("{"); - _indent++; - Line($"dict_{index} = new StringArray.Builder().Build();"); - Line($"idx_{index} = new Int16Array.Builder().AppendNull().Build();"); + Line("}, 1);"); + _indent--; Line("}"); } - else - { - Line($"var dict_{index} = new StringArray.Builder().Append({access}.ToString()).Build();"); - Line($"var idx_{index} = new Int16Array.Builder().Append(0).Build();"); - } - Line($"var {varName} = new DictionaryArray(new DictionaryType(Int16Type.Default, StringType.Default, false), idx_{index}, dict_{index});"); - } - - private void EmitListArrayBuilder(PropertyModel prop, int index, string varName, string access) - { - var elemType = prop.Type.ElementType!; - if (NeedsManualListBuilder(elemType)) - { - EmitManualListArray(elemType, index, varName, access, prop.FieldName); - return; - } - var arrowElemType = GetArrowTypeExpression(elemType); - Line($"var lb_{index} = new ListArray.Builder({arrowElemType});"); - Line($"lb_{index}.Append();"); - EmitListValuePopulation(elemType, $"lb_{index}", access, index); - Line($"var {varName} = lb_{index}.Build();"); - } - - private void EmitSetArrayBuilder(PropertyModel prop, int index, string varName, string access) - { - var elemType = prop.Type.ElementType!; - if (NeedsManualListBuilder(elemType)) + private void EmitArrayBuilder(PropertyModel prop, int index) { - EmitManualListArray(elemType, index, varName, access, prop.FieldName); - return; - } - var arrowElemType = GetArrowTypeExpression(elemType); + var varName = $"arr_{index}"; + var access = $"value.{prop.PropertyName}"; - Line($"var lb_{index} = new ListArray.Builder({arrowElemType});"); - Line($"lb_{index}.Append();"); - EmitListValuePopulation(elemType, $"lb_{index}", access, index); - Line($"var {varName} = lb_{index}.Build();"); - } - - /// - /// Returns true for element types that cannot use ListArray.Builder because - /// ArrowArrayBuilderFactory.Build() does not support them (extension types like Guid). - /// - private static bool NeedsManualListBuilder(TypeInfo elemType) - { - return elemType.Kind == TypeKind2.Guid - || elemType.Kind == TypeKind2.Enum - || elemType.Kind == TypeKind2.NestedRecord; - } - - /// - /// Returns true for element types that need manual deserialization (can't use GetElementReader). - /// - private static bool NeedsManualListDeserialize(TypeInfo elemType) - { - return NeedsManualListBuilder(elemType) - || elemType.Kind == TypeKind2.List - || elemType.Kind == TypeKind2.Array - || elemType.Kind == TypeKind2.Dictionary; - } + if (prop.IsNullable) + { + EmitNullableArrayBuilder(prop, index, varName, access); + return; + } - /// - /// Builds a single-row ListArray manually for element types unsupported by ArrowArrayBuilderFactory. - /// - private void EmitManualListArray(TypeInfo elemType, int index, string varName, string access, string fieldName) - { - var arrowElemType = GetArrowTypeExpression(elemType); - switch (elemType.Kind) - { - case TypeKind2.Guid: - Line($"var guidList_{index} = new System.Collections.Generic.List({access});"); - Line($"var guidValues_{index} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildGuidArray(guidList_{index});"); - Line($"var offsets_{index} = new ArrowBuffer.Builder();"); - Line($"offsets_{index}.Append(0);"); - Line($"offsets_{index}.Append(guidList_{index}.Count);"); - Line($"var {varName} = new ListArray(new ListType(new Field(\"item\", {arrowElemType}, false)), 1, offsets_{index}.Build(), guidValues_{index}, ArrowBuffer.Empty);"); - break; - case TypeKind2.Enum: - { - // Enum list elements: dictionary-encoded (matching Python schema) - Line($"var enumDict_{index} = new StringArray.Builder();"); - Line($"var enumIdx_{index} = new Int16Array.Builder();"); - Line($"var enumMap_{index} = new System.Collections.Generic.Dictionary();"); - Line($"foreach (var e_{index} in {access}) {{"); - Line($" var es_{index} = e_{index}.ToString();"); - Line($" if (!enumMap_{index}.TryGetValue(es_{index}, out var ei_{index})) {{ ei_{index} = (short)enumMap_{index}.Count; enumMap_{index}[es_{index}] = ei_{index}; }}"); - Line($" enumIdx_{index}.Append(ei_{index});"); - Line($"}}"); - Line($"foreach (var kv in enumMap_{index}.OrderBy(kv => kv.Value)) enumDict_{index}.Append(kv.Key);"); - Line($"var enumDictArr_{index} = new DictionaryArray(new DictionaryType(Int16Type.Default, StringType.Default, false), enumIdx_{index}.Build(), enumDict_{index}.Build());"); - Line($"var offsets_{index} = new ArrowBuffer.Builder();"); - Line($"offsets_{index}.Append(0);"); - Line($"offsets_{index}.Append(enumDictArr_{index}.Length);"); - Line($"var {varName} = new ListArray(ArrowSchema.GetFieldByName(\"{fieldName}\").DataType, 1, offsets_{index}.Build(), enumDictArr_{index}, ArrowBuffer.Empty);"); - break; - } - case TypeKind2.NestedRecord: - { - // Nested record list elements: serialize via child's ToRecordBatch, wrap in StructArray - var typeName = elemType.FullTypeName; - Line($"var nestedList_{index} = new System.Collections.Generic.List<{typeName}>({access});"); - Line($"var nestedBatch_{index} = {typeName}.ToRecordBatch((IReadOnlyList<{typeName}>)nestedList_{index});"); - Line($"var nestedCols_{index} = new IArrowArray[nestedBatch_{index}.ColumnCount];"); - Line($"for (int f = 0; f < nestedBatch_{index}.ColumnCount; f++) nestedCols_{index}[f] = nestedBatch_{index}.Column(f);"); - Line($"var structValues_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), nestedList_{index}.Count, nestedCols_{index}, ArrowBuffer.Empty);"); - Line($"var offsets_{index} = new ArrowBuffer.Builder();"); - Line($"offsets_{index}.Append(0);"); - Line($"offsets_{index}.Append(nestedList_{index}.Count);"); - Line($"var {varName} = new ListArray(new ListType(new Field(\"item\", new StructType({typeName}.ArrowSchema.FieldsList), false)), 1, offsets_{index}.Build(), structValues_{index}, ArrowBuffer.Empty);"); - break; + switch (prop.Type.Kind) + { + case ArrowTypeKind.String: + { + var si = GetStringArrayInfo(prop); + Line($"var {varName} = new {si.BuilderType}().Append({access}).Build();"); + break; + } + case ArrowTypeKind.Bool: + { + var bl = GetBoolArrayInfo(prop); + Line($"var {varName} = new {bl.BuilderType}().Append({access}).Build();"); + break; + } + case ArrowTypeKind.Byte: + Line($"var {varName} = new UInt8Array.Builder().Append({access}).Build();"); + break; + case ArrowTypeKind.SByte: + Line($"var {varName} = new Int8Array.Builder().Append({access}).Build();"); + break; + case ArrowTypeKind.Int16: + Line($"var {varName} = new Int16Array.Builder().Append({access}).Build();"); + break; + case ArrowTypeKind.UInt16: + Line($"var {varName} = new UInt16Array.Builder().Append({access}).Build();"); + break; + case ArrowTypeKind.Int32: + Line($"var {varName} = new Int32Array.Builder().Append({access}).Build();"); + break; + case ArrowTypeKind.UInt32: + Line($"var {varName} = new UInt32Array.Builder().Append({access}).Build();"); + break; + case ArrowTypeKind.Int64: + Line($"var {varName} = new Int64Array.Builder().Append({access}).Build();"); + break; + case ArrowTypeKind.UInt64: + Line($"var {varName} = new UInt64Array.Builder().Append({access}).Build();"); + break; + case ArrowTypeKind.Float: + Line($"var {varName} = new FloatArray.Builder().Append({access}).Build();"); + break; + case ArrowTypeKind.Double: + Line($"var {varName} = new DoubleArray.Builder().Append({access}).Build();"); + break; + case ArrowTypeKind.Binary: + { + var bi = GetBinaryArrayInfo(prop); + Line($"var {varName} = new {bi.BuilderType}().Append({BinaryToSpan(prop, access)}).Build();"); + break; + } + case ArrowTypeKind.DateTime: + Line($"var {varName} = new TimestampArray.Builder({GetTimestampTypeExpr(prop)}).Append(Apache.Arrow.Serialization.ArrowArrayHelper.{TimestampConvertMethod(prop)}({access})).Build();"); + break; + case ArrowTypeKind.DateTimeOffset: + Line($"var {varName} = new TimestampArray.Builder({GetTimestampTypeExpr(prop)}).Append(Apache.Arrow.Serialization.ArrowArrayHelper.{TimestampConvertMethod(prop)}({access})).Build();"); + break; + case ArrowTypeKind.DateOnly: + { + var di = GetDateArrayInfo(prop); + Line($"var {varName} = new {di.BuilderType}().Append({access}).Build();"); + break; + } + case ArrowTypeKind.TimeOnly: + { + var ti = GetTimeArrayInfo(prop); + Line($"var {varName} = new {ti.BuilderType}({ti.TypeExpr}).Append({access}).Build();"); + break; + } + case ArrowTypeKind.TimeSpan: + Line($"var {varName} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildDurationArray({access});"); + break; + case ArrowTypeKind.Decimal: + Line($"var {varName} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildDecimalArray({access});"); + break; + case ArrowTypeKind.Guid: + Line($"var {varName} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildGuidArray({access});"); + break; + case ArrowTypeKind.Half: + Line($"var {varName} = new HalfFloatArray.Builder().Append({access}).Build();"); + break; + case ArrowTypeKind.Enum: + EmitEnumArrayBuilder(prop, index, varName, access); + break; + case ArrowTypeKind.List: + case ArrowTypeKind.Array: + EmitListArrayBuilder(prop, index, varName, access); + break; + case ArrowTypeKind.Set: + EmitSetArrayBuilder(prop, index, varName, access); + break; + case ArrowTypeKind.Dictionary: + EmitMapArrayBuilder(prop, index, varName, access); + break; + case ArrowTypeKind.NestedRecord: + EmitNestedRecordSerializer(prop, index, varName, access); + break; + case ArrowTypeKind.Custom: + Line($"var {varName} = new {prop.ConverterTypeName}().ToArray({access});"); + break; + default: + Line($"// TODO: Unsupported type {prop.Type.Kind} for {prop.PropertyName}"); + Line($"IArrowArray {varName} = new NullArray(1);"); + break; + } + } + + private void EmitNullableArrayBuilder(PropertyModel prop, int index, string varName, string access) + { + switch (prop.Type.Kind) + { + case ArrowTypeKind.String: + { + var si = GetStringArrayInfo(prop); + Line($"var {varName} = new {si.BuilderType}().Append({access}).Build();"); + break; + } + case ArrowTypeKind.Bool: + { + var bl = GetBoolArrayInfo(prop); + Line($"var b_{index} = new {bl.BuilderType}();"); + Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(v_{index}); else b_{index}.AppendNull();"); + Line($"var {varName} = b_{index}.Build();"); + break; + } + case ArrowTypeKind.Int32: + Line($"var b_{index} = new Int32Array.Builder();"); + Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(v_{index}); else b_{index}.AppendNull();"); + Line($"var {varName} = b_{index}.Build();"); + break; + case ArrowTypeKind.Int64: + Line($"var b_{index} = new Int64Array.Builder();"); + Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(v_{index}); else b_{index}.AppendNull();"); + Line($"var {varName} = b_{index}.Build();"); + break; + case ArrowTypeKind.Float: + Line($"var b_{index} = new FloatArray.Builder();"); + Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(v_{index}); else b_{index}.AppendNull();"); + Line($"var {varName} = b_{index}.Build();"); + break; + case ArrowTypeKind.Double: + Line($"var b_{index} = new DoubleArray.Builder();"); + Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(v_{index}); else b_{index}.AppendNull();"); + Line($"var {varName} = b_{index}.Build();"); + break; + case ArrowTypeKind.DateTime: + Line($"var b_{index} = new TimestampArray.Builder({GetTimestampTypeExpr(prop)});"); + Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(Apache.Arrow.Serialization.ArrowArrayHelper.{TimestampConvertMethod(prop)}(v_{index})); else b_{index}.AppendNull();"); + Line($"var {varName} = b_{index}.Build();"); + break; + case ArrowTypeKind.DateTimeOffset: + Line($"var b_{index} = new TimestampArray.Builder({GetTimestampTypeExpr(prop)});"); + Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(Apache.Arrow.Serialization.ArrowArrayHelper.{TimestampConvertMethod(prop)}(v_{index})); else b_{index}.AppendNull();"); + Line($"var {varName} = b_{index}.Build();"); + break; + case ArrowTypeKind.DateOnly: + { + var di = GetDateArrayInfo(prop); + Line($"var b_{index} = new {di.BuilderType}();"); + Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(v_{index}); else b_{index}.AppendNull();"); + Line($"var {varName} = b_{index}.Build();"); + break; + } + case ArrowTypeKind.TimeOnly: + { + var ti = GetTimeArrayInfo(prop); + Line($"var b_{index} = new {ti.BuilderType}({ti.TypeExpr});"); + Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(v_{index}); else b_{index}.AppendNull();"); + Line($"var {varName} = b_{index}.Build();"); + break; + } + case ArrowTypeKind.TimeSpan: + Line($"var {varName} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildDurationArray({access});"); + break; + case ArrowTypeKind.Decimal: + Line($"var {varName} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildDecimalArray({access});"); + break; + case ArrowTypeKind.Guid: + Line($"var {varName} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildGuidArray({access});"); + break; + case ArrowTypeKind.Half: + Line($"var b_{index} = new HalfFloatArray.Builder();"); + Line($"if ({access} is {{ }} v_{index}) b_{index}.Append(v_{index}); else b_{index}.AppendNull();"); + Line($"var {varName} = b_{index}.Build();"); + break; + case ArrowTypeKind.Enum: + EmitEnumArrayBuilder(prop, index, varName, access); + break; + case ArrowTypeKind.NestedRecord: + EmitNestedRecordSerializer(prop, index, varName, access); + break; + case ArrowTypeKind.Custom: + Line($"IArrowArray {varName};"); + Line($"if ({access} is {{ }} cv_{index})"); + Line($" {varName} = new {prop.ConverterTypeName}().ToArray(cv_{index});"); + Line($"else"); + Line($" {varName} = new NullArray(1);"); + break; + case ArrowTypeKind.List: + case ArrowTypeKind.Array: + case ArrowTypeKind.Set: + { + Line($"IArrowArray {varName};"); + Line($"if ({access} != null)"); + Line("{"); + _indent++; + var tempVar = $"_nonnull_{index}"; + if (prop.Type.Kind == ArrowTypeKind.Set) + EmitSetArrayBuilder(prop, index, tempVar, access); + else + EmitListArrayBuilder(prop, index, tempVar, access); + Line($"{varName} = {tempVar};"); + _indent--; + Line("}"); + Line("else"); + Line("{"); + _indent++; + Line($"{varName} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildNullArray(ArrowSchema.GetFieldByName(\"{prop.FieldName}\").DataType, 1);"); + _indent--; + Line("}"); + break; + } + case ArrowTypeKind.Dictionary: + { + Line($"IArrowArray {varName};"); + Line($"if ({access} != null)"); + Line("{"); + _indent++; + var tempVar = $"_nonnull_{index}"; + EmitMapArrayBuilder(prop, index, tempVar, access); + Line($"{varName} = {tempVar};"); + _indent--; + Line("}"); + Line("else"); + Line("{"); + _indent++; + Line($"{varName} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildNullArray(ArrowSchema.GetFieldByName(\"{prop.FieldName}\").DataType, 1);"); + _indent--; + Line("}"); + break; + } + default: + // Fallback: use the non-nullable path (many builders accept null for ref types) + EmitArrayBuilder(new PropertyModel + { + PropertyName = prop.PropertyName, + FieldName = prop.FieldName, + Type = prop.Type, + IsNullable = false, + }, index); + break; } } - } - private void EmitListValuePopulation(TypeInfo elemType, string builderVar, string access, int index, int depth = 0) - { - switch (elemType.Kind) + private void EmitEnumArrayBuilder(PropertyModel prop, int index, string varName, string access) { - case TypeKind2.String: - { - var bt = GetArrayBuilderType(elemType); - Line($"foreach (var item_{index} in {access}) (({bt}){builderVar}.ValueBuilder).Append(item_{index});"); - break; - } - case TypeKind2.Byte: - Line($"foreach (var item_{index} in {access}) ((UInt8Array.Builder){builderVar}.ValueBuilder).Append(item_{index});"); - break; - case TypeKind2.SByte: - Line($"foreach (var item_{index} in {access}) ((Int8Array.Builder){builderVar}.ValueBuilder).Append(item_{index});"); - break; - case TypeKind2.Int16: - Line($"foreach (var item_{index} in {access}) ((Int16Array.Builder){builderVar}.ValueBuilder).Append(item_{index});"); - break; - case TypeKind2.UInt16: - Line($"foreach (var item_{index} in {access}) ((UInt16Array.Builder){builderVar}.ValueBuilder).Append(item_{index});"); - break; - case TypeKind2.Int32: - Line($"foreach (var item_{index} in {access}) ((Int32Array.Builder){builderVar}.ValueBuilder).Append(item_{index});"); - break; - case TypeKind2.UInt32: - Line($"foreach (var item_{index} in {access}) ((UInt32Array.Builder){builderVar}.ValueBuilder).Append(item_{index});"); - break; - case TypeKind2.Int64: - Line($"foreach (var item_{index} in {access}) ((Int64Array.Builder){builderVar}.ValueBuilder).Append(item_{index});"); - break; - case TypeKind2.UInt64: - Line($"foreach (var item_{index} in {access}) ((UInt64Array.Builder){builderVar}.ValueBuilder).Append(item_{index});"); - break; - case TypeKind2.Float: - Line($"foreach (var item_{index} in {access}) ((FloatArray.Builder){builderVar}.ValueBuilder).Append(item_{index});"); - break; - case TypeKind2.Double: - Line($"foreach (var item_{index} in {access}) ((DoubleArray.Builder){builderVar}.ValueBuilder).Append(item_{index});"); - break; - case TypeKind2.Half: - Line($"foreach (var item_{index} in {access}) ((HalfFloatArray.Builder){builderVar}.ValueBuilder).Append(item_{index});"); - break; - case TypeKind2.Bool: - Line($"foreach (var item_{index} in {access}) ((BooleanArray.Builder){builderVar}.ValueBuilder).Append(item_{index});"); - break; - case TypeKind2.Decimal: - Line($"foreach (var item_{index} in {access}) ((Decimal128Array.Builder){builderVar}.ValueBuilder).Append(item_{index});"); - break; - case TypeKind2.DateTime: - { - var hasTz = TypeInfoHasTimezone(elemType); - var convert = hasTz ? "ToUtcDateTimeOffset" : "ToWallClockDateTimeOffset"; - Line($"foreach (var item_{index} in {access}) ((TimestampArray.Builder){builderVar}.ValueBuilder).Append(Apache.Arrow.Serialization.ArrowArrayHelper.{convert}(item_{index}));"); - break; - } - case TypeKind2.DateTimeOffset: - { - var hasTz = TypeInfoHasTimezone(elemType); - var convert = hasTz ? "ToUtcDateTimeOffset" : "ToWallClockDateTimeOffset"; - Line($"foreach (var item_{index} in {access}) ((TimestampArray.Builder){builderVar}.ValueBuilder).Append(Apache.Arrow.Serialization.ArrowArrayHelper.{convert}(item_{index}));"); - break; - } - case TypeKind2.DateOnly: - { - var arrayType = GetDateArrayBuilderType(elemType); - Line($"foreach (var item_{index} in {access}) (({arrayType}){builderVar}.ValueBuilder).Append(item_{index});"); - break; - } - case TypeKind2.TimeOnly: - { - var arrayType = GetTimeArrayBuilderType(elemType); - Line($"foreach (var item_{index} in {access}) (({arrayType}){builderVar}.ValueBuilder).Append(item_{index});"); - break; - } - case TypeKind2.TimeSpan: - Line($"foreach (var item_{index} in {access}) ((DurationArray.Builder){builderVar}.ValueBuilder).Append(item_{index});"); - break; - case TypeKind2.Guid: - Line($"foreach (var item_{index} in {access}) ((GuidArray.Builder){builderVar}.ValueBuilder).Append(item_{index});"); - break; - case TypeKind2.Binary: - Line($"foreach (var item_{index} in {access}) ((BinaryArray.Builder){builderVar}.ValueBuilder).Append((ReadOnlySpan)item_{index});"); - break; - case TypeKind2.Enum: - // For list of enums, we need to build dictionary-encoded values - // Simplified: store as string list for now - Line($"foreach (var item_{index} in {access}) ((StringArray.Builder){builderVar}.ValueBuilder).Append(item_{index}.ToString());"); - break; - case TypeKind2.List: - case TypeKind2.Array: - { - // Nested list: ValueBuilder is itself a ListArray.Builder - var innerElemType = elemType.ElementType!; - var innerBuilderCast = $"((ListArray.Builder){builderVar}.ValueBuilder)"; - var loopVar = depth == 0 ? $"inner_{index}" : $"inner_{index}_{depth}"; - Line($"foreach (var {loopVar} in {access})"); + // Dictionary-encoded string array: indices + dictionary + // For a single value, build a StringArray for the dictionary and Int16Array for indices + if (prop.IsNullable) + { + Line($"StringArray dict_{index};"); + Line($"Int16Array idx_{index};"); + Line($"if ({access} is {{ }} ev_{index})"); Line("{"); _indent++; - Line($"{innerBuilderCast}.Append();"); - EmitListValuePopulation(innerElemType, innerBuilderCast, loopVar, index, depth + 1); + Line($"dict_{index} = new StringArray.Builder().Append(ev_{index}.ToString()).Build();"); + Line($"idx_{index} = new Int16Array.Builder().Append(0).Build();"); _indent--; Line("}"); - break; - } - case TypeKind2.Dictionary: - { - // Dict values inside a list: ValueBuilder is a MapArray.Builder - var keyType = elemType.KeyType!; - var valueType = elemType.ValueType!; - var innerMapBuilder = $"((MapArray.Builder){builderVar}.ValueBuilder)"; - var loopVar = depth == 0 ? $"dictItem_{index}" : $"dictItem_{index}_{depth}"; - var kvVar = depth == 0 ? $"mkv_{index}" : $"mkv_{index}_{depth}"; - Line($"foreach (var {loopVar} in {access})"); - Line("{"); - _indent++; - Line($"{innerMapBuilder}.Append();"); - Line($"foreach (var {kvVar} in {loopVar})"); + Line("else"); Line("{"); _indent++; - EmitMapKeyAppend(keyType, innerMapBuilder, $"{kvVar}.Key", index, depth + 1); - EmitMapValueAppend(valueType, innerMapBuilder, $"{kvVar}.Value", index, depth + 1); - _indent--; - Line("}"); + Line($"dict_{index} = new StringArray.Builder().Build();"); + Line($"idx_{index} = new Int16Array.Builder().AppendNull().Build();"); _indent--; Line("}"); - break; } - default: - Line($"// TODO: List element type {elemType.Kind} not yet supported for population"); - break; + else + { + Line($"var dict_{index} = new StringArray.Builder().Append({access}.ToString()).Build();"); + Line($"var idx_{index} = new Int16Array.Builder().Append(0).Build();"); + } + Line($"var {varName} = new DictionaryArray(new DictionaryType(Int16Type.Default, StringType.Default, false), idx_{index}, dict_{index});"); } - } - - /// - /// Returns true for value types that cannot use MapArray.Builder because - /// ArrowArrayBuilderFactory.Build() does not support them. - /// - private static bool NeedsManualMapBuilder(TypeInfo valueType) - { - return valueType.Kind == TypeKind2.Guid - || valueType.Kind == TypeKind2.Enum - || valueType.Kind == TypeKind2.NestedRecord; - } - private void EmitMapArrayBuilder(PropertyModel prop, int index, string varName, string access) - { - var keyType = prop.Type.KeyType!; - var valueType = prop.Type.ValueType!; - - if (NeedsManualMapBuilder(valueType)) + private void EmitListArrayBuilder(PropertyModel prop, int index, string varName, string access) { - EmitManualMapArray(prop, index, varName, access); - return; - } + var elemType = prop.Type.ElementType!; + if (NeedsManualListBuilder(elemType)) + { + EmitManualListArray(elemType, index, varName, access, prop.FieldName); + return; + } + var arrowElemType = GetArrowTypeExpression(elemType); - var mapType = GetArrowTypeExpression(prop.Type); - Line($"var mb_{index} = new MapArray.Builder({mapType});"); - Line($"mb_{index}.Append();"); - Line($"foreach (var kv_{index} in {access})"); - Line("{"); - _indent++; - EmitMapKeyAppend(keyType, $"mb_{index}", $"kv_{index}.Key", index); - EmitMapValueAppend(valueType, $"mb_{index}", $"kv_{index}.Value", index); - _indent--; - Line("}"); - Line($"var {varName} = mb_{index}.Build();"); - } + Line($"var lb_{index} = new ListArray.Builder({arrowElemType});"); + Line($"lb_{index}.Append();"); + EmitListValuePopulation(elemType, $"lb_{index}", access, index); + Line($"var {varName} = lb_{index}.Build();"); + } - /// - /// Builds a single-row MapArray manually for value types unsupported by ArrowArrayBuilderFactory. - /// - private void EmitManualMapArray(PropertyModel prop, int index, string varName, string access) - { - var keyType = prop.Type.KeyType!; - var valueType = prop.Type.ValueType!; - var mapType = GetArrowTypeExpression(prop.Type); - - // Build keys - var keyBuilderType = GetArrayBuilderType(keyType); - Line($"var mk_{index} = new {keyBuilderType}();"); - Line($"foreach (var kv_{index} in {access}) mk_{index}.Append(kv_{index}.Key);"); - Line($"var mkArr_{index} = mk_{index}.Build();"); - - // Build values based on type - switch (valueType.Kind) + private void EmitSetArrayBuilder(PropertyModel prop, int index, string varName, string access) { - case TypeKind2.Guid: - Line($"var mvItems_{index} = new System.Collections.Generic.List();"); - Line($"foreach (var kv_{index}v in {access}) mvItems_{index}.Add(kv_{index}v.Value);"); - Line($"var mvArr_{index} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildGuidArray(mvItems_{index});"); - break; - case TypeKind2.Enum: - { - Line($"var mvDict_{index} = new StringArray.Builder();"); - Line($"var mvIdx_{index} = new Int16Array.Builder();"); - Line($"var mvMap_{index} = new System.Collections.Generic.Dictionary();"); - Line($"foreach (var kv_{index}v in {access})"); - Line("{"); - _indent++; - Line($"var es_{index} = kv_{index}v.Value.ToString();"); - Line($"if (!mvMap_{index}.TryGetValue(es_{index}, out var ei_{index})) {{ ei_{index} = (short)mvMap_{index}.Count; mvMap_{index}[es_{index}] = ei_{index}; }}"); - Line($"mvIdx_{index}.Append(ei_{index});"); - _indent--; - Line("}"); - Line($"foreach (var kv in mvMap_{index}.OrderBy(kv => kv.Value)) mvDict_{index}.Append(kv.Key);"); - Line($"var mvArr_{index} = new DictionaryArray(new DictionaryType(Int16Type.Default, StringType.Default, false), mvIdx_{index}.Build(), mvDict_{index}.Build());"); - break; - } - case TypeKind2.NestedRecord: + var elemType = prop.Type.ElementType!; + if (NeedsManualListBuilder(elemType)) { - var typeName = valueType.FullTypeName; - Line($"var mvItems_{index} = new System.Collections.Generic.List<{typeName}>();"); - Line($"foreach (var kv_{index}v in {access}) mvItems_{index}.Add(kv_{index}v.Value);"); - Line($"IArrowArray mvArr_{index};"); - Line($"if (mvItems_{index}.Count > 0)"); - Line("{"); - _indent++; - Line($"var nb_{index} = {typeName}.ToRecordBatch((IReadOnlyList<{typeName}>)mvItems_{index});"); - Line($"var ca_{index} = new IArrowArray[nb_{index}.ColumnCount];"); - Line($"for (int f = 0; f < nb_{index}.ColumnCount; f++) ca_{index}[f] = nb_{index}.Column(f);"); - Line($"mvArr_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), mvItems_{index}.Count, ca_{index}, ArrowBuffer.Empty);"); - _indent--; - Line("}"); - Line($"else mvArr_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), 0, System.Array.Empty(), ArrowBuffer.Empty);"); - break; + EmitManualListArray(elemType, index, varName, access, prop.FieldName); + return; } + var arrowElemType = GetArrowTypeExpression(elemType); + + Line($"var lb_{index} = new ListArray.Builder({arrowElemType});"); + Line($"lb_{index}.Append();"); + EmitListValuePopulation(elemType, $"lb_{index}", access, index); + Line($"var {varName} = lb_{index}.Build();"); } - // Construct MapArray manually: offsets, StructArray(keys, values), then MapArray - Line($"var mOffsets_{index} = new ArrowBuffer.Builder();"); - Line($"mOffsets_{index}.Append(0);"); - Line($"mOffsets_{index}.Append(mkArr_{index}.Length);"); - Line($"var mEntries_{index} = new StructArray(((MapType){mapType}).KeyValueType, mkArr_{index}.Length, new IArrowArray[] {{ mkArr_{index}, mvArr_{index} }}, ArrowBuffer.Empty);"); - Line($"var {varName} = new MapArray({mapType}, 1, mOffsets_{index}.Build(), mEntries_{index}, ArrowBuffer.Empty);"); - } + /// + /// Returns true for element types that cannot use ListArray.Builder because + /// ArrowArrayBuilderFactory.Build() does not support them (extension types like Guid). + /// + private static bool NeedsManualListBuilder(TypeInfo elemType) + { + return elemType.Kind == ArrowTypeKind.Guid + || elemType.Kind == ArrowTypeKind.Enum + || elemType.Kind == ArrowTypeKind.NestedRecord; + } - private void EmitMapKeyAppend(TypeInfo keyType, string builderVar, string access, int index, int depth = 0) - { - var builderType = GetArrayBuilderType(keyType); - var appendExpr = GetMapAppendExpression(keyType, builderType, $"{builderVar}.KeyBuilder", access); - Line(appendExpr); - } + /// + /// Returns true for element types that need manual deserialization (can't use GetElementReader). + /// + private static bool NeedsManualListDeserialize(TypeInfo elemType) + { + return NeedsManualListBuilder(elemType) + || elemType.Kind == ArrowTypeKind.List + || elemType.Kind == ArrowTypeKind.Array + || elemType.Kind == ArrowTypeKind.Dictionary; + } - private void EmitMapValueAppend(TypeInfo valueType, string builderVar, string access, int index, int depth = 0) - { - switch (valueType.Kind) + /// + /// Builds a single-row ListArray manually for element types unsupported by ArrowArrayBuilderFactory. + /// + private void EmitManualListArray(TypeInfo elemType, int index, string varName, string access, string fieldName) { - case TypeKind2.List: - case TypeKind2.Array: - case TypeKind2.Set: - { - // List value in map: ValueBuilder is a ListArray.Builder - var innerListBuilder = $"((ListArray.Builder){builderVar}.ValueBuilder)"; - Line($"{innerListBuilder}.Append();"); - EmitListValuePopulation(valueType.ElementType!, innerListBuilder, access, index, depth + 1); - break; - } - case TypeKind2.Dictionary: - { - // Nested map value: ValueBuilder is a MapArray.Builder - var innerMapBuilder = $"((MapArray.Builder){builderVar}.ValueBuilder)"; - var innerKvVar = depth == 0 ? $"mvkv_{index}" : $"mvkv_{index}_{depth}"; - Line($"{innerMapBuilder}.Append();"); - Line($"foreach (var {innerKvVar} in {access})"); - Line("{"); - _indent++; - EmitMapKeyAppend(valueType.KeyType!, innerMapBuilder, $"{innerKvVar}.Key", index, depth + 1); - EmitMapValueAppend(valueType.ValueType!, innerMapBuilder, $"{innerKvVar}.Value", index, depth + 1); - _indent--; - Line("}"); - break; - } - default: + var arrowElemType = GetArrowTypeExpression(elemType); + switch (elemType.Kind) { - var builderType = GetArrayBuilderType(valueType); - var appendExpr = GetMapAppendExpression(valueType, builderType, $"{builderVar}.ValueBuilder", access); - Line(appendExpr); - break; + case ArrowTypeKind.Guid: + Line($"var guidList_{index} = new System.Collections.Generic.List({access});"); + Line($"var guidValues_{index} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildGuidArray(guidList_{index});"); + Line($"var offsets_{index} = new ArrowBuffer.Builder();"); + Line($"offsets_{index}.Append(0);"); + Line($"offsets_{index}.Append(guidList_{index}.Count);"); + Line($"var {varName} = new ListArray(new ListType(new Field(\"item\", {arrowElemType}, false)), 1, offsets_{index}.Build(), guidValues_{index}, ArrowBuffer.Empty);"); + break; + case ArrowTypeKind.Enum: + { + // Enum list elements: dictionary-encoded (matching Python schema) + Line($"var enumDict_{index} = new StringArray.Builder();"); + Line($"var enumIdx_{index} = new Int16Array.Builder();"); + Line($"var enumMap_{index} = new System.Collections.Generic.Dictionary();"); + Line($"foreach (var e_{index} in {access}) {{"); + Line($" var es_{index} = e_{index}.ToString();"); + Line($" if (!enumMap_{index}.TryGetValue(es_{index}, out var ei_{index})) {{ ei_{index} = (short)enumMap_{index}.Count; enumMap_{index}[es_{index}] = ei_{index}; }}"); + Line($" enumIdx_{index}.Append(ei_{index});"); + Line($"}}"); + Line($"foreach (var kv in enumMap_{index}.OrderBy(kv => kv.Value)) enumDict_{index}.Append(kv.Key);"); + Line($"var enumDictArr_{index} = new DictionaryArray(new DictionaryType(Int16Type.Default, StringType.Default, false), enumIdx_{index}.Build(), enumDict_{index}.Build());"); + Line($"var offsets_{index} = new ArrowBuffer.Builder();"); + Line($"offsets_{index}.Append(0);"); + Line($"offsets_{index}.Append(enumDictArr_{index}.Length);"); + Line($"var {varName} = new ListArray(ArrowSchema.GetFieldByName(\"{fieldName}\").DataType, 1, offsets_{index}.Build(), enumDictArr_{index}, ArrowBuffer.Empty);"); + break; + } + case ArrowTypeKind.NestedRecord: + { + // Nested record list elements: serialize via child's ToRecordBatch, wrap in StructArray + var typeName = elemType.FullTypeName; + Line($"var nestedList_{index} = new System.Collections.Generic.List<{typeName}>({access});"); + Line($"var nestedBatch_{index} = {typeName}.ToRecordBatch((IReadOnlyList<{typeName}>)nestedList_{index});"); + Line($"var nestedCols_{index} = new IArrowArray[nestedBatch_{index}.ColumnCount];"); + Line($"for (int f = 0; f < nestedBatch_{index}.ColumnCount; f++) nestedCols_{index}[f] = nestedBatch_{index}.Column(f);"); + Line($"var structValues_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), nestedList_{index}.Count, nestedCols_{index}, ArrowBuffer.Empty);"); + Line($"var offsets_{index} = new ArrowBuffer.Builder();"); + Line($"offsets_{index}.Append(0);"); + Line($"offsets_{index}.Append(nestedList_{index}.Count);"); + Line($"var {varName} = new ListArray(new ListType(new Field(\"item\", new StructType({typeName}.ArrowSchema.FieldsList), false)), 1, offsets_{index}.Build(), structValues_{index}, ArrowBuffer.Empty);"); + break; + } } } - } - private string GetMapAppendExpression(TypeInfo type, string builderType, string builderAccess, string valueAccess) - { - switch (type.Kind) + private void EmitListValuePopulation(TypeInfo elemType, string builderVar, string access, int index, int depth = 0) { - case TypeKind2.DateTime: - case TypeKind2.DateTimeOffset: - { - var convert = TypeInfoHasTimezone(type) ? "ToUtcDateTimeOffset" : "ToWallClockDateTimeOffset"; - return $"(({builderType}){builderAccess}).Append(Apache.Arrow.Serialization.ArrowArrayHelper.{convert}({valueAccess}));"; - } - case TypeKind2.Binary: - return $"(({builderType}){builderAccess}).Append((ReadOnlySpan){valueAccess});"; - case TypeKind2.Enum: - return $"(({builderType}){builderAccess}).Append({valueAccess}.ToString());"; - default: - return $"(({builderType}){builderAccess}).Append({valueAccess});"; + switch (elemType.Kind) + { + case ArrowTypeKind.String: + { + var bt = GetArrayBuilderType(elemType); + Line($"foreach (var item_{index} in {access}) (({bt}){builderVar}.ValueBuilder).Append(item_{index});"); + break; + } + case ArrowTypeKind.Byte: + Line($"foreach (var item_{index} in {access}) ((UInt8Array.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case ArrowTypeKind.SByte: + Line($"foreach (var item_{index} in {access}) ((Int8Array.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case ArrowTypeKind.Int16: + Line($"foreach (var item_{index} in {access}) ((Int16Array.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case ArrowTypeKind.UInt16: + Line($"foreach (var item_{index} in {access}) ((UInt16Array.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case ArrowTypeKind.Int32: + Line($"foreach (var item_{index} in {access}) ((Int32Array.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case ArrowTypeKind.UInt32: + Line($"foreach (var item_{index} in {access}) ((UInt32Array.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case ArrowTypeKind.Int64: + Line($"foreach (var item_{index} in {access}) ((Int64Array.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case ArrowTypeKind.UInt64: + Line($"foreach (var item_{index} in {access}) ((UInt64Array.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case ArrowTypeKind.Float: + Line($"foreach (var item_{index} in {access}) ((FloatArray.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case ArrowTypeKind.Double: + Line($"foreach (var item_{index} in {access}) ((DoubleArray.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case ArrowTypeKind.Half: + Line($"foreach (var item_{index} in {access}) ((HalfFloatArray.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case ArrowTypeKind.Bool: + Line($"foreach (var item_{index} in {access}) ((BooleanArray.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case ArrowTypeKind.Decimal: + Line($"foreach (var item_{index} in {access}) ((Decimal128Array.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case ArrowTypeKind.DateTime: + { + var hasTz = TypeInfoHasTimezone(elemType); + var convert = hasTz ? "ToUtcDateTimeOffset" : "ToWallClockDateTimeOffset"; + Line($"foreach (var item_{index} in {access}) ((TimestampArray.Builder){builderVar}.ValueBuilder).Append(Apache.Arrow.Serialization.ArrowArrayHelper.{convert}(item_{index}));"); + break; + } + case ArrowTypeKind.DateTimeOffset: + { + var hasTz = TypeInfoHasTimezone(elemType); + var convert = hasTz ? "ToUtcDateTimeOffset" : "ToWallClockDateTimeOffset"; + Line($"foreach (var item_{index} in {access}) ((TimestampArray.Builder){builderVar}.ValueBuilder).Append(Apache.Arrow.Serialization.ArrowArrayHelper.{convert}(item_{index}));"); + break; + } + case ArrowTypeKind.DateOnly: + { + var arrayType = GetDateArrayBuilderType(elemType); + Line($"foreach (var item_{index} in {access}) (({arrayType}){builderVar}.ValueBuilder).Append(item_{index});"); + break; + } + case ArrowTypeKind.TimeOnly: + { + var arrayType = GetTimeArrayBuilderType(elemType); + Line($"foreach (var item_{index} in {access}) (({arrayType}){builderVar}.ValueBuilder).Append(item_{index});"); + break; + } + case ArrowTypeKind.TimeSpan: + Line($"foreach (var item_{index} in {access}) ((DurationArray.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case ArrowTypeKind.Guid: + Line($"foreach (var item_{index} in {access}) ((GuidArray.Builder){builderVar}.ValueBuilder).Append(item_{index});"); + break; + case ArrowTypeKind.Binary: + Line($"foreach (var item_{index} in {access}) ((BinaryArray.Builder){builderVar}.ValueBuilder).Append((ReadOnlySpan)item_{index});"); + break; + case ArrowTypeKind.Enum: + // For list of enums, we need to build dictionary-encoded values + // Simplified: store as string list for now + Line($"foreach (var item_{index} in {access}) ((StringArray.Builder){builderVar}.ValueBuilder).Append(item_{index}.ToString());"); + break; + case ArrowTypeKind.List: + case ArrowTypeKind.Array: + { + // Nested list: ValueBuilder is itself a ListArray.Builder + var innerElemType = elemType.ElementType!; + var innerBuilderCast = $"((ListArray.Builder){builderVar}.ValueBuilder)"; + var loopVar = depth == 0 ? $"inner_{index}" : $"inner_{index}_{depth}"; + Line($"foreach (var {loopVar} in {access})"); + Line("{"); + _indent++; + Line($"{innerBuilderCast}.Append();"); + EmitListValuePopulation(innerElemType, innerBuilderCast, loopVar, index, depth + 1); + _indent--; + Line("}"); + break; + } + case ArrowTypeKind.Dictionary: + { + // Dict values inside a list: ValueBuilder is a MapArray.Builder + var keyType = elemType.KeyType!; + var valueType = elemType.ValueType!; + var innerMapBuilder = $"((MapArray.Builder){builderVar}.ValueBuilder)"; + var loopVar = depth == 0 ? $"dictItem_{index}" : $"dictItem_{index}_{depth}"; + var kvVar = depth == 0 ? $"mkv_{index}" : $"mkv_{index}_{depth}"; + Line($"foreach (var {loopVar} in {access})"); + Line("{"); + _indent++; + Line($"{innerMapBuilder}.Append();"); + Line($"foreach (var {kvVar} in {loopVar})"); + Line("{"); + _indent++; + EmitMapKeyAppend(keyType, innerMapBuilder, $"{kvVar}.Key", index, depth + 1); + EmitMapValueAppend(valueType, innerMapBuilder, $"{kvVar}.Value", index, depth + 1); + _indent--; + Line("}"); + _indent--; + Line("}"); + break; + } + default: + Line($"// TODO: List element type {elemType.Kind} not yet supported for population"); + break; + } } - } - private void EmitNestedRecordSerializer(PropertyModel prop, int index, string varName, string access) - { - var typeName = prop.Type.FullTypeName; + /// + /// Returns true for value types that cannot use MapArray.Builder because + /// ArrowArrayBuilderFactory.Build() does not support them. + /// + private static bool NeedsManualMapBuilder(TypeInfo valueType) + { + return valueType.Kind == ArrowTypeKind.Guid + || valueType.Kind == ArrowTypeKind.Enum + || valueType.Kind == ArrowTypeKind.NestedRecord; + } - if (prop.IsNullable) + private void EmitMapArrayBuilder(PropertyModel prop, int index, string varName, string access) { - Line($"IArrowArray {varName};"); - Line($"if ({access} is {{ }} nv_{index})"); - Line("{"); - _indent++; - Line($"var nestedBatch_{index} = {typeName}.ToRecordBatch(nv_{index});"); - Line($"var nestedArrays_{index} = new IArrowArray[nestedBatch_{index}.ColumnCount];"); - Line($"for (int i = 0; i < nestedBatch_{index}.ColumnCount; i++) nestedArrays_{index}[i] = nestedBatch_{index}.Column(i);"); - Line($"{varName} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), 1, nestedArrays_{index}, ArrowBuffer.Empty);"); - _indent--; - Line("}"); - Line("else"); + var keyType = prop.Type.KeyType!; + var valueType = prop.Type.ValueType!; + + if (NeedsManualMapBuilder(valueType)) + { + EmitManualMapArray(prop, index, varName, access); + return; + } + + var mapType = GetArrowTypeExpression(prop.Type); + Line($"var mb_{index} = new MapArray.Builder({mapType});"); + Line($"mb_{index}.Append();"); + Line($"foreach (var kv_{index} in {access})"); Line("{"); _indent++; - Line($"var nullArrays_{index} = new IArrowArray[{typeName}.ArrowSchema.FieldsList.Count];"); - Line($"for (int i = 0; i < nullArrays_{index}.Length; i++)"); - Line($" nullArrays_{index}[i] = Apache.Arrow.Serialization.ArrowArrayHelper.BuildNullArray({typeName}.ArrowSchema.FieldsList[i].DataType, 1);"); - Line($"var nullBitmap_{index} = new ArrowBuffer.BitmapBuilder();"); - Line($"nullBitmap_{index}.Append(false);"); - Line($"{varName} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), 1, nullArrays_{index}, nullBitmap_{index}.Build(), 1);"); + EmitMapKeyAppend(keyType, $"mb_{index}", $"kv_{index}.Key", index); + EmitMapValueAppend(valueType, $"mb_{index}", $"kv_{index}.Value", index); _indent--; Line("}"); + Line($"var {varName} = mb_{index}.Build();"); } - else + + /// + /// Builds a single-row MapArray manually for value types unsupported by ArrowArrayBuilderFactory. + /// + private void EmitManualMapArray(PropertyModel prop, int index, string varName, string access) { - Line($"var nestedBatch_{index} = {typeName}.ToRecordBatch({access});"); - Line($"var nestedArrays_{index} = new IArrowArray[nestedBatch_{index}.ColumnCount];"); - Line($"for (int i = 0; i < nestedBatch_{index}.ColumnCount; i++) nestedArrays_{index}[i] = nestedBatch_{index}.Column(i);"); - Line($"var {varName} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), 1, nestedArrays_{index}, ArrowBuffer.Empty);"); - } - } + var keyType = prop.Type.KeyType!; + var valueType = prop.Type.ValueType!; + var mapType = GetArrowTypeExpression(prop.Type); - private void EmitDeserialize() - { - var newKw = _model.HasArrowSerializableBase ? "new " : ""; - Line($"public static {newKw}{_model.TypeName} FromRecordBatch(RecordBatch batch)"); - Line("{"); - _indent++; - - Line("if (batch.Length == 0)"); - Line(" throw new InvalidOperationException(\"Cannot deserialize from empty RecordBatch.\");"); - Line("if (batch.Length > 1)"); - Line(" throw new InvalidOperationException($\"Expected single-row RecordBatch, got {batch.Length} rows.\");"); - Line(); - - for (int i = 0; i < _model.Properties.Count; i++) - { - var prop = _model.Properties[i]; - EmitDeserializeProperty(prop, i); - } + // Build keys + var keyBuilderType = GetArrayBuilderType(keyType); + Line($"var mk_{index} = new {keyBuilderType}();"); + Line($"foreach (var kv_{index} in {access}) mk_{index}.Append(kv_{index}.Key);"); + Line($"var mkArr_{index} = mk_{index}.Build();"); - Line(); - if (_model.HasSerializationCallback) - { - EmitObjectConstruction("var result = ", ";"); - Line("((Apache.Arrow.Serialization.IArrowSerializationCallback)result).OnAfterDeserialize();"); - Line("return result;"); + // Build values based on type + switch (valueType.Kind) + { + case ArrowTypeKind.Guid: + Line($"var mvItems_{index} = new System.Collections.Generic.List();"); + Line($"foreach (var kv_{index}v in {access}) mvItems_{index}.Add(kv_{index}v.Value);"); + Line($"var mvArr_{index} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildGuidArray(mvItems_{index});"); + break; + case ArrowTypeKind.Enum: + { + Line($"var mvDict_{index} = new StringArray.Builder();"); + Line($"var mvIdx_{index} = new Int16Array.Builder();"); + Line($"var mvMap_{index} = new System.Collections.Generic.Dictionary();"); + Line($"foreach (var kv_{index}v in {access})"); + Line("{"); + _indent++; + Line($"var es_{index} = kv_{index}v.Value.ToString();"); + Line($"if (!mvMap_{index}.TryGetValue(es_{index}, out var ei_{index})) {{ ei_{index} = (short)mvMap_{index}.Count; mvMap_{index}[es_{index}] = ei_{index}; }}"); + Line($"mvIdx_{index}.Append(ei_{index});"); + _indent--; + Line("}"); + Line($"foreach (var kv in mvMap_{index}.OrderBy(kv => kv.Value)) mvDict_{index}.Append(kv.Key);"); + Line($"var mvArr_{index} = new DictionaryArray(new DictionaryType(Int16Type.Default, StringType.Default, false), mvIdx_{index}.Build(), mvDict_{index}.Build());"); + break; + } + case ArrowTypeKind.NestedRecord: + { + var typeName = valueType.FullTypeName; + Line($"var mvItems_{index} = new System.Collections.Generic.List<{typeName}>();"); + Line($"foreach (var kv_{index}v in {access}) mvItems_{index}.Add(kv_{index}v.Value);"); + Line($"IArrowArray mvArr_{index};"); + Line($"if (mvItems_{index}.Count > 0)"); + Line("{"); + _indent++; + Line($"var nb_{index} = {typeName}.ToRecordBatch((IReadOnlyList<{typeName}>)mvItems_{index});"); + Line($"var ca_{index} = new IArrowArray[nb_{index}.ColumnCount];"); + Line($"for (int f = 0; f < nb_{index}.ColumnCount; f++) ca_{index}[f] = nb_{index}.Column(f);"); + Line($"mvArr_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), mvItems_{index}.Count, ca_{index}, ArrowBuffer.Empty);"); + _indent--; + Line("}"); + Line($"else mvArr_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), 0, System.Array.Empty(), ArrowBuffer.Empty);"); + break; + } + } + + // Construct MapArray manually: offsets, StructArray(keys, values), then MapArray + Line($"var mOffsets_{index} = new ArrowBuffer.Builder();"); + Line($"mOffsets_{index}.Append(0);"); + Line($"mOffsets_{index}.Append(mkArr_{index}.Length);"); + Line($"var mEntries_{index} = new StructArray(((MapType){mapType}).KeyValueType, mkArr_{index}.Length, new IArrowArray[] {{ mkArr_{index}, mvArr_{index} }}, ArrowBuffer.Empty);"); + Line($"var {varName} = new MapArray({mapType}, 1, mOffsets_{index}.Build(), mEntries_{index}, ArrowBuffer.Empty);"); } - else + + private void EmitMapKeyAppend(TypeInfo keyType, string builderVar, string access, int index, int depth = 0) { - EmitObjectConstruction("return ", ";"); + var builderType = GetArrayBuilderType(keyType); + var appendExpr = GetMapAppendExpression(keyType, builderType, $"{builderVar}.KeyBuilder", access); + Line(appendExpr); } - _indent--; - Line("}"); - } - - /// - /// Emits either constructor call or object initializer to create an instance. - /// Uses constructor when ConstructorParams is set (field-based types), otherwise object initializer. - /// - private void EmitObjectConstruction(string prefix, string suffix) - { - if (_model.ConstructorParams != null) + private void EmitMapValueAppend(TypeInfo valueType, string builderVar, string access, int index, int depth = 0) { - // Constructor-based: new TypeName(paramName: prop_N, ...) - var propByName = new Dictionary(StringComparer.OrdinalIgnoreCase); - for (int i = 0; i < _model.Properties.Count; i++) - propByName[_model.Properties[i].PropertyName] = i; - - var args = new List(); - foreach (var param in _model.ConstructorParams) + switch (valueType.Kind) { - if (propByName.TryGetValue(param.Name, out int idx)) - args.Add($"{param.Name}: prop_{idx}"); - else if (param.HasDefaultValue) - continue; // use default + case ArrowTypeKind.List: + case ArrowTypeKind.Array: + case ArrowTypeKind.Set: + { + // List value in map: ValueBuilder is a ListArray.Builder + var innerListBuilder = $"((ListArray.Builder){builderVar}.ValueBuilder)"; + Line($"{innerListBuilder}.Append();"); + EmitListValuePopulation(valueType.ElementType!, innerListBuilder, access, index, depth + 1); + break; + } + case ArrowTypeKind.Dictionary: + { + // Nested map value: ValueBuilder is a MapArray.Builder + var innerMapBuilder = $"((MapArray.Builder){builderVar}.ValueBuilder)"; + var innerKvVar = depth == 0 ? $"mvkv_{index}" : $"mvkv_{index}_{depth}"; + Line($"{innerMapBuilder}.Append();"); + Line($"foreach (var {innerKvVar} in {access})"); + Line("{"); + _indent++; + EmitMapKeyAppend(valueType.KeyType!, innerMapBuilder, $"{innerKvVar}.Key", index, depth + 1); + EmitMapValueAppend(valueType.ValueType!, innerMapBuilder, $"{innerKvVar}.Value", index, depth + 1); + _indent--; + Line("}"); + break; + } + default: + { + var builderType = GetArrayBuilderType(valueType); + var appendExpr = GetMapAppendExpression(valueType, builderType, $"{builderVar}.ValueBuilder", access); + Line(appendExpr); + break; + } } - - Line($"{prefix}new {_model.TypeName}({string.Join(", ", args)}){suffix}"); } - else + + private string GetMapAppendExpression(TypeInfo type, string builderType, string builderAccess, string valueAccess) { - // Object initializer: new TypeName { Prop = prop_N, ... } - Line($"{prefix}new {_model.TypeName}"); - Line("{"); - _indent++; - for (int i = 0; i < _model.Properties.Count; i++) + switch (type.Kind) { - var prop = _model.Properties[i]; - var comma = i < _model.Properties.Count - 1 ? "," : ""; - Line($"{prop.PropertyName} = prop_{i}{comma}"); + case ArrowTypeKind.DateTime: + case ArrowTypeKind.DateTimeOffset: + { + var convert = TypeInfoHasTimezone(type) ? "ToUtcDateTimeOffset" : "ToWallClockDateTimeOffset"; + return $"(({builderType}){builderAccess}).Append(Apache.Arrow.Serialization.ArrowArrayHelper.{convert}({valueAccess}));"; + } + case ArrowTypeKind.Binary: + return $"(({builderType}){builderAccess}).Append((ReadOnlySpan){valueAccess});"; + case ArrowTypeKind.Enum: + return $"(({builderType}){builderAccess}).Append({valueAccess}.ToString());"; + default: + return $"(({builderType}){builderAccess}).Append({valueAccess});"; } - _indent--; - Line($"}}{suffix}"); } - } - - private void EmitDeserializeProperty(PropertyModel prop, int index) - { - var colAccess = $"batch.Column(\"{prop.FieldName}\")"; - switch (prop.Type.Kind) + private void EmitNestedRecordSerializer(PropertyModel prop, int index, string varName, string access) { - case TypeKind2.String: - { - var si = GetStringArrayInfo(prop); - if (prop.IsNullable) - Line($"var prop_{index} = (({colAccess}) is {si.ArrayType} sa_{index} && !sa_{index}.IsNull(0)) ? sa_{index}.GetString(0) : null;"); - else - Line($"var prop_{index} = (({si.ArrayType}){colAccess}).GetString(0)!;"); - break; - } - case TypeKind2.Bool: - { - var bl = GetBoolArrayInfo(prop); - EmitPrimitiveDeserialize(prop, index, colAccess, bl.ArrayType, "GetValue"); - break; - } - case TypeKind2.Byte: - EmitPrimitiveDeserialize(prop, index, colAccess, "UInt8Array", "GetValue"); - break; - case TypeKind2.SByte: - EmitPrimitiveDeserialize(prop, index, colAccess, "Int8Array", "GetValue"); - break; - case TypeKind2.Int16: - EmitPrimitiveDeserialize(prop, index, colAccess, "Int16Array", "GetValue"); - break; - case TypeKind2.UInt16: - EmitPrimitiveDeserialize(prop, index, colAccess, "UInt16Array", "GetValue"); - break; - case TypeKind2.Int32: - EmitPrimitiveDeserialize(prop, index, colAccess, "Int32Array", "GetValue"); - break; - case TypeKind2.UInt32: - EmitPrimitiveDeserialize(prop, index, colAccess, "UInt32Array", "GetValue"); - break; - case TypeKind2.Int64: - EmitPrimitiveDeserialize(prop, index, colAccess, "Int64Array", "GetValue"); - break; - case TypeKind2.UInt64: - EmitPrimitiveDeserialize(prop, index, colAccess, "UInt64Array", "GetValue"); - break; - case TypeKind2.Float: - EmitPrimitiveDeserialize(prop, index, colAccess, "FloatArray", "GetValue"); - break; - case TypeKind2.Double: - EmitPrimitiveDeserialize(prop, index, colAccess, "DoubleArray", "GetValue"); - break; - case TypeKind2.Binary: - { - var bi = GetBinaryArrayInfo(prop); - if (IsReadOnlyMemoryByte(prop)) - Line($"ReadOnlyMemory prop_{index} = (({bi.ArrayType}){colAccess}).GetBytes(0).ToArray();"); - else - Line($"var prop_{index} = (({bi.ArrayType}){colAccess}).GetBytes(0).ToArray();"); - break; - } - case TypeKind2.DateTime: - { - var dtProp = TimestampHasTimezone(prop) ? "UtcDateTime" : "DateTime"; - if (prop.IsNullable) - Line($"var prop_{index} = ((TimestampArray){colAccess}).IsNull(0) ? (System.DateTime?)null : ((TimestampArray){colAccess}).GetTimestamp(0)!.Value.{dtProp};"); - else - Line($"var prop_{index} = ((TimestampArray){colAccess}).GetTimestamp(0)!.Value.{dtProp};"); - break; - } - case TypeKind2.DateTimeOffset: - if (prop.IsNullable) - Line($"var prop_{index} = ((TimestampArray){colAccess}).IsNull(0) ? (System.DateTimeOffset?)null : ((TimestampArray){colAccess}).GetTimestamp(0)!.Value;"); - else - Line($"var prop_{index} = ((TimestampArray){colAccess}).GetTimestamp(0)!.Value;"); - break; - case TypeKind2.DateOnly: + var typeName = prop.Type.FullTypeName; + + if (prop.IsNullable) { - var di = GetDateArrayInfo(prop); - if (prop.IsNullable) - Line($"var prop_{index} = (({di.ArrayType}){colAccess}).IsNull(0) ? (System.DateOnly?)null : (({di.ArrayType}){colAccess}).GetDateOnly(0);"); - else - Line($"var prop_{index} = (({di.ArrayType}){colAccess}).GetDateOnly(0)!.Value;"); - break; + Line($"IArrowArray {varName};"); + Line($"if ({access} is {{ }} nv_{index})"); + Line("{"); + _indent++; + Line($"var nestedBatch_{index} = {typeName}.ToRecordBatch(nv_{index});"); + Line($"var nestedArrays_{index} = new IArrowArray[nestedBatch_{index}.ColumnCount];"); + Line($"for (int i = 0; i < nestedBatch_{index}.ColumnCount; i++) nestedArrays_{index}[i] = nestedBatch_{index}.Column(i);"); + Line($"{varName} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), 1, nestedArrays_{index}, ArrowBuffer.Empty);"); + _indent--; + Line("}"); + Line("else"); + Line("{"); + _indent++; + Line($"var nullArrays_{index} = new IArrowArray[{typeName}.ArrowSchema.FieldsList.Count];"); + Line($"for (int i = 0; i < nullArrays_{index}.Length; i++)"); + Line($" nullArrays_{index}[i] = Apache.Arrow.Serialization.ArrowArrayHelper.BuildNullArray({typeName}.ArrowSchema.FieldsList[i].DataType, 1);"); + Line($"var nullBitmap_{index} = new ArrowBuffer.BitmapBuilder();"); + Line($"nullBitmap_{index}.Append(false);"); + Line($"{varName} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), 1, nullArrays_{index}, nullBitmap_{index}.Build(), 1);"); + _indent--; + Line("}"); } - case TypeKind2.TimeOnly: + else { - var ti = GetTimeArrayInfo(prop); - if (prop.IsNullable) - Line($"var prop_{index} = (({ti.ArrayType}){colAccess}).GetTime(0);"); - else - Line($"var prop_{index} = (({ti.ArrayType}){colAccess}).GetTime(0)!.Value;"); - break; + Line($"var nestedBatch_{index} = {typeName}.ToRecordBatch({access});"); + Line($"var nestedArrays_{index} = new IArrowArray[nestedBatch_{index}.ColumnCount];"); + Line($"for (int i = 0; i < nestedBatch_{index}.ColumnCount; i++) nestedArrays_{index}[i] = nestedBatch_{index}.Column(i);"); + Line($"var {varName} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), 1, nestedArrays_{index}, ArrowBuffer.Empty);"); } - case TypeKind2.TimeSpan: - if (prop.IsNullable) - Line($"var prop_{index} = ((DurationArray){colAccess}).IsNull(0) ? (System.TimeSpan?)null : Apache.Arrow.Serialization.ArrowArrayHelper.ReadDuration((DurationArray){colAccess}, 0);"); - else - Line($"var prop_{index} = Apache.Arrow.Serialization.ArrowArrayHelper.ReadDuration((DurationArray){colAccess}, 0);"); - break; - case TypeKind2.Decimal: - if (prop.IsNullable) - Line($"var prop_{index} = ((Decimal128Array){colAccess}).IsNull(0) ? (decimal?)null : ((Decimal128Array){colAccess}).GetValue(0).Value;"); - else - Line($"var prop_{index} = ((Decimal128Array){colAccess}).GetValue(0).Value;"); - break; - case TypeKind2.Guid: - if (prop.IsNullable) - Line($"var prop_{index} = ((GuidArray){colAccess}).GetGuid(0);"); - else - Line($"var prop_{index} = ((GuidArray){colAccess}).GetGuid(0)!.Value;"); - break; - case TypeKind2.Half: - EmitPrimitiveDeserialize(prop, index, colAccess, "HalfFloatArray", "GetValue"); - break; - case TypeKind2.Enum: - EmitEnumDeserialize(prop, index, colAccess); - break; - case TypeKind2.List: - case TypeKind2.Array: - EmitListDeserialize(prop, index, colAccess); - break; - case TypeKind2.Set: - EmitSetDeserialize(prop, index, colAccess); - break; - case TypeKind2.Dictionary: - EmitDictDeserialize(prop, index, colAccess); - break; - case TypeKind2.NestedRecord: - EmitNestedRecordDeserialize(prop, index, colAccess); - break; - case TypeKind2.Custom: - if (prop.IsNullable) - Line($"var prop_{index} = {colAccess}.IsNull(0) ? ({prop.Type.FullTypeName}?)null : new {prop.ConverterTypeName}().FromArray({colAccess}, 0);"); - else - Line($"var prop_{index} = new {prop.ConverterTypeName}().FromArray({colAccess}, 0);"); - break; - default: - Line($"object? prop_{index} = null; // TODO: Unsupported type {prop.Type.Kind}"); - break; - } - } - - private void EmitPrimitiveDeserialize(PropertyModel prop, int index, string colAccess, string arrayType, string getMethod) - { - if (prop.IsNullable) - Line($"var prop_{index} = (({arrayType}){colAccess}).IsNull(0) ? null : ({prop.Type.FullTypeName}?)(({arrayType}){colAccess}).{getMethod}(0);"); - else - Line($"var prop_{index} = (({arrayType}){colAccess}).{getMethod}(0).Value;"); - } - - private void EmitEnumDeserialize(PropertyModel prop, int index, string colAccess) - { - var enumType = prop.Type.FullTypeName; - if (prop.IsNullable) - { - Line($"var dictArr_{index} = (DictionaryArray){colAccess};"); - Line($"{enumType}? prop_{index} = dictArr_{index}.IsNull(0) ? null : System.Enum.Parse<{enumType}>(((StringArray)dictArr_{index}.Dictionary).GetString((int)((Int16Array)dictArr_{index}.Indices).GetValue(0).Value)!);"); - } - else - { - Line($"var dictArr_{index} = (DictionaryArray){colAccess};"); - Line($"var prop_{index} = System.Enum.Parse<{enumType}>(((StringArray)dictArr_{index}.Dictionary).GetString((int)((Int16Array)dictArr_{index}.Indices).GetValue(0).Value)!);"); } - } - - private void EmitListDeserialize(PropertyModel prop, int index, string colAccess) - { - var elemType = prop.Type.ElementType!; - var listArr = $"listArr_{index}"; - Line($"var {listArr} = (ListArray){colAccess};"); - if (prop.IsNullable) + private void EmitDeserialize() { - var collTypeName = prop.Type.Kind == TypeKind2.Array - ? $"{elemType.FullTypeName}[]" - : $"System.Collections.Generic.List<{elemType.FullTypeName}>"; - Line($"{collTypeName}? prop_{index};"); - Line($"if ({listArr}.IsNull(0))"); - Line($" prop_{index} = null;"); - Line("else"); + var newKw = _model.HasArrowSerializableBase ? "new " : ""; + Line($"public static {newKw}{_model.TypeName} FromRecordBatch(RecordBatch batch)"); Line("{"); _indent++; - EmitListDeserializeInner(prop, elemType, index, listArr, isSet: false, assignTarget: $"prop_{index}"); - _indent--; - Line("}"); - } - else - { - EmitListDeserializeInner(prop, elemType, index, listArr, isSet: false, assignTarget: null); - } - } - /// - /// Emits the inner list deserialization logic (non-null path). - /// If assignTarget is null, declares var prop_{index}; otherwise assigns to assignTarget. - /// - private void EmitListDeserializeInner(PropertyModel prop, TypeInfo elemType, int index, string listArr, bool isSet, string? assignTarget) - { - var declPrefix = assignTarget != null ? $"{assignTarget}" : $"var prop_{index}"; - if (NeedsManualListDeserialize(elemType)) - { - EmitManualListDeserialize(prop, elemType, index, listArr, isSet, assignTarget); - } - else - { - Line($"var {listArr}_values = {listArr}.GetSlicedValues(0);"); - var elemReader = GetElementReader(elemType, $"{listArr}_values"); - if (isSet) - Line($"{declPrefix} = new System.Collections.Generic.HashSet<{elemType.FullTypeName}>({elemReader});"); - else if (prop.Type.Kind == TypeKind2.Array) - Line($"{declPrefix} = {elemReader}.ToArray();"); - else - Line($"{declPrefix} = new System.Collections.Generic.List<{elemType.FullTypeName}>({elemReader});"); - } - } + Line("if (batch.Length == 0)"); + Line(" throw new InvalidOperationException(\"Cannot deserialize from empty RecordBatch.\");"); + Line("if (batch.Length > 1)"); + Line(" throw new InvalidOperationException($\"Expected single-row RecordBatch, got {batch.Length} rows.\");"); + Line(); - private void EmitSetDeserialize(PropertyModel prop, int index, string colAccess) - { - var elemType = prop.Type.ElementType!; - var listArr = $"listArr_{index}"; - Line($"var {listArr} = (ListArray){colAccess};"); + for (int i = 0; i < _model.Properties.Count; i++) + { + var prop = _model.Properties[i]; + EmitDeserializeProperty(prop, i); + } + + Line(); + if (_model.HasSerializationCallback) + { + EmitObjectConstruction("var result = ", ";"); + Line("((Apache.Arrow.Serialization.IArrowSerializationCallback)result).OnAfterDeserialize();"); + Line("return result;"); + } + else + { + EmitObjectConstruction("return ", ";"); + } - if (prop.IsNullable) - { - var collTypeName = $"System.Collections.Generic.HashSet<{elemType.FullTypeName}>"; - Line($"{collTypeName}? prop_{index};"); - Line($"if ({listArr}.IsNull(0))"); - Line($" prop_{index} = null;"); - Line("else"); - Line("{"); - _indent++; - EmitListDeserializeInner(prop, elemType, index, listArr, isSet: true, assignTarget: $"prop_{index}"); _indent--; Line("}"); } - else - { - EmitListDeserializeInner(prop, elemType, index, listArr, isSet: true, assignTarget: null); - } - } - /// - /// Emits deserialization for single-row list/set with extension type elements (e.g. Guid) - /// that cannot use GetSlicedValues (which fails for ExtensionArray). - /// - private void EmitManualListDeserialize(PropertyModel prop, TypeInfo elemType, int index, string listArr, bool isSet, string? assignTarget = null) - { - // When assignTarget is set, assign to it; otherwise declare var prop_{index} - string PropDecl(string expr) => assignTarget != null ? $"{assignTarget} = {expr};" : $"var prop_{index} = {expr};"; - - // Read offsets and access the full values array directly - Line($"var {listArr}_start = {listArr}.ValueOffsets[0];"); - Line($"var {listArr}_len = {listArr}.GetValueLength(0);"); - switch (elemType.Kind) + /// + /// Emits either constructor call or object initializer to create an instance. + /// Uses constructor when ConstructorParams is set (field-based types), otherwise object initializer. + /// + private void EmitObjectConstruction(string prefix, string suffix) { - case TypeKind2.Guid: - { - Line($"var {listArr}_allValues = (GuidArray){listArr}.Values;"); - var readExpr = $"Enumerable.Range({listArr}_start, {listArr}_len).Select(i => {listArr}_allValues.GetGuid(i)!.Value)"; - if (isSet) - Line(PropDecl($"new System.Collections.Generic.HashSet<{elemType.FullTypeName}>({readExpr})")); - else if (prop.Type.Kind == TypeKind2.Array) - Line(PropDecl($"{readExpr}.ToArray()")); - else - Line(PropDecl($"new System.Collections.Generic.List<{elemType.FullTypeName}>({readExpr})")); - break; - } - case TypeKind2.Enum: + if (_model.ConstructorParams != null) { - // Enum list elements are dictionary-encoded: Dictionary(Int16, Utf8) - Line($"var {listArr}_dictArr = (DictionaryArray){listArr}.Values;"); - Line($"var {listArr}_dictValues = (StringArray){listArr}_dictArr.Dictionary;"); - var readExpr = $"Enumerable.Range({listArr}_start, {listArr}_len).Select(i => System.Enum.Parse<{elemType.FullTypeName}>({listArr}_dictValues.GetString(((Int16Array){listArr}_dictArr.Indices).GetValue(i)!.Value)!))"; - if (isSet) - Line(PropDecl($"new System.Collections.Generic.HashSet<{elemType.FullTypeName}>({readExpr})")); - else if (prop.Type.Kind == TypeKind2.Array) - Line(PropDecl($"{readExpr}.ToArray()")); - else - Line(PropDecl($"new System.Collections.Generic.List<{elemType.FullTypeName}>({readExpr})")); - break; + // Constructor-based: new TypeName(paramName: prop_N, ...) + var propByName = new Dictionary(StringComparer.OrdinalIgnoreCase); + for (int i = 0; i < _model.Properties.Count; i++) + propByName[_model.Properties[i].PropertyName] = i; + + var args = new List(); + foreach (var param in _model.ConstructorParams) + { + if (propByName.TryGetValue(param.Name, out int idx)) + args.Add($"{param.Name}: prop_{idx}"); + else if (param.HasDefaultValue) + continue; // use default + } + + Line($"{prefix}new {_model.TypeName}({string.Join(", ", args)}){suffix}"); } - case TypeKind2.NestedRecord: + else { - // Nested record list elements: read from StructArray via child's FromRecordBatch - var typeName = elemType.FullTypeName; - Line($"var {listArr}_struct = (StructArray){listArr}.Values;"); - Line($"var {listArr}_result = new System.Collections.Generic.List<{typeName}>({listArr}_len);"); - Line($"for (int i = {listArr}_start; i < {listArr}_start + {listArr}_len; i++)"); - Line("{"); - _indent++; - Line($"var childArrays_{index} = new IArrowArray[{listArr}_struct.Fields.Count];"); - Line($"for (int f = 0; f < {listArr}_struct.Fields.Count; f++) childArrays_{index}[f] = {listArr}_struct.Fields[f].Data.Slice(i, 1) is var sd_{index} ? ArrowArrayFactory.BuildArray(sd_{index}) : throw new System.InvalidOperationException();"); - Line($"var childBatch_{index} = new RecordBatch({typeName}.ArrowSchema, childArrays_{index}, 1);"); - Line($"{listArr}_result.Add({typeName}.FromRecordBatch(childBatch_{index}));"); - _indent--; - Line("}"); - if (isSet) - Line(PropDecl($"new System.Collections.Generic.HashSet<{typeName}>({listArr}_result)")); - else if (prop.Type.Kind == TypeKind2.Array) - Line(PropDecl($"{listArr}_result.ToArray()")); - else - Line(PropDecl($"{listArr}_result")); - break; - } - case TypeKind2.List: - case TypeKind2.Array: - { - // Nested list elements: read inner lists from ListArray values - var innerElemType = elemType.ElementType!; - var innerTypeName = elemType.FullTypeName; - Line($"var {listArr}_innerArr = (ListArray){listArr}.Values;"); - Line($"var {listArr}_result = new System.Collections.Generic.List<{innerTypeName}>({listArr}_len);"); - Line($"for (int i = {listArr}_start; i < {listArr}_start + {listArr}_len; i++)"); + // Object initializer: new TypeName { Prop = prop_N, ... } + Line($"{prefix}new {_model.TypeName}"); Line("{"); _indent++; - if (innerElemType.Kind == TypeKind2.Dictionary) - { - // Dict inner elements: read each dict from inner list's MapArray values - var keyType = innerElemType.KeyType!; - var valType = innerElemType.ValueType!; - var dictTypeName = $"System.Collections.Generic.Dictionary<{keyType.FullTypeName}, {valType.FullTypeName}>"; - Line($"var {listArr}_mapSlice = (MapArray){listArr}_innerArr.GetSlicedValues(i);"); - Line($"var {listArr}_innerResult = new System.Collections.Generic.List<{dictTypeName}>({listArr}_mapSlice.Length);"); - Line($"for (int j_{index} = 0; j_{index} < {listArr}_mapSlice.Length; j_{index}++)"); - Line("{"); - _indent++; - Line($"var kvs_{index} = (StructArray){listArr}_mapSlice.GetSlicedValues(j_{index});"); - Line($"var ks_{index} = kvs_{index}.Fields[0];"); - Line($"var vs_{index} = kvs_{index}.Fields[1];"); - Line($"var d_{index} = new {dictTypeName}();"); - Line($"for (int k_{index} = 0; k_{index} < kvs_{index}.Length; k_{index}++)"); - Line("{"); - _indent++; - var dKeyReader = GetScalarReader(keyType, $"ks_{index}", $"k_{index}"); - EmitDictValueRead(valType, $"d_{index}", dKeyReader, $"vs_{index}", $"k_{index}", index); - _indent--; - Line("}"); - Line($"{listArr}_innerResult.Add(d_{index});"); - _indent--; - Line("}"); - Line($"{listArr}_result.Add({listArr}_innerResult);"); - } - else + for (int i = 0; i < _model.Properties.Count; i++) { - Line($"var {listArr}_innerValues = {listArr}_innerArr.GetSlicedValues(i);"); - var innerElemReader = GetElementReader(innerElemType, $"{listArr}_innerValues"); - if (elemType.Kind == TypeKind2.Array) - Line($"{listArr}_result.Add({innerElemReader}.ToArray());"); - else - Line($"{listArr}_result.Add(new {innerTypeName}({innerElemReader}));"); + var prop = _model.Properties[i]; + var comma = i < _model.Properties.Count - 1 ? "," : ""; + Line($"{prop.PropertyName} = prop_{i}{comma}"); } _indent--; - Line("}"); - if (isSet) - Line(PropDecl($"new System.Collections.Generic.HashSet<{innerTypeName}>({listArr}_result)")); - else if (prop.Type.Kind == TypeKind2.Array) - Line(PropDecl($"{listArr}_result.ToArray()")); - else - Line(PropDecl($"{listArr}_result")); - break; - } - case TypeKind2.Dictionary: - { - // Dict elements inside a list: read each dict from MapArray values - var keyType = elemType.KeyType!; - var valType = elemType.ValueType!; - var dictTypeName = $"System.Collections.Generic.Dictionary<{keyType.FullTypeName}, {valType.FullTypeName}>"; - Line($"var mapArr_{index} = (MapArray){listArr}.Values;"); - Line($"var {listArr}_result = new System.Collections.Generic.List<{dictTypeName}>({listArr}_len);"); - Line($"for (int i = {listArr}_start; i < {listArr}_start + {listArr}_len; i++)"); - Line("{"); - _indent++; - Line($"var kvs_{index} = (StructArray)mapArr_{index}.GetSlicedValues(i);"); - Line($"var ks_{index} = kvs_{index}.Fields[0];"); - Line($"var vs_{index} = kvs_{index}.Fields[1];"); - Line($"var d_{index} = new {dictTypeName}();"); - Line($"for (int j_{index} = 0; j_{index} < kvs_{index}.Length; j_{index}++)"); - Line("{"); - _indent++; - var dKeyReader = GetScalarReader(keyType, $"ks_{index}", $"j_{index}"); - EmitDictValueRead(valType, $"d_{index}", dKeyReader, $"vs_{index}", $"j_{index}", index); - _indent--; - Line("}"); - Line($"{listArr}_result.Add(d_{index});"); - _indent--; - Line("}"); - if (isSet) - Line(PropDecl($"new System.Collections.Generic.HashSet<{dictTypeName}>({listArr}_result)")); - else if (prop.Type.Kind == TypeKind2.Array) - Line(PropDecl($"{listArr}_result.ToArray()")); - else - Line(PropDecl($"{listArr}_result")); - break; + Line($"}}{suffix}"); } } - } - private string GetElementReader(TypeInfo elemType, string valuesVar) - { - var stringArrayType = GetArrayReadType(elemType); - return elemType.Kind switch + private void EmitDeserializeProperty(PropertyModel prop, int index) { - TypeKind2.String => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => (({stringArrayType}){valuesVar}).GetString(i)!)", - TypeKind2.Byte => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((UInt8Array){valuesVar}).GetValue(i).Value)", - TypeKind2.SByte => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((Int8Array){valuesVar}).GetValue(i).Value)", - TypeKind2.Int16 => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((Int16Array){valuesVar}).GetValue(i).Value)", - TypeKind2.UInt16 => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((UInt16Array){valuesVar}).GetValue(i).Value)", - TypeKind2.Int32 => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((Int32Array){valuesVar}).GetValue(i).Value)", - TypeKind2.UInt32 => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((UInt32Array){valuesVar}).GetValue(i).Value)", - TypeKind2.Int64 => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((Int64Array){valuesVar}).GetValue(i).Value)", - TypeKind2.UInt64 => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((UInt64Array){valuesVar}).GetValue(i).Value)", - TypeKind2.Float => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((FloatArray){valuesVar}).GetValue(i).Value)", - TypeKind2.Double => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((DoubleArray){valuesVar}).GetValue(i).Value)", - TypeKind2.Half => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((HalfFloatArray){valuesVar}).GetValue(i).Value)", - TypeKind2.Bool => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((BooleanArray){valuesVar}).GetValue(i).Value)", - TypeKind2.Decimal => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((Decimal128Array){valuesVar}).GetValue(i).Value)", - TypeKind2.DateTime => TypeInfoHasTimezone(elemType) - ? $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((TimestampArray){valuesVar}).GetTimestamp(i)!.Value.UtcDateTime)" - : $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((TimestampArray){valuesVar}).GetTimestamp(i)!.Value.DateTime)", - TypeKind2.DateTimeOffset => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((TimestampArray){valuesVar}).GetTimestamp(i)!.Value)", - TypeKind2.DateOnly => GetDateArrayReadType(elemType) == "Date64Array" - ? $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((Date64Array){valuesVar}).GetDateOnly(i).Value)" - : $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((Date32Array){valuesVar}).GetDateOnly(i).Value)", - TypeKind2.TimeOnly => GetTimeArrayReadType(elemType) == "Time32Array" - ? $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((Time32Array){valuesVar}).GetTime(i).Value)" - : $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((Time64Array){valuesVar}).GetTime(i).Value)", - TypeKind2.TimeSpan => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => Apache.Arrow.Serialization.ArrowArrayHelper.ReadDuration((DurationArray){valuesVar}, i))", - TypeKind2.Guid => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((GuidArray){valuesVar}).GetGuid(i)!.Value)", - TypeKind2.Binary => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((BinaryArray){valuesVar}).GetBytes(i).ToArray())", - TypeKind2.Enum => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => System.Enum.Parse<{elemType.FullTypeName}>(((StringArray){valuesVar}).GetString(i)!))", - _ => $"System.Array.Empty<{elemType.FullTypeName}>()", - }; - } + var colAccess = $"batch.Column(\"{prop.FieldName}\")"; - private void EmitDictDeserialize(PropertyModel prop, int index, string colAccess) - { - var keyType = prop.Type.KeyType!; - var valueType = prop.Type.ValueType!; - var dictTypeName = $"System.Collections.Generic.Dictionary<{keyType.FullTypeName}, {valueType.FullTypeName}>"; + switch (prop.Type.Kind) + { + case ArrowTypeKind.String: + { + var si = GetStringArrayInfo(prop); + if (prop.IsNullable) + Line($"var prop_{index} = (({colAccess}) is {si.ArrayType} sa_{index} && !sa_{index}.IsNull(0)) ? sa_{index}.GetString(0) : null;"); + else + Line($"var prop_{index} = (({si.ArrayType}){colAccess}).GetString(0)!;"); + break; + } + case ArrowTypeKind.Bool: + { + var bl = GetBoolArrayInfo(prop); + EmitPrimitiveDeserialize(prop, index, colAccess, bl.ArrayType, "GetValue"); + break; + } + case ArrowTypeKind.Byte: + EmitPrimitiveDeserialize(prop, index, colAccess, "UInt8Array", "GetValue"); + break; + case ArrowTypeKind.SByte: + EmitPrimitiveDeserialize(prop, index, colAccess, "Int8Array", "GetValue"); + break; + case ArrowTypeKind.Int16: + EmitPrimitiveDeserialize(prop, index, colAccess, "Int16Array", "GetValue"); + break; + case ArrowTypeKind.UInt16: + EmitPrimitiveDeserialize(prop, index, colAccess, "UInt16Array", "GetValue"); + break; + case ArrowTypeKind.Int32: + EmitPrimitiveDeserialize(prop, index, colAccess, "Int32Array", "GetValue"); + break; + case ArrowTypeKind.UInt32: + EmitPrimitiveDeserialize(prop, index, colAccess, "UInt32Array", "GetValue"); + break; + case ArrowTypeKind.Int64: + EmitPrimitiveDeserialize(prop, index, colAccess, "Int64Array", "GetValue"); + break; + case ArrowTypeKind.UInt64: + EmitPrimitiveDeserialize(prop, index, colAccess, "UInt64Array", "GetValue"); + break; + case ArrowTypeKind.Float: + EmitPrimitiveDeserialize(prop, index, colAccess, "FloatArray", "GetValue"); + break; + case ArrowTypeKind.Double: + EmitPrimitiveDeserialize(prop, index, colAccess, "DoubleArray", "GetValue"); + break; + case ArrowTypeKind.Binary: + { + var bi = GetBinaryArrayInfo(prop); + if (IsReadOnlyMemoryByte(prop)) + Line($"ReadOnlyMemory prop_{index} = (({bi.ArrayType}){colAccess}).GetBytes(0).ToArray();"); + else + Line($"var prop_{index} = (({bi.ArrayType}){colAccess}).GetBytes(0).ToArray();"); + break; + } + case ArrowTypeKind.DateTime: + { + var dtProp = TimestampHasTimezone(prop) ? "UtcDateTime" : "DateTime"; + if (prop.IsNullable) + Line($"var prop_{index} = ((TimestampArray){colAccess}).IsNull(0) ? (System.DateTime?)null : ((TimestampArray){colAccess}).GetTimestamp(0)!.Value.{dtProp};"); + else + Line($"var prop_{index} = ((TimestampArray){colAccess}).GetTimestamp(0)!.Value.{dtProp};"); + break; + } + case ArrowTypeKind.DateTimeOffset: + if (prop.IsNullable) + Line($"var prop_{index} = ((TimestampArray){colAccess}).IsNull(0) ? (System.DateTimeOffset?)null : ((TimestampArray){colAccess}).GetTimestamp(0)!.Value;"); + else + Line($"var prop_{index} = ((TimestampArray){colAccess}).GetTimestamp(0)!.Value;"); + break; + case ArrowTypeKind.DateOnly: + { + var di = GetDateArrayInfo(prop); + if (prop.IsNullable) + Line($"var prop_{index} = (({di.ArrayType}){colAccess}).IsNull(0) ? (System.DateOnly?)null : (({di.ArrayType}){colAccess}).GetDateOnly(0);"); + else + Line($"var prop_{index} = (({di.ArrayType}){colAccess}).GetDateOnly(0)!.Value;"); + break; + } + case ArrowTypeKind.TimeOnly: + { + var ti = GetTimeArrayInfo(prop); + if (prop.IsNullable) + Line($"var prop_{index} = (({ti.ArrayType}){colAccess}).GetTime(0);"); + else + Line($"var prop_{index} = (({ti.ArrayType}){colAccess}).GetTime(0)!.Value;"); + break; + } + case ArrowTypeKind.TimeSpan: + if (prop.IsNullable) + Line($"var prop_{index} = ((DurationArray){colAccess}).IsNull(0) ? (System.TimeSpan?)null : Apache.Arrow.Serialization.ArrowArrayHelper.ReadDuration((DurationArray){colAccess}, 0);"); + else + Line($"var prop_{index} = Apache.Arrow.Serialization.ArrowArrayHelper.ReadDuration((DurationArray){colAccess}, 0);"); + break; + case ArrowTypeKind.Decimal: + if (prop.IsNullable) + Line($"var prop_{index} = ((Decimal128Array){colAccess}).IsNull(0) ? (decimal?)null : ((Decimal128Array){colAccess}).GetValue(0).Value;"); + else + Line($"var prop_{index} = ((Decimal128Array){colAccess}).GetValue(0).Value;"); + break; + case ArrowTypeKind.Guid: + if (prop.IsNullable) + Line($"var prop_{index} = ((GuidArray){colAccess}).GetGuid(0);"); + else + Line($"var prop_{index} = ((GuidArray){colAccess}).GetGuid(0)!.Value;"); + break; + case ArrowTypeKind.Half: + EmitPrimitiveDeserialize(prop, index, colAccess, "HalfFloatArray", "GetValue"); + break; + case ArrowTypeKind.Enum: + EmitEnumDeserialize(prop, index, colAccess); + break; + case ArrowTypeKind.List: + case ArrowTypeKind.Array: + EmitListDeserialize(prop, index, colAccess); + break; + case ArrowTypeKind.Set: + EmitSetDeserialize(prop, index, colAccess); + break; + case ArrowTypeKind.Dictionary: + EmitDictDeserialize(prop, index, colAccess); + break; + case ArrowTypeKind.NestedRecord: + EmitNestedRecordDeserialize(prop, index, colAccess); + break; + case ArrowTypeKind.Custom: + if (prop.IsNullable) + Line($"var prop_{index} = {colAccess}.IsNull(0) ? ({prop.Type.FullTypeName}?)null : new {prop.ConverterTypeName}().FromArray({colAccess}, 0);"); + else + Line($"var prop_{index} = new {prop.ConverterTypeName}().FromArray({colAccess}, 0);"); + break; + default: + Line($"object? prop_{index} = null; // TODO: Unsupported type {prop.Type.Kind}"); + break; + } + } - Line($"var mapArr_{index} = (MapArray){colAccess};"); - if (prop.IsNullable) + private void EmitPrimitiveDeserialize(PropertyModel prop, int index, string colAccess, string arrayType, string getMethod) { - Line($"{dictTypeName}? prop_{index};"); - Line($"if (mapArr_{index}.IsNull(0))"); - Line($" prop_{index} = null;"); - Line("else"); - Line("{"); - _indent++; - Line($"prop_{index} = new {dictTypeName}();"); + if (prop.IsNullable) + Line($"var prop_{index} = (({arrayType}){colAccess}).IsNull(0) ? null : ({prop.Type.FullTypeName}?)(({arrayType}){colAccess}).{getMethod}(0);"); + else + Line($"var prop_{index} = (({arrayType}){colAccess}).{getMethod}(0).Value;"); } - else + + private void EmitEnumDeserialize(PropertyModel prop, int index, string colAccess) { - Line($"var prop_{index} = new {dictTypeName}();"); - Line($"if (!mapArr_{index}.IsNull(0))"); - Line("{"); - _indent++; + var enumType = prop.Type.FullTypeName; + if (prop.IsNullable) + { + Line($"var dictArr_{index} = (DictionaryArray){colAccess};"); + Line($"{enumType}? prop_{index} = dictArr_{index}.IsNull(0) ? null : System.Enum.Parse<{enumType}>(((StringArray)dictArr_{index}.Dictionary).GetString((int)((Int16Array)dictArr_{index}.Indices).GetValue(0).Value)!);"); + } + else + { + Line($"var dictArr_{index} = (DictionaryArray){colAccess};"); + Line($"var prop_{index} = System.Enum.Parse<{enumType}>(((StringArray)dictArr_{index}.Dictionary).GetString((int)((Int16Array)dictArr_{index}.Indices).GetValue(0).Value)!);"); + } } - Line($"var kvStruct_{index} = (StructArray)mapArr_{index}.GetSlicedValues(0);"); - Line($"var keys_{index} = kvStruct_{index}.Fields[0];"); - Line($"var vals_{index} = kvStruct_{index}.Fields[1];"); - Line($"for (int i = 0; i < kvStruct_{index}.Length; i++)"); - Line("{"); - _indent++; - var keyReader = GetScalarReader(keyType, $"keys_{index}", "i"); - EmitDictValueRead(valueType, $"prop_{index}", keyReader, $"vals_{index}", "i", index); - _indent--; - Line("}"); - _indent--; - Line("}"); - } - /// - /// Emits code to read a dict value and assign it to dictVar[keyExpr]. - /// Handles scalar values via GetScalarReader, and recurses for List/Dict values. - /// - private void EmitDictValueRead(TypeInfo valueType, string dictVar, string keyExpr, string valsArrayVar, string indexVar, int propIndex, int depth = 0) - { - var d = depth > 0 ? $"_{depth}" : ""; - switch (valueType.Kind) + private void EmitListDeserialize(PropertyModel prop, int index, string colAccess) { - case TypeKind2.List: - case TypeKind2.Array: - case TypeKind2.Set: - { - var elemType = valueType.ElementType!; - var listValsVar = $"lv_{propIndex}{d}"; - Line($"var {listValsVar} = ((ListArray){valsArrayVar}).GetSlicedValues({indexVar});"); - var elemReader = GetElementReader(elemType, listValsVar); - Line($"{dictVar}[{keyExpr}] = new System.Collections.Generic.List<{elemType.FullTypeName}>({elemReader});"); - break; - } - case TypeKind2.Dictionary: - { - var innerKeyType = valueType.KeyType!; - var innerValType = valueType.ValueType!; - var innerKvsVar = $"ikvs_{propIndex}{d}"; - var innerKeysVar = $"iks_{propIndex}{d}"; - var innerValsVar = $"ivs_{propIndex}{d}"; - var innerDictVar = $"id_{propIndex}{d}"; - var innerIdxVar = $"ii_{propIndex}{d}"; - Line($"var {innerKvsVar} = (StructArray)((MapArray){valsArrayVar}).GetSlicedValues({indexVar});"); - Line($"var {innerKeysVar} = {innerKvsVar}.Fields[0];"); - Line($"var {innerValsVar} = {innerKvsVar}.Fields[1];"); - Line($"var {innerDictVar} = new System.Collections.Generic.Dictionary<{innerKeyType.FullTypeName}, {innerValType.FullTypeName}>();"); - Line($"for (int {innerIdxVar} = 0; {innerIdxVar} < {innerKvsVar}.Length; {innerIdxVar}++)"); + var elemType = prop.Type.ElementType!; + var listArr = $"listArr_{index}"; + Line($"var {listArr} = (ListArray){colAccess};"); + + if (prop.IsNullable) + { + var collTypeName = prop.Type.Kind == ArrowTypeKind.Array + ? $"{elemType.FullTypeName}[]" + : $"System.Collections.Generic.List<{elemType.FullTypeName}>"; + Line($"{collTypeName}? prop_{index};"); + Line($"if ({listArr}.IsNull(0))"); + Line($" prop_{index} = null;"); + Line("else"); Line("{"); _indent++; - var innerKeyExpr = GetScalarReader(innerKeyType, innerKeysVar, innerIdxVar); - EmitDictValueRead(innerValType, innerDictVar, innerKeyExpr, innerValsVar, innerIdxVar, propIndex, depth + 1); + EmitListDeserializeInner(prop, elemType, index, listArr, isSet: false, assignTarget: $"prop_{index}"); _indent--; Line("}"); - Line($"{dictVar}[{keyExpr}] = {innerDictVar};"); - break; } - case TypeKind2.Enum: + else { - // Enum values in maps are dictionary-encoded: Dictionary(Int16, Utf8) - var enumType = valueType.FullTypeName; - var dvVar = $"dv_{propIndex}{d}"; - Line($"var {dvVar} = (DictionaryArray){valsArrayVar};"); - Line($"{dictVar}[{keyExpr}] = System.Enum.Parse<{enumType}>(((StringArray){dvVar}.Dictionary).GetString((int)((Int16Array){dvVar}.Indices).GetValue({indexVar}).Value)!);"); - break; + EmitListDeserializeInner(prop, elemType, index, listArr, isSet: false, assignTarget: null); } - case TypeKind2.NestedRecord: + } + + /// + /// Emits the inner list deserialization logic (non-null path). + /// If assignTarget is null, declares var prop_{index}; otherwise assigns to assignTarget. + /// + private void EmitListDeserializeInner(PropertyModel prop, TypeInfo elemType, int index, string listArr, bool isSet, string? assignTarget) + { + var declPrefix = assignTarget != null ? $"{assignTarget}" : $"var prop_{index}"; + if (NeedsManualListDeserialize(elemType)) { - var typeName = valueType.FullTypeName; - var sVar = $"sv_{propIndex}{d}"; - Line($"var {sVar} = (StructArray){valsArrayVar};"); - Line($"var ca_{propIndex}{d} = new IArrowArray[{sVar}.Fields.Count];"); - Line($"for (int f = 0; f < {sVar}.Fields.Count; f++) ca_{propIndex}{d}[f] = ArrowArrayFactory.BuildArray({sVar}.Fields[f].Data.Slice({indexVar}, 1));"); - Line($"{dictVar}[{keyExpr}] = {typeName}.FromRecordBatch(new RecordBatch({typeName}.ArrowSchema, ca_{propIndex}{d}, 1));"); - break; + EmitManualListDeserialize(prop, elemType, index, listArr, isSet, assignTarget); } - default: + else { - var valueReader = GetScalarReader(valueType, valsArrayVar, indexVar); - Line($"{dictVar}[{keyExpr}] = {valueReader};"); - break; + Line($"var {listArr}_values = {listArr}.GetSlicedValues(0);"); + var elemReader = GetElementReader(elemType, $"{listArr}_values"); + if (isSet) + Line($"{declPrefix} = new System.Collections.Generic.HashSet<{elemType.FullTypeName}>({elemReader});"); + else if (prop.Type.Kind == ArrowTypeKind.Array) + Line($"{declPrefix} = {elemReader}.ToArray();"); + else + Line($"{declPrefix} = new System.Collections.Generic.List<{elemType.FullTypeName}>({elemReader});"); } } - } - - private string GetScalarReader(TypeInfo type, string arrayVar, string indexVar) - { - var stringArrayType = GetArrayReadType(type); - return type.Kind switch - { - TypeKind2.String => $"(({stringArrayType}){arrayVar}).GetString({indexVar})!", - TypeKind2.Byte => $"((UInt8Array){arrayVar}).GetValue({indexVar}).Value", - TypeKind2.SByte => $"((Int8Array){arrayVar}).GetValue({indexVar}).Value", - TypeKind2.Int16 => $"((Int16Array){arrayVar}).GetValue({indexVar}).Value", - TypeKind2.UInt16 => $"((UInt16Array){arrayVar}).GetValue({indexVar}).Value", - TypeKind2.Int32 => $"((Int32Array){arrayVar}).GetValue({indexVar}).Value", - TypeKind2.UInt32 => $"((UInt32Array){arrayVar}).GetValue({indexVar}).Value", - TypeKind2.Int64 => $"((Int64Array){arrayVar}).GetValue({indexVar}).Value", - TypeKind2.UInt64 => $"((UInt64Array){arrayVar}).GetValue({indexVar}).Value", - TypeKind2.Float => $"((FloatArray){arrayVar}).GetValue({indexVar}).Value", - TypeKind2.Double => $"((DoubleArray){arrayVar}).GetValue({indexVar}).Value", - TypeKind2.Half => $"((HalfFloatArray){arrayVar}).GetValue({indexVar}).Value", - TypeKind2.Bool => $"((BooleanArray){arrayVar}).GetValue({indexVar}).Value", - TypeKind2.Decimal => $"((Decimal128Array){arrayVar}).GetValue({indexVar}).Value", - TypeKind2.DateTime => TypeInfoHasTimezone(type) - ? $"((TimestampArray){arrayVar}).GetTimestamp({indexVar})!.Value.UtcDateTime" - : $"((TimestampArray){arrayVar}).GetTimestamp({indexVar})!.Value.DateTime", - TypeKind2.DateTimeOffset => $"((TimestampArray){arrayVar}).GetTimestamp({indexVar})!.Value", - TypeKind2.DateOnly => GetDateArrayReadType(type) == "Date64Array" - ? $"((Date64Array){arrayVar}).GetDateOnly({indexVar}).Value" - : $"((Date32Array){arrayVar}).GetDateOnly({indexVar}).Value", - TypeKind2.TimeOnly => GetTimeArrayReadType(type) == "Time32Array" - ? $"((Time32Array){arrayVar}).GetTime({indexVar}).Value" - : $"((Time64Array){arrayVar}).GetTime({indexVar}).Value", - TypeKind2.TimeSpan => $"Apache.Arrow.Serialization.ArrowArrayHelper.ReadDuration((DurationArray){arrayVar}, {indexVar})", - TypeKind2.Guid => $"((GuidArray){arrayVar}).GetGuid({indexVar})!.Value", - TypeKind2.Binary => $"((BinaryArray){arrayVar}).GetBytes({indexVar}).ToArray()", - TypeKind2.Enum => $"System.Enum.Parse<{type.FullTypeName}>(((StringArray){arrayVar}).GetString({indexVar})!)", - _ => $"default({type.FullTypeName})", - }; - } - - private void EmitNestedRecordDeserialize(PropertyModel prop, int index, string colAccess) - { - var typeName = prop.Type.FullTypeName; - - if (prop.IsNullable) - { - Line($"var structArr_{index} = (StructArray){colAccess};"); - Line($"{typeName}? prop_{index};"); - Line($"if (structArr_{index}.IsNull(0))"); - Line("{"); - _indent++; - Line($"prop_{index} = null;"); - _indent--; - Line("}"); - Line("else"); - Line("{"); - _indent++; - Line($"var nestedArrays_{index} = new IArrowArray[structArr_{index}.Fields.Count];"); - Line($"for (int i = 0; i < structArr_{index}.Fields.Count; i++) nestedArrays_{index}[i] = structArr_{index}.Fields[i];"); - Line($"var nestedBatch_{index} = new RecordBatch({typeName}.ArrowSchema, nestedArrays_{index}, 1);"); - Line($"prop_{index} = {typeName}.FromRecordBatch(nestedBatch_{index});"); - _indent--; - Line("}"); - } - else - { - Line($"var structArr_{index} = (StructArray){colAccess};"); - Line($"var nestedArrays_{index} = new IArrowArray[structArr_{index}.Fields.Count];"); - Line($"for (int i = 0; i < structArr_{index}.Fields.Count; i++) nestedArrays_{index}[i] = structArr_{index}.Fields[i];"); - Line($"var nestedBatch_{index} = new RecordBatch({typeName}.ArrowSchema, nestedArrays_{index}, 1);"); - Line($"var prop_{index} = {typeName}.FromRecordBatch(nestedBatch_{index});"); - } - } - - private void EmitMultiRowSerialize() - { - Line($"public static RecordBatch ToRecordBatch(IReadOnlyList<{_model.TypeName}> values)"); - Line("{"); - _indent++; - - Line("var count = values.Count;"); - Line(); - - // Create a builder for each column - for (int i = 0; i < _model.Properties.Count; i++) - { - var prop = _model.Properties[i]; - EmitMultiRowBuilderDecl(prop, i); - } - - Line(); - Line("for (int row = 0; row < count; row++)"); - Line("{"); - _indent++; - Line("var item = values[row];"); - if (_model.HasSerializationCallback) - Line("((Apache.Arrow.Serialization.IArrowSerializationCallback)item).OnBeforeSerialize();"); - - for (int i = 0; i < _model.Properties.Count; i++) + private void EmitSetDeserialize(PropertyModel prop, int index, string colAccess) { - var prop = _model.Properties[i]; - EmitMultiRowBuilderAppend(prop, i); - } + var elemType = prop.Type.ElementType!; + var listArr = $"listArr_{index}"; + Line($"var {listArr} = (ListArray){colAccess};"); - _indent--; - Line("}"); - Line(); - - // Build arrays and return RecordBatch - for (int i = 0; i < _model.Properties.Count; i++) - { - var prop = _model.Properties[i]; - EmitMultiRowBuilderBuild(prop, i); + if (prop.IsNullable) + { + var collTypeName = $"System.Collections.Generic.HashSet<{elemType.FullTypeName}>"; + Line($"{collTypeName}? prop_{index};"); + Line($"if ({listArr}.IsNull(0))"); + Line($" prop_{index} = null;"); + Line("else"); + Line("{"); + _indent++; + EmitListDeserializeInner(prop, elemType, index, listArr, isSet: true, assignTarget: $"prop_{index}"); + _indent--; + Line("}"); + } + else + { + EmitListDeserializeInner(prop, elemType, index, listArr, isSet: true, assignTarget: null); + } } - Line(); - Line("return new RecordBatch(_arrowSchema, new IArrowArray[]"); - Line("{"); - _indent++; - for (int i = 0; i < _model.Properties.Count; i++) + /// + /// Emits deserialization for single-row list/set with extension type elements (e.g. Guid) + /// that cannot use GetSlicedValues (which fails for ExtensionArray). + /// + private void EmitManualListDeserialize(PropertyModel prop, TypeInfo elemType, int index, string listArr, bool isSet, string? assignTarget = null) { - var comma = i < _model.Properties.Count - 1 ? "," : ""; - Line($"arr_{i}{comma}"); - } - _indent--; - Line("}, count);"); - - _indent--; - Line("}"); - } + // When assignTarget is set, assign to it; otherwise declare var prop_{index} + string PropDecl(string expr) => assignTarget != null ? $"{assignTarget} = {expr};" : $"var prop_{index} = {expr};"; - private void EmitMultiRowBuilderDecl(PropertyModel prop, int index) - { - switch (prop.Type.Kind) - { - case TypeKind2.String: - { - var si = GetStringArrayInfo(prop); - Line($"var bld_{index} = new {si.BuilderType}();"); - break; - } - case TypeKind2.Bool: - { - var bl = GetBoolArrayInfo(prop); - Line($"var bld_{index} = new {bl.BuilderType}();"); - break; - } - case TypeKind2.Byte: - Line($"var bld_{index} = new UInt8Array.Builder();"); - break; - case TypeKind2.SByte: - Line($"var bld_{index} = new Int8Array.Builder();"); - break; - case TypeKind2.Int16: - Line($"var bld_{index} = new Int16Array.Builder();"); - break; - case TypeKind2.UInt16: - Line($"var bld_{index} = new UInt16Array.Builder();"); - break; - case TypeKind2.Int32: - Line($"var bld_{index} = new Int32Array.Builder();"); - break; - case TypeKind2.UInt32: - Line($"var bld_{index} = new UInt32Array.Builder();"); - break; - case TypeKind2.Int64: - Line($"var bld_{index} = new Int64Array.Builder();"); - break; - case TypeKind2.UInt64: - Line($"var bld_{index} = new UInt64Array.Builder();"); - break; - case TypeKind2.Float: - Line($"var bld_{index} = new FloatArray.Builder();"); - break; - case TypeKind2.Double: - Line($"var bld_{index} = new DoubleArray.Builder();"); - break; - case TypeKind2.Binary: - { - var bi = GetBinaryArrayInfo(prop); - Line($"var bld_{index} = new {bi.BuilderType}();"); - break; - } - case TypeKind2.DateTime: - case TypeKind2.DateTimeOffset: - Line($"var bld_{index} = new TimestampArray.Builder({GetTimestampTypeExpr(prop)});"); - break; - case TypeKind2.DateOnly: - { - var di = GetDateArrayInfo(prop); - Line($"var bld_{index} = new {di.BuilderType}();"); - break; - } - case TypeKind2.TimeOnly: - { - var ti = GetTimeArrayInfo(prop); - Line($"var bld_{index} = new {ti.BuilderType}({ti.TypeExpr});"); - break; - } - case TypeKind2.TimeSpan: - case TypeKind2.Decimal: - Line($"var bld_{index}_items = new List<{prop.Type.FullTypeName}{(prop.IsNullable ? "?" : "")}>(count);"); - break; - case TypeKind2.Guid: - Line($"var bld_{index}_items = new List<{prop.Type.FullTypeName}{(prop.IsNullable ? "?" : "")}>(count);"); - break; - case TypeKind2.Half: - Line($"var bld_{index} = new HalfFloatArray.Builder();"); - break; - case TypeKind2.Enum: - Line($"var bld_{index}_dict = new StringArray.Builder();"); - Line($"var bld_{index}_idx = new Int16Array.Builder();"); - Line($"var bld_{index}_map = new Dictionary();"); - break; - case TypeKind2.List: - case TypeKind2.Array: - case TypeKind2.Set: - { - var elemType = prop.Type.ElementType!; - if (NeedsManualListBuilder(elemType)) - { - // Collect raw items + offsets for manual ListArray construction - Line($"var bld_{index}_items = new List<{elemType.FullTypeName}>();"); - Line($"var bld_{index}_offsets = new ArrowBuffer.Builder();"); - if (prop.IsNullable) - Line($"var bld_{index}_nulls = new ArrowBuffer.BitmapBuilder();"); - } - else - { - var arrowElemType = GetArrowTypeExpression(elemType); - Line($"var bld_{index} = new ListArray.Builder({arrowElemType});"); - } - break; - } - case TypeKind2.Dictionary: + // Read offsets and access the full values array directly + Line($"var {listArr}_start = {listArr}.ValueOffsets[0];"); + Line($"var {listArr}_len = {listArr}.GetValueLength(0);"); + switch (elemType.Kind) { - var valueType = prop.Type.ValueType!; - if (NeedsManualMapBuilder(valueType)) - { - var keyType = prop.Type.KeyType!; - var keyBuilderType = GetArrayBuilderType(keyType); - Line($"var bld_{index}_keys = new {keyBuilderType}();"); - switch (valueType.Kind) - { - case TypeKind2.Guid: - Line($"var bld_{index}_vals = new System.Collections.Generic.List();"); - break; - case TypeKind2.Enum: - Line($"var bld_{index}_valDict = new StringArray.Builder();"); - Line($"var bld_{index}_valIdx = new Int16Array.Builder();"); - Line($"var bld_{index}_valMap = new System.Collections.Generic.Dictionary();"); - break; - case TypeKind2.NestedRecord: - Line($"var bld_{index}_vals = new System.Collections.Generic.List<{valueType.FullTypeName}>();"); - break; - } - Line($"var bld_{index}_offsets = new ArrowBuffer.Builder();"); - if (prop.IsNullable) - Line($"var bld_{index}_nulls = new ArrowBuffer.BitmapBuilder();"); - } - else - { - var mapType = GetArrowTypeExpression(prop.Type); - Line($"var bld_{index} = new MapArray.Builder({mapType});"); - } - break; - } - case TypeKind2.NestedRecord: - { - var typeName = prop.Type.FullTypeName; - Line($"var bld_{index}_items = new List<{typeName}{(prop.IsNullable ? "?" : "")}>(count);"); - break; - } - case TypeKind2.Custom: - Line($"var bld_{index}_items = new List<{prop.Type.FullTypeName}>(count);"); - break; - default: - Line($"// TODO: Unsupported multi-row builder for {prop.Type.Kind}"); - break; - } - } - - private void EmitMultiRowBuilderAppend(PropertyModel prop, int index) - { - var access = $"item.{prop.PropertyName}"; - - switch (prop.Type.Kind) - { - case TypeKind2.String: - Line($"bld_{index}.Append({access});"); - break; - case TypeKind2.Bool: - if (prop.IsNullable) - { - Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(v_{index}); else bld_{index}.AppendNull();"); - } - else - Line($"bld_{index}.Append({access});"); - break; - case TypeKind2.Byte: - case TypeKind2.SByte: - case TypeKind2.Int16: - case TypeKind2.UInt16: - case TypeKind2.Int32: - case TypeKind2.UInt32: - case TypeKind2.Int64: - case TypeKind2.UInt64: - case TypeKind2.Float: - case TypeKind2.Double: - if (prop.IsNullable) - { - Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(v_{index}); else bld_{index}.AppendNull();"); - } - else - Line($"bld_{index}.Append({access});"); - break; - case TypeKind2.Binary: - Line($"bld_{index}.Append({BinaryToSpan(prop, access)});"); - break; - case TypeKind2.DateTime: - if (prop.IsNullable) - { - Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(Apache.Arrow.Serialization.ArrowArrayHelper.{TimestampConvertMethod(prop)}(v_{index})); else bld_{index}.AppendNull();"); - } - else - Line($"bld_{index}.Append(Apache.Arrow.Serialization.ArrowArrayHelper.{TimestampConvertMethod(prop)}({access}));"); - break; - case TypeKind2.DateTimeOffset: - if (prop.IsNullable) - { - Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(Apache.Arrow.Serialization.ArrowArrayHelper.{TimestampConvertMethod(prop)}(v_{index})); else bld_{index}.AppendNull();"); - } - else - Line($"bld_{index}.Append(Apache.Arrow.Serialization.ArrowArrayHelper.{TimestampConvertMethod(prop)}({access}));"); - break; - case TypeKind2.DateOnly: - if (prop.IsNullable) - { - Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(v_{index}); else bld_{index}.AppendNull();"); - } - else - Line($"bld_{index}.Append({access});"); - break; - case TypeKind2.TimeOnly: - if (prop.IsNullable) - { - Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(v_{index}); else bld_{index}.AppendNull();"); - } - else - Line($"bld_{index}.Append({access});"); - break; - case TypeKind2.TimeSpan: - case TypeKind2.Decimal: - Line($"bld_{index}_items.Add({access});"); - break; - case TypeKind2.Guid: - Line($"bld_{index}_items.Add({access});"); - break; - case TypeKind2.Half: - if (prop.IsNullable) - { - Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(v_{index}); else bld_{index}.AppendNull();"); - } - else - Line($"bld_{index}.Append({access});"); - break; - case TypeKind2.Enum: - if (prop.IsNullable) - { - Line($"if ({access} is {{ }} ev_{index})"); - Line("{"); - _indent++; - Line($"var es_{index} = ev_{index}.ToString();"); - Line($"if (!bld_{index}_map.TryGetValue(es_{index}, out var ei_{index})) {{ ei_{index} = (short)bld_{index}_map.Count; bld_{index}_map[es_{index}] = ei_{index}; }}"); - Line($"bld_{index}_idx.Append(ei_{index});"); - _indent--; - Line("}"); - Line($"else bld_{index}_idx.AppendNull();"); - } - else - { - Line($"var es_{index} = {access}.ToString();"); - Line($"if (!bld_{index}_map.TryGetValue(es_{index}, out var ei_{index})) {{ ei_{index} = (short)bld_{index}_map.Count; bld_{index}_map[es_{index}] = ei_{index}; }}"); - Line($"bld_{index}_idx.Append(ei_{index});"); - } - break; - case TypeKind2.List: - case TypeKind2.Array: - case TypeKind2.Set: - { - var elemType = prop.Type.ElementType!; - if (prop.IsNullable) - { - Line($"if ({access} != null)"); - Line("{"); - _indent++; - } - if (NeedsManualListBuilder(elemType)) - { - Line($"bld_{index}_offsets.Append(bld_{index}_items.Count);"); - Line($"bld_{index}_items.AddRange({access});"); - if (prop.IsNullable) Line($"bld_{index}_nulls.Append(true);"); - } - else - { - Line($"bld_{index}.Append();"); - EmitListValuePopulation(elemType, $"bld_{index}", access, index); - } - if (prop.IsNullable) - { - _indent--; - Line("}"); - Line("else"); - Line("{"); - _indent++; - if (NeedsManualListBuilder(elemType)) + case ArrowTypeKind.Guid: { - Line($"bld_{index}_offsets.Append(bld_{index}_items.Count);"); - Line($"bld_{index}_nulls.Append(false);"); + Line($"var {listArr}_allValues = (GuidArray){listArr}.Values;"); + var readExpr = $"Enumerable.Range({listArr}_start, {listArr}_len).Select(i => {listArr}_allValues.GetGuid(i)!.Value)"; + if (isSet) + Line(PropDecl($"new System.Collections.Generic.HashSet<{elemType.FullTypeName}>({readExpr})")); + else if (prop.Type.Kind == ArrowTypeKind.Array) + Line(PropDecl($"{readExpr}.ToArray()")); + else + Line(PropDecl($"new System.Collections.Generic.List<{elemType.FullTypeName}>({readExpr})")); + break; } - else + case ArrowTypeKind.Enum: { - Line($"bld_{index}.AppendNull();"); + // Enum list elements are dictionary-encoded: Dictionary(Int16, Utf8) + Line($"var {listArr}_dictArr = (DictionaryArray){listArr}.Values;"); + Line($"var {listArr}_dictValues = (StringArray){listArr}_dictArr.Dictionary;"); + var readExpr = $"Enumerable.Range({listArr}_start, {listArr}_len).Select(i => System.Enum.Parse<{elemType.FullTypeName}>({listArr}_dictValues.GetString(((Int16Array){listArr}_dictArr.Indices).GetValue(i)!.Value)!))"; + if (isSet) + Line(PropDecl($"new System.Collections.Generic.HashSet<{elemType.FullTypeName}>({readExpr})")); + else if (prop.Type.Kind == ArrowTypeKind.Array) + Line(PropDecl($"{readExpr}.ToArray()")); + else + Line(PropDecl($"new System.Collections.Generic.List<{elemType.FullTypeName}>({readExpr})")); + break; } - _indent--; - Line("}"); - } - break; - } - case TypeKind2.Dictionary: - { - var keyType = prop.Type.KeyType!; - var valueType = prop.Type.ValueType!; - if (NeedsManualMapBuilder(valueType)) - { - if (prop.IsNullable) + case ArrowTypeKind.NestedRecord: { - Line($"if ({access} != null)"); + // Nested record list elements: read from StructArray via child's FromRecordBatch + var typeName = elemType.FullTypeName; + Line($"var {listArr}_struct = (StructArray){listArr}.Values;"); + Line($"var {listArr}_result = new System.Collections.Generic.List<{typeName}>({listArr}_len);"); + Line($"for (int i = {listArr}_start; i < {listArr}_start + {listArr}_len; i++)"); Line("{"); _indent++; + Line($"var childArrays_{index} = new IArrowArray[{listArr}_struct.Fields.Count];"); + Line($"for (int f = 0; f < {listArr}_struct.Fields.Count; f++) childArrays_{index}[f] = {listArr}_struct.Fields[f].Data.Slice(i, 1) is var sd_{index} ? ArrowArrayFactory.BuildArray(sd_{index}) : throw new System.InvalidOperationException();"); + Line($"var childBatch_{index} = new RecordBatch({typeName}.ArrowSchema, childArrays_{index}, 1);"); + Line($"{listArr}_result.Add({typeName}.FromRecordBatch(childBatch_{index}));"); + _indent--; + Line("}"); + if (isSet) + Line(PropDecl($"new System.Collections.Generic.HashSet<{typeName}>({listArr}_result)")); + else if (prop.Type.Kind == ArrowTypeKind.Array) + Line(PropDecl($"{listArr}_result.ToArray()")); + else + Line(PropDecl($"{listArr}_result")); + break; } - Line($"bld_{index}_offsets.Append(bld_{index}_keys.Length);"); - Line($"foreach (var kv_{index} in {access})"); - Line("{"); - _indent++; - Line($"bld_{index}_keys.Append(kv_{index}.Key);"); - switch (valueType.Kind) - { - case TypeKind2.Guid: - Line($"bld_{index}_vals.Add(kv_{index}.Value);"); - break; - case TypeKind2.Enum: + case ArrowTypeKind.List: + case ArrowTypeKind.Array: + { + // Nested list elements: read inner lists from ListArray values + var innerElemType = elemType.ElementType!; + var innerTypeName = elemType.FullTypeName; + Line($"var {listArr}_innerArr = (ListArray){listArr}.Values;"); + Line($"var {listArr}_result = new System.Collections.Generic.List<{innerTypeName}>({listArr}_len);"); + Line($"for (int i = {listArr}_start; i < {listArr}_start + {listArr}_len; i++)"); + Line("{"); + _indent++; + if (innerElemType.Kind == ArrowTypeKind.Dictionary) { - Line($"var es_{index} = kv_{index}.Value.ToString();"); - Line($"if (!bld_{index}_valMap.TryGetValue(es_{index}, out var ei_{index})) {{ ei_{index} = (short)bld_{index}_valMap.Count; bld_{index}_valMap[es_{index}] = ei_{index}; }}"); - Line($"bld_{index}_valIdx.Append(ei_{index});"); - break; + // Dict inner elements: read each dict from inner list's MapArray values + var keyType = innerElemType.KeyType!; + var valType = innerElemType.ValueType!; + var dictTypeName = $"System.Collections.Generic.Dictionary<{keyType.FullTypeName}, {valType.FullTypeName}>"; + Line($"var {listArr}_mapSlice = (MapArray){listArr}_innerArr.GetSlicedValues(i);"); + Line($"var {listArr}_innerResult = new System.Collections.Generic.List<{dictTypeName}>({listArr}_mapSlice.Length);"); + Line($"for (int j_{index} = 0; j_{index} < {listArr}_mapSlice.Length; j_{index}++)"); + Line("{"); + _indent++; + Line($"var kvs_{index} = (StructArray){listArr}_mapSlice.GetSlicedValues(j_{index});"); + Line($"var ks_{index} = kvs_{index}.Fields[0];"); + Line($"var vs_{index} = kvs_{index}.Fields[1];"); + Line($"var d_{index} = new {dictTypeName}();"); + Line($"for (int k_{index} = 0; k_{index} < kvs_{index}.Length; k_{index}++)"); + Line("{"); + _indent++; + var dKeyReader = GetScalarReader(keyType, $"ks_{index}", $"k_{index}"); + EmitDictValueRead(valType, $"d_{index}", dKeyReader, $"vs_{index}", $"k_{index}", index); + _indent--; + Line("}"); + Line($"{listArr}_innerResult.Add(d_{index});"); + _indent--; + Line("}"); + Line($"{listArr}_result.Add({listArr}_innerResult);"); + } + else + { + Line($"var {listArr}_innerValues = {listArr}_innerArr.GetSlicedValues(i);"); + var innerElemReader = GetElementReader(innerElemType, $"{listArr}_innerValues"); + if (elemType.Kind == ArrowTypeKind.Array) + Line($"{listArr}_result.Add({innerElemReader}.ToArray());"); + else + Line($"{listArr}_result.Add(new {innerTypeName}({innerElemReader}));"); } - case TypeKind2.NestedRecord: - Line($"bld_{index}_vals.Add(kv_{index}.Value);"); - break; - } - _indent--; - Line("}"); - if (prop.IsNullable) - { - Line($"bld_{index}_nulls.Append(true);"); _indent--; Line("}"); - Line("else"); + if (isSet) + Line(PropDecl($"new System.Collections.Generic.HashSet<{innerTypeName}>({listArr}_result)")); + else if (prop.Type.Kind == ArrowTypeKind.Array) + Line(PropDecl($"{listArr}_result.ToArray()")); + else + Line(PropDecl($"{listArr}_result")); + break; + } + case ArrowTypeKind.Dictionary: + { + // Dict elements inside a list: read each dict from MapArray values + var keyType = elemType.KeyType!; + var valType = elemType.ValueType!; + var dictTypeName = $"System.Collections.Generic.Dictionary<{keyType.FullTypeName}, {valType.FullTypeName}>"; + Line($"var mapArr_{index} = (MapArray){listArr}.Values;"); + Line($"var {listArr}_result = new System.Collections.Generic.List<{dictTypeName}>({listArr}_len);"); + Line($"for (int i = {listArr}_start; i < {listArr}_start + {listArr}_len; i++)"); + Line("{"); + _indent++; + Line($"var kvs_{index} = (StructArray)mapArr_{index}.GetSlicedValues(i);"); + Line($"var ks_{index} = kvs_{index}.Fields[0];"); + Line($"var vs_{index} = kvs_{index}.Fields[1];"); + Line($"var d_{index} = new {dictTypeName}();"); + Line($"for (int j_{index} = 0; j_{index} < kvs_{index}.Length; j_{index}++)"); Line("{"); _indent++; - Line($"bld_{index}_offsets.Append(bld_{index}_keys.Length);"); - Line($"bld_{index}_nulls.Append(false);"); + var dKeyReader = GetScalarReader(keyType, $"ks_{index}", $"j_{index}"); + EmitDictValueRead(valType, $"d_{index}", dKeyReader, $"vs_{index}", $"j_{index}", index); _indent--; Line("}"); + Line($"{listArr}_result.Add(d_{index});"); + _indent--; + Line("}"); + if (isSet) + Line(PropDecl($"new System.Collections.Generic.HashSet<{dictTypeName}>({listArr}_result)")); + else if (prop.Type.Kind == ArrowTypeKind.Array) + Line(PropDecl($"{listArr}_result.ToArray()")); + else + Line(PropDecl($"{listArr}_result")); + break; } - } - else - { - if (prop.IsNullable) + } + } + + private string GetElementReader(TypeInfo elemType, string valuesVar) + { + var stringArrayType = GetArrayReadType(elemType); + return elemType.Kind switch + { + ArrowTypeKind.String => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => (({stringArrayType}){valuesVar}).GetString(i)!)", + ArrowTypeKind.Byte => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((UInt8Array){valuesVar}).GetValue(i).Value)", + ArrowTypeKind.SByte => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((Int8Array){valuesVar}).GetValue(i).Value)", + ArrowTypeKind.Int16 => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((Int16Array){valuesVar}).GetValue(i).Value)", + ArrowTypeKind.UInt16 => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((UInt16Array){valuesVar}).GetValue(i).Value)", + ArrowTypeKind.Int32 => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((Int32Array){valuesVar}).GetValue(i).Value)", + ArrowTypeKind.UInt32 => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((UInt32Array){valuesVar}).GetValue(i).Value)", + ArrowTypeKind.Int64 => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((Int64Array){valuesVar}).GetValue(i).Value)", + ArrowTypeKind.UInt64 => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((UInt64Array){valuesVar}).GetValue(i).Value)", + ArrowTypeKind.Float => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((FloatArray){valuesVar}).GetValue(i).Value)", + ArrowTypeKind.Double => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((DoubleArray){valuesVar}).GetValue(i).Value)", + ArrowTypeKind.Half => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((HalfFloatArray){valuesVar}).GetValue(i).Value)", + ArrowTypeKind.Bool => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((BooleanArray){valuesVar}).GetValue(i).Value)", + ArrowTypeKind.Decimal => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((Decimal128Array){valuesVar}).GetValue(i).Value)", + ArrowTypeKind.DateTime => TypeInfoHasTimezone(elemType) + ? $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((TimestampArray){valuesVar}).GetTimestamp(i)!.Value.UtcDateTime)" + : $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((TimestampArray){valuesVar}).GetTimestamp(i)!.Value.DateTime)", + ArrowTypeKind.DateTimeOffset => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((TimestampArray){valuesVar}).GetTimestamp(i)!.Value)", + ArrowTypeKind.DateOnly => GetDateArrayReadType(elemType) == "Date64Array" + ? $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((Date64Array){valuesVar}).GetDateOnly(i).Value)" + : $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((Date32Array){valuesVar}).GetDateOnly(i).Value)", + ArrowTypeKind.TimeOnly => GetTimeArrayReadType(elemType) == "Time32Array" + ? $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((Time32Array){valuesVar}).GetTime(i).Value)" + : $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((Time64Array){valuesVar}).GetTime(i).Value)", + ArrowTypeKind.TimeSpan => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => Apache.Arrow.Serialization.ArrowArrayHelper.ReadDuration((DurationArray){valuesVar}, i))", + ArrowTypeKind.Guid => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((GuidArray){valuesVar}).GetGuid(i)!.Value)", + ArrowTypeKind.Binary => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((BinaryArray){valuesVar}).GetBytes(i).ToArray())", + ArrowTypeKind.Enum => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => System.Enum.Parse<{elemType.FullTypeName}>(((StringArray){valuesVar}).GetString(i)!))", + _ => $"System.Array.Empty<{elemType.FullTypeName}>()", + }; + } + + private void EmitDictDeserialize(PropertyModel prop, int index, string colAccess) + { + var keyType = prop.Type.KeyType!; + var valueType = prop.Type.ValueType!; + var dictTypeName = $"System.Collections.Generic.Dictionary<{keyType.FullTypeName}, {valueType.FullTypeName}>"; + + Line($"var mapArr_{index} = (MapArray){colAccess};"); + if (prop.IsNullable) + { + Line($"{dictTypeName}? prop_{index};"); + Line($"if (mapArr_{index}.IsNull(0))"); + Line($" prop_{index} = null;"); + Line("else"); + Line("{"); + _indent++; + Line($"prop_{index} = new {dictTypeName}();"); + } + else + { + Line($"var prop_{index} = new {dictTypeName}();"); + Line($"if (!mapArr_{index}.IsNull(0))"); + Line("{"); + _indent++; + } + Line($"var kvStruct_{index} = (StructArray)mapArr_{index}.GetSlicedValues(0);"); + Line($"var keys_{index} = kvStruct_{index}.Fields[0];"); + Line($"var vals_{index} = kvStruct_{index}.Fields[1];"); + Line($"for (int i = 0; i < kvStruct_{index}.Length; i++)"); + Line("{"); + _indent++; + var keyReader = GetScalarReader(keyType, $"keys_{index}", "i"); + EmitDictValueRead(valueType, $"prop_{index}", keyReader, $"vals_{index}", "i", index); + _indent--; + Line("}"); + _indent--; + Line("}"); + } + + /// + /// Emits code to read a dict value and assign it to dictVar[keyExpr]. + /// Handles scalar values via GetScalarReader, and recurses for List/Dict values. + /// + private void EmitDictValueRead(TypeInfo valueType, string dictVar, string keyExpr, string valsArrayVar, string indexVar, int propIndex, int depth = 0) + { + var d = depth > 0 ? $"_{depth}" : ""; + switch (valueType.Kind) + { + case ArrowTypeKind.List: + case ArrowTypeKind.Array: + case ArrowTypeKind.Set: { - Line($"if ({access} != null)"); - Line("{"); - _indent++; + var elemType = valueType.ElementType!; + var listValsVar = $"lv_{propIndex}{d}"; + Line($"var {listValsVar} = ((ListArray){valsArrayVar}).GetSlicedValues({indexVar});"); + var elemReader = GetElementReader(elemType, listValsVar); + Line($"{dictVar}[{keyExpr}] = new System.Collections.Generic.List<{elemType.FullTypeName}>({elemReader});"); + break; } - Line($"bld_{index}.Append();"); - Line($"foreach (var kv_{index} in {access})"); - Line("{"); - _indent++; - EmitMapKeyAppend(keyType, $"bld_{index}", $"kv_{index}.Key", index); - EmitMapValueAppend(valueType, $"bld_{index}", $"kv_{index}.Value", index); - _indent--; - Line("}"); - if (prop.IsNullable) + case ArrowTypeKind.Dictionary: { + var innerKeyType = valueType.KeyType!; + var innerValType = valueType.ValueType!; + var innerKvsVar = $"ikvs_{propIndex}{d}"; + var innerKeysVar = $"iks_{propIndex}{d}"; + var innerValsVar = $"ivs_{propIndex}{d}"; + var innerDictVar = $"id_{propIndex}{d}"; + var innerIdxVar = $"ii_{propIndex}{d}"; + Line($"var {innerKvsVar} = (StructArray)((MapArray){valsArrayVar}).GetSlicedValues({indexVar});"); + Line($"var {innerKeysVar} = {innerKvsVar}.Fields[0];"); + Line($"var {innerValsVar} = {innerKvsVar}.Fields[1];"); + Line($"var {innerDictVar} = new System.Collections.Generic.Dictionary<{innerKeyType.FullTypeName}, {innerValType.FullTypeName}>();"); + Line($"for (int {innerIdxVar} = 0; {innerIdxVar} < {innerKvsVar}.Length; {innerIdxVar}++)"); + Line("{"); + _indent++; + var innerKeyExpr = GetScalarReader(innerKeyType, innerKeysVar, innerIdxVar); + EmitDictValueRead(innerValType, innerDictVar, innerKeyExpr, innerValsVar, innerIdxVar, propIndex, depth + 1); _indent--; Line("}"); - Line($"else bld_{index}.AppendNull();"); + Line($"{dictVar}[{keyExpr}] = {innerDictVar};"); + break; } - } - break; - } - case TypeKind2.NestedRecord: - Line($"bld_{index}_items.Add({access});"); - break; - case TypeKind2.Custom: - Line($"bld_{index}_items.Add({access});"); - break; - default: - Line($"// TODO: Unsupported multi-row append for {prop.Type.Kind}"); - break; + case ArrowTypeKind.Enum: + { + // Enum values in maps are dictionary-encoded: Dictionary(Int16, Utf8) + var enumType = valueType.FullTypeName; + var dvVar = $"dv_{propIndex}{d}"; + Line($"var {dvVar} = (DictionaryArray){valsArrayVar};"); + Line($"{dictVar}[{keyExpr}] = System.Enum.Parse<{enumType}>(((StringArray){dvVar}.Dictionary).GetString((int)((Int16Array){dvVar}.Indices).GetValue({indexVar}).Value)!);"); + break; + } + case ArrowTypeKind.NestedRecord: + { + var typeName = valueType.FullTypeName; + var sVar = $"sv_{propIndex}{d}"; + Line($"var {sVar} = (StructArray){valsArrayVar};"); + Line($"var ca_{propIndex}{d} = new IArrowArray[{sVar}.Fields.Count];"); + Line($"for (int f = 0; f < {sVar}.Fields.Count; f++) ca_{propIndex}{d}[f] = ArrowArrayFactory.BuildArray({sVar}.Fields[f].Data.Slice({indexVar}, 1));"); + Line($"{dictVar}[{keyExpr}] = {typeName}.FromRecordBatch(new RecordBatch({typeName}.ArrowSchema, ca_{propIndex}{d}, 1));"); + break; + } + default: + { + var valueReader = GetScalarReader(valueType, valsArrayVar, indexVar); + Line($"{dictVar}[{keyExpr}] = {valueReader};"); + break; + } + } } - } - private void EmitMultiRowBuilderBuild(PropertyModel prop, int index) - { - switch (prop.Type.Kind) + private string GetScalarReader(TypeInfo type, string arrayVar, string indexVar) + { + var stringArrayType = GetArrayReadType(type); + return type.Kind switch + { + ArrowTypeKind.String => $"(({stringArrayType}){arrayVar}).GetString({indexVar})!", + ArrowTypeKind.Byte => $"((UInt8Array){arrayVar}).GetValue({indexVar}).Value", + ArrowTypeKind.SByte => $"((Int8Array){arrayVar}).GetValue({indexVar}).Value", + ArrowTypeKind.Int16 => $"((Int16Array){arrayVar}).GetValue({indexVar}).Value", + ArrowTypeKind.UInt16 => $"((UInt16Array){arrayVar}).GetValue({indexVar}).Value", + ArrowTypeKind.Int32 => $"((Int32Array){arrayVar}).GetValue({indexVar}).Value", + ArrowTypeKind.UInt32 => $"((UInt32Array){arrayVar}).GetValue({indexVar}).Value", + ArrowTypeKind.Int64 => $"((Int64Array){arrayVar}).GetValue({indexVar}).Value", + ArrowTypeKind.UInt64 => $"((UInt64Array){arrayVar}).GetValue({indexVar}).Value", + ArrowTypeKind.Float => $"((FloatArray){arrayVar}).GetValue({indexVar}).Value", + ArrowTypeKind.Double => $"((DoubleArray){arrayVar}).GetValue({indexVar}).Value", + ArrowTypeKind.Half => $"((HalfFloatArray){arrayVar}).GetValue({indexVar}).Value", + ArrowTypeKind.Bool => $"((BooleanArray){arrayVar}).GetValue({indexVar}).Value", + ArrowTypeKind.Decimal => $"((Decimal128Array){arrayVar}).GetValue({indexVar}).Value", + ArrowTypeKind.DateTime => TypeInfoHasTimezone(type) + ? $"((TimestampArray){arrayVar}).GetTimestamp({indexVar})!.Value.UtcDateTime" + : $"((TimestampArray){arrayVar}).GetTimestamp({indexVar})!.Value.DateTime", + ArrowTypeKind.DateTimeOffset => $"((TimestampArray){arrayVar}).GetTimestamp({indexVar})!.Value", + ArrowTypeKind.DateOnly => GetDateArrayReadType(type) == "Date64Array" + ? $"((Date64Array){arrayVar}).GetDateOnly({indexVar}).Value" + : $"((Date32Array){arrayVar}).GetDateOnly({indexVar}).Value", + ArrowTypeKind.TimeOnly => GetTimeArrayReadType(type) == "Time32Array" + ? $"((Time32Array){arrayVar}).GetTime({indexVar}).Value" + : $"((Time64Array){arrayVar}).GetTime({indexVar}).Value", + ArrowTypeKind.TimeSpan => $"Apache.Arrow.Serialization.ArrowArrayHelper.ReadDuration((DurationArray){arrayVar}, {indexVar})", + ArrowTypeKind.Guid => $"((GuidArray){arrayVar}).GetGuid({indexVar})!.Value", + ArrowTypeKind.Binary => $"((BinaryArray){arrayVar}).GetBytes({indexVar}).ToArray()", + ArrowTypeKind.Enum => $"System.Enum.Parse<{type.FullTypeName}>(((StringArray){arrayVar}).GetString({indexVar})!)", + _ => $"default({type.FullTypeName})", + }; + } + + private void EmitNestedRecordDeserialize(PropertyModel prop, int index, string colAccess) + { + var typeName = prop.Type.FullTypeName; + + if (prop.IsNullable) + { + Line($"var structArr_{index} = (StructArray){colAccess};"); + Line($"{typeName}? prop_{index};"); + Line($"if (structArr_{index}.IsNull(0))"); + Line("{"); + _indent++; + Line($"prop_{index} = null;"); + _indent--; + Line("}"); + Line("else"); + Line("{"); + _indent++; + Line($"var nestedArrays_{index} = new IArrowArray[structArr_{index}.Fields.Count];"); + Line($"for (int i = 0; i < structArr_{index}.Fields.Count; i++) nestedArrays_{index}[i] = structArr_{index}.Fields[i];"); + Line($"var nestedBatch_{index} = new RecordBatch({typeName}.ArrowSchema, nestedArrays_{index}, 1);"); + Line($"prop_{index} = {typeName}.FromRecordBatch(nestedBatch_{index});"); + _indent--; + Line("}"); + } + else + { + Line($"var structArr_{index} = (StructArray){colAccess};"); + Line($"var nestedArrays_{index} = new IArrowArray[structArr_{index}.Fields.Count];"); + Line($"for (int i = 0; i < structArr_{index}.Fields.Count; i++) nestedArrays_{index}[i] = structArr_{index}.Fields[i];"); + Line($"var nestedBatch_{index} = new RecordBatch({typeName}.ArrowSchema, nestedArrays_{index}, 1);"); + Line($"var prop_{index} = {typeName}.FromRecordBatch(nestedBatch_{index});"); + } + } + + private void EmitMultiRowSerialize() { - case TypeKind2.Enum: + Line($"public static RecordBatch ToRecordBatch(IReadOnlyList<{_model.TypeName}> values)"); + Line("{"); + _indent++; + + Line("var count = values.Count;"); + Line(); + + // Create a builder for each column + for (int i = 0; i < _model.Properties.Count; i++) { - Line($"var dictValues_{index} = new StringArray.Builder();"); - Line($"var sortedEntries_{index} = bld_{index}_map.OrderBy(kv => kv.Value).Select(kv => kv.Key);"); - Line($"foreach (var e in sortedEntries_{index}) dictValues_{index}.Append(e);"); - Line($"var arr_{index} = new DictionaryArray(new DictionaryType(Int16Type.Default, StringType.Default, false), bld_{index}_idx.Build(), dictValues_{index}.Build());"); - break; + var prop = _model.Properties[i]; + EmitMultiRowBuilderDecl(prop, i); } - case TypeKind2.NestedRecord: + + Line(); + Line("for (int row = 0; row < count; row++)"); + Line("{"); + _indent++; + Line("var item = values[row];"); + + if (_model.HasSerializationCallback) + Line("((Apache.Arrow.Serialization.IArrowSerializationCallback)item).OnBeforeSerialize();"); + + for (int i = 0; i < _model.Properties.Count; i++) { - var typeName = prop.Type.FullTypeName; - if (prop.IsNullable) - { - // For nullable nested: substitute nulls with the first non-null item (value doesn't matter for null rows) - Line($"IArrowArray arr_{index};"); - Line("{"); - _indent++; - Line($"var firstNonNull_{index} = bld_{index}_items.FirstOrDefault(x => x != null);"); - Line($"var nullBitmap_{index} = new ArrowBuffer.BitmapBuilder();"); - Line($"int nullCount_{index} = 0;"); - Line($"if (firstNonNull_{index} != null)"); - Line("{"); - _indent++; - Line($"var resolved_{index} = new List<{typeName}>(count);"); - Line($"for (int r = 0; r < count; r++)"); - Line("{"); - _indent++; - Line($"if (bld_{index}_items[r] != null) {{ resolved_{index}.Add(bld_{index}_items[r]!); nullBitmap_{index}.Append(true); }}"); - Line($"else {{ resolved_{index}.Add(firstNonNull_{index}!); nullBitmap_{index}.Append(false); nullCount_{index}++; }}"); - _indent--; - Line("}"); - Line($"var nb_{index} = {typeName}.ToRecordBatch((IReadOnlyList<{typeName}>)resolved_{index});"); - Line($"var ca_{index} = new IArrowArray[nb_{index}.ColumnCount];"); - Line($"for (int f = 0; f < nb_{index}.ColumnCount; f++) ca_{index}[f] = nb_{index}.Column(f);"); - Line($"arr_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), count, ca_{index}, nullBitmap_{index}.Build(), nullCount_{index});"); - _indent--; - Line("}"); - Line("else"); - Line("{"); - _indent++; - Line($"for (int r = 0; r < count; r++) {{ nullBitmap_{index}.Append(false); nullCount_{index}++; }}"); - Line($"var nc_{index} = new IArrowArray[{typeName}.ArrowSchema.FieldsList.Count];"); - Line($"for (int f = 0; f < nc_{index}.Length; f++)"); - Line($" nc_{index}[f] = Apache.Arrow.Serialization.ArrowArrayHelper.BuildNullArray({typeName}.ArrowSchema.FieldsList[f].DataType, count);"); - Line($"arr_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), count, nc_{index}, nullBitmap_{index}.Build(), nullCount_{index});"); - _indent--; - Line("}"); - _indent--; - Line("}"); - } - else - { - // Non-nullable: directly use multi-row serializer - Line($"var nb_{index} = {typeName}.ToRecordBatch((IReadOnlyList<{typeName}>)bld_{index}_items);"); - Line($"var ca_{index} = new IArrowArray[nb_{index}.ColumnCount];"); - Line($"for (int f = 0; f < nb_{index}.ColumnCount; f++) ca_{index}[f] = nb_{index}.Column(f);"); - Line($"var arr_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), count, ca_{index}, ArrowBuffer.Empty);"); - } - break; - } - case TypeKind2.TimeOnly: - Line($"var arr_{index} = bld_{index}.Build();"); - break; - case TypeKind2.TimeSpan: - Line($"var arr_{index} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildDurationArray(bld_{index}_items);"); - break; - case TypeKind2.Decimal: - Line($"var arr_{index} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildDecimalArray(bld_{index}_items);"); - break; - case TypeKind2.Guid: - Line($"var arr_{index} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildGuidArray(bld_{index}_items);"); - break; - case TypeKind2.Custom: - Line($"var arr_{index} = new {prop.ConverterTypeName}().ToArray(bld_{index}_items);"); - break; - case TypeKind2.List: - case TypeKind2.Array: - case TypeKind2.Set: - { - var elemType = prop.Type.ElementType!; - if (NeedsManualListBuilder(elemType)) - { - var arrowElemType = GetArrowTypeExpression(elemType); - // Finalize the offsets with the total item count - Line($"bld_{index}_offsets.Append(bld_{index}_items.Count);"); - switch (elemType.Kind) + var prop = _model.Properties[i]; + EmitMultiRowBuilderAppend(prop, i); + } + + _indent--; + Line("}"); + Line(); + + // Build arrays and return RecordBatch + for (int i = 0; i < _model.Properties.Count; i++) + { + var prop = _model.Properties[i]; + EmitMultiRowBuilderBuild(prop, i); + } + + Line(); + Line("return new RecordBatch(_arrowSchema, new IArrowArray[]"); + Line("{"); + _indent++; + for (int i = 0; i < _model.Properties.Count; i++) + { + var comma = i < _model.Properties.Count - 1 ? "," : ""; + Line($"arr_{i}{comma}"); + } + _indent--; + Line("}, count);"); + + _indent--; + Line("}"); + } + + private void EmitMultiRowBuilderDecl(PropertyModel prop, int index) + { + switch (prop.Type.Kind) + { + case ArrowTypeKind.String: { - case TypeKind2.Guid: + var si = GetStringArrayInfo(prop); + Line($"var bld_{index} = new {si.BuilderType}();"); + break; + } + case ArrowTypeKind.Bool: + { + var bl = GetBoolArrayInfo(prop); + Line($"var bld_{index} = new {bl.BuilderType}();"); + break; + } + case ArrowTypeKind.Byte: + Line($"var bld_{index} = new UInt8Array.Builder();"); + break; + case ArrowTypeKind.SByte: + Line($"var bld_{index} = new Int8Array.Builder();"); + break; + case ArrowTypeKind.Int16: + Line($"var bld_{index} = new Int16Array.Builder();"); + break; + case ArrowTypeKind.UInt16: + Line($"var bld_{index} = new UInt16Array.Builder();"); + break; + case ArrowTypeKind.Int32: + Line($"var bld_{index} = new Int32Array.Builder();"); + break; + case ArrowTypeKind.UInt32: + Line($"var bld_{index} = new UInt32Array.Builder();"); + break; + case ArrowTypeKind.Int64: + Line($"var bld_{index} = new Int64Array.Builder();"); + break; + case ArrowTypeKind.UInt64: + Line($"var bld_{index} = new UInt64Array.Builder();"); + break; + case ArrowTypeKind.Float: + Line($"var bld_{index} = new FloatArray.Builder();"); + break; + case ArrowTypeKind.Double: + Line($"var bld_{index} = new DoubleArray.Builder();"); + break; + case ArrowTypeKind.Binary: + { + var bi = GetBinaryArrayInfo(prop); + Line($"var bld_{index} = new {bi.BuilderType}();"); + break; + } + case ArrowTypeKind.DateTime: + case ArrowTypeKind.DateTimeOffset: + Line($"var bld_{index} = new TimestampArray.Builder({GetTimestampTypeExpr(prop)});"); + break; + case ArrowTypeKind.DateOnly: + { + var di = GetDateArrayInfo(prop); + Line($"var bld_{index} = new {di.BuilderType}();"); + break; + } + case ArrowTypeKind.TimeOnly: + { + var ti = GetTimeArrayInfo(prop); + Line($"var bld_{index} = new {ti.BuilderType}({ti.TypeExpr});"); + break; + } + case ArrowTypeKind.TimeSpan: + case ArrowTypeKind.Decimal: + Line($"var bld_{index}_items = new List<{prop.Type.FullTypeName}{(prop.IsNullable ? "?" : "")}>(count);"); + break; + case ArrowTypeKind.Guid: + Line($"var bld_{index}_items = new List<{prop.Type.FullTypeName}{(prop.IsNullable ? "?" : "")}>(count);"); + break; + case ArrowTypeKind.Half: + Line($"var bld_{index} = new HalfFloatArray.Builder();"); + break; + case ArrowTypeKind.Enum: + Line($"var bld_{index}_dict = new StringArray.Builder();"); + Line($"var bld_{index}_idx = new Int16Array.Builder();"); + Line($"var bld_{index}_map = new Dictionary();"); + break; + case ArrowTypeKind.List: + case ArrowTypeKind.Array: + case ArrowTypeKind.Set: + { + var elemType = prop.Type.ElementType!; + if (NeedsManualListBuilder(elemType)) + { + // Collect raw items + offsets for manual ListArray construction + Line($"var bld_{index}_items = new List<{elemType.FullTypeName}>();"); + Line($"var bld_{index}_offsets = new ArrowBuffer.Builder();"); + if (prop.IsNullable) + Line($"var bld_{index}_nulls = new ArrowBuffer.BitmapBuilder();"); + } + else { - Line($"var guidValues_{index} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildGuidArray(bld_{index}_items);"); - var guidNullBuf = prop.IsNullable ? $"bld_{index}_nulls.Build()" : "ArrowBuffer.Empty"; - Line($"var arr_{index} = new ListArray(new ListType(new Field(\"item\", {arrowElemType}, false)), count, bld_{index}_offsets.Build(), guidValues_{index}, {guidNullBuf});"); - break; + var arrowElemType = GetArrowTypeExpression(elemType); + Line($"var bld_{index} = new ListArray.Builder({arrowElemType});"); } - case TypeKind2.Enum: + break; + } + case ArrowTypeKind.Dictionary: + { + var valueType = prop.Type.ValueType!; + if (NeedsManualMapBuilder(valueType)) { - // Dictionary-encode enum values: Dictionary(Int16, Utf8) - Line($"var enumDict_{index} = new StringArray.Builder();"); - Line($"var enumIdx_{index} = new Int16Array.Builder();"); - Line($"var enumMap_{index} = new System.Collections.Generic.Dictionary();"); - Line($"foreach (var e in bld_{index}_items) {{"); - Line($" var es = e.ToString();"); - Line($" if (!enumMap_{index}.TryGetValue(es, out var ei)) {{ ei = (short)enumMap_{index}.Count; enumMap_{index}[es] = ei; }}"); - Line($" enumIdx_{index}.Append(ei);"); - Line($"}}"); - Line($"foreach (var kv in enumMap_{index}.OrderBy(kv => kv.Value)) enumDict_{index}.Append(kv.Key);"); - Line($"var enumDictArr_{index} = new DictionaryArray(new DictionaryType(Int16Type.Default, StringType.Default, false), enumIdx_{index}.Build(), enumDict_{index}.Build());"); - var enumNullBuf = prop.IsNullable ? $"bld_{index}_nulls.Build()" : "ArrowBuffer.Empty"; - Line($"var arr_{index} = new ListArray(ArrowSchema.GetFieldByName(\"{prop.FieldName}\").DataType, count, bld_{index}_offsets.Build(), enumDictArr_{index}, {enumNullBuf});"); - break; + var keyType = prop.Type.KeyType!; + var keyBuilderType = GetArrayBuilderType(keyType); + Line($"var bld_{index}_keys = new {keyBuilderType}();"); + switch (valueType.Kind) + { + case ArrowTypeKind.Guid: + Line($"var bld_{index}_vals = new System.Collections.Generic.List();"); + break; + case ArrowTypeKind.Enum: + Line($"var bld_{index}_valDict = new StringArray.Builder();"); + Line($"var bld_{index}_valIdx = new Int16Array.Builder();"); + Line($"var bld_{index}_valMap = new System.Collections.Generic.Dictionary();"); + break; + case ArrowTypeKind.NestedRecord: + Line($"var bld_{index}_vals = new System.Collections.Generic.List<{valueType.FullTypeName}>();"); + break; + } + Line($"var bld_{index}_offsets = new ArrowBuffer.Builder();"); + if (prop.IsNullable) + Line($"var bld_{index}_nulls = new ArrowBuffer.BitmapBuilder();"); } - case TypeKind2.NestedRecord: + else { - var typeName = elemType.FullTypeName; - Line($"IArrowArray structValues_{index};"); - Line($"if (bld_{index}_items.Count > 0)"); + var mapType = GetArrowTypeExpression(prop.Type); + Line($"var bld_{index} = new MapArray.Builder({mapType});"); + } + break; + } + case ArrowTypeKind.NestedRecord: + { + var typeName = prop.Type.FullTypeName; + Line($"var bld_{index}_items = new List<{typeName}{(prop.IsNullable ? "?" : "")}>(count);"); + break; + } + case ArrowTypeKind.Custom: + Line($"var bld_{index}_items = new List<{prop.Type.FullTypeName}>(count);"); + break; + default: + Line($"// TODO: Unsupported multi-row builder for {prop.Type.Kind}"); + break; + } + } + + private void EmitMultiRowBuilderAppend(PropertyModel prop, int index) + { + var access = $"item.{prop.PropertyName}"; + + switch (prop.Type.Kind) + { + case ArrowTypeKind.String: + Line($"bld_{index}.Append({access});"); + break; + case ArrowTypeKind.Bool: + if (prop.IsNullable) + { + Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(v_{index}); else bld_{index}.AppendNull();"); + } + else + Line($"bld_{index}.Append({access});"); + break; + case ArrowTypeKind.Byte: + case ArrowTypeKind.SByte: + case ArrowTypeKind.Int16: + case ArrowTypeKind.UInt16: + case ArrowTypeKind.Int32: + case ArrowTypeKind.UInt32: + case ArrowTypeKind.Int64: + case ArrowTypeKind.UInt64: + case ArrowTypeKind.Float: + case ArrowTypeKind.Double: + if (prop.IsNullable) + { + Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(v_{index}); else bld_{index}.AppendNull();"); + } + else + Line($"bld_{index}.Append({access});"); + break; + case ArrowTypeKind.Binary: + Line($"bld_{index}.Append({BinaryToSpan(prop, access)});"); + break; + case ArrowTypeKind.DateTime: + if (prop.IsNullable) + { + Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(Apache.Arrow.Serialization.ArrowArrayHelper.{TimestampConvertMethod(prop)}(v_{index})); else bld_{index}.AppendNull();"); + } + else + Line($"bld_{index}.Append(Apache.Arrow.Serialization.ArrowArrayHelper.{TimestampConvertMethod(prop)}({access}));"); + break; + case ArrowTypeKind.DateTimeOffset: + if (prop.IsNullable) + { + Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(Apache.Arrow.Serialization.ArrowArrayHelper.{TimestampConvertMethod(prop)}(v_{index})); else bld_{index}.AppendNull();"); + } + else + Line($"bld_{index}.Append(Apache.Arrow.Serialization.ArrowArrayHelper.{TimestampConvertMethod(prop)}({access}));"); + break; + case ArrowTypeKind.DateOnly: + if (prop.IsNullable) + { + Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(v_{index}); else bld_{index}.AppendNull();"); + } + else + Line($"bld_{index}.Append({access});"); + break; + case ArrowTypeKind.TimeOnly: + if (prop.IsNullable) + { + Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(v_{index}); else bld_{index}.AppendNull();"); + } + else + Line($"bld_{index}.Append({access});"); + break; + case ArrowTypeKind.TimeSpan: + case ArrowTypeKind.Decimal: + Line($"bld_{index}_items.Add({access});"); + break; + case ArrowTypeKind.Guid: + Line($"bld_{index}_items.Add({access});"); + break; + case ArrowTypeKind.Half: + if (prop.IsNullable) + { + Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(v_{index}); else bld_{index}.AppendNull();"); + } + else + Line($"bld_{index}.Append({access});"); + break; + case ArrowTypeKind.Enum: + if (prop.IsNullable) + { + Line($"if ({access} is {{ }} ev_{index})"); + Line("{"); + _indent++; + Line($"var es_{index} = ev_{index}.ToString();"); + Line($"if (!bld_{index}_map.TryGetValue(es_{index}, out var ei_{index})) {{ ei_{index} = (short)bld_{index}_map.Count; bld_{index}_map[es_{index}] = ei_{index}; }}"); + Line($"bld_{index}_idx.Append(ei_{index});"); + _indent--; + Line("}"); + Line($"else bld_{index}_idx.AppendNull();"); + } + else + { + Line($"var es_{index} = {access}.ToString();"); + Line($"if (!bld_{index}_map.TryGetValue(es_{index}, out var ei_{index})) {{ ei_{index} = (short)bld_{index}_map.Count; bld_{index}_map[es_{index}] = ei_{index}; }}"); + Line($"bld_{index}_idx.Append(ei_{index});"); + } + break; + case ArrowTypeKind.List: + case ArrowTypeKind.Array: + case ArrowTypeKind.Set: + { + var elemType = prop.Type.ElementType!; + if (prop.IsNullable) + { + Line($"if ({access} != null)"); Line("{"); _indent++; - Line($"var nb_{index} = {typeName}.ToRecordBatch((IReadOnlyList<{typeName}>)bld_{index}_items);"); - Line($"var ca_{index} = new IArrowArray[nb_{index}.ColumnCount];"); - Line($"for (int f = 0; f < nb_{index}.ColumnCount; f++) ca_{index}[f] = nb_{index}.Column(f);"); - Line($"structValues_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), bld_{index}_items.Count, ca_{index}, ArrowBuffer.Empty);"); + } + if (NeedsManualListBuilder(elemType)) + { + Line($"bld_{index}_offsets.Append(bld_{index}_items.Count);"); + Line($"bld_{index}_items.AddRange({access});"); + if (prop.IsNullable) Line($"bld_{index}_nulls.Append(true);"); + } + else + { + Line($"bld_{index}.Append();"); + EmitListValuePopulation(elemType, $"bld_{index}", access, index); + } + if (prop.IsNullable) + { _indent--; Line("}"); Line("else"); Line("{"); _indent++; - Line($"structValues_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), 0, System.Array.Empty(), ArrowBuffer.Empty);"); + if (NeedsManualListBuilder(elemType)) + { + Line($"bld_{index}_offsets.Append(bld_{index}_items.Count);"); + Line($"bld_{index}_nulls.Append(false);"); + } + else + { + Line($"bld_{index}.AppendNull();"); + } _indent--; Line("}"); - var nestedNullBuf = prop.IsNullable ? $"bld_{index}_nulls.Build()" : "ArrowBuffer.Empty"; - Line($"var arr_{index} = new ListArray(new ListType(new Field(\"item\", new StructType({typeName}.ArrowSchema.FieldsList), false)), count, bld_{index}_offsets.Build(), structValues_{index}, {nestedNullBuf});"); - break; } + break; } - } - else - { - Line($"var arr_{index} = bld_{index}.Build();"); - } - break; - } - case TypeKind2.Dictionary: - { - var valueType = prop.Type.ValueType!; - if (NeedsManualMapBuilder(valueType)) - { - var mapType = GetArrowTypeExpression(prop.Type); - // Finalize offsets - Line($"bld_{index}_offsets.Append(bld_{index}_keys.Length);"); - Line($"var mkArr_{index} = bld_{index}_keys.Build();"); - - // Build value array - switch (valueType.Kind) - { - case TypeKind2.Guid: - Line($"var mvArr_{index} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildGuidArray(bld_{index}_vals);"); - break; - case TypeKind2.Enum: + case ArrowTypeKind.Dictionary: + { + var keyType = prop.Type.KeyType!; + var valueType = prop.Type.ValueType!; + if (NeedsManualMapBuilder(valueType)) { - Line($"foreach (var kv in bld_{index}_valMap.OrderBy(kv => kv.Value)) bld_{index}_valDict.Append(kv.Key);"); - Line($"var mvArr_{index} = new DictionaryArray(new DictionaryType(Int16Type.Default, StringType.Default, false), bld_{index}_valIdx.Build(), bld_{index}_valDict.Build());"); - break; + if (prop.IsNullable) + { + Line($"if ({access} != null)"); + Line("{"); + _indent++; + } + Line($"bld_{index}_offsets.Append(bld_{index}_keys.Length);"); + Line($"foreach (var kv_{index} in {access})"); + Line("{"); + _indent++; + Line($"bld_{index}_keys.Append(kv_{index}.Key);"); + switch (valueType.Kind) + { + case ArrowTypeKind.Guid: + Line($"bld_{index}_vals.Add(kv_{index}.Value);"); + break; + case ArrowTypeKind.Enum: + { + Line($"var es_{index} = kv_{index}.Value.ToString();"); + Line($"if (!bld_{index}_valMap.TryGetValue(es_{index}, out var ei_{index})) {{ ei_{index} = (short)bld_{index}_valMap.Count; bld_{index}_valMap[es_{index}] = ei_{index}; }}"); + Line($"bld_{index}_valIdx.Append(ei_{index});"); + break; + } + case ArrowTypeKind.NestedRecord: + Line($"bld_{index}_vals.Add(kv_{index}.Value);"); + break; + } + _indent--; + Line("}"); + if (prop.IsNullable) + { + Line($"bld_{index}_nulls.Append(true);"); + _indent--; + Line("}"); + Line("else"); + Line("{"); + _indent++; + Line($"bld_{index}_offsets.Append(bld_{index}_keys.Length);"); + Line($"bld_{index}_nulls.Append(false);"); + _indent--; + Line("}"); + } } - case TypeKind2.NestedRecord: + else { - var typeName = valueType.FullTypeName; - Line($"IArrowArray mvArr_{index};"); - Line($"if (bld_{index}_vals.Count > 0)"); + if (prop.IsNullable) + { + Line($"if ({access} != null)"); + Line("{"); + _indent++; + } + Line($"bld_{index}.Append();"); + Line($"foreach (var kv_{index} in {access})"); Line("{"); _indent++; - Line($"var nb_{index} = {typeName}.ToRecordBatch((IReadOnlyList<{typeName}>)bld_{index}_vals);"); - Line($"var ca_{index} = new IArrowArray[nb_{index}.ColumnCount];"); - Line($"for (int f = 0; f < nb_{index}.ColumnCount; f++) ca_{index}[f] = nb_{index}.Column(f);"); - Line($"mvArr_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), bld_{index}_vals.Count, ca_{index}, ArrowBuffer.Empty);"); + EmitMapKeyAppend(keyType, $"bld_{index}", $"kv_{index}.Key", index); + EmitMapValueAppend(valueType, $"bld_{index}", $"kv_{index}.Value", index); _indent--; Line("}"); - Line($"else mvArr_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), 0, System.Array.Empty(), ArrowBuffer.Empty);"); - break; + if (prop.IsNullable) + { + _indent--; + Line("}"); + Line($"else bld_{index}.AppendNull();"); + } } + break; } - - // Construct MapArray manually - var nullBuf = prop.IsNullable ? $"bld_{index}_nulls.Build()" : "ArrowBuffer.Empty"; - Line($"var mEntries_{index} = new StructArray(((MapType){mapType}).KeyValueType, mkArr_{index}.Length, new IArrowArray[] {{ mkArr_{index}, mvArr_{index} }}, ArrowBuffer.Empty);"); - Line($"var arr_{index} = new MapArray({mapType}, count, bld_{index}_offsets.Build(), mEntries_{index}, {nullBuf});"); - } - else - { - Line($"var arr_{index} = bld_{index}.Build();"); - } - break; + case ArrowTypeKind.NestedRecord: + Line($"bld_{index}_items.Add({access});"); + break; + case ArrowTypeKind.Custom: + Line($"bld_{index}_items.Add({access});"); + break; + default: + Line($"// TODO: Unsupported multi-row append for {prop.Type.Kind}"); + break; } - default: - Line($"var arr_{index} = bld_{index}.Build();"); - break; } - } - private void EmitMultiRowDeserialize() - { - var newKw = _model.HasArrowSerializableBase ? "new " : ""; - Line($"public static {newKw}IReadOnlyList<{_model.TypeName}> ListFromRecordBatch(RecordBatch batch)"); - Line("{"); - _indent++; - Line("var count = batch.Length;"); - Line($"var result = new List<{_model.TypeName}>(count);"); - Line(); - - // Get column references - for (int i = 0; i < _model.Properties.Count; i++) + private void EmitMultiRowBuilderBuild(PropertyModel prop, int index) { - var prop = _model.Properties[i]; - EmitMultiRowColumnRef(prop, i); - } + switch (prop.Type.Kind) + { + case ArrowTypeKind.Enum: + { + Line($"var dictValues_{index} = new StringArray.Builder();"); + Line($"var sortedEntries_{index} = bld_{index}_map.OrderBy(kv => kv.Value).Select(kv => kv.Key);"); + Line($"foreach (var e in sortedEntries_{index}) dictValues_{index}.Append(e);"); + Line($"var arr_{index} = new DictionaryArray(new DictionaryType(Int16Type.Default, StringType.Default, false), bld_{index}_idx.Build(), dictValues_{index}.Build());"); + break; + } + case ArrowTypeKind.NestedRecord: + { + var typeName = prop.Type.FullTypeName; + if (prop.IsNullable) + { + // For nullable nested: substitute nulls with the first non-null item (value doesn't matter for null rows) + Line($"IArrowArray arr_{index};"); + Line("{"); + _indent++; + Line($"var firstNonNull_{index} = bld_{index}_items.FirstOrDefault(x => x != null);"); + Line($"var nullBitmap_{index} = new ArrowBuffer.BitmapBuilder();"); + Line($"int nullCount_{index} = 0;"); + Line($"if (firstNonNull_{index} != null)"); + Line("{"); + _indent++; + Line($"var resolved_{index} = new List<{typeName}>(count);"); + Line($"for (int r = 0; r < count; r++)"); + Line("{"); + _indent++; + Line($"if (bld_{index}_items[r] != null) {{ resolved_{index}.Add(bld_{index}_items[r]!); nullBitmap_{index}.Append(true); }}"); + Line($"else {{ resolved_{index}.Add(firstNonNull_{index}!); nullBitmap_{index}.Append(false); nullCount_{index}++; }}"); + _indent--; + Line("}"); + Line($"var nb_{index} = {typeName}.ToRecordBatch((IReadOnlyList<{typeName}>)resolved_{index});"); + Line($"var ca_{index} = new IArrowArray[nb_{index}.ColumnCount];"); + Line($"for (int f = 0; f < nb_{index}.ColumnCount; f++) ca_{index}[f] = nb_{index}.Column(f);"); + Line($"arr_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), count, ca_{index}, nullBitmap_{index}.Build(), nullCount_{index});"); + _indent--; + Line("}"); + Line("else"); + Line("{"); + _indent++; + Line($"for (int r = 0; r < count; r++) {{ nullBitmap_{index}.Append(false); nullCount_{index}++; }}"); + Line($"var nc_{index} = new IArrowArray[{typeName}.ArrowSchema.FieldsList.Count];"); + Line($"for (int f = 0; f < nc_{index}.Length; f++)"); + Line($" nc_{index}[f] = Apache.Arrow.Serialization.ArrowArrayHelper.BuildNullArray({typeName}.ArrowSchema.FieldsList[f].DataType, count);"); + Line($"arr_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), count, nc_{index}, nullBitmap_{index}.Build(), nullCount_{index});"); + _indent--; + Line("}"); + _indent--; + Line("}"); + } + else + { + // Non-nullable: directly use multi-row serializer + Line($"var nb_{index} = {typeName}.ToRecordBatch((IReadOnlyList<{typeName}>)bld_{index}_items);"); + Line($"var ca_{index} = new IArrowArray[nb_{index}.ColumnCount];"); + Line($"for (int f = 0; f < nb_{index}.ColumnCount; f++) ca_{index}[f] = nb_{index}.Column(f);"); + Line($"var arr_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), count, ca_{index}, ArrowBuffer.Empty);"); + } + break; + } + case ArrowTypeKind.TimeOnly: + Line($"var arr_{index} = bld_{index}.Build();"); + break; + case ArrowTypeKind.TimeSpan: + Line($"var arr_{index} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildDurationArray(bld_{index}_items);"); + break; + case ArrowTypeKind.Decimal: + Line($"var arr_{index} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildDecimalArray(bld_{index}_items);"); + break; + case ArrowTypeKind.Guid: + Line($"var arr_{index} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildGuidArray(bld_{index}_items);"); + break; + case ArrowTypeKind.Custom: + Line($"var arr_{index} = new {prop.ConverterTypeName}().ToArray(bld_{index}_items);"); + break; + case ArrowTypeKind.List: + case ArrowTypeKind.Array: + case ArrowTypeKind.Set: + { + var elemType = prop.Type.ElementType!; + if (NeedsManualListBuilder(elemType)) + { + var arrowElemType = GetArrowTypeExpression(elemType); + // Finalize the offsets with the total item count + Line($"bld_{index}_offsets.Append(bld_{index}_items.Count);"); + switch (elemType.Kind) + { + case ArrowTypeKind.Guid: + { + Line($"var guidValues_{index} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildGuidArray(bld_{index}_items);"); + var guidNullBuf = prop.IsNullable ? $"bld_{index}_nulls.Build()" : "ArrowBuffer.Empty"; + Line($"var arr_{index} = new ListArray(new ListType(new Field(\"item\", {arrowElemType}, false)), count, bld_{index}_offsets.Build(), guidValues_{index}, {guidNullBuf});"); + break; + } + case ArrowTypeKind.Enum: + { + // Dictionary-encode enum values: Dictionary(Int16, Utf8) + Line($"var enumDict_{index} = new StringArray.Builder();"); + Line($"var enumIdx_{index} = new Int16Array.Builder();"); + Line($"var enumMap_{index} = new System.Collections.Generic.Dictionary();"); + Line($"foreach (var e in bld_{index}_items) {{"); + Line($" var es = e.ToString();"); + Line($" if (!enumMap_{index}.TryGetValue(es, out var ei)) {{ ei = (short)enumMap_{index}.Count; enumMap_{index}[es] = ei; }}"); + Line($" enumIdx_{index}.Append(ei);"); + Line($"}}"); + Line($"foreach (var kv in enumMap_{index}.OrderBy(kv => kv.Value)) enumDict_{index}.Append(kv.Key);"); + Line($"var enumDictArr_{index} = new DictionaryArray(new DictionaryType(Int16Type.Default, StringType.Default, false), enumIdx_{index}.Build(), enumDict_{index}.Build());"); + var enumNullBuf = prop.IsNullable ? $"bld_{index}_nulls.Build()" : "ArrowBuffer.Empty"; + Line($"var arr_{index} = new ListArray(ArrowSchema.GetFieldByName(\"{prop.FieldName}\").DataType, count, bld_{index}_offsets.Build(), enumDictArr_{index}, {enumNullBuf});"); + break; + } + case ArrowTypeKind.NestedRecord: + { + var typeName = elemType.FullTypeName; + Line($"IArrowArray structValues_{index};"); + Line($"if (bld_{index}_items.Count > 0)"); + Line("{"); + _indent++; + Line($"var nb_{index} = {typeName}.ToRecordBatch((IReadOnlyList<{typeName}>)bld_{index}_items);"); + Line($"var ca_{index} = new IArrowArray[nb_{index}.ColumnCount];"); + Line($"for (int f = 0; f < nb_{index}.ColumnCount; f++) ca_{index}[f] = nb_{index}.Column(f);"); + Line($"structValues_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), bld_{index}_items.Count, ca_{index}, ArrowBuffer.Empty);"); + _indent--; + Line("}"); + Line("else"); + Line("{"); + _indent++; + Line($"structValues_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), 0, System.Array.Empty(), ArrowBuffer.Empty);"); + _indent--; + Line("}"); + var nestedNullBuf = prop.IsNullable ? $"bld_{index}_nulls.Build()" : "ArrowBuffer.Empty"; + Line($"var arr_{index} = new ListArray(new ListType(new Field(\"item\", new StructType({typeName}.ArrowSchema.FieldsList), false)), count, bld_{index}_offsets.Build(), structValues_{index}, {nestedNullBuf});"); + break; + } + } + } + else + { + Line($"var arr_{index} = bld_{index}.Build();"); + } + break; + } + case ArrowTypeKind.Dictionary: + { + var valueType = prop.Type.ValueType!; + if (NeedsManualMapBuilder(valueType)) + { + var mapType = GetArrowTypeExpression(prop.Type); + // Finalize offsets + Line($"bld_{index}_offsets.Append(bld_{index}_keys.Length);"); + Line($"var mkArr_{index} = bld_{index}_keys.Build();"); - Line(); - Line("for (int row = 0; row < count; row++)"); - Line("{"); - _indent++; + // Build value array + switch (valueType.Kind) + { + case ArrowTypeKind.Guid: + Line($"var mvArr_{index} = Apache.Arrow.Serialization.ArrowArrayHelper.BuildGuidArray(bld_{index}_vals);"); + break; + case ArrowTypeKind.Enum: + { + Line($"foreach (var kv in bld_{index}_valMap.OrderBy(kv => kv.Value)) bld_{index}_valDict.Append(kv.Key);"); + Line($"var mvArr_{index} = new DictionaryArray(new DictionaryType(Int16Type.Default, StringType.Default, false), bld_{index}_valIdx.Build(), bld_{index}_valDict.Build());"); + break; + } + case ArrowTypeKind.NestedRecord: + { + var typeName = valueType.FullTypeName; + Line($"IArrowArray mvArr_{index};"); + Line($"if (bld_{index}_vals.Count > 0)"); + Line("{"); + _indent++; + Line($"var nb_{index} = {typeName}.ToRecordBatch((IReadOnlyList<{typeName}>)bld_{index}_vals);"); + Line($"var ca_{index} = new IArrowArray[nb_{index}.ColumnCount];"); + Line($"for (int f = 0; f < nb_{index}.ColumnCount; f++) ca_{index}[f] = nb_{index}.Column(f);"); + Line($"mvArr_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), bld_{index}_vals.Count, ca_{index}, ArrowBuffer.Empty);"); + _indent--; + Line("}"); + Line($"else mvArr_{index} = new StructArray(new StructType({typeName}.ArrowSchema.FieldsList), 0, System.Array.Empty(), ArrowBuffer.Empty);"); + break; + } + } - // Read each property from the column at 'row' - for (int i = 0; i < _model.Properties.Count; i++) - { - var prop = _model.Properties[i]; - EmitMultiRowReadProperty(prop, i); + // Construct MapArray manually + var nullBuf = prop.IsNullable ? $"bld_{index}_nulls.Build()" : "ArrowBuffer.Empty"; + Line($"var mEntries_{index} = new StructArray(((MapType){mapType}).KeyValueType, mkArr_{index}.Length, new IArrowArray[] {{ mkArr_{index}, mvArr_{index} }}, ArrowBuffer.Empty);"); + Line($"var arr_{index} = new MapArray({mapType}, count, bld_{index}_offsets.Build(), mEntries_{index}, {nullBuf});"); + } + else + { + Line($"var arr_{index} = bld_{index}.Build();"); + } + break; + } + default: + Line($"var arr_{index} = bld_{index}.Build();"); + break; + } } - // Construct the record - if (_model.HasSerializationCallback) - { - EmitObjectConstruction("var item = ", ";"); - Line("((Apache.Arrow.Serialization.IArrowSerializationCallback)item).OnAfterDeserialize();"); - Line("result.Add(item);"); - } - else + private void EmitMultiRowDeserialize() { - EmitObjectConstruction("result.Add(", ");"); - } - - _indent--; - Line("}"); - Line(); - Line("return result;"); + var newKw = _model.HasArrowSerializableBase ? "new " : ""; + Line($"public static {newKw}IReadOnlyList<{_model.TypeName}> ListFromRecordBatch(RecordBatch batch)"); + Line("{"); + _indent++; + Line("var count = batch.Length;"); + Line($"var result = new List<{_model.TypeName}>(count);"); + Line(); - _indent--; - Line("}"); - } + // Get column references + for (int i = 0; i < _model.Properties.Count; i++) + { + var prop = _model.Properties[i]; + EmitMultiRowColumnRef(prop, i); + } - private void EmitMultiRowColumnRef(PropertyModel prop, int index) - { - var colAccess = $"batch.Column(\"{prop.FieldName}\")"; + Line(); + Line("for (int row = 0; row < count; row++)"); + Line("{"); + _indent++; - switch (prop.Type.Kind) - { - case TypeKind2.String: - { - var si = GetStringArrayInfo(prop); - Line($"var col_{index} = ({si.ArrayType}){colAccess};"); - break; - } - case TypeKind2.Bool: - { - var bl = GetBoolArrayInfo(prop); - Line($"var col_{index} = ({bl.ArrayType}){colAccess};"); - break; - } - case TypeKind2.Byte: - Line($"var col_{index} = (UInt8Array){colAccess};"); - break; - case TypeKind2.SByte: - Line($"var col_{index} = (Int8Array){colAccess};"); - break; - case TypeKind2.Int16: - Line($"var col_{index} = (Int16Array){colAccess};"); - break; - case TypeKind2.UInt16: - Line($"var col_{index} = (UInt16Array){colAccess};"); - break; - case TypeKind2.Int32: - Line($"var col_{index} = (Int32Array){colAccess};"); - break; - case TypeKind2.UInt32: - Line($"var col_{index} = (UInt32Array){colAccess};"); - break; - case TypeKind2.Int64: - Line($"var col_{index} = (Int64Array){colAccess};"); - break; - case TypeKind2.UInt64: - Line($"var col_{index} = (UInt64Array){colAccess};"); - break; - case TypeKind2.Float: - Line($"var col_{index} = (FloatArray){colAccess};"); - break; - case TypeKind2.Double: - Line($"var col_{index} = (DoubleArray){colAccess};"); - break; - case TypeKind2.Binary: - { - var bi = GetBinaryArrayInfo(prop); - Line($"var col_{index} = ({bi.ArrayType}){colAccess};"); - break; - } - case TypeKind2.DateTime: - case TypeKind2.DateTimeOffset: - Line($"var col_{index} = (TimestampArray){colAccess};"); - break; - case TypeKind2.DateOnly: - { - var di = GetDateArrayInfo(prop); - Line($"var col_{index} = ({di.ArrayType}){colAccess};"); - break; - } - case TypeKind2.TimeOnly: - { - var ti = GetTimeArrayInfo(prop); - Line($"var col_{index} = ({ti.ArrayType}){colAccess};"); - break; - } - case TypeKind2.TimeSpan: - Line($"var col_{index} = (DurationArray){colAccess};"); - break; - case TypeKind2.Decimal: - Line($"var col_{index} = (Decimal128Array){colAccess};"); - break; - case TypeKind2.Guid: - Line($"var col_{index} = (GuidArray){colAccess};"); - break; - case TypeKind2.Half: - Line($"var col_{index} = (HalfFloatArray){colAccess};"); - break; - case TypeKind2.Enum: - Line($"var col_{index} = (DictionaryArray){colAccess};"); - break; - case TypeKind2.List: - case TypeKind2.Array: - case TypeKind2.Set: - Line($"var col_{index} = (ListArray){colAccess};"); - break; - case TypeKind2.Dictionary: - Line($"var col_{index} = (MapArray){colAccess};"); - break; - case TypeKind2.NestedRecord: - Line($"var col_{index} = (StructArray){colAccess};"); - break; - case TypeKind2.Custom: - Line($"var col_{index} = {colAccess};"); - break; - default: - Line($"var col_{index} = {colAccess};"); - break; - } - } + // Read each property from the column at 'row' + for (int i = 0; i < _model.Properties.Count; i++) + { + var prop = _model.Properties[i]; + EmitMultiRowReadProperty(prop, i); + } - private void EmitMultiRowReadProperty(PropertyModel prop, int index) - { - switch (prop.Type.Kind) - { - case TypeKind2.String: - if (prop.IsNullable) - Line($"var prop_{index} = col_{index}.IsNull(row) ? null : col_{index}.GetString(row);"); - else - Line($"var prop_{index} = col_{index}.GetString(row)!;"); - break; - case TypeKind2.Bool: - case TypeKind2.Byte: - case TypeKind2.SByte: - case TypeKind2.Int16: - case TypeKind2.UInt16: - case TypeKind2.Int32: - case TypeKind2.UInt32: - case TypeKind2.Int64: - case TypeKind2.UInt64: - case TypeKind2.Float: - case TypeKind2.Double: - if (prop.IsNullable) - Line($"var prop_{index} = col_{index}.IsNull(row) ? ({prop.Type.FullTypeName}?)null : col_{index}.GetValue(row).Value;"); - else - Line($"var prop_{index} = col_{index}.GetValue(row).Value;"); - break; - case TypeKind2.Binary: - if (IsReadOnlyMemoryByte(prop)) - Line($"ReadOnlyMemory prop_{index} = col_{index}.GetBytes(row).ToArray();"); - else - Line($"var prop_{index} = col_{index}.GetBytes(row).ToArray();"); - break; - case TypeKind2.DateTime: + // Construct the record + if (_model.HasSerializationCallback) { - var dtProp = TimestampHasTimezone(prop) ? "UtcDateTime" : "DateTime"; - if (prop.IsNullable) - Line($"var prop_{index} = col_{index}.IsNull(row) ? (System.DateTime?)null : col_{index}.GetTimestamp(row)!.Value.{dtProp};"); - else - Line($"var prop_{index} = col_{index}.GetTimestamp(row)!.Value.{dtProp};"); - break; + EmitObjectConstruction("var item = ", ";"); + Line("((Apache.Arrow.Serialization.IArrowSerializationCallback)item).OnAfterDeserialize();"); + Line("result.Add(item);"); } - case TypeKind2.DateTimeOffset: - if (prop.IsNullable) - Line($"var prop_{index} = col_{index}.IsNull(row) ? (System.DateTimeOffset?)null : col_{index}.GetTimestamp(row)!.Value;"); - else - Line($"var prop_{index} = col_{index}.GetTimestamp(row)!.Value;"); - break; - case TypeKind2.DateOnly: - if (prop.IsNullable) - Line($"var prop_{index} = col_{index}.IsNull(row) ? (System.DateOnly?)null : col_{index}.GetDateOnly(row);"); - else - Line($"var prop_{index} = col_{index}.GetDateOnly(row)!.Value;"); - break; - case TypeKind2.TimeOnly: - if (prop.IsNullable) - Line($"var prop_{index} = col_{index}.GetTime(row);"); - else - Line($"var prop_{index} = col_{index}.GetTime(row)!.Value;"); - break; - case TypeKind2.TimeSpan: - if (prop.IsNullable) - Line($"var prop_{index} = col_{index}.IsNull(row) ? (System.TimeSpan?)null : Apache.Arrow.Serialization.ArrowArrayHelper.ReadDuration(col_{index}, row);"); - else - Line($"var prop_{index} = Apache.Arrow.Serialization.ArrowArrayHelper.ReadDuration(col_{index}, row);"); - break; - case TypeKind2.Decimal: - if (prop.IsNullable) - Line($"var prop_{index} = col_{index}.IsNull(row) ? (decimal?)null : ((Decimal128Array)col_{index}).GetValue(row).Value;"); - else - Line($"var prop_{index} = ((Decimal128Array)col_{index}).GetValue(row).Value;"); - break; - case TypeKind2.Guid: - if (prop.IsNullable) - Line($"var prop_{index} = col_{index}.GetGuid(row);"); - else - Line($"var prop_{index} = col_{index}.GetGuid(row)!.Value;"); - break; - case TypeKind2.Half: - if (prop.IsNullable) - Line($"var prop_{index} = col_{index}.IsNull(row) ? (System.Half?)null : col_{index}.GetValue(row).Value;"); - else - Line($"var prop_{index} = col_{index}.GetValue(row).Value;"); - break; - case TypeKind2.Enum: + else { - var enumType = prop.Type.FullTypeName; - if (prop.IsNullable) - Line($"var prop_{index} = col_{index}.IsNull(row) ? ({enumType}?)null : System.Enum.Parse<{enumType}>(((StringArray)col_{index}.Dictionary).GetString((int)((Int16Array)col_{index}.Indices).GetValue(row).Value)!);"); - else - Line($"var prop_{index} = System.Enum.Parse<{enumType}>(((StringArray)col_{index}.Dictionary).GetString((int)((Int16Array)col_{index}.Indices).GetValue(row).Value)!);"); - break; + EmitObjectConstruction("result.Add(", ");"); } - case TypeKind2.List: - case TypeKind2.Array: - case TypeKind2.Set: + + _indent--; + Line("}"); + Line(); + Line("return result;"); + + _indent--; + Line("}"); + } + + private void EmitMultiRowColumnRef(PropertyModel prop, int index) + { + var colAccess = $"batch.Column(\"{prop.FieldName}\")"; + + switch (prop.Type.Kind) { - var elemType = prop.Type.ElementType!; - // Nullable list: wrap in IsNull check - if (prop.IsNullable) - { - var collTypeName = prop.Type.Kind switch - { - TypeKind2.Array => $"{elemType.FullTypeName}[]", - TypeKind2.Set => $"System.Collections.Generic.HashSet<{elemType.FullTypeName}>", - _ => $"System.Collections.Generic.List<{elemType.FullTypeName}>" - }; - Line($"{collTypeName}? prop_{index};"); - Line($"if (col_{index}.IsNull(row))"); - Line($" prop_{index} = null;"); - Line("else"); - Line("{"); - _indent++; - } - if (NeedsManualListDeserialize(elemType)) - { - // Extension types can't use GetSlicedValues; use offsets + Values directly - Line($"var listStart_{index} = col_{index}.ValueOffsets[row];"); - Line($"var listLen_{index} = col_{index}.GetValueLength(row);"); - var multiDeclPrefix = prop.IsNullable ? $"prop_{index}" : $"var prop_{index}"; - switch (elemType.Kind) + case ArrowTypeKind.String: + { + var si = GetStringArrayInfo(prop); + Line($"var col_{index} = ({si.ArrayType}){colAccess};"); + break; + } + case ArrowTypeKind.Bool: + { + var bl = GetBoolArrayInfo(prop); + Line($"var col_{index} = ({bl.ArrayType}){colAccess};"); + break; + } + case ArrowTypeKind.Byte: + Line($"var col_{index} = (UInt8Array){colAccess};"); + break; + case ArrowTypeKind.SByte: + Line($"var col_{index} = (Int8Array){colAccess};"); + break; + case ArrowTypeKind.Int16: + Line($"var col_{index} = (Int16Array){colAccess};"); + break; + case ArrowTypeKind.UInt16: + Line($"var col_{index} = (UInt16Array){colAccess};"); + break; + case ArrowTypeKind.Int32: + Line($"var col_{index} = (Int32Array){colAccess};"); + break; + case ArrowTypeKind.UInt32: + Line($"var col_{index} = (UInt32Array){colAccess};"); + break; + case ArrowTypeKind.Int64: + Line($"var col_{index} = (Int64Array){colAccess};"); + break; + case ArrowTypeKind.UInt64: + Line($"var col_{index} = (UInt64Array){colAccess};"); + break; + case ArrowTypeKind.Float: + Line($"var col_{index} = (FloatArray){colAccess};"); + break; + case ArrowTypeKind.Double: + Line($"var col_{index} = (DoubleArray){colAccess};"); + break; + case ArrowTypeKind.Binary: + { + var bi = GetBinaryArrayInfo(prop); + Line($"var col_{index} = ({bi.ArrayType}){colAccess};"); + break; + } + case ArrowTypeKind.DateTime: + case ArrowTypeKind.DateTimeOffset: + Line($"var col_{index} = (TimestampArray){colAccess};"); + break; + case ArrowTypeKind.DateOnly: + { + var di = GetDateArrayInfo(prop); + Line($"var col_{index} = ({di.ArrayType}){colAccess};"); + break; + } + case ArrowTypeKind.TimeOnly: + { + var ti = GetTimeArrayInfo(prop); + Line($"var col_{index} = ({ti.ArrayType}){colAccess};"); + break; + } + case ArrowTypeKind.TimeSpan: + Line($"var col_{index} = (DurationArray){colAccess};"); + break; + case ArrowTypeKind.Decimal: + Line($"var col_{index} = (Decimal128Array){colAccess};"); + break; + case ArrowTypeKind.Guid: + Line($"var col_{index} = (GuidArray){colAccess};"); + break; + case ArrowTypeKind.Half: + Line($"var col_{index} = (HalfFloatArray){colAccess};"); + break; + case ArrowTypeKind.Enum: + Line($"var col_{index} = (DictionaryArray){colAccess};"); + break; + case ArrowTypeKind.List: + case ArrowTypeKind.Array: + case ArrowTypeKind.Set: + Line($"var col_{index} = (ListArray){colAccess};"); + break; + case ArrowTypeKind.Dictionary: + Line($"var col_{index} = (MapArray){colAccess};"); + break; + case ArrowTypeKind.NestedRecord: + Line($"var col_{index} = (StructArray){colAccess};"); + break; + case ArrowTypeKind.Custom: + Line($"var col_{index} = {colAccess};"); + break; + default: + Line($"var col_{index} = {colAccess};"); + break; + } + } + + private void EmitMultiRowReadProperty(PropertyModel prop, int index) + { + switch (prop.Type.Kind) + { + case ArrowTypeKind.String: + if (prop.IsNullable) + Line($"var prop_{index} = col_{index}.IsNull(row) ? null : col_{index}.GetString(row);"); + else + Line($"var prop_{index} = col_{index}.GetString(row)!;"); + break; + case ArrowTypeKind.Bool: + case ArrowTypeKind.Byte: + case ArrowTypeKind.SByte: + case ArrowTypeKind.Int16: + case ArrowTypeKind.UInt16: + case ArrowTypeKind.Int32: + case ArrowTypeKind.UInt32: + case ArrowTypeKind.Int64: + case ArrowTypeKind.UInt64: + case ArrowTypeKind.Float: + case ArrowTypeKind.Double: + if (prop.IsNullable) + Line($"var prop_{index} = col_{index}.IsNull(row) ? ({prop.Type.FullTypeName}?)null : col_{index}.GetValue(row).Value;"); + else + Line($"var prop_{index} = col_{index}.GetValue(row).Value;"); + break; + case ArrowTypeKind.Binary: + if (IsReadOnlyMemoryByte(prop)) + Line($"ReadOnlyMemory prop_{index} = col_{index}.GetBytes(row).ToArray();"); + else + Line($"var prop_{index} = col_{index}.GetBytes(row).ToArray();"); + break; + case ArrowTypeKind.DateTime: + { + var dtProp = TimestampHasTimezone(prop) ? "UtcDateTime" : "DateTime"; + if (prop.IsNullable) + Line($"var prop_{index} = col_{index}.IsNull(row) ? (System.DateTime?)null : col_{index}.GetTimestamp(row)!.Value.{dtProp};"); + else + Line($"var prop_{index} = col_{index}.GetTimestamp(row)!.Value.{dtProp};"); + break; + } + case ArrowTypeKind.DateTimeOffset: + if (prop.IsNullable) + Line($"var prop_{index} = col_{index}.IsNull(row) ? (System.DateTimeOffset?)null : col_{index}.GetTimestamp(row)!.Value;"); + else + Line($"var prop_{index} = col_{index}.GetTimestamp(row)!.Value;"); + break; + case ArrowTypeKind.DateOnly: + if (prop.IsNullable) + Line($"var prop_{index} = col_{index}.IsNull(row) ? (System.DateOnly?)null : col_{index}.GetDateOnly(row);"); + else + Line($"var prop_{index} = col_{index}.GetDateOnly(row)!.Value;"); + break; + case ArrowTypeKind.TimeOnly: + if (prop.IsNullable) + Line($"var prop_{index} = col_{index}.GetTime(row);"); + else + Line($"var prop_{index} = col_{index}.GetTime(row)!.Value;"); + break; + case ArrowTypeKind.TimeSpan: + if (prop.IsNullable) + Line($"var prop_{index} = col_{index}.IsNull(row) ? (System.TimeSpan?)null : Apache.Arrow.Serialization.ArrowArrayHelper.ReadDuration(col_{index}, row);"); + else + Line($"var prop_{index} = Apache.Arrow.Serialization.ArrowArrayHelper.ReadDuration(col_{index}, row);"); + break; + case ArrowTypeKind.Decimal: + if (prop.IsNullable) + Line($"var prop_{index} = col_{index}.IsNull(row) ? (decimal?)null : ((Decimal128Array)col_{index}).GetValue(row).Value;"); + else + Line($"var prop_{index} = ((Decimal128Array)col_{index}).GetValue(row).Value;"); + break; + case ArrowTypeKind.Guid: + if (prop.IsNullable) + Line($"var prop_{index} = col_{index}.GetGuid(row);"); + else + Line($"var prop_{index} = col_{index}.GetGuid(row)!.Value;"); + break; + case ArrowTypeKind.Half: + if (prop.IsNullable) + Line($"var prop_{index} = col_{index}.IsNull(row) ? (System.Half?)null : col_{index}.GetValue(row).Value;"); + else + Line($"var prop_{index} = col_{index}.GetValue(row).Value;"); + break; + case ArrowTypeKind.Enum: { - case TypeKind2.Guid: + var enumType = prop.Type.FullTypeName; + if (prop.IsNullable) + Line($"var prop_{index} = col_{index}.IsNull(row) ? ({enumType}?)null : System.Enum.Parse<{enumType}>(((StringArray)col_{index}.Dictionary).GetString((int)((Int16Array)col_{index}.Indices).GetValue(row).Value)!);"); + else + Line($"var prop_{index} = System.Enum.Parse<{enumType}>(((StringArray)col_{index}.Dictionary).GetString((int)((Int16Array)col_{index}.Indices).GetValue(row).Value)!);"); + break; + } + case ArrowTypeKind.List: + case ArrowTypeKind.Array: + case ArrowTypeKind.Set: + { + var elemType = prop.Type.ElementType!; + // Nullable list: wrap in IsNull check + if (prop.IsNullable) { - Line($"var guidArr_{index} = (GuidArray)col_{index}.Values;"); - var readExpr = $"Enumerable.Range(listStart_{index}, listLen_{index}).Select(i => guidArr_{index}.GetGuid(i)!.Value)"; - if (prop.Type.Kind == TypeKind2.Array) - Line($"{multiDeclPrefix} = {readExpr}.ToArray();"); - else if (prop.Type.Kind == TypeKind2.Set) - Line($"{multiDeclPrefix} = new System.Collections.Generic.HashSet<{elemType.FullTypeName}>({readExpr});"); - else - Line($"{multiDeclPrefix} = new System.Collections.Generic.List<{elemType.FullTypeName}>({readExpr});"); - break; + var collTypeName = prop.Type.Kind switch + { + ArrowTypeKind.Array => $"{elemType.FullTypeName}[]", + ArrowTypeKind.Set => $"System.Collections.Generic.HashSet<{elemType.FullTypeName}>", + _ => $"System.Collections.Generic.List<{elemType.FullTypeName}>" + }; + Line($"{collTypeName}? prop_{index};"); + Line($"if (col_{index}.IsNull(row))"); + Line($" prop_{index} = null;"); + Line("else"); + Line("{"); + _indent++; + } + if (NeedsManualListDeserialize(elemType)) + { + // Extension types can't use GetSlicedValues; use offsets + Values directly + Line($"var listStart_{index} = col_{index}.ValueOffsets[row];"); + Line($"var listLen_{index} = col_{index}.GetValueLength(row);"); + var multiDeclPrefix = prop.IsNullable ? $"prop_{index}" : $"var prop_{index}"; + switch (elemType.Kind) + { + case ArrowTypeKind.Guid: + { + Line($"var guidArr_{index} = (GuidArray)col_{index}.Values;"); + var readExpr = $"Enumerable.Range(listStart_{index}, listLen_{index}).Select(i => guidArr_{index}.GetGuid(i)!.Value)"; + if (prop.Type.Kind == ArrowTypeKind.Array) + Line($"{multiDeclPrefix} = {readExpr}.ToArray();"); + else if (prop.Type.Kind == ArrowTypeKind.Set) + Line($"{multiDeclPrefix} = new System.Collections.Generic.HashSet<{elemType.FullTypeName}>({readExpr});"); + else + Line($"{multiDeclPrefix} = new System.Collections.Generic.List<{elemType.FullTypeName}>({readExpr});"); + break; + } + case ArrowTypeKind.Enum: + { + // Dictionary-encoded enum values: Dictionary(Int16, Utf8) + Line($"var enumDictArr_{index} = (DictionaryArray)col_{index}.Values;"); + Line($"var enumDictValues_{index} = (StringArray)enumDictArr_{index}.Dictionary;"); + var readExpr2 = $"Enumerable.Range(listStart_{index}, listLen_{index}).Select(i => System.Enum.Parse<{elemType.FullTypeName}>(enumDictValues_{index}.GetString(((Int16Array)enumDictArr_{index}.Indices).GetValue(i)!.Value)!))"; + if (prop.Type.Kind == ArrowTypeKind.Array) + Line($"{multiDeclPrefix} = {readExpr2}.ToArray();"); + else if (prop.Type.Kind == ArrowTypeKind.Set) + Line($"{multiDeclPrefix} = new System.Collections.Generic.HashSet<{elemType.FullTypeName}>({readExpr2});"); + else + Line($"{multiDeclPrefix} = new System.Collections.Generic.List<{elemType.FullTypeName}>({readExpr2});"); + break; + } + case ArrowTypeKind.NestedRecord: + { + var typeName = elemType.FullTypeName; + Line($"var structArr_{index} = (StructArray)col_{index}.Values;"); + Line($"{multiDeclPrefix} = new System.Collections.Generic.List<{typeName}>(listLen_{index});"); + Line($"for (int i = listStart_{index}; i < listStart_{index} + listLen_{index}; i++)"); + Line("{"); + _indent++; + Line($"var childArrays_{index} = new IArrowArray[structArr_{index}.Fields.Count];"); + Line($"for (int f = 0; f < structArr_{index}.Fields.Count; f++) childArrays_{index}[f] = ArrowArrayFactory.BuildArray(structArr_{index}.Fields[f].Data.Slice(i, 1));"); + Line($"prop_{index}.Add({typeName}.FromRecordBatch(new RecordBatch({typeName}.ArrowSchema, childArrays_{index}, 1)));"); + _indent--; + Line("}"); + break; + } + case ArrowTypeKind.List: + case ArrowTypeKind.Array: + { + // Nested list elements: read inner lists from ListArray values + var innerElemType = elemType.ElementType!; + var innerTypeName = elemType.FullTypeName; + Line($"var innerArr_{index} = (ListArray)col_{index}.Values;"); + Line($"{multiDeclPrefix} = new System.Collections.Generic.List<{innerTypeName}>(listLen_{index});"); + Line($"for (int i = listStart_{index}; i < listStart_{index} + listLen_{index}; i++)"); + Line("{"); + _indent++; + if (innerElemType.Kind == ArrowTypeKind.Dictionary) + { + // Dict inner elements: read each dict from inner list's MapArray + var keyType2 = innerElemType.KeyType!; + var valType2 = innerElemType.ValueType!; + var dictTypeName2 = $"System.Collections.Generic.Dictionary<{keyType2.FullTypeName}, {valType2.FullTypeName}>"; + Line($"var mapSlice_{index} = (MapArray)innerArr_{index}.GetSlicedValues(i);"); + Line($"var innerResult_{index} = new System.Collections.Generic.List<{dictTypeName2}>(mapSlice_{index}.Length);"); + Line($"for (int j_{index} = 0; j_{index} < mapSlice_{index}.Length; j_{index}++)"); + Line("{"); + _indent++; + Line($"var kvs_{index} = (StructArray)mapSlice_{index}.GetSlicedValues(j_{index});"); + Line($"var ks_{index} = kvs_{index}.Fields[0];"); + Line($"var vs_{index} = kvs_{index}.Fields[1];"); + Line($"var d_{index} = new {dictTypeName2}();"); + Line($"for (int k_{index} = 0; k_{index} < kvs_{index}.Length; k_{index}++)"); + Line("{"); + _indent++; + var dKeyReader2 = GetScalarReader(keyType2, $"ks_{index}", $"k_{index}"); + EmitDictValueRead(valType2, $"d_{index}", dKeyReader2, $"vs_{index}", $"k_{index}", index); + _indent--; + Line("}"); + Line($"innerResult_{index}.Add(d_{index});"); + _indent--; + Line("}"); + Line($"prop_{index}.Add(innerResult_{index});"); + } + else + { + Line($"var innerValues_{index} = innerArr_{index}.GetSlicedValues(i);"); + var innerElemReader = GetElementReader(innerElemType, $"innerValues_{index}"); + if (elemType.Kind == ArrowTypeKind.Array) + Line($"prop_{index}.Add({innerElemReader}.ToArray());"); + else + Line($"prop_{index}.Add(new {innerTypeName}({innerElemReader}));"); + } + _indent--; + Line("}"); + break; + } + case ArrowTypeKind.Dictionary: + { + // Dict elements inside a list: read each dict from MapArray values + var keyType = elemType.KeyType!; + var valType = elemType.ValueType!; + var dictTypeName = $"System.Collections.Generic.Dictionary<{keyType.FullTypeName}, {valType.FullTypeName}>"; + Line($"var mapArr_{index} = (MapArray)col_{index}.Values;"); + Line($"{multiDeclPrefix} = new System.Collections.Generic.List<{dictTypeName}>(listLen_{index});"); + Line($"for (int i = listStart_{index}; i < listStart_{index} + listLen_{index}; i++)"); + Line("{"); + _indent++; + Line($"var kvs_{index} = (StructArray)mapArr_{index}.GetSlicedValues(i);"); + Line($"var ks_{index} = kvs_{index}.Fields[0];"); + Line($"var vs_{index} = kvs_{index}.Fields[1];"); + Line($"var d_{index} = new {dictTypeName}();"); + Line($"for (int j_{index} = 0; j_{index} < kvs_{index}.Length; j_{index}++)"); + Line("{"); + _indent++; + var dKeyReader = GetScalarReader(keyType, $"ks_{index}", $"j_{index}"); + EmitDictValueRead(valType, $"d_{index}", dKeyReader, $"vs_{index}", $"j_{index}", index); + _indent--; + Line("}"); + Line($"prop_{index}.Add(d_{index});"); + _indent--; + Line("}"); + break; + } + } } - case TypeKind2.Enum: + else { - // Dictionary-encoded enum values: Dictionary(Int16, Utf8) - Line($"var enumDictArr_{index} = (DictionaryArray)col_{index}.Values;"); - Line($"var enumDictValues_{index} = (StringArray)enumDictArr_{index}.Dictionary;"); - var readExpr2 = $"Enumerable.Range(listStart_{index}, listLen_{index}).Select(i => System.Enum.Parse<{elemType.FullTypeName}>(enumDictValues_{index}.GetString(((Int16Array)enumDictArr_{index}.Indices).GetValue(i)!.Value)!))"; - if (prop.Type.Kind == TypeKind2.Array) - Line($"{multiDeclPrefix} = {readExpr2}.ToArray();"); - else if (prop.Type.Kind == TypeKind2.Set) - Line($"{multiDeclPrefix} = new System.Collections.Generic.HashSet<{elemType.FullTypeName}>({readExpr2});"); + var nonManualDeclPrefix = prop.IsNullable ? $"prop_{index}" : $"var prop_{index}"; + Line($"var listValues_{index} = col_{index}.GetSlicedValues(row);"); + var elemReader = GetElementReader(elemType, $"listValues_{index}"); + if (prop.Type.Kind == ArrowTypeKind.Array) + Line($"{nonManualDeclPrefix} = {elemReader}.ToArray();"); + else if (prop.Type.Kind == ArrowTypeKind.Set) + Line($"{nonManualDeclPrefix} = new System.Collections.Generic.HashSet<{elemType.FullTypeName}>({elemReader});"); else - Line($"{multiDeclPrefix} = new System.Collections.Generic.List<{elemType.FullTypeName}>({readExpr2});"); - break; + Line($"{nonManualDeclPrefix} = new System.Collections.Generic.List<{elemType.FullTypeName}>({elemReader});"); } - case TypeKind2.NestedRecord: + if (prop.IsNullable) { - var typeName = elemType.FullTypeName; - Line($"var structArr_{index} = (StructArray)col_{index}.Values;"); - Line($"{multiDeclPrefix} = new System.Collections.Generic.List<{typeName}>(listLen_{index});"); - Line($"for (int i = listStart_{index}; i < listStart_{index} + listLen_{index}; i++)"); - Line("{"); - _indent++; - Line($"var childArrays_{index} = new IArrowArray[structArr_{index}.Fields.Count];"); - Line($"for (int f = 0; f < structArr_{index}.Fields.Count; f++) childArrays_{index}[f] = ArrowArrayFactory.BuildArray(structArr_{index}.Fields[f].Data.Slice(i, 1));"); - Line($"prop_{index}.Add({typeName}.FromRecordBatch(new RecordBatch({typeName}.ArrowSchema, childArrays_{index}, 1)));"); _indent--; Line("}"); - break; } - case TypeKind2.List: - case TypeKind2.Array: + break; + } + case ArrowTypeKind.Dictionary: + { + var keyType = prop.Type.KeyType!; + var valueType = prop.Type.ValueType!; + var dictTypeName2 = $"System.Collections.Generic.Dictionary<{keyType.FullTypeName}, {valueType.FullTypeName}>"; + if (prop.IsNullable) { - // Nested list elements: read inner lists from ListArray values - var innerElemType = elemType.ElementType!; - var innerTypeName = elemType.FullTypeName; - Line($"var innerArr_{index} = (ListArray)col_{index}.Values;"); - Line($"{multiDeclPrefix} = new System.Collections.Generic.List<{innerTypeName}>(listLen_{index});"); - Line($"for (int i = listStart_{index}; i < listStart_{index} + listLen_{index}; i++)"); + Line($"{dictTypeName2}? prop_{index};"); + Line($"if (col_{index}.IsNull(row))"); + Line($" prop_{index} = null;"); + Line("else"); Line("{"); _indent++; - if (innerElemType.Kind == TypeKind2.Dictionary) - { - // Dict inner elements: read each dict from inner list's MapArray - var keyType2 = innerElemType.KeyType!; - var valType2 = innerElemType.ValueType!; - var dictTypeName2 = $"System.Collections.Generic.Dictionary<{keyType2.FullTypeName}, {valType2.FullTypeName}>"; - Line($"var mapSlice_{index} = (MapArray)innerArr_{index}.GetSlicedValues(i);"); - Line($"var innerResult_{index} = new System.Collections.Generic.List<{dictTypeName2}>(mapSlice_{index}.Length);"); - Line($"for (int j_{index} = 0; j_{index} < mapSlice_{index}.Length; j_{index}++)"); - Line("{"); - _indent++; - Line($"var kvs_{index} = (StructArray)mapSlice_{index}.GetSlicedValues(j_{index});"); - Line($"var ks_{index} = kvs_{index}.Fields[0];"); - Line($"var vs_{index} = kvs_{index}.Fields[1];"); - Line($"var d_{index} = new {dictTypeName2}();"); - Line($"for (int k_{index} = 0; k_{index} < kvs_{index}.Length; k_{index}++)"); - Line("{"); - _indent++; - var dKeyReader2 = GetScalarReader(keyType2, $"ks_{index}", $"k_{index}"); - EmitDictValueRead(valType2, $"d_{index}", dKeyReader2, $"vs_{index}", $"k_{index}", index); - _indent--; - Line("}"); - Line($"innerResult_{index}.Add(d_{index});"); - _indent--; - Line("}"); - Line($"prop_{index}.Add(innerResult_{index});"); - } - else - { - Line($"var innerValues_{index} = innerArr_{index}.GetSlicedValues(i);"); - var innerElemReader = GetElementReader(innerElemType, $"innerValues_{index}"); - if (elemType.Kind == TypeKind2.Array) - Line($"prop_{index}.Add({innerElemReader}.ToArray());"); - else - Line($"prop_{index}.Add(new {innerTypeName}({innerElemReader}));"); - } - _indent--; - Line("}"); - break; + Line($"prop_{index} = new {dictTypeName2}();"); } - case TypeKind2.Dictionary: + else { - // Dict elements inside a list: read each dict from MapArray values - var keyType = elemType.KeyType!; - var valType = elemType.ValueType!; - var dictTypeName = $"System.Collections.Generic.Dictionary<{keyType.FullTypeName}, {valType.FullTypeName}>"; - Line($"var mapArr_{index} = (MapArray)col_{index}.Values;"); - Line($"{multiDeclPrefix} = new System.Collections.Generic.List<{dictTypeName}>(listLen_{index});"); - Line($"for (int i = listStart_{index}; i < listStart_{index} + listLen_{index}; i++)"); + Line($"var prop_{index} = new {dictTypeName2}();"); + Line($"if (!col_{index}.IsNull(row))"); Line("{"); _indent++; - Line($"var kvs_{index} = (StructArray)mapArr_{index}.GetSlicedValues(i);"); - Line($"var ks_{index} = kvs_{index}.Fields[0];"); - Line($"var vs_{index} = kvs_{index}.Fields[1];"); - Line($"var d_{index} = new {dictTypeName}();"); - Line($"for (int j_{index} = 0; j_{index} < kvs_{index}.Length; j_{index}++)"); + } + Line($"var kvStruct_{index} = (StructArray)col_{index}.GetSlicedValues(row);"); + Line($"var keys_{index} = kvStruct_{index}.Fields[0];"); + Line($"var vals_{index} = kvStruct_{index}.Fields[1];"); + Line($"for (int i = 0; i < kvStruct_{index}.Length; i++)"); + Line("{"); + _indent++; + var keyReader = GetScalarReader(keyType, $"keys_{index}", "i"); + EmitDictValueRead(valueType, $"prop_{index}", keyReader, $"vals_{index}", "i", index); + _indent--; + Line("}"); + _indent--; + Line("}"); + break; + } + case ArrowTypeKind.NestedRecord: + { + var typeName = prop.Type.FullTypeName; + if (prop.IsNullable) + { + Line($"{typeName}? prop_{index};"); + Line($"if (col_{index}.IsNull(row))"); Line("{"); _indent++; - var dKeyReader = GetScalarReader(keyType, $"ks_{index}", $"j_{index}"); - EmitDictValueRead(valType, $"d_{index}", dKeyReader, $"vs_{index}", $"j_{index}", index); + Line($"prop_{index} = null;"); _indent--; Line("}"); - Line($"prop_{index}.Add(d_{index});"); + Line("else"); + Line("{"); + _indent++; + EmitMultiRowNestedRead(typeName, index, declareVar: false); _indent--; Line("}"); - break; } + else + { + EmitMultiRowNestedRead(typeName, index, declareVar: true); + } + break; } - } - else - { - var nonManualDeclPrefix = prop.IsNullable ? $"prop_{index}" : $"var prop_{index}"; - Line($"var listValues_{index} = col_{index}.GetSlicedValues(row);"); - var elemReader = GetElementReader(elemType, $"listValues_{index}"); - if (prop.Type.Kind == TypeKind2.Array) - Line($"{nonManualDeclPrefix} = {elemReader}.ToArray();"); - else if (prop.Type.Kind == TypeKind2.Set) - Line($"{nonManualDeclPrefix} = new System.Collections.Generic.HashSet<{elemType.FullTypeName}>({elemReader});"); + case ArrowTypeKind.Custom: + if (prop.IsNullable) + Line($"var prop_{index} = col_{index}.IsNull(row) ? ({prop.Type.FullTypeName}?)null : new {prop.ConverterTypeName}().FromArray(col_{index}, row);"); else - Line($"{nonManualDeclPrefix} = new System.Collections.Generic.List<{elemType.FullTypeName}>({elemReader});"); - } - if (prop.IsNullable) - { - _indent--; - Line("}"); - } - break; - } - case TypeKind2.Dictionary: - { - var keyType = prop.Type.KeyType!; - var valueType = prop.Type.ValueType!; - var dictTypeName2 = $"System.Collections.Generic.Dictionary<{keyType.FullTypeName}, {valueType.FullTypeName}>"; - if (prop.IsNullable) - { - Line($"{dictTypeName2}? prop_{index};"); - Line($"if (col_{index}.IsNull(row))"); - Line($" prop_{index} = null;"); - Line("else"); - Line("{"); - _indent++; - Line($"prop_{index} = new {dictTypeName2}();"); - } - else - { - Line($"var prop_{index} = new {dictTypeName2}();"); - Line($"if (!col_{index}.IsNull(row))"); - Line("{"); - _indent++; - } - Line($"var kvStruct_{index} = (StructArray)col_{index}.GetSlicedValues(row);"); - Line($"var keys_{index} = kvStruct_{index}.Fields[0];"); - Line($"var vals_{index} = kvStruct_{index}.Fields[1];"); - Line($"for (int i = 0; i < kvStruct_{index}.Length; i++)"); - Line("{"); - _indent++; - var keyReader = GetScalarReader(keyType, $"keys_{index}", "i"); - EmitDictValueRead(valueType, $"prop_{index}", keyReader, $"vals_{index}", "i", index); - _indent--; - Line("}"); - _indent--; - Line("}"); - break; - } - case TypeKind2.NestedRecord: - { - var typeName = prop.Type.FullTypeName; - if (prop.IsNullable) - { - Line($"{typeName}? prop_{index};"); - Line($"if (col_{index}.IsNull(row))"); - Line("{"); - _indent++; - Line($"prop_{index} = null;"); - _indent--; - Line("}"); - Line("else"); - Line("{"); - _indent++; - EmitMultiRowNestedRead(typeName, index, declareVar: false); - _indent--; - Line("}"); - } - else - { - EmitMultiRowNestedRead(typeName, index, declareVar: true); - } - break; + Line($"var prop_{index} = new {prop.ConverterTypeName}().FromArray(col_{index}, row);"); + break; + default: + Line($"object? prop_{index} = null; // TODO: Unsupported type {prop.Type.Kind}"); + break; } - case TypeKind2.Custom: - if (prop.IsNullable) - Line($"var prop_{index} = col_{index}.IsNull(row) ? ({prop.Type.FullTypeName}?)null : new {prop.ConverterTypeName}().FromArray(col_{index}, row);"); - else - Line($"var prop_{index} = new {prop.ConverterTypeName}().FromArray(col_{index}, row);"); - break; - default: - Line($"object? prop_{index} = null; // TODO: Unsupported type {prop.Type.Kind}"); - break; } - } - - private void EmitMultiRowNestedRead(string typeName, int index, bool declareVar) - { - Line($"var na_{index} = new IArrowArray[col_{index}.Fields.Count];"); - Line($"for (int f = 0; f < col_{index}.Fields.Count; f++)"); - Line("{"); - _indent++; - Line($"na_{index}[f] = ArrowArrayFactory.BuildArray(col_{index}.Fields[f].Data.Slice(row, 1));"); - _indent--; - Line("}"); - var decl = declareVar ? "var " : ""; - Line($"{decl}prop_{index} = {typeName}.FromRecordBatch(new RecordBatch({typeName}.ArrowSchema, na_{index}, 1));"); - } - - // --- Arrow type expression helpers --- - - private string GetTypeKeyword() - { - if (_model.IsRecord && _model.IsValueType) return "record struct"; - if (_model.IsRecord) return "record"; - if (_model.IsValueType) return "struct"; - return "class"; - } - - private static string TimestampConvertMethod(PropertyModel prop) => - TimestampHasTimezone(prop) ? "ToUtcDateTimeOffset" : "ToWallClockDateTimeOffset"; - - private static bool IsReadOnlyMemoryByte(PropertyModel prop) => - prop.Type.FullTypeName.Contains("ReadOnlyMemory"); - private static string BinaryToSpan(PropertyModel prop, string access) => - IsReadOnlyMemoryByte(prop) ? $"{access}.Span" : $"(ReadOnlySpan){access}"; - - /// - /// Like TimestampHasTimezone but works on TypeInfo directly (for element types in collections). - /// - private static bool TypeInfoHasTimezone(TypeInfo type) - { - var over = type.ArrowTypeOverride; - if (over == null) - return true; // default is UTC - var lower = over.Trim().ToLowerInvariant(); - if (!lower.StartsWith("timestamp")) - return true; - var start = lower.IndexOfAny(new[] { '[', '(' }); - if (start < 0) - return false; - var end = lower.IndexOfAny(new[] { ']', ')' }, start); - if (end < 0) - return false; - var inner = lower.Substring(start + 1, end - start - 1); - return inner.Contains(","); - } + private void EmitMultiRowNestedRead(string typeName, int index, bool declareVar) + { + Line($"var na_{index} = new IArrowArray[col_{index}.Fields.Count];"); + Line($"for (int f = 0; f < col_{index}.Fields.Count; f++)"); + Line("{"); + _indent++; + Line($"na_{index}[f] = ArrowArrayFactory.BuildArray(col_{index}.Fields[f].Data.Slice(row, 1));"); + _indent--; + Line("}"); + var decl = declareVar ? "var " : ""; + Line($"{decl}prop_{index} = {typeName}.FromRecordBatch(new RecordBatch({typeName}.ArrowSchema, na_{index}, 1));"); + } - private static string GetDateArrayBuilderType(TypeInfo type) - { - var over = type.ArrowTypeOverride?.Trim().ToLowerInvariant(); - if (over != null && over.StartsWith("date64")) - return "Date64Array.Builder"; - return "Date32Array.Builder"; - } + // --- Arrow type expression helpers --- - private static string GetTimeArrayBuilderType(TypeInfo type) - { - var over = type.ArrowTypeOverride?.Trim().ToLowerInvariant(); - if (over != null && over.StartsWith("time32")) - return "Time32Array.Builder"; - return "Time64Array.Builder"; - } + private string GetTypeKeyword() + { + if (_model.IsRecord && _model.IsValueType) return "record struct"; + if (_model.IsRecord) return "record"; + if (_model.IsValueType) return "struct"; + return "class"; + } - private static string GetDateArrayReadType(TypeInfo type) - { - var over = type.ArrowTypeOverride?.Trim().ToLowerInvariant(); - if (over != null && over.StartsWith("date64")) - return "Date64Array"; - return "Date32Array"; - } + private static string TimestampConvertMethod(PropertyModel prop) => + TimestampHasTimezone(prop) ? "ToUtcDateTimeOffset" : "ToWallClockDateTimeOffset"; - private static string GetTimeArrayReadType(TypeInfo type) - { - var over = type.ArrowTypeOverride?.Trim().ToLowerInvariant(); - if (over != null && over.StartsWith("time32")) - return "Time32Array"; - return "Time64Array"; - } + private static bool IsReadOnlyMemoryByte(PropertyModel prop) => + prop.Type.FullTypeName.Contains("ReadOnlyMemory"); - private string GetTimestampTypeExpr(PropertyModel prop) => GetArrowTypeExpression(prop.Type); + private static string BinaryToSpan(PropertyModel prop, string access) => + IsReadOnlyMemoryByte(prop) ? $"{access}.Span" : $"(ReadOnlySpan){access}"; - /// - /// Returns true if the timestamp type has a timezone (default is UTC). - /// [ArrowType("timestamp[us]")] → no timezone. - /// No override or [ArrowType("timestamp[ns, UTC]")] → has timezone. - /// - private static bool TimestampHasTimezone(PropertyModel prop) - { - var over = prop.Type.ArrowTypeOverride; - if (over == null) - return true; // default is UTC - var lower = over.Trim().ToLowerInvariant(); - if (!lower.StartsWith("timestamp")) - return true; // not a timestamp override, use default - // Check if there's a comma (unit, tz) in the bracket params - var start = lower.IndexOfAny(new[] { '[', '(' }); - if (start < 0) - return false; // just "timestamp" with no params — no timezone - var end = lower.IndexOfAny(new[] { ']', ')' }, start); - if (end < 0) - return false; - var inner = lower.Substring(start + 1, end - start - 1); - return inner.Contains(","); - } + /// + /// Like TimestampHasTimezone but works on TypeInfo directly (for element types in collections). + /// + private static bool TypeInfoHasTimezone(TypeInfo type) + { + var over = type.ArrowTypeOverride; + if (over == null) + return true; // default is UTC + var lower = over.Trim().ToLowerInvariant(); + if (!lower.StartsWith("timestamp")) + return true; + var start = lower.IndexOfAny(new[] { '[', '(' }); + if (start < 0) + return false; + var end = lower.IndexOfAny(new[] { ']', ')' }, start); + if (end < 0) + return false; + var inner = lower.Substring(start + 1, end - start - 1); + return inner.Contains(","); + } - /// - /// Returns ("Time64Array", "Time64Type", "new Time64Type(TimeUnit.Microsecond)") or - /// ("Time32Array", "Time32Type", "new Time32Type(TimeUnit.Millisecond)") based on override. - /// - private static (string ArrayType, string BuilderType, string TypeExpr) GetTimeArrayInfo(PropertyModel prop) - { - var over = prop.Type.ArrowTypeOverride?.Trim().ToLowerInvariant(); - if (over != null && over.StartsWith("time32")) + private static string GetDateArrayBuilderType(TypeInfo type) { - var unit = ParseUnitParam(over, "time32", "Millisecond"); - return ("Time32Array", "Time32Array.Builder", $"new Time32Type(TimeUnit.{unit})"); + var over = type.ArrowTypeOverride?.Trim().ToLowerInvariant(); + if (over != null && over.StartsWith("date64")) + return "Date64Array.Builder"; + return "Date32Array.Builder"; } - // Default: time64 - if (over != null && over.StartsWith("time64")) + + private static string GetTimeArrayBuilderType(TypeInfo type) { - var unit = ParseUnitParam(over, "time64", "Microsecond"); - return ("Time64Array", "Time64Array.Builder", $"new Time64Type(TimeUnit.{unit})"); + var over = type.ArrowTypeOverride?.Trim().ToLowerInvariant(); + if (over != null && over.StartsWith("time32")) + return "Time32Array.Builder"; + return "Time64Array.Builder"; } - return ("Time64Array", "Time64Array.Builder", "new Time64Type(TimeUnit.Microsecond)"); - } - private static (string ArrayType, string BuilderType) GetDateArrayInfo(PropertyModel prop) - { - var over = prop.Type.ArrowTypeOverride?.Trim().ToLowerInvariant(); - if (over != null && over.StartsWith("date64")) - return ("Date64Array", "Date64Array.Builder"); - return ("Date32Array", "Date32Array.Builder"); - } + private static string GetDateArrayReadType(TypeInfo type) + { + var over = type.ArrowTypeOverride?.Trim().ToLowerInvariant(); + if (over != null && over.StartsWith("date64")) + return "Date64Array"; + return "Date32Array"; + } - private static bool IsStringView(PropertyModel prop) => - prop.Type.ArrowTypeOverride != null && - (prop.Type.ArrowTypeOverride.Trim().ToLowerInvariant() is "string_view" or "utf8_view" or "utf8view" or "stringview"); + private static string GetTimeArrayReadType(TypeInfo type) + { + var over = type.ArrowTypeOverride?.Trim().ToLowerInvariant(); + if (over != null && over.StartsWith("time32")) + return "Time32Array"; + return "Time64Array"; + } - private static (string ArrayType, string BuilderType) GetStringArrayInfo(PropertyModel prop) => - IsStringView(prop) - ? ("StringViewArray", "StringViewArray.Builder") - : ("StringArray", "StringArray.Builder"); + private string GetTimestampTypeExpr(PropertyModel prop) => GetArrowTypeExpression(prop.Type); - private static bool IsBool8(PropertyModel prop) => - prop.Type.ArrowTypeOverride != null && - (prop.Type.ArrowTypeOverride.Trim().ToLowerInvariant() is "bool8" or "arrow.bool8"); + /// + /// Returns true if the timestamp type has a timezone (default is UTC). + /// [ArrowType("timestamp[us]")] → no timezone. + /// No override or [ArrowType("timestamp[ns, UTC]")] → has timezone. + /// + private static bool TimestampHasTimezone(PropertyModel prop) + { + var over = prop.Type.ArrowTypeOverride; + if (over == null) + return true; // default is UTC + var lower = over.Trim().ToLowerInvariant(); + if (!lower.StartsWith("timestamp")) + return true; // not a timestamp override, use default + // Check if there's a comma (unit, tz) in the bracket params + var start = lower.IndexOfAny(new[] { '[', '(' }); + if (start < 0) + return false; // just "timestamp" with no params — no timezone + var end = lower.IndexOfAny(new[] { ']', ')' }, start); + if (end < 0) + return false; + var inner = lower.Substring(start + 1, end - start - 1); + return inner.Contains(","); + } + + /// + /// Returns ("Time64Array", "Time64Type", "new Time64Type(TimeUnit.Microsecond)") or + /// ("Time32Array", "Time32Type", "new Time32Type(TimeUnit.Millisecond)") based on override. + /// + private static (string ArrayType, string BuilderType, string TypeExpr) GetTimeArrayInfo(PropertyModel prop) + { + var over = prop.Type.ArrowTypeOverride?.Trim().ToLowerInvariant(); + if (over != null && over.StartsWith("time32")) + { + var unit = ParseUnitParam(over, "time32", "Millisecond"); + return ("Time32Array", "Time32Array.Builder", $"new Time32Type(TimeUnit.{unit})"); + } + // Default: time64 + if (over != null && over.StartsWith("time64")) + { + var unit = ParseUnitParam(over, "time64", "Microsecond"); + return ("Time64Array", "Time64Array.Builder", $"new Time64Type(TimeUnit.{unit})"); + } + return ("Time64Array", "Time64Array.Builder", "new Time64Type(TimeUnit.Microsecond)"); + } - private static (string ArrayType, string BuilderType) GetBoolArrayInfo(PropertyModel prop) => - IsBool8(prop) - ? ("Bool8Array", "Bool8Array.Builder") - : ("BooleanArray", "BooleanArray.Builder"); + private static (string ArrayType, string BuilderType) GetDateArrayInfo(PropertyModel prop) + { + var over = prop.Type.ArrowTypeOverride?.Trim().ToLowerInvariant(); + if (over != null && over.StartsWith("date64")) + return ("Date64Array", "Date64Array.Builder"); + return ("Date32Array", "Date32Array.Builder"); + } - private static bool IsBinaryView(PropertyModel prop) => - prop.Type.ArrowTypeOverride != null && - (prop.Type.ArrowTypeOverride.Trim().ToLowerInvariant() is "binary_view" or "binaryview"); + private static bool IsStringView(PropertyModel prop) => + prop.Type.ArrowTypeOverride != null && + (prop.Type.ArrowTypeOverride.Trim().ToLowerInvariant() is "string_view" or "utf8_view" or "utf8view" or "stringview"); - private static (string ArrayType, string BuilderType) GetBinaryArrayInfo(PropertyModel prop) => - IsBinaryView(prop) - ? ("BinaryViewArray", "BinaryViewArray.Builder") - : ("BinaryArray", "BinaryArray.Builder"); + private static (string ArrayType, string BuilderType) GetStringArrayInfo(PropertyModel prop) => + IsStringView(prop) + ? ("StringViewArray", "StringViewArray.Builder") + : ("StringArray", "StringArray.Builder"); - private string GetArrowTypeExpression(TypeInfo type) - { - if (type.ArrowTypeOverride != null) - return ParseArrowTypeOverride(type.ArrowTypeOverride); + private static bool IsBool8(PropertyModel prop) => + prop.Type.ArrowTypeOverride != null && + (prop.Type.ArrowTypeOverride.Trim().ToLowerInvariant() is "bool8" or "arrow.bool8"); - return type.Kind switch - { - TypeKind2.String => "StringType.Default", - TypeKind2.Bool => "BooleanType.Default", - TypeKind2.Byte => "UInt8Type.Default", - TypeKind2.SByte => "Int8Type.Default", - TypeKind2.Int16 => "Int16Type.Default", - TypeKind2.UInt16 => "UInt16Type.Default", - TypeKind2.Int32 => "Int32Type.Default", - TypeKind2.UInt32 => "UInt32Type.Default", - TypeKind2.Int64 => "Int64Type.Default", - TypeKind2.UInt64 => "UInt64Type.Default", - TypeKind2.Float => "FloatType.Default", - TypeKind2.Double => "DoubleType.Default", - TypeKind2.Decimal => "new Decimal128Type(38, 18)", - TypeKind2.Binary => "BinaryType.Default", - TypeKind2.DateTime => "new TimestampType(TimeUnit.Microsecond, \"UTC\")", - TypeKind2.DateTimeOffset => "new TimestampType(TimeUnit.Microsecond, \"UTC\")", - TypeKind2.DateOnly => "Date32Type.Default", - TypeKind2.TimeOnly => "new Time64Type(TimeUnit.Microsecond)", - TypeKind2.TimeSpan => "DurationType.Microsecond", - TypeKind2.Guid => "GuidType.Default", - TypeKind2.Half => "HalfFloatType.Default", - TypeKind2.Enum => "new DictionaryType(Int16Type.Default, StringType.Default, false)", - TypeKind2.List or TypeKind2.Array or TypeKind2.Set => - $"new ListType(new Field(\"item\", {GetListElementTypeExpression(type.ElementType!)}, {(type.ElementType!.IsNullable ? "true" : "false")}))", - TypeKind2.Dictionary => - $"new MapType(new Field(\"key\", {GetArrowTypeExpression(type.KeyType!)}, false), new Field(\"value\", {GetArrowTypeExpression(type.ValueType!)}, true))", - TypeKind2.NestedRecord => - $"new StructType({type.FullTypeName}.ArrowSchema.FieldsList)", - _ => "StringType.Default /* unsupported */", - }; - } + private static (string ArrayType, string BuilderType) GetBoolArrayInfo(PropertyModel prop) => + IsBool8(prop) + ? ("Bool8Array", "Bool8Array.Builder") + : ("BooleanArray", "BooleanArray.Builder"); - /// - /// Returns the Arrow type expression for list element types. - /// Enums inside lists use DictionaryType (matching Python's schema). - /// - private string GetListElementTypeExpression(TypeInfo elemType) - { - return GetArrowTypeExpression(elemType); - } + private static bool IsBinaryView(PropertyModel prop) => + prop.Type.ArrowTypeOverride != null && + (prop.Type.ArrowTypeOverride.Trim().ToLowerInvariant() is "binary_view" or "binaryview"); - private static string ParseArrowTypeOverride(string typeName) - { - var lower = typeName.ToLowerInvariant().Trim(); - var original = typeName.Trim(); + private static (string ArrayType, string BuilderType) GetBinaryArrayInfo(PropertyModel prop) => + IsBinaryView(prop) + ? ("BinaryViewArray", "BinaryViewArray.Builder") + : ("BinaryArray", "BinaryArray.Builder"); - // Parameterized types: timestamp[unit, tz], decimal128(precision, scale), - // time64[unit], time32[unit], duration[unit] - if (lower.StartsWith("timestamp")) - { - var (unit, tz) = ParseTimestampParams(original, "timestamp"); - return $"new TimestampType(TimeUnit.{unit}, \"{tz}\")"; - } - if (lower.StartsWith("decimal128") || lower.StartsWith("decimal")) - { - var (precision, scale) = ParseDecimalParams(lower); - return $"new Decimal128Type({precision}, {scale})"; - } - if (lower.StartsWith("time64")) - { - var unit = ParseUnitParam(lower, "time64", "Microsecond"); - return $"new Time64Type(TimeUnit.{unit})"; + private string GetArrowTypeExpression(TypeInfo type) + { + if (type.ArrowTypeOverride != null) + return ParseArrowTypeOverride(type.ArrowTypeOverride); + + return type.Kind switch + { + ArrowTypeKind.String => "StringType.Default", + ArrowTypeKind.Bool => "BooleanType.Default", + ArrowTypeKind.Byte => "UInt8Type.Default", + ArrowTypeKind.SByte => "Int8Type.Default", + ArrowTypeKind.Int16 => "Int16Type.Default", + ArrowTypeKind.UInt16 => "UInt16Type.Default", + ArrowTypeKind.Int32 => "Int32Type.Default", + ArrowTypeKind.UInt32 => "UInt32Type.Default", + ArrowTypeKind.Int64 => "Int64Type.Default", + ArrowTypeKind.UInt64 => "UInt64Type.Default", + ArrowTypeKind.Float => "FloatType.Default", + ArrowTypeKind.Double => "DoubleType.Default", + ArrowTypeKind.Decimal => "new Decimal128Type(38, 18)", + ArrowTypeKind.Binary => "BinaryType.Default", + ArrowTypeKind.DateTime => "new TimestampType(TimeUnit.Microsecond, \"UTC\")", + ArrowTypeKind.DateTimeOffset => "new TimestampType(TimeUnit.Microsecond, \"UTC\")", + ArrowTypeKind.DateOnly => "Date32Type.Default", + ArrowTypeKind.TimeOnly => "new Time64Type(TimeUnit.Microsecond)", + ArrowTypeKind.TimeSpan => "DurationType.Microsecond", + ArrowTypeKind.Guid => "GuidType.Default", + ArrowTypeKind.Half => "HalfFloatType.Default", + ArrowTypeKind.Enum => "new DictionaryType(Int16Type.Default, StringType.Default, false)", + ArrowTypeKind.List or ArrowTypeKind.Array or ArrowTypeKind.Set => + $"new ListType(new Field(\"item\", {GetListElementTypeExpression(type.ElementType!)}, {(type.ElementType!.IsNullable ? "true" : "false")}))", + ArrowTypeKind.Dictionary => + $"new MapType(new Field(\"key\", {GetArrowTypeExpression(type.KeyType!)}, false), new Field(\"value\", {GetArrowTypeExpression(type.ValueType!)}, true))", + ArrowTypeKind.NestedRecord => + $"new StructType({type.FullTypeName}.ArrowSchema.FieldsList)", + _ => "StringType.Default /* unsupported */", + }; } - if (lower.StartsWith("time32")) + + /// + /// Returns the Arrow type expression for list element types. + /// Enums inside lists use DictionaryType (matching Python's schema). + /// + private string GetListElementTypeExpression(TypeInfo elemType) { - var unit = ParseUnitParam(lower, "time32", "Millisecond"); - return $"new Time32Type(TimeUnit.{unit})"; + return GetArrowTypeExpression(elemType); } - if (lower.StartsWith("duration")) + + private static string ParseArrowTypeOverride(string typeName) { - var unit = ParseUnitParam(lower, "duration", "Microsecond"); - return $"DurationType.{unit}"; + var lower = typeName.ToLowerInvariant().Trim(); + var original = typeName.Trim(); + + // Parameterized types: timestamp[unit, tz], decimal128(precision, scale), + // time64[unit], time32[unit], duration[unit] + if (lower.StartsWith("timestamp")) + { + var (unit, tz) = ParseTimestampParams(original, "timestamp"); + return $"new TimestampType(TimeUnit.{unit}, \"{tz}\")"; + } + if (lower.StartsWith("decimal128") || lower.StartsWith("decimal")) + { + var (precision, scale) = ParseDecimalParams(lower); + return $"new Decimal128Type({precision}, {scale})"; + } + if (lower.StartsWith("time64")) + { + var unit = ParseUnitParam(lower, "time64", "Microsecond"); + return $"new Time64Type(TimeUnit.{unit})"; + } + if (lower.StartsWith("time32")) + { + var unit = ParseUnitParam(lower, "time32", "Millisecond"); + return $"new Time32Type(TimeUnit.{unit})"; + } + if (lower.StartsWith("duration")) + { + var unit = ParseUnitParam(lower, "duration", "Microsecond"); + return $"DurationType.{unit}"; + } + + return lower switch + { + "int8" => "Int8Type.Default", + "int16" => "Int16Type.Default", + "int32" => "Int32Type.Default", + "int64" => "Int64Type.Default", + "uint8" => "UInt8Type.Default", + "uint16" => "UInt16Type.Default", + "uint32" => "UInt32Type.Default", + "uint64" => "UInt64Type.Default", + "float16" or "halffloat" => "HalfFloatType.Default", + "float32" or "float" => "FloatType.Default", + "float64" or "double" => "DoubleType.Default", + "bool" or "boolean" => "BooleanType.Default", + "string" or "utf8" => "StringType.Default", + "binary" => "BinaryType.Default", + "date32" => "Date32Type.Default", + "date64" => "Date64Type.Default", + "string_view" or "utf8_view" or "utf8view" or "stringview" => "StringViewType.Default", + "binary_view" or "binaryview" => "BinaryViewType.Default", + "guid" => "GuidType.Default", + "bool8" or "arrow.bool8" => "Bool8Type.Default", + _ => $"StringType.Default /* unrecognized override: {typeName} */", + }; } - return lower switch + private static (string Unit, string Timezone) ParseTimestampParams(string input, string prefix) { - "int8" => "Int8Type.Default", - "int16" => "Int16Type.Default", - "int32" => "Int32Type.Default", - "int64" => "Int64Type.Default", - "uint8" => "UInt8Type.Default", - "uint16" => "UInt16Type.Default", - "uint32" => "UInt32Type.Default", - "uint64" => "UInt64Type.Default", - "float16" or "halffloat" => "HalfFloatType.Default", - "float32" or "float" => "FloatType.Default", - "float64" or "double" => "DoubleType.Default", - "bool" or "boolean" => "BooleanType.Default", - "string" or "utf8" => "StringType.Default", - "binary" => "BinaryType.Default", - "date32" => "Date32Type.Default", - "date64" => "Date64Type.Default", - "string_view" or "utf8_view" or "utf8view" or "stringview" => "StringViewType.Default", - "binary_view" or "binaryview" => "BinaryViewType.Default", - "guid" => "GuidType.Default", - "bool8" or "arrow.bool8" => "Bool8Type.Default", - _ => $"StringType.Default /* unrecognized override: {typeName} */", - }; - } + var unit = "Microsecond"; + var tz = "UTC"; + var inner = ExtractBracketParams(input, prefix); + if (inner != null) + { + var parts = inner.Split(','); + if (parts.Length >= 1) unit = NormalizeTimeUnit(parts[0].Trim()); + if (parts.Length >= 2) tz = parts[1].Trim().Trim('"', '\''); + } + return (unit, tz); + } - private static (string Unit, string Timezone) ParseTimestampParams(string input, string prefix) - { - var unit = "Microsecond"; - var tz = "UTC"; - var inner = ExtractBracketParams(input, prefix); - if (inner != null) + private static (int Precision, int Scale) ParseDecimalParams(string input) { - var parts = inner.Split(','); - if (parts.Length >= 1) unit = NormalizeTimeUnit(parts[0].Trim()); - if (parts.Length >= 2) tz = parts[1].Trim().Trim('"', '\''); + var precision = 38; + var scale = 18; + // Match decimal128(p, s) or decimal(p, s) + var start = input.IndexOf('('); + var end = input.IndexOf(')'); + if (start >= 0 && end > start) + { + var parts = input.Substring(start + 1, end - start - 1).Split(','); + if (parts.Length >= 1 && int.TryParse(parts[0].Trim(), out var p)) precision = p; + if (parts.Length >= 2 && int.TryParse(parts[1].Trim(), out var s)) scale = s; + } + return (precision, scale); } - return (unit, tz); - } - private static (int Precision, int Scale) ParseDecimalParams(string input) - { - var precision = 38; - var scale = 18; - // Match decimal128(p, s) or decimal(p, s) - var start = input.IndexOf('('); - var end = input.IndexOf(')'); - if (start >= 0 && end > start) + private static string ParseUnitParam(string input, string prefix, string defaultUnit) { - var parts = input.Substring(start + 1, end - start - 1).Split(','); - if (parts.Length >= 1 && int.TryParse(parts[0].Trim(), out var p)) precision = p; - if (parts.Length >= 2 && int.TryParse(parts[1].Trim(), out var s)) scale = s; + var inner = ExtractBracketParams(input, prefix); + return inner != null ? NormalizeTimeUnit(inner.Trim()) : defaultUnit; } - return (precision, scale); - } - - private static string ParseUnitParam(string input, string prefix, string defaultUnit) - { - var inner = ExtractBracketParams(input, prefix); - return inner != null ? NormalizeTimeUnit(inner.Trim()) : defaultUnit; - } - private static string ExtractBracketParams(string input, string prefix) - { - // Supports both [] and () delimiters: timestamp[us, UTC] or timestamp(us, UTC) - var rest = input.Substring(prefix.Length).Trim(); - if (rest.Length >= 2) + private static string ExtractBracketParams(string input, string prefix) { - if ((rest[0] == '[' && rest[rest.Length - 1] == ']') || - (rest[0] == '(' && rest[rest.Length - 1] == ')')) - return rest.Substring(1, rest.Length - 2); + // Supports both [] and () delimiters: timestamp[us, UTC] or timestamp(us, UTC) + var rest = input.Substring(prefix.Length).Trim(); + if (rest.Length >= 2) + { + if ((rest[0] == '[' && rest[rest.Length - 1] == ']') || + (rest[0] == '(' && rest[rest.Length - 1] == ')')) + return rest.Substring(1, rest.Length - 2); + } + return null!; } - return null!; - } - private static string NormalizeTimeUnit(string unit) - { - switch (unit.ToLowerInvariant()) + private static string NormalizeTimeUnit(string unit) { - case "s": case "second": case "seconds": return "Second"; - case "ms": case "millisecond": case "milliseconds": return "Millisecond"; - case "us": case "microsecond": case "microseconds": case "μs": return "Microsecond"; - case "ns": case "nanosecond": case "nanoseconds": return "Nanosecond"; - default: return "Microsecond"; + switch (unit.ToLowerInvariant()) + { + case "s": case "second": case "seconds": return "Second"; + case "ms": case "millisecond": case "milliseconds": return "Millisecond"; + case "us": case "microsecond": case "microseconds": case "μs": return "Microsecond"; + case "ns": case "nanosecond": case "nanoseconds": return "Nanosecond"; + default: return "Microsecond"; + } } - } - private static string GetArrayBuilderType(TypeInfo type) - { - // Check for view type overrides - if (type.ArrowTypeOverride != null) + private static string GetArrayBuilderType(TypeInfo type) { - var ov = type.ArrowTypeOverride.ToLowerInvariant().Replace("_", ""); - if (ov is "stringview" or "utf8view") return "StringViewArray.Builder"; - if (ov is "binaryview") return "BinaryViewArray.Builder"; + // Check for view type overrides + if (type.ArrowTypeOverride != null) + { + var ov = type.ArrowTypeOverride.ToLowerInvariant().Replace("_", ""); + if (ov is "stringview" or "utf8view") return "StringViewArray.Builder"; + if (ov is "binaryview") return "BinaryViewArray.Builder"; + } + return type.Kind switch + { + ArrowTypeKind.String => "StringArray.Builder", + ArrowTypeKind.Bool => "BooleanArray.Builder", + ArrowTypeKind.Byte => "UInt8Array.Builder", + ArrowTypeKind.SByte => "Int8Array.Builder", + ArrowTypeKind.Int16 => "Int16Array.Builder", + ArrowTypeKind.UInt16 => "UInt16Array.Builder", + ArrowTypeKind.Int32 => "Int32Array.Builder", + ArrowTypeKind.UInt32 => "UInt32Array.Builder", + ArrowTypeKind.Int64 => "Int64Array.Builder", + ArrowTypeKind.UInt64 => "UInt64Array.Builder", + ArrowTypeKind.Float => "FloatArray.Builder", + ArrowTypeKind.Double => "DoubleArray.Builder", + ArrowTypeKind.Half => "HalfFloatArray.Builder", + ArrowTypeKind.Decimal => "Decimal128Array.Builder", + ArrowTypeKind.DateTime or ArrowTypeKind.DateTimeOffset => "TimestampArray.Builder", + ArrowTypeKind.DateOnly => GetDateArrayBuilderType(type), + ArrowTypeKind.TimeOnly => GetTimeArrayBuilderType(type), + ArrowTypeKind.TimeSpan => "DurationArray.Builder", + ArrowTypeKind.Guid => "GuidArray.Builder", + ArrowTypeKind.Binary => "BinaryArray.Builder", + ArrowTypeKind.Enum => "StringArray.Builder", + _ => $"StringArray.Builder /* fallback for {type.Kind} */", + }; } - return type.Kind switch - { - TypeKind2.String => "StringArray.Builder", - TypeKind2.Bool => "BooleanArray.Builder", - TypeKind2.Byte => "UInt8Array.Builder", - TypeKind2.SByte => "Int8Array.Builder", - TypeKind2.Int16 => "Int16Array.Builder", - TypeKind2.UInt16 => "UInt16Array.Builder", - TypeKind2.Int32 => "Int32Array.Builder", - TypeKind2.UInt32 => "UInt32Array.Builder", - TypeKind2.Int64 => "Int64Array.Builder", - TypeKind2.UInt64 => "UInt64Array.Builder", - TypeKind2.Float => "FloatArray.Builder", - TypeKind2.Double => "DoubleArray.Builder", - TypeKind2.Half => "HalfFloatArray.Builder", - TypeKind2.Decimal => "Decimal128Array.Builder", - TypeKind2.DateTime or TypeKind2.DateTimeOffset => "TimestampArray.Builder", - TypeKind2.DateOnly => GetDateArrayBuilderType(type), - TypeKind2.TimeOnly => GetTimeArrayBuilderType(type), - TypeKind2.TimeSpan => "DurationArray.Builder", - TypeKind2.Guid => "GuidArray.Builder", - TypeKind2.Binary => "BinaryArray.Builder", - TypeKind2.Enum => "StringArray.Builder", - _ => $"StringArray.Builder /* fallback for {type.Kind} */", - }; - } - /// - /// Returns the Arrow array type name for reading, respecting view type overrides. - /// - private static string GetArrayReadType(TypeInfo type) - { - if (type.ArrowTypeOverride != null) + /// + /// Returns the Arrow array type name for reading, respecting view type overrides. + /// + private static string GetArrayReadType(TypeInfo type) { - var ov = type.ArrowTypeOverride.ToLowerInvariant().Replace("_", ""); - if (ov is "stringview" or "utf8view") return "StringViewArray"; - if (ov is "binaryview") return "BinaryViewArray"; + if (type.ArrowTypeOverride != null) + { + var ov = type.ArrowTypeOverride.ToLowerInvariant().Replace("_", ""); + if (ov is "stringview" or "utf8view") return "StringViewArray"; + if (ov is "binaryview") return "BinaryViewArray"; + } + return type.Kind switch + { + ArrowTypeKind.String => "StringArray", + _ => null!, // Not used for non-string types + }; } - return type.Kind switch - { - TypeKind2.String => "StringArray", - _ => null!, // Not used for non-string types - }; - } - // --- Output helpers --- + // --- Output helpers --- - private void Line(string text = "") - { - if (string.IsNullOrEmpty(text)) + private void Line(string text = "") { - _sb.AppendLine(); - return; + if (string.IsNullOrEmpty(text)) + { + _sb.AppendLine(); + return; + } + _sb.Append(' ', _indent * 4); + _sb.AppendLine(text); } - _sb.Append(' ', _indent * 4); - _sb.AppendLine(text); } -} } // namespace diff --git a/src/Apache.Arrow.Serialization.Generator/JsonSchemaEmitter.cs b/src/Apache.Arrow.Serialization.Generator/JsonSchemaEmitter.cs index 10381d4f..49907ffd 100644 --- a/src/Apache.Arrow.Serialization.Generator/JsonSchemaEmitter.cs +++ b/src/Apache.Arrow.Serialization.Generator/JsonSchemaEmitter.cs @@ -20,176 +20,176 @@ namespace Apache.Arrow.Serialization.Generator { -/// -/// Emits a JSON schema descriptor as a static ArrowSchemaJson property. -/// The JSON describes the Arrow schema for cross-language codegen tools. -/// -internal static class JsonSchemaEmitter -{ - public static void Emit(StringBuilder sb, TypeModel model) + /// + /// Emits a JSON schema descriptor as a static ArrowSchemaJson property. + /// The JSON describes the Arrow schema for cross-language codegen tools. + /// + internal static class JsonSchemaEmitter { - var json = BuildJson(model); - - sb.AppendLine("// "); - sb.AppendLine("#nullable enable"); - sb.AppendLine(); - - if (model.Namespace != null) + public static void Emit(StringBuilder sb, TypeModel model) { - sb.AppendLine($"namespace {model.Namespace};"); - sb.AppendLine(); - } - - var typeKeyword = GetTypeKeyword(model); - sb.AppendLine($"partial {typeKeyword} {model.TypeName}"); - sb.AppendLine("{"); - sb.AppendLine($" public static string ArrowSchemaJson => @\"{json.Replace("\"", "\"\"")}\";"); - sb.AppendLine("}"); - } + var json = BuildJson(model); - private static string BuildJson(TypeModel model) - { - var sb = new StringBuilder(); - sb.AppendLine("{"); - sb.AppendLine($" \"type\": \"{JsonEscape(model.FullTypeName)}\","); - sb.AppendLine($" \"namespace\": {(model.Namespace != null ? $"\"{JsonEscape(model.Namespace)}\"" : "null")},"); - sb.AppendLine($" \"typeName\": \"{JsonEscape(model.TypeName)}\","); - - // Metadata - sb.Append(" \"metadata\": {"); - if (model.Metadata.Count > 0) - { + sb.AppendLine("// "); + sb.AppendLine("#nullable enable"); sb.AppendLine(); - for (int i = 0; i < model.Metadata.Count; i++) + + if (model.Namespace != null) { - var kv = model.Metadata[i]; - var comma = i < model.Metadata.Count - 1 ? "," : ""; - sb.AppendLine($" \"{JsonEscape(kv.Key)}\": \"{JsonEscape(kv.Value)}\"{comma}"); + sb.AppendLine($"namespace {model.Namespace};"); + sb.AppendLine(); } - sb.AppendLine(" },"); - } - else - { - sb.AppendLine("},"); + + var typeKeyword = GetTypeKeyword(model); + sb.AppendLine($"partial {typeKeyword} {model.TypeName}"); + sb.AppendLine("{"); + sb.AppendLine($" public static string ArrowSchemaJson => @\"{json.Replace("\"", "\"\"")}\";"); + sb.AppendLine("}"); } - // Fields - sb.AppendLine(" \"fields\": ["); - for (int i = 0; i < model.Properties.Count; i++) + private static string BuildJson(TypeModel model) { - var prop = model.Properties[i]; - var comma = i < model.Properties.Count - 1 ? "," : ""; - EmitField(sb, prop); - // Remove trailing newline, add comma - sb.Length -= sb.ToString().EndsWith("\r\n") ? 2 : 1; - sb.AppendLine(comma); - } - sb.AppendLine(" ]"); - sb.Append("}"); + var sb = new StringBuilder(); + sb.AppendLine("{"); + sb.AppendLine($" \"type\": \"{JsonEscape(model.FullTypeName)}\","); + sb.AppendLine($" \"namespace\": {(model.Namespace != null ? $"\"{JsonEscape(model.Namespace)}\"" : "null")},"); + sb.AppendLine($" \"typeName\": \"{JsonEscape(model.TypeName)}\","); + + // Metadata + sb.Append(" \"metadata\": {"); + if (model.Metadata.Count > 0) + { + sb.AppendLine(); + for (int i = 0; i < model.Metadata.Count; i++) + { + var kv = model.Metadata[i]; + var comma = i < model.Metadata.Count - 1 ? "," : ""; + sb.AppendLine($" \"{JsonEscape(kv.Key)}\": \"{JsonEscape(kv.Value)}\"{comma}"); + } + sb.AppendLine(" },"); + } + else + { + sb.AppendLine("},"); + } - return sb.ToString(); - } + // Fields + sb.AppendLine(" \"fields\": ["); + for (int i = 0; i < model.Properties.Count; i++) + { + var prop = model.Properties[i]; + var comma = i < model.Properties.Count - 1 ? "," : ""; + EmitField(sb, prop); + // Remove trailing newline, add comma + sb.Length -= sb.ToString().EndsWith("\r\n") ? 2 : 1; + sb.AppendLine(comma); + } + sb.AppendLine(" ]"); + sb.Append("}"); - private static void EmitField(StringBuilder sb, PropertyModel prop) - { - sb.AppendLine(" {"); - sb.AppendLine($" \"name\": \"{JsonEscape(prop.FieldName)}\","); - sb.AppendLine($" \"propertyName\": \"{JsonEscape(prop.PropertyName)}\","); - sb.AppendLine($" \"arrowType\": \"{JsonEscape(GetArrowTypeName(prop.Type))}\","); - sb.AppendLine($" \"typeKind\": \"{prop.Type.Kind}\","); - sb.AppendLine($" \"nullable\": {(prop.IsNullable ? "true" : "false")},"); - sb.AppendLine($" \"csharpType\": \"{JsonEscape(prop.Type.FullTypeName)}\""); - - if (prop.Type.ElementType != null) - { - // Replace last newline to add comma - sb.Length -= sb.ToString().EndsWith("\r\n") ? 2 : 1; - sb.AppendLine(","); - sb.AppendLine($" \"elementType\": \"{JsonEscape(GetArrowTypeName(prop.Type.ElementType))}\""); + return sb.ToString(); } - if (prop.Type.KeyType != null) + private static void EmitField(StringBuilder sb, PropertyModel prop) { - sb.Length -= sb.ToString().EndsWith("\r\n") ? 2 : 1; - sb.AppendLine(","); - sb.AppendLine($" \"keyType\": \"{JsonEscape(GetArrowTypeName(prop.Type.KeyType))}\","); - sb.AppendLine($" \"valueType\": \"{JsonEscape(GetArrowTypeName(prop.Type.ValueType!))}\""); - } + sb.AppendLine(" {"); + sb.AppendLine($" \"name\": \"{JsonEscape(prop.FieldName)}\","); + sb.AppendLine($" \"propertyName\": \"{JsonEscape(prop.PropertyName)}\","); + sb.AppendLine($" \"arrowType\": \"{JsonEscape(GetArrowTypeName(prop.Type))}\","); + sb.AppendLine($" \"typeKind\": \"{prop.Type.Kind}\","); + sb.AppendLine($" \"nullable\": {(prop.IsNullable ? "true" : "false")},"); + sb.AppendLine($" \"csharpType\": \"{JsonEscape(prop.Type.FullTypeName)}\""); + + if (prop.Type.ElementType != null) + { + // Replace last newline to add comma + sb.Length -= sb.ToString().EndsWith("\r\n") ? 2 : 1; + sb.AppendLine(","); + sb.AppendLine($" \"elementType\": \"{JsonEscape(GetArrowTypeName(prop.Type.ElementType))}\""); + } - if (prop.ConverterTypeName != null) - { - sb.Length -= sb.ToString().EndsWith("\r\n") ? 2 : 1; - sb.AppendLine(","); - sb.AppendLine($" \"converter\": \"{JsonEscape(prop.ConverterTypeName)}\""); - } + if (prop.Type.KeyType != null) + { + sb.Length -= sb.ToString().EndsWith("\r\n") ? 2 : 1; + sb.AppendLine(","); + sb.AppendLine($" \"keyType\": \"{JsonEscape(GetArrowTypeName(prop.Type.KeyType))}\","); + sb.AppendLine($" \"valueType\": \"{JsonEscape(GetArrowTypeName(prop.Type.ValueType!))}\""); + } - if (prop.Metadata.Count > 0) - { - sb.Length -= sb.ToString().EndsWith("\r\n") ? 2 : 1; - sb.AppendLine(","); - sb.Append(" \"metadata\": {"); - sb.AppendLine(); - for (int j = 0; j < prop.Metadata.Count; j++) + if (prop.ConverterTypeName != null) + { + sb.Length -= sb.ToString().EndsWith("\r\n") ? 2 : 1; + sb.AppendLine(","); + sb.AppendLine($" \"converter\": \"{JsonEscape(prop.ConverterTypeName)}\""); + } + + if (prop.Metadata.Count > 0) { - var kv = prop.Metadata[j]; - var mComma = j < prop.Metadata.Count - 1 ? "," : ""; - sb.AppendLine($" \"{JsonEscape(kv.Key)}\": \"{JsonEscape(kv.Value)}\"{mComma}"); + sb.Length -= sb.ToString().EndsWith("\r\n") ? 2 : 1; + sb.AppendLine(","); + sb.Append(" \"metadata\": {"); + sb.AppendLine(); + for (int j = 0; j < prop.Metadata.Count; j++) + { + var kv = prop.Metadata[j]; + var mComma = j < prop.Metadata.Count - 1 ? "," : ""; + sb.AppendLine($" \"{JsonEscape(kv.Key)}\": \"{JsonEscape(kv.Value)}\"{mComma}"); + } + sb.AppendLine(" }"); } - sb.AppendLine(" }"); + + sb.AppendLine(" }"); } - sb.AppendLine(" }"); - } + private static string GetArrowTypeName(TypeInfo type) + { + if (type.ArrowTypeOverride != null) + return type.ArrowTypeOverride; - private static string GetArrowTypeName(TypeInfo type) - { - if (type.ArrowTypeOverride != null) - return type.ArrowTypeOverride; + return type.Kind switch + { + ArrowTypeKind.String => "utf8", + ArrowTypeKind.Bool => "bool", + ArrowTypeKind.Byte => "uint8", + ArrowTypeKind.SByte => "int8", + ArrowTypeKind.Int16 => "int16", + ArrowTypeKind.UInt16 => "uint16", + ArrowTypeKind.Int32 => "int32", + ArrowTypeKind.UInt32 => "uint32", + ArrowTypeKind.Int64 => "int64", + ArrowTypeKind.UInt64 => "uint64", + ArrowTypeKind.Float => "float32", + ArrowTypeKind.Double => "float64", + ArrowTypeKind.Decimal => "decimal128(38,18)", + ArrowTypeKind.Binary => "binary", + ArrowTypeKind.DateTime => "timestamp[us, UTC]", + ArrowTypeKind.DateTimeOffset => "timestamp[us, UTC]", + ArrowTypeKind.DateOnly => "date32", + ArrowTypeKind.TimeOnly => "time64[us]", + ArrowTypeKind.TimeSpan => "duration[us]", + ArrowTypeKind.Guid => "fixed_size_binary(16)", + ArrowTypeKind.Half => "float16", + ArrowTypeKind.Enum => "dictionary", + ArrowTypeKind.List => $"list<{GetArrowTypeName(type.ElementType!)}>", + ArrowTypeKind.Array => $"list<{GetArrowTypeName(type.ElementType!)}>", + ArrowTypeKind.Set => $"list<{GetArrowTypeName(type.ElementType!)}>", + ArrowTypeKind.Dictionary => $"map<{GetArrowTypeName(type.KeyType!)}, {GetArrowTypeName(type.ValueType!)}>", + ArrowTypeKind.NestedRecord => $"struct<{type.FullTypeName}>", + ArrowTypeKind.Custom => "custom", + _ => "unknown", + }; + } - return type.Kind switch + private static string GetTypeKeyword(TypeModel model) { - TypeKind2.String => "utf8", - TypeKind2.Bool => "bool", - TypeKind2.Byte => "uint8", - TypeKind2.SByte => "int8", - TypeKind2.Int16 => "int16", - TypeKind2.UInt16 => "uint16", - TypeKind2.Int32 => "int32", - TypeKind2.UInt32 => "uint32", - TypeKind2.Int64 => "int64", - TypeKind2.UInt64 => "uint64", - TypeKind2.Float => "float32", - TypeKind2.Double => "float64", - TypeKind2.Decimal => "decimal128(38,18)", - TypeKind2.Binary => "binary", - TypeKind2.DateTime => "timestamp[us, UTC]", - TypeKind2.DateTimeOffset => "timestamp[us, UTC]", - TypeKind2.DateOnly => "date32", - TypeKind2.TimeOnly => "time64[us]", - TypeKind2.TimeSpan => "duration[us]", - TypeKind2.Guid => "fixed_size_binary(16)", - TypeKind2.Half => "float16", - TypeKind2.Enum => "dictionary", - TypeKind2.List => $"list<{GetArrowTypeName(type.ElementType!)}>", - TypeKind2.Array => $"list<{GetArrowTypeName(type.ElementType!)}>", - TypeKind2.Set => $"list<{GetArrowTypeName(type.ElementType!)}>", - TypeKind2.Dictionary => $"map<{GetArrowTypeName(type.KeyType!)}, {GetArrowTypeName(type.ValueType!)}>", - TypeKind2.NestedRecord => $"struct<{type.FullTypeName}>", - TypeKind2.Custom => "custom", - _ => "unknown", - }; - } + if (model.IsRecord && model.IsValueType) return "record struct"; + if (model.IsRecord) return "record"; + if (model.IsValueType) return "struct"; + return "class"; + } - private static string GetTypeKeyword(TypeModel model) - { - if (model.IsRecord && model.IsValueType) return "record struct"; - if (model.IsRecord) return "record"; - if (model.IsValueType) return "struct"; - return "class"; + private static string JsonEscape(string s) => + s.Replace("\\", "\\\\").Replace("\"", "\\\"").Replace("\n", "\\n").Replace("\r", "\\r"); } - private static string JsonEscape(string s) => - s.Replace("\\", "\\\\").Replace("\"", "\\\"").Replace("\n", "\\n").Replace("\r", "\\r"); -} - } // namespace diff --git a/src/Apache.Arrow.Serialization.Generator/Models.cs b/src/Apache.Arrow.Serialization.Generator/Models.cs index 27092ae9..13124260 100644 --- a/src/Apache.Arrow.Serialization.Generator/Models.cs +++ b/src/Apache.Arrow.Serialization.Generator/Models.cs @@ -19,7 +19,7 @@ namespace Apache.Arrow.Serialization.Generator { - internal enum TypeKind2 + internal enum ArrowTypeKind { Unknown, String, @@ -54,7 +54,7 @@ internal enum TypeKind2 internal class TypeInfo { - public TypeKind2 Kind { get; set; } + public ArrowTypeKind Kind { get; set; } public string FullTypeName { get; set; } = ""; public bool IsNullable { get; set; } public string? ArrowTypeOverride { get; set; } diff --git a/src/Apache.Arrow.Serialization.Generator/PolymorphicCodeEmitter.cs b/src/Apache.Arrow.Serialization.Generator/PolymorphicCodeEmitter.cs index 51d18a88..3e8b4fd0 100644 --- a/src/Apache.Arrow.Serialization.Generator/PolymorphicCodeEmitter.cs +++ b/src/Apache.Arrow.Serialization.Generator/PolymorphicCodeEmitter.cs @@ -22,795 +22,801 @@ namespace Apache.Arrow.Serialization.Generator { -internal class PolymorphicCodeEmitter -{ - private readonly StringBuilder _sb; - private readonly PolymorphicModel _model; - private int _indent; - - public PolymorphicCodeEmitter(StringBuilder sb, PolymorphicModel model) + internal class PolymorphicCodeEmitter { - _sb = sb; - _model = model; - } + private readonly StringBuilder _sb; + private readonly PolymorphicModel _model; + private int _indent; - public void Emit() - { - Line("// "); - Line("#nullable enable"); - Line("#pragma warning disable CS8629 // Nullable value type may be null"); - Line(); - Line("using System.Collections.Generic;"); - Line("using System.Linq;"); - Line("using Apache.Arrow;"); - Line("using Apache.Arrow.Arrays;"); - Line("using Apache.Arrow.Types;"); - Line("using Apache.Arrow.Serialization;"); - Line(); - - if (_model.Namespace != null) + public PolymorphicCodeEmitter(StringBuilder sb, PolymorphicModel model) { - Line($"namespace {_model.Namespace};"); - Line(); + _sb = sb; + _model = model; } - var typeKeyword = _model.IsInterface ? "interface" : (_model.IsRecord ? "record" : "class"); - Line($"partial {typeKeyword} {_model.TypeName} : IArrowSerializer<{_model.TypeName}>"); - Line("{"); - _indent++; - - EmitSchemaField(); - Line(); - EmitSerialize(); - Line(); - EmitDeserialize(); - Line(); - EmitMultiRowSerialize(); - Line(); - EmitMultiRowDeserialize(); - - _indent--; - Line("}"); - } - - private void EmitSchemaField() - { - Line($"private static readonly Schema _arrowSchema = new Schema.Builder()"); - _indent++; - - // Discriminator field - Line($".Field(new Field(\"{Escape(_model.TypeDiscriminatorFieldName)}\", StringType.Default, false))"); - - // Union properties — all nullable - foreach (var prop in _model.UnionProperties) + public void Emit() { - var arrowType = CodeEmitterHelpers.GetArrowTypeExpression(prop.Type); - Line($".Field(new Field(\"{Escape(prop.FieldName)}\", {arrowType}, true))"); - } + Line("// "); + Line("#nullable enable"); + Line("#pragma warning disable CS8629 // Nullable value type may be null"); + Line("#pragma warning disable CS0162 // Unreachable code after NotSupportedException throws"); + Line(); + Line("using System.Collections.Generic;"); + Line("using System.Linq;"); + Line("using Apache.Arrow;"); + Line("using Apache.Arrow.Arrays;"); + Line("using Apache.Arrow.Types;"); + Line("using Apache.Arrow.Serialization;"); + Line(); - _indent--; - Line($".Build();"); - Line(); - Line($"public static Schema ArrowSchema => _arrowSchema;"); - } + if (_model.Namespace != null) + { + Line($"namespace {_model.Namespace};"); + Line(); + } - private void EmitSerialize() - { - Line($"public static RecordBatch ToRecordBatch({_model.TypeName} value)"); - Line("{"); - _indent++; + var typeKeyword = _model.IsInterface ? "interface" : (_model.IsRecord ? "record" : "class"); + Line($"partial {typeKeyword} {_model.TypeName} : IArrowSerializer<{_model.TypeName}>"); + Line("{"); + _indent++; - // Build discriminator column - Line("var discriminatorBuilder = new StringArray.Builder();"); + EmitSchemaField(); + Line(); + EmitSerialize(); + Line(); + EmitDeserialize(); + Line(); + EmitMultiRowSerialize(); + Line(); + EmitMultiRowDeserialize(); - // Build union property builders - for (int i = 0; i < _model.UnionProperties.Count; i++) - { - var prop = _model.UnionProperties[i]; - EmitBuilderDeclaration(prop, i); + _indent--; + Line("}"); } - Line(); - // Switch on concrete type - Line("switch (value)"); - Line("{"); - _indent++; - - foreach (var dt in _model.DerivedTypes) + private void EmitSchemaField() { - Line($"case {dt.FullTypeName} v_{dt.TypeName}:"); - Line("{"); + Line($"private static readonly Schema _arrowSchema = new Schema.Builder()"); _indent++; - Line($"discriminatorBuilder.Append(\"{Escape(dt.TypeDiscriminator)}\");"); - // For each union property, either append the value or null - for (int i = 0; i < _model.UnionProperties.Count; i++) + // Discriminator field + Line($".Field(new Field(\"{Escape(_model.TypeDiscriminatorFieldName)}\", StringType.Default, false))"); + + // Union properties — all nullable + foreach (var prop in _model.UnionProperties) { - var unionProp = _model.UnionProperties[i]; - var derivedProp = FindProperty(dt, unionProp.FieldName); - if (derivedProp != null) - { - EmitAppendValue(derivedProp, i, $"v_{dt.TypeName}.{derivedProp.PropertyName}"); - } - else - { - EmitAppendNull(unionProp, i); - } + var arrowType = CodeEmitterHelpers.GetArrowTypeExpression(prop.Type); + Line($".Field(new Field(\"{Escape(prop.FieldName)}\", {arrowType}, true))"); } - Line("break;"); _indent--; - Line("}"); + Line($".Build();"); + Line(); + Line($"public static Schema ArrowSchema => _arrowSchema;"); } - Line($"default:"); - _indent++; - Line($"throw new System.ArgumentException($\"Unknown derived type: {{value.GetType().Name}}\");"); - _indent--; - - _indent--; - Line("}"); - - Line(); - // Build arrays and RecordBatch - Line("var columns = new IArrowArray[]"); - Line("{"); - _indent++; - Line("discriminatorBuilder.Build(),"); - for (int i = 0; i < _model.UnionProperties.Count; i++) + private void EmitSerialize() { - EmitBuildColumn(i); - } - _indent--; - Line("};"); - Line("return new RecordBatch(_arrowSchema, columns, 1);"); + Line($"public static RecordBatch ToRecordBatch({_model.TypeName} value)"); + Line("{"); + _indent++; - _indent--; - Line("}"); - } + // Build discriminator column + Line("var discriminatorBuilder = new StringArray.Builder();"); - private void EmitDeserialize() - { - Line($"public static {_model.TypeName} FromRecordBatch(RecordBatch batch)"); - Line("{"); - _indent++; - - Line("var discriminator = ((StringArray)batch.Column(0)).GetString(0)!;"); - Line("switch (discriminator)"); - Line("{"); - _indent++; + // Build union property builders + for (int i = 0; i < _model.UnionProperties.Count; i++) + { + var prop = _model.UnionProperties[i]; + EmitBuilderDeclaration(prop, i); + } - foreach (var dt in _model.DerivedTypes) - { - Line($"case \"{Escape(dt.TypeDiscriminator)}\":"); + Line(); + // Switch on concrete type + Line("switch (value)"); Line("{"); _indent++; - // Read each property for this derived type from the batch columns - for (int i = 0; i < dt.Properties.Count; i++) + foreach (var dt in _model.DerivedTypes) { - var prop = dt.Properties[i]; - var unionIndex = FindUnionPropertyIndex(prop.FieldName); - var colIndex = unionIndex + 1; // +1 for discriminator - EmitReadProperty(prop, i, colIndex); + Line($"case {dt.FullTypeName} v_{dt.TypeName}:"); + Line("{"); + _indent++; + Line($"discriminatorBuilder.Append(\"{Escape(dt.TypeDiscriminator)}\");"); + + // For each union property, either append the value or null + for (int i = 0; i < _model.UnionProperties.Count; i++) + { + var unionProp = _model.UnionProperties[i]; + var derivedProp = FindProperty(dt, unionProp.FieldName); + if (derivedProp != null) + { + EmitAppendValue(derivedProp, i, $"v_{dt.TypeName}.{derivedProp.PropertyName}"); + } + else + { + EmitAppendNull(unionProp, i); + } + } + + Line("break;"); + _indent--; + Line("}"); } - // Construct the derived type - Line($"return new {dt.FullTypeName}"); + Line($"default:"); + _indent++; + Line($"throw new System.ArgumentException($\"Unknown derived type: {{value.GetType().Name}}\");"); + _indent--; + + _indent--; + Line("}"); + + Line(); + // Build arrays and RecordBatch + Line("var columns = new IArrowArray[]"); Line("{"); _indent++; - for (int i = 0; i < dt.Properties.Count; i++) + Line("discriminatorBuilder.Build(),"); + for (int i = 0; i < _model.UnionProperties.Count; i++) { - var prop = dt.Properties[i]; - Line($"{prop.PropertyName} = prop_{i},"); + EmitBuildColumn(i); } _indent--; Line("};"); + Line("return new RecordBatch(_arrowSchema, columns, 1);"); _indent--; Line("}"); } - Line("default:"); - _indent++; - Line("throw new System.ArgumentException($\"Unknown type discriminator: {discriminator}\");"); - _indent--; - - _indent--; - Line("}"); - - _indent--; - Line("}"); - } - - private void EmitMultiRowSerialize() - { - Line($"public static RecordBatch ToRecordBatch(IReadOnlyList<{_model.TypeName}> values)"); - Line("{"); - _indent++; - - Line("var discriminatorBuilder = new StringArray.Builder();"); - for (int i = 0; i < _model.UnionProperties.Count; i++) + private void EmitDeserialize() { - var prop = _model.UnionProperties[i]; - EmitBuilderDeclaration(prop, i); - } - - Line(); - Line("foreach (var value in values)"); - Line("{"); - _indent++; - - Line("switch (value)"); - Line("{"); - _indent++; + Line($"public static {_model.TypeName} FromRecordBatch(RecordBatch batch)"); + Line("{"); + _indent++; - foreach (var dt in _model.DerivedTypes) - { - Line($"case {dt.FullTypeName} v_{dt.TypeName}:"); + Line("var discriminator = ((StringArray)batch.Column(0)).GetString(0)!;"); + Line("switch (discriminator)"); Line("{"); _indent++; - Line($"discriminatorBuilder.Append(\"{Escape(dt.TypeDiscriminator)}\");"); - for (int i = 0; i < _model.UnionProperties.Count; i++) + foreach (var dt in _model.DerivedTypes) { - var unionProp = _model.UnionProperties[i]; - var derivedProp = FindProperty(dt, unionProp.FieldName); - if (derivedProp != null) + Line($"case \"{Escape(dt.TypeDiscriminator)}\":"); + Line("{"); + _indent++; + + // Read each property for this derived type from the batch columns + for (int i = 0; i < dt.Properties.Count; i++) { - EmitAppendValue(derivedProp, i, $"v_{dt.TypeName}.{derivedProp.PropertyName}"); + var prop = dt.Properties[i]; + var unionIndex = FindUnionPropertyIndex(prop.FieldName); + var colIndex = unionIndex + 1; // +1 for discriminator + EmitReadProperty(prop, i, colIndex); } - else + + // Construct the derived type + Line($"return new {dt.FullTypeName}"); + Line("{"); + _indent++; + for (int i = 0; i < dt.Properties.Count; i++) { - EmitAppendNull(unionProp, i); + var prop = dt.Properties[i]; + Line($"{prop.PropertyName} = prop_{i},"); } + _indent--; + Line("};"); + + _indent--; + Line("}"); } - Line("break;"); + Line("default:"); + _indent++; + Line("throw new System.ArgumentException($\"Unknown type discriminator: {discriminator}\");"); _indent--; - Line("}"); - } - - Line($"default:"); - _indent++; - Line($"throw new System.ArgumentException($\"Unknown derived type: {{value.GetType().Name}}\");"); - _indent--; - - _indent--; - Line("}"); - _indent--; - Line("}"); - - Line(); - Line("var columns = new IArrowArray[]"); - Line("{"); - _indent++; - Line("discriminatorBuilder.Build(),"); - for (int i = 0; i < _model.UnionProperties.Count; i++) - { - EmitBuildColumn(i); - } - _indent--; - Line("};"); - Line("return new RecordBatch(_arrowSchema, columns, values.Count);"); - - _indent--; - Line("}"); - } + _indent--; + Line("}"); - private void EmitBuildColumn(int index) - { - var prop = _model.UnionProperties[index]; - if (prop.Type.Kind == TypeKind2.Enum) - { - // Build DictionaryArray from index + dict builders - Line($"new DictionaryArray(new DictionaryType(Int16Type.Default, StringType.Default, false), bld_{index}_idx.Build(), new StringArray.Builder().AppendRange(bld_{index}_dict).Build()),"); - } - else - { - Line($"bld_{index}.Build(),"); + _indent--; + Line("}"); } - } - private void EmitMultiRowDeserialize() - { - Line($"public static IReadOnlyList<{_model.TypeName}> ListFromRecordBatch(RecordBatch batch)"); - Line("{"); - _indent++; - - Line("var discriminatorCol = (StringArray)batch.Column(0);"); - - // Cast all union property columns - for (int i = 0; i < _model.UnionProperties.Count; i++) + private void EmitMultiRowSerialize() { - var prop = _model.UnionProperties[i]; - var colIndex = i + 1; - var castType = CodeEmitterHelpers.GetArrayCastType(prop); - Line($"var col_{i} = ({castType})batch.Column({colIndex});"); - } + Line($"public static RecordBatch ToRecordBatch(IReadOnlyList<{_model.TypeName}> values)"); + Line("{"); + _indent++; - Line(); - Line($"var result = new List<{_model.TypeName}>(batch.Length);"); - Line("for (int row = 0; row < batch.Length; row++)"); - Line("{"); - _indent++; + Line("var discriminatorBuilder = new StringArray.Builder();"); + for (int i = 0; i < _model.UnionProperties.Count; i++) + { + var prop = _model.UnionProperties[i]; + EmitBuilderDeclaration(prop, i); + } - Line("var discriminator = discriminatorCol.GetString(row)!;"); - Line("switch (discriminator)"); - Line("{"); - _indent++; + Line(); + Line("foreach (var value in values)"); + Line("{"); + _indent++; - foreach (var dt in _model.DerivedTypes) - { - Line($"case \"{Escape(dt.TypeDiscriminator)}\":"); + Line("switch (value)"); Line("{"); _indent++; - for (int i = 0; i < dt.Properties.Count; i++) + foreach (var dt in _model.DerivedTypes) { - var prop = dt.Properties[i]; - var unionIndex = FindUnionPropertyIndex(prop.FieldName); - EmitMultiRowReadProperty(prop, i, unionIndex); + Line($"case {dt.FullTypeName} v_{dt.TypeName}:"); + Line("{"); + _indent++; + Line($"discriminatorBuilder.Append(\"{Escape(dt.TypeDiscriminator)}\");"); + + for (int i = 0; i < _model.UnionProperties.Count; i++) + { + var unionProp = _model.UnionProperties[i]; + var derivedProp = FindProperty(dt, unionProp.FieldName); + if (derivedProp != null) + { + EmitAppendValue(derivedProp, i, $"v_{dt.TypeName}.{derivedProp.PropertyName}"); + } + else + { + EmitAppendNull(unionProp, i); + } + } + + Line("break;"); + _indent--; + Line("}"); } - Line($"result.Add(new {dt.FullTypeName}"); + Line($"default:"); + _indent++; + Line($"throw new System.ArgumentException($\"Unknown derived type: {{value.GetType().Name}}\");"); + _indent--; + + _indent--; + Line("}"); + + _indent--; + Line("}"); + + Line(); + Line("var columns = new IArrowArray[]"); Line("{"); _indent++; - for (int i = 0; i < dt.Properties.Count; i++) + Line("discriminatorBuilder.Build(),"); + for (int i = 0; i < _model.UnionProperties.Count; i++) { - var prop = dt.Properties[i]; - Line($"{prop.PropertyName} = prop_{i},"); + EmitBuildColumn(i); } _indent--; - Line("});"); - Line("break;"); + Line("};"); + Line("return new RecordBatch(_arrowSchema, columns, values.Count);"); _indent--; Line("}"); } - Line("default:"); - _indent++; - Line("throw new System.ArgumentException($\"Unknown type discriminator: {discriminator}\");"); - _indent--; + private void EmitBuildColumn(int index) + { + var prop = _model.UnionProperties[index]; + if (prop.Type.Kind == ArrowTypeKind.Enum) + { + // Build DictionaryArray from index + dict builders + Line($"new DictionaryArray(new DictionaryType(Int16Type.Default, StringType.Default, false), bld_{index}_idx.Build(), new StringArray.Builder().AppendRange(bld_{index}_dict).Build()),"); + } + else + { + Line($"bld_{index}.Build(),"); + } + } - _indent--; - Line("}"); + private void EmitMultiRowDeserialize() + { + Line($"public static IReadOnlyList<{_model.TypeName}> ListFromRecordBatch(RecordBatch batch)"); + Line("{"); + _indent++; - _indent--; - Line("}"); - Line("return result;"); + Line("var discriminatorCol = (StringArray)batch.Column(0);"); - _indent--; - Line("}"); - } + // Cast all union property columns + for (int i = 0; i < _model.UnionProperties.Count; i++) + { + var prop = _model.UnionProperties[i]; + var colIndex = i + 1; + var castType = CodeEmitterHelpers.GetArrayCastType(prop); + Line($"var col_{i} = ({castType})batch.Column({colIndex});"); + } - // --- Helpers --- + Line(); + Line($"var result = new List<{_model.TypeName}>(batch.Length);"); + Line("for (int row = 0; row < batch.Length; row++)"); + Line("{"); + _indent++; - private void EmitBuilderDeclaration(PropertyModel prop, int index) - { - if (prop.Type.Kind == TypeKind2.Enum) - { - // Enum uses Dictionary(Int16, Utf8) — built manually - Line($"var bld_{index}_idx = new Int16Array.Builder();"); - Line($"var bld_{index}_dict = new List();"); - } - else - { - var builderType = CodeEmitterHelpers.GetNullableBuilderDeclaration(prop); - Line($"var bld_{index} = {builderType};"); - } - } + Line("var discriminator = discriminatorCol.GetString(row)!;"); + Line("switch (discriminator)"); + Line("{"); + _indent++; - private void EmitAppendValue(PropertyModel prop, int index, string access) - { - // For non-nullable properties in the derived type, we still need to handle them - // as nullable in the union schema - switch (prop.Type.Kind) - { - case TypeKind2.String: - if (prop.IsNullable) - Line($"if ({access} != null) bld_{index}.Append({access}); else bld_{index}.AppendNull();"); - else - Line($"bld_{index}.Append({access});"); - break; - case TypeKind2.Bool: - case TypeKind2.Byte: - case TypeKind2.SByte: - case TypeKind2.Int16: - case TypeKind2.UInt16: - case TypeKind2.Int32: - case TypeKind2.UInt32: - case TypeKind2.Int64: - case TypeKind2.UInt64: - case TypeKind2.Float: - case TypeKind2.Double: - case TypeKind2.Half: - if (prop.IsNullable) - Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(v_{index}); else bld_{index}.AppendNull();"); - else - Line($"bld_{index}.Append({access});"); - break; - case TypeKind2.DateTime: - case TypeKind2.DateTimeOffset: - if (prop.IsNullable) - Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(Apache.Arrow.Serialization.ArrowArrayHelper.ToUtcDateTimeOffset(v_{index})); else bld_{index}.AppendNull();"); - else - Line($"bld_{index}.Append(Apache.Arrow.Serialization.ArrowArrayHelper.ToUtcDateTimeOffset({access}));"); - break; - case TypeKind2.DateOnly: - if (prop.IsNullable) - Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(v_{index}); else bld_{index}.AppendNull();"); - else - Line($"bld_{index}.Append({access});"); - break; - case TypeKind2.Decimal: - if (prop.IsNullable) - Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(v_{index}); else bld_{index}.AppendNull();"); - else - Line($"bld_{index}.Append({access});"); - break; - case TypeKind2.Enum: - if (prop.IsNullable) - Line($"if ({access} is {{ }} v_{index}) {{ bld_{index}_idx.Append((short)bld_{index}_dict.Count); bld_{index}_dict.Add(v_{index}.ToString()); }} else bld_{index}_idx.AppendNull();"); - else + foreach (var dt in _model.DerivedTypes) + { + Line($"case \"{Escape(dt.TypeDiscriminator)}\":"); + Line("{"); + _indent++; + + for (int i = 0; i < dt.Properties.Count; i++) { - Line($"bld_{index}_idx.Append((short)bld_{index}_dict.Count);"); - Line($"bld_{index}_dict.Add({access}.ToString());"); + var prop = dt.Properties[i]; + var unionIndex = FindUnionPropertyIndex(prop.FieldName); + EmitMultiRowReadProperty(prop, i, unionIndex); } - break; - default: - // For complex types (Binary, Guid, TimeOnly, TimeSpan, nested, collections), - // fall back to AppendNull for now — these are less common in polymorphic scenarios - Line($"bld_{index}.AppendNull(); // TODO: complex type {prop.Type.Kind}"); - break; + + Line($"result.Add(new {dt.FullTypeName}"); + Line("{"); + _indent++; + for (int i = 0; i < dt.Properties.Count; i++) + { + var prop = dt.Properties[i]; + Line($"{prop.PropertyName} = prop_{i},"); + } + _indent--; + Line("});"); + Line("break;"); + + _indent--; + Line("}"); + } + + Line("default:"); + _indent++; + Line("throw new System.ArgumentException($\"Unknown type discriminator: {discriminator}\");"); + _indent--; + + _indent--; + Line("}"); + + _indent--; + Line("}"); + Line("return result;"); + + _indent--; + Line("}"); } - } - private void EmitAppendNull(PropertyModel prop, int index) - { - if (prop.Type.Kind == TypeKind2.Enum) - Line($"bld_{index}_idx.AppendNull();"); - else - Line($"bld_{index}.AppendNull();"); - } + // --- Helpers --- - private void EmitReadProperty(PropertyModel prop, int propIndex, int colIndex) - { - var col = $"batch.Column({colIndex})"; - switch (prop.Type.Kind) + private void EmitBuilderDeclaration(PropertyModel prop, int index) { - case TypeKind2.String: - if (prop.IsNullable) - Line($"var prop_{propIndex} = ((StringArray){col}).IsNull(0) ? null : ((StringArray){col}).GetString(0);"); - else - Line($"var prop_{propIndex} = ((StringArray){col}).GetString(0)!;"); - break; - case TypeKind2.Bool: - case TypeKind2.Byte: - case TypeKind2.SByte: - case TypeKind2.Int16: - case TypeKind2.UInt16: - case TypeKind2.Int32: - case TypeKind2.UInt32: - case TypeKind2.Int64: - case TypeKind2.UInt64: - case TypeKind2.Float: - case TypeKind2.Double: - case TypeKind2.Half: - { - var castType = CodeEmitterHelpers.GetArrayCastType(prop); - if (prop.IsNullable) - Line($"var prop_{propIndex} = (({castType}){col}).IsNull(0) ? ({prop.Type.FullTypeName}?)null : (({castType}){col}).GetValue(0).Value;"); - else - Line($"var prop_{propIndex} = (({castType}){col}).GetValue(0).Value;"); - break; - } - case TypeKind2.DateTime: + if (prop.Type.Kind == ArrowTypeKind.Enum) { - if (prop.IsNullable) - Line($"var prop_{propIndex} = ((TimestampArray){col}).IsNull(0) ? (System.DateTime?)null : ((TimestampArray){col}).GetTimestamp(0)!.Value.UtcDateTime;"); - else - Line($"var prop_{propIndex} = ((TimestampArray){col}).GetTimestamp(0)!.Value.UtcDateTime;"); - break; + // Enum uses Dictionary(Int16, Utf8) — built manually. + // The lookup deduplicates repeated values so each name appears once in the dictionary. + Line($"var bld_{index}_idx = new Int16Array.Builder();"); + Line($"var bld_{index}_dict = new List();"); + Line($"var bld_{index}_lookup = new Dictionary();"); } - case TypeKind2.DateTimeOffset: + else { - if (prop.IsNullable) - Line($"var prop_{propIndex} = ((TimestampArray){col}).IsNull(0) ? (System.DateTimeOffset?)null : ((TimestampArray){col}).GetTimestamp(0)!.Value;"); - else - Line($"var prop_{propIndex} = ((TimestampArray){col}).GetTimestamp(0)!.Value;"); - break; + var builderType = CodeEmitterHelpers.GetNullableBuilderDeclaration(prop); + Line($"var bld_{index} = {builderType};"); } - case TypeKind2.DateOnly: + } + + private void EmitAppendValue(PropertyModel prop, int index, string access) + { + // For non-nullable properties in the derived type, we still need to handle them + // as nullable in the union schema + switch (prop.Type.Kind) { - var castType = CodeEmitterHelpers.GetArrayCastType(prop); - if (prop.IsNullable) - Line($"var prop_{propIndex} = (({castType}){col}).IsNull(0) ? (System.DateOnly?)null : (({castType}){col}).GetDateOnly(0);"); - else - Line($"var prop_{propIndex} = (({castType}){col}).GetDateOnly(0)!.Value;"); - break; + case ArrowTypeKind.String: + if (prop.IsNullable) + Line($"if ({access} != null) bld_{index}.Append({access}); else bld_{index}.AppendNull();"); + else + Line($"bld_{index}.Append({access});"); + break; + case ArrowTypeKind.Bool: + case ArrowTypeKind.Byte: + case ArrowTypeKind.SByte: + case ArrowTypeKind.Int16: + case ArrowTypeKind.UInt16: + case ArrowTypeKind.Int32: + case ArrowTypeKind.UInt32: + case ArrowTypeKind.Int64: + case ArrowTypeKind.UInt64: + case ArrowTypeKind.Float: + case ArrowTypeKind.Double: + case ArrowTypeKind.Half: + if (prop.IsNullable) + Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(v_{index}); else bld_{index}.AppendNull();"); + else + Line($"bld_{index}.Append({access});"); + break; + case ArrowTypeKind.DateTime: + case ArrowTypeKind.DateTimeOffset: + if (prop.IsNullable) + Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(Apache.Arrow.Serialization.ArrowArrayHelper.ToUtcDateTimeOffset(v_{index})); else bld_{index}.AppendNull();"); + else + Line($"bld_{index}.Append(Apache.Arrow.Serialization.ArrowArrayHelper.ToUtcDateTimeOffset({access}));"); + break; + case ArrowTypeKind.DateOnly: + if (prop.IsNullable) + Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(v_{index}); else bld_{index}.AppendNull();"); + else + Line($"bld_{index}.Append({access});"); + break; + case ArrowTypeKind.Decimal: + if (prop.IsNullable) + Line($"if ({access} is {{ }} v_{index}) bld_{index}.Append(v_{index}); else bld_{index}.AppendNull();"); + else + Line($"bld_{index}.Append({access});"); + break; + case ArrowTypeKind.Enum: + if (prop.IsNullable) + { + Line($"if ({access} is {{ }} v_{index}) {{ var name_{index} = v_{index}.ToString(); if (!bld_{index}_lookup.TryGetValue(name_{index}, out var dictIdx_{index})) {{ dictIdx_{index} = (short)bld_{index}_dict.Count; bld_{index}_lookup[name_{index}] = dictIdx_{index}; bld_{index}_dict.Add(name_{index}); }} bld_{index}_idx.Append(dictIdx_{index}); }} else bld_{index}_idx.AppendNull();"); + } + else + { + Line($"var name_{index} = {access}.ToString();"); + Line($"if (!bld_{index}_lookup.TryGetValue(name_{index}, out var dictIdx_{index})) {{ dictIdx_{index} = (short)bld_{index}_dict.Count; bld_{index}_lookup[name_{index}] = dictIdx_{index}; bld_{index}_dict.Add(name_{index}); }}"); + Line($"bld_{index}_idx.Append(dictIdx_{index});"); + } + break; + default: + // Complex types (Binary, Guid, TimeOnly, TimeSpan, nested records, collections) + // are not supported in polymorphic hierarchies — fail loudly instead of losing data. + Line($"throw new System.NotSupportedException(\"Property '{prop.PropertyName}' of type kind {prop.Type.Kind} is not supported in polymorphic Arrow serialization.\");"); + break; } - case TypeKind2.Decimal: + } + + private void EmitAppendNull(PropertyModel prop, int index) + { + if (prop.Type.Kind == ArrowTypeKind.Enum) + Line($"bld_{index}_idx.AppendNull();"); + else + Line($"bld_{index}.AppendNull();"); + } + + private void EmitReadProperty(PropertyModel prop, int propIndex, int colIndex) + { + var col = $"batch.Column({colIndex})"; + switch (prop.Type.Kind) { - if (prop.IsNullable) - Line($"var prop_{propIndex} = ((Decimal128Array){col}).IsNull(0) ? (decimal?)null : ((Decimal128Array){col}).GetValue(0).Value;"); - else - Line($"var prop_{propIndex} = ((Decimal128Array){col}).GetValue(0)!.Value;"); - break; + case ArrowTypeKind.String: + if (prop.IsNullable) + Line($"var prop_{propIndex} = ((StringArray){col}).IsNull(0) ? null : ((StringArray){col}).GetString(0);"); + else + Line($"var prop_{propIndex} = ((StringArray){col}).GetString(0)!;"); + break; + case ArrowTypeKind.Bool: + case ArrowTypeKind.Byte: + case ArrowTypeKind.SByte: + case ArrowTypeKind.Int16: + case ArrowTypeKind.UInt16: + case ArrowTypeKind.Int32: + case ArrowTypeKind.UInt32: + case ArrowTypeKind.Int64: + case ArrowTypeKind.UInt64: + case ArrowTypeKind.Float: + case ArrowTypeKind.Double: + case ArrowTypeKind.Half: + { + var castType = CodeEmitterHelpers.GetArrayCastType(prop); + if (prop.IsNullable) + Line($"var prop_{propIndex} = (({castType}){col}).IsNull(0) ? ({prop.Type.FullTypeName}?)null : (({castType}){col}).GetValue(0).Value;"); + else + Line($"var prop_{propIndex} = (({castType}){col}).GetValue(0).Value;"); + break; + } + case ArrowTypeKind.DateTime: + { + if (prop.IsNullable) + Line($"var prop_{propIndex} = ((TimestampArray){col}).IsNull(0) ? (System.DateTime?)null : ((TimestampArray){col}).GetTimestamp(0)!.Value.UtcDateTime;"); + else + Line($"var prop_{propIndex} = ((TimestampArray){col}).GetTimestamp(0)!.Value.UtcDateTime;"); + break; + } + case ArrowTypeKind.DateTimeOffset: + { + if (prop.IsNullable) + Line($"var prop_{propIndex} = ((TimestampArray){col}).IsNull(0) ? (System.DateTimeOffset?)null : ((TimestampArray){col}).GetTimestamp(0)!.Value;"); + else + Line($"var prop_{propIndex} = ((TimestampArray){col}).GetTimestamp(0)!.Value;"); + break; + } + case ArrowTypeKind.DateOnly: + { + var castType = CodeEmitterHelpers.GetArrayCastType(prop); + if (prop.IsNullable) + Line($"var prop_{propIndex} = (({castType}){col}).IsNull(0) ? (System.DateOnly?)null : (({castType}){col}).GetDateOnly(0);"); + else + Line($"var prop_{propIndex} = (({castType}){col}).GetDateOnly(0)!.Value;"); + break; + } + case ArrowTypeKind.Decimal: + { + if (prop.IsNullable) + Line($"var prop_{propIndex} = ((Decimal128Array){col}).IsNull(0) ? (decimal?)null : ((Decimal128Array){col}).GetValue(0).Value;"); + else + Line($"var prop_{propIndex} = ((Decimal128Array){col}).GetValue(0)!.Value;"); + break; + } + case ArrowTypeKind.Enum: + { + var enumType = prop.Type.FullTypeName; + if (prop.IsNullable) + { + Line($"{enumType}? prop_{propIndex} = null;"); + Line($"{{ var da = (DictionaryArray){col}; if (!da.IsNull(0)) {{ var idx = ((Int16Array)da.Indices).GetValue(0).Value; var name = ((StringArray)da.Dictionary).GetString(idx); prop_{propIndex} = System.Enum.Parse<{enumType}>(name!); }} }}"); + } + else + { + Line($"var da_{propIndex} = (DictionaryArray){col};"); + Line($"var idx_{propIndex} = ((Int16Array)da_{propIndex}.Indices).GetValue(0).Value;"); + Line($"var prop_{propIndex} = System.Enum.Parse<{enumType}>(((StringArray)da_{propIndex}.Dictionary).GetString(idx_{propIndex})!);"); + } + break; + } + default: + Line($"var prop_{propIndex} = Apache.Arrow.Serialization.ArrowArrayHelper.ThrowNotSupported<{prop.Type.FullTypeName}>(\"Property '{prop.PropertyName}' of type kind {prop.Type.Kind} is not supported in polymorphic Arrow serialization.\");"); + break; } - case TypeKind2.Enum: - { - var enumType = prop.Type.FullTypeName; - if (prop.IsNullable) - { - Line($"{enumType}? prop_{propIndex} = null;"); - Line($"{{ var da = (DictionaryArray){col}; if (!da.IsNull(0)) {{ var idx = ((Int16Array)da.Indices).GetValue(0).Value; var name = ((StringArray)da.Dictionary).GetString(idx); prop_{propIndex} = System.Enum.Parse<{enumType}>(name!); }} }}"); - } - else - { - Line($"var da_{propIndex} = (DictionaryArray){col};"); - Line($"var idx_{propIndex} = ((Int16Array)da_{propIndex}.Indices).GetValue(0).Value;"); - Line($"var prop_{propIndex} = System.Enum.Parse<{enumType}>(((StringArray)da_{propIndex}.Dictionary).GetString(idx_{propIndex})!);"); - } - break; - } - default: - Line($"object? prop_{propIndex} = null; // TODO: unsupported type {prop.Type.Kind}"); - break; } - } - private void EmitMultiRowReadProperty(PropertyModel prop, int propIndex, int unionIndex) - { - var col = $"col_{unionIndex}"; - switch (prop.Type.Kind) + private void EmitMultiRowReadProperty(PropertyModel prop, int propIndex, int unionIndex) { - case TypeKind2.String: - if (prop.IsNullable) - Line($"var prop_{propIndex} = {col}.IsNull(row) ? null : {col}.GetString(row);"); - else - Line($"var prop_{propIndex} = {col}.GetString(row)!;"); - break; - case TypeKind2.Bool: - case TypeKind2.Byte: - case TypeKind2.SByte: - case TypeKind2.Int16: - case TypeKind2.UInt16: - case TypeKind2.Int32: - case TypeKind2.UInt32: - case TypeKind2.Int64: - case TypeKind2.UInt64: - case TypeKind2.Float: - case TypeKind2.Double: - case TypeKind2.Half: - if (prop.IsNullable) - Line($"var prop_{propIndex} = {col}.IsNull(row) ? ({prop.Type.FullTypeName}?)null : {col}.GetValue(row).Value;"); - else - Line($"var prop_{propIndex} = {col}.GetValue(row).Value;"); - break; - case TypeKind2.DateTime: - if (prop.IsNullable) - Line($"var prop_{propIndex} = {col}.IsNull(row) ? (System.DateTime?)null : {col}.GetTimestamp(row)!.Value.UtcDateTime;"); - else - Line($"var prop_{propIndex} = {col}.GetTimestamp(row)!.Value.UtcDateTime;"); - break; - case TypeKind2.DateTimeOffset: - if (prop.IsNullable) - Line($"var prop_{propIndex} = {col}.IsNull(row) ? (System.DateTimeOffset?)null : {col}.GetTimestamp(row)!.Value;"); - else - Line($"var prop_{propIndex} = {col}.GetTimestamp(row)!.Value;"); - break; - case TypeKind2.DateOnly: - if (prop.IsNullable) - Line($"var prop_{propIndex} = {col}.IsNull(row) ? (System.DateOnly?)null : {col}.GetDateOnly(row);"); - else - Line($"var prop_{propIndex} = {col}.GetDateOnly(row)!.Value;"); - break; - case TypeKind2.Decimal: - if (prop.IsNullable) - Line($"var prop_{propIndex} = {col}.IsNull(row) ? (decimal?)null : {col}.GetValue(row).Value;"); - else - Line($"var prop_{propIndex} = {col}.GetValue(row)!.Value;"); - break; - case TypeKind2.Enum: + var col = $"col_{unionIndex}"; + switch (prop.Type.Kind) { - var enumType = prop.Type.FullTypeName; - if (prop.IsNullable) - { - Line($"{enumType}? prop_{propIndex} = null;"); - Line($"if (!{col}.IsNull(row)) {{ var idx = ((Int16Array){col}.Indices).GetValue(row).Value; var name = ((StringArray){col}.Dictionary).GetString(idx); prop_{propIndex} = System.Enum.Parse<{enumType}>(name!); }}"); - } - else - { - Line($"var idx_{propIndex} = ((Int16Array){col}.Indices).GetValue(row).Value;"); - Line($"var prop_{propIndex} = System.Enum.Parse<{enumType}>(((StringArray){col}.Dictionary).GetString(idx_{propIndex})!);"); - } - break; + case ArrowTypeKind.String: + if (prop.IsNullable) + Line($"var prop_{propIndex} = {col}.IsNull(row) ? null : {col}.GetString(row);"); + else + Line($"var prop_{propIndex} = {col}.GetString(row)!;"); + break; + case ArrowTypeKind.Bool: + case ArrowTypeKind.Byte: + case ArrowTypeKind.SByte: + case ArrowTypeKind.Int16: + case ArrowTypeKind.UInt16: + case ArrowTypeKind.Int32: + case ArrowTypeKind.UInt32: + case ArrowTypeKind.Int64: + case ArrowTypeKind.UInt64: + case ArrowTypeKind.Float: + case ArrowTypeKind.Double: + case ArrowTypeKind.Half: + if (prop.IsNullable) + Line($"var prop_{propIndex} = {col}.IsNull(row) ? ({prop.Type.FullTypeName}?)null : {col}.GetValue(row).Value;"); + else + Line($"var prop_{propIndex} = {col}.GetValue(row).Value;"); + break; + case ArrowTypeKind.DateTime: + if (prop.IsNullable) + Line($"var prop_{propIndex} = {col}.IsNull(row) ? (System.DateTime?)null : {col}.GetTimestamp(row)!.Value.UtcDateTime;"); + else + Line($"var prop_{propIndex} = {col}.GetTimestamp(row)!.Value.UtcDateTime;"); + break; + case ArrowTypeKind.DateTimeOffset: + if (prop.IsNullable) + Line($"var prop_{propIndex} = {col}.IsNull(row) ? (System.DateTimeOffset?)null : {col}.GetTimestamp(row)!.Value;"); + else + Line($"var prop_{propIndex} = {col}.GetTimestamp(row)!.Value;"); + break; + case ArrowTypeKind.DateOnly: + if (prop.IsNullable) + Line($"var prop_{propIndex} = {col}.IsNull(row) ? (System.DateOnly?)null : {col}.GetDateOnly(row);"); + else + Line($"var prop_{propIndex} = {col}.GetDateOnly(row)!.Value;"); + break; + case ArrowTypeKind.Decimal: + if (prop.IsNullable) + Line($"var prop_{propIndex} = {col}.IsNull(row) ? (decimal?)null : {col}.GetValue(row).Value;"); + else + Line($"var prop_{propIndex} = {col}.GetValue(row)!.Value;"); + break; + case ArrowTypeKind.Enum: + { + var enumType = prop.Type.FullTypeName; + if (prop.IsNullable) + { + Line($"{enumType}? prop_{propIndex} = null;"); + Line($"if (!{col}.IsNull(row)) {{ var idx = ((Int16Array){col}.Indices).GetValue(row).Value; var name = ((StringArray){col}.Dictionary).GetString(idx); prop_{propIndex} = System.Enum.Parse<{enumType}>(name!); }}"); + } + else + { + Line($"var idx_{propIndex} = ((Int16Array){col}.Indices).GetValue(row).Value;"); + Line($"var prop_{propIndex} = System.Enum.Parse<{enumType}>(((StringArray){col}.Dictionary).GetString(idx_{propIndex})!);"); + } + break; + } + default: + Line($"var prop_{propIndex} = Apache.Arrow.Serialization.ArrowArrayHelper.ThrowNotSupported<{prop.Type.FullTypeName}>(\"Property '{prop.PropertyName}' of type kind {prop.Type.Kind} is not supported in polymorphic Arrow serialization.\");"); + break; } - default: - Line($"object? prop_{propIndex} = null; // TODO: unsupported type {prop.Type.Kind}"); - break; } - } - private static PropertyModel? FindProperty(DerivedTypeInfo dt, string fieldName) - { - foreach (var prop in dt.Properties) + private static PropertyModel? FindProperty(DerivedTypeInfo dt, string fieldName) { - if (prop.FieldName == fieldName) - return prop; + foreach (var prop in dt.Properties) + { + if (prop.FieldName == fieldName) + return prop; + } + return null; } - return null; - } - private int FindUnionPropertyIndex(string fieldName) - { - for (int i = 0; i < _model.UnionProperties.Count; i++) + private int FindUnionPropertyIndex(string fieldName) { - if (_model.UnionProperties[i].FieldName == fieldName) - return i; + for (int i = 0; i < _model.UnionProperties.Count; i++) + { + if (_model.UnionProperties[i].FieldName == fieldName) + return i; + } + return -1; } - return -1; - } - private static string Escape(string s) => s.Replace("\\", "\\\\").Replace("\"", "\\\""); + private static string Escape(string s) => s.Replace("\\", "\\\\").Replace("\"", "\\\""); - private void Line(string text = "") - { - if (string.IsNullOrEmpty(text)) + private void Line(string text = "") { - _sb.AppendLine(); - return; + if (string.IsNullOrEmpty(text)) + { + _sb.AppendLine(); + return; + } + _sb.Append(' ', _indent * 4); + _sb.AppendLine(text); } - _sb.Append(' ', _indent * 4); - _sb.AppendLine(text); } -} -/// -/// Shared helpers used by both CodeEmitter and PolymorphicCodeEmitter. -/// -internal static class CodeEmitterHelpers -{ /// - /// Returns the Arrow type expression for a TypeInfo (e.g., "Int32Type.Default"). + /// Shared helpers used by both CodeEmitter and PolymorphicCodeEmitter. /// - public static string GetArrowTypeExpression(TypeInfo type) + internal static class CodeEmitterHelpers { - if (type.ArrowTypeOverride != null) - return ParseArrowTypeOverride(type.ArrowTypeOverride); - - return type.Kind switch + /// + /// Returns the Arrow type expression for a TypeInfo (e.g., "Int32Type.Default"). + /// + public static string GetArrowTypeExpression(TypeInfo type) { - TypeKind2.String => "StringType.Default", - TypeKind2.Bool => "BooleanType.Default", - TypeKind2.Byte => "UInt8Type.Default", - TypeKind2.SByte => "Int8Type.Default", - TypeKind2.Int16 => "Int16Type.Default", - TypeKind2.UInt16 => "UInt16Type.Default", - TypeKind2.Int32 => "Int32Type.Default", - TypeKind2.UInt32 => "UInt32Type.Default", - TypeKind2.Int64 => "Int64Type.Default", - TypeKind2.UInt64 => "UInt64Type.Default", - TypeKind2.Float => "FloatType.Default", - TypeKind2.Double => "DoubleType.Default", - TypeKind2.Decimal => "new Decimal128Type(38, 18)", - TypeKind2.Binary => "BinaryType.Default", - TypeKind2.DateTime => "new TimestampType(TimeUnit.Microsecond, \"UTC\")", - TypeKind2.DateTimeOffset => "new TimestampType(TimeUnit.Microsecond, \"UTC\")", - TypeKind2.DateOnly => "Date32Type.Default", - TypeKind2.TimeOnly => "new Time64Type(TimeUnit.Microsecond)", - TypeKind2.TimeSpan => "DurationType.Microsecond", - TypeKind2.Guid => "GuidType.Default", - TypeKind2.Half => "HalfFloatType.Default", - TypeKind2.Enum => "new DictionaryType(Int16Type.Default, StringType.Default, false)", - TypeKind2.List or TypeKind2.Array or TypeKind2.Set => - $"new ListType(new Field(\"item\", {GetArrowTypeExpression(type.ElementType!)}, {(type.ElementType!.IsNullable ? "true" : "false")}))", - TypeKind2.Dictionary => - $"new MapType(new Field(\"key\", {GetArrowTypeExpression(type.KeyType!)}, false), new Field(\"value\", {GetArrowTypeExpression(type.ValueType!)}, true))", - TypeKind2.NestedRecord => - $"new StructType({type.FullTypeName}.ArrowSchema.FieldsList)", - _ => "StringType.Default /* unsupported */", - }; - } + if (type.ArrowTypeOverride != null) + return ParseArrowTypeOverride(type.ArrowTypeOverride); - private static string ParseArrowTypeOverride(string typeName) - { - var lower = typeName.ToLowerInvariant().Trim(); - return lower switch + return type.Kind switch + { + ArrowTypeKind.String => "StringType.Default", + ArrowTypeKind.Bool => "BooleanType.Default", + ArrowTypeKind.Byte => "UInt8Type.Default", + ArrowTypeKind.SByte => "Int8Type.Default", + ArrowTypeKind.Int16 => "Int16Type.Default", + ArrowTypeKind.UInt16 => "UInt16Type.Default", + ArrowTypeKind.Int32 => "Int32Type.Default", + ArrowTypeKind.UInt32 => "UInt32Type.Default", + ArrowTypeKind.Int64 => "Int64Type.Default", + ArrowTypeKind.UInt64 => "UInt64Type.Default", + ArrowTypeKind.Float => "FloatType.Default", + ArrowTypeKind.Double => "DoubleType.Default", + ArrowTypeKind.Decimal => "new Decimal128Type(38, 18)", + ArrowTypeKind.Binary => "BinaryType.Default", + ArrowTypeKind.DateTime => "new TimestampType(TimeUnit.Microsecond, \"UTC\")", + ArrowTypeKind.DateTimeOffset => "new TimestampType(TimeUnit.Microsecond, \"UTC\")", + ArrowTypeKind.DateOnly => "Date32Type.Default", + ArrowTypeKind.TimeOnly => "new Time64Type(TimeUnit.Microsecond)", + ArrowTypeKind.TimeSpan => "DurationType.Microsecond", + ArrowTypeKind.Guid => "GuidType.Default", + ArrowTypeKind.Half => "HalfFloatType.Default", + ArrowTypeKind.Enum => "new DictionaryType(Int16Type.Default, StringType.Default, false)", + ArrowTypeKind.List or ArrowTypeKind.Array or ArrowTypeKind.Set => + $"new ListType(new Field(\"item\", {GetArrowTypeExpression(type.ElementType!)}, {(type.ElementType!.IsNullable ? "true" : "false")}))", + ArrowTypeKind.Dictionary => + $"new MapType(new Field(\"key\", {GetArrowTypeExpression(type.KeyType!)}, false), new Field(\"value\", {GetArrowTypeExpression(type.ValueType!)}, true))", + ArrowTypeKind.NestedRecord => + $"new StructType({type.FullTypeName}.ArrowSchema.FieldsList)", + _ => "StringType.Default /* unsupported */", + }; + } + + private static string ParseArrowTypeOverride(string typeName) { - "date32" => "Date32Type.Default", - "date64" => "Date64Type.Default", - _ => $"StringType.Default /* unrecognized: {typeName} */", - }; - } + var lower = typeName.ToLowerInvariant().Trim(); + return lower switch + { + "date32" => "Date32Type.Default", + "date64" => "Date64Type.Default", + _ => $"StringType.Default /* unrecognized: {typeName} */", + }; + } - /// - /// Returns the Arrow array type for casting (e.g., "Int32Array", "StringArray"). - /// - public static string GetArrayCastType(PropertyModel prop) - { - return prop.Type.Kind switch + /// + /// Returns the Arrow array type for casting (e.g., "Int32Array", "StringArray"). + /// + public static string GetArrayCastType(PropertyModel prop) { - TypeKind2.String => "StringArray", - TypeKind2.Bool => "BooleanArray", - TypeKind2.Byte => "UInt8Array", - TypeKind2.SByte => "Int8Array", - TypeKind2.Int16 => "Int16Array", - TypeKind2.UInt16 => "UInt16Array", - TypeKind2.Int32 => "Int32Array", - TypeKind2.UInt32 => "UInt32Array", - TypeKind2.Int64 => "Int64Array", - TypeKind2.UInt64 => "UInt64Array", - TypeKind2.Float => "FloatArray", - TypeKind2.Double => "DoubleArray", - TypeKind2.Half => "HalfFloatArray", - TypeKind2.Decimal => "Decimal128Array", - TypeKind2.DateTime => "TimestampArray", - TypeKind2.DateTimeOffset => "TimestampArray", - TypeKind2.DateOnly => "Date32Array", - TypeKind2.TimeOnly => "Time64Array", - TypeKind2.TimeSpan => "DurationArray", - TypeKind2.Guid => "GuidArray", - TypeKind2.Binary => "BinaryArray", - TypeKind2.Enum => "DictionaryArray", - TypeKind2.List or TypeKind2.Array or TypeKind2.Set => "ListArray", - TypeKind2.Dictionary => "MapArray", - TypeKind2.NestedRecord => "StructArray", - _ => "IArrowArray", - }; - } + return prop.Type.Kind switch + { + ArrowTypeKind.String => "StringArray", + ArrowTypeKind.Bool => "BooleanArray", + ArrowTypeKind.Byte => "UInt8Array", + ArrowTypeKind.SByte => "Int8Array", + ArrowTypeKind.Int16 => "Int16Array", + ArrowTypeKind.UInt16 => "UInt16Array", + ArrowTypeKind.Int32 => "Int32Array", + ArrowTypeKind.UInt32 => "UInt32Array", + ArrowTypeKind.Int64 => "Int64Array", + ArrowTypeKind.UInt64 => "UInt64Array", + ArrowTypeKind.Float => "FloatArray", + ArrowTypeKind.Double => "DoubleArray", + ArrowTypeKind.Half => "HalfFloatArray", + ArrowTypeKind.Decimal => "Decimal128Array", + ArrowTypeKind.DateTime => "TimestampArray", + ArrowTypeKind.DateTimeOffset => "TimestampArray", + ArrowTypeKind.DateOnly => "Date32Array", + ArrowTypeKind.TimeOnly => "Time64Array", + ArrowTypeKind.TimeSpan => "DurationArray", + ArrowTypeKind.Guid => "GuidArray", + ArrowTypeKind.Binary => "BinaryArray", + ArrowTypeKind.Enum => "DictionaryArray", + ArrowTypeKind.List or ArrowTypeKind.Array or ArrowTypeKind.Set => "ListArray", + ArrowTypeKind.Dictionary => "MapArray", + ArrowTypeKind.NestedRecord => "StructArray", + _ => "IArrowArray", + }; + } - /// - /// Returns a "new SomeBuilder()" expression for nullable builders. - /// - public static string GetNullableBuilderDeclaration(PropertyModel prop) - { - switch (prop.Type.Kind) + /// + /// Returns a "new SomeBuilder()" expression for nullable builders. + /// + public static string GetNullableBuilderDeclaration(PropertyModel prop) { - case TypeKind2.String: - return "new StringArray.Builder()"; - case TypeKind2.Bool: - return "new BooleanArray.Builder()"; - case TypeKind2.Byte: - return "new UInt8Array.Builder()"; - case TypeKind2.SByte: - return "new Int8Array.Builder()"; - case TypeKind2.Int16: - return "new Int16Array.Builder()"; - case TypeKind2.UInt16: - return "new UInt16Array.Builder()"; - case TypeKind2.Int32: - return "new Int32Array.Builder()"; - case TypeKind2.UInt32: - return "new UInt32Array.Builder()"; - case TypeKind2.Int64: - return "new Int64Array.Builder()"; - case TypeKind2.UInt64: - return "new UInt64Array.Builder()"; - case TypeKind2.Float: - return "new FloatArray.Builder()"; - case TypeKind2.Double: - return "new DoubleArray.Builder()"; - case TypeKind2.Half: - return "new HalfFloatArray.Builder()"; - case TypeKind2.Decimal: - return "new Decimal128Array.Builder(new Decimal128Type(38, 18))"; - case TypeKind2.DateTime: - case TypeKind2.DateTimeOffset: - return "new TimestampArray.Builder(new TimestampType(TimeUnit.Microsecond, \"UTC\"))"; - case TypeKind2.DateOnly: - return "new Date32Array.Builder()"; - case TypeKind2.TimeOnly: - return "new Time64Array.Builder(Time64Type.Default)"; - case TypeKind2.TimeSpan: - return "new DurationArray.Builder(DurationType.Microsecond)"; - default: - return $"new StringArray.Builder() /* fallback for {prop.Type.Kind} */"; + switch (prop.Type.Kind) + { + case ArrowTypeKind.String: + return "new StringArray.Builder()"; + case ArrowTypeKind.Bool: + return "new BooleanArray.Builder()"; + case ArrowTypeKind.Byte: + return "new UInt8Array.Builder()"; + case ArrowTypeKind.SByte: + return "new Int8Array.Builder()"; + case ArrowTypeKind.Int16: + return "new Int16Array.Builder()"; + case ArrowTypeKind.UInt16: + return "new UInt16Array.Builder()"; + case ArrowTypeKind.Int32: + return "new Int32Array.Builder()"; + case ArrowTypeKind.UInt32: + return "new UInt32Array.Builder()"; + case ArrowTypeKind.Int64: + return "new Int64Array.Builder()"; + case ArrowTypeKind.UInt64: + return "new UInt64Array.Builder()"; + case ArrowTypeKind.Float: + return "new FloatArray.Builder()"; + case ArrowTypeKind.Double: + return "new DoubleArray.Builder()"; + case ArrowTypeKind.Half: + return "new HalfFloatArray.Builder()"; + case ArrowTypeKind.Decimal: + return "new Decimal128Array.Builder(new Decimal128Type(38, 18))"; + case ArrowTypeKind.DateTime: + case ArrowTypeKind.DateTimeOffset: + return "new TimestampArray.Builder(new TimestampType(TimeUnit.Microsecond, \"UTC\"))"; + case ArrowTypeKind.DateOnly: + return "new Date32Array.Builder()"; + case ArrowTypeKind.TimeOnly: + return "new Time64Array.Builder(Time64Type.Default)"; + case ArrowTypeKind.TimeSpan: + return "new DurationArray.Builder(DurationType.Microsecond)"; + default: + return $"new StringArray.Builder() /* fallback for {prop.Type.Kind} */"; + } } } -} } // namespace diff --git a/src/Apache.Arrow.Serialization/Apache.Arrow.Serialization.csproj b/src/Apache.Arrow.Serialization/Apache.Arrow.Serialization.csproj index 031efaf9..1a5d3263 100644 --- a/src/Apache.Arrow.Serialization/Apache.Arrow.Serialization.csproj +++ b/src/Apache.Arrow.Serialization/Apache.Arrow.Serialization.csproj @@ -5,8 +5,13 @@ enable enable Source-generated Apache Arrow serialization for .NET. Provides [ArrowSerializable] attribute and IArrowSerializer<T> interface for compile-time Arrow schema derivation, serialization, and deserialization. + README.md + + + + @@ -17,4 +22,13 @@ ReferenceOutputAssembly="false" /> + + + + + diff --git a/src/Apache.Arrow.Serialization/ArrowArrayHelper.cs b/src/Apache.Arrow.Serialization/ArrowArrayHelper.cs index 397e5170..74db0db6 100644 --- a/src/Apache.Arrow.Serialization/ArrowArrayHelper.cs +++ b/src/Apache.Arrow.Serialization/ArrowArrayHelper.cs @@ -33,187 +33,187 @@ public static IArrowArray BuildNullArray(IArrowType type, int length) switch (type) { case BooleanType: - { - var b = new BooleanArray.Builder(); - for (int i = 0; i < length; i++) b.AppendNull(); - return b.Build(); - } + { + var b = new BooleanArray.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } case Bool8Type: - { - var b = new Bool8Array.Builder(); - for (int i = 0; i < length; i++) b.AppendNull(); - return b.Build(); - } + { + var b = new Bool8Array.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } case Int8Type: - { - var b = new Int8Array.Builder(); - for (int i = 0; i < length; i++) b.AppendNull(); - return b.Build(); - } + { + var b = new Int8Array.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } case UInt8Type: - { - var b = new UInt8Array.Builder(); - for (int i = 0; i < length; i++) b.AppendNull(); - return b.Build(); - } + { + var b = new UInt8Array.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } case Int16Type: - { - var b = new Int16Array.Builder(); - for (int i = 0; i < length; i++) b.AppendNull(); - return b.Build(); - } + { + var b = new Int16Array.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } case UInt16Type: - { - var b = new UInt16Array.Builder(); - for (int i = 0; i < length; i++) b.AppendNull(); - return b.Build(); - } + { + var b = new UInt16Array.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } case Int32Type: - { - var b = new Int32Array.Builder(); - for (int i = 0; i < length; i++) b.AppendNull(); - return b.Build(); - } + { + var b = new Int32Array.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } case UInt32Type: - { - var b = new UInt32Array.Builder(); - for (int i = 0; i < length; i++) b.AppendNull(); - return b.Build(); - } + { + var b = new UInt32Array.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } case Int64Type: - { - var b = new Int64Array.Builder(); - for (int i = 0; i < length; i++) b.AppendNull(); - return b.Build(); - } + { + var b = new Int64Array.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } case UInt64Type: - { - var b = new UInt64Array.Builder(); - for (int i = 0; i < length; i++) b.AppendNull(); - return b.Build(); - } + { + var b = new UInt64Array.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } case HalfFloatType: - { - var b = new HalfFloatArray.Builder(); - for (int i = 0; i < length; i++) b.AppendNull(); - return b.Build(); - } + { + var b = new HalfFloatArray.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } case FloatType: - { - var b = new FloatArray.Builder(); - for (int i = 0; i < length; i++) b.AppendNull(); - return b.Build(); - } + { + var b = new FloatArray.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } case DoubleType: - { - var b = new DoubleArray.Builder(); - for (int i = 0; i < length; i++) b.AppendNull(); - return b.Build(); - } + { + var b = new DoubleArray.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } case Decimal128Type dt: - { - var b = new Decimal128Array.Builder(dt); - for (int i = 0; i < length; i++) b.AppendNull(); - return b.Build(); - } + { + var b = new Decimal128Array.Builder(dt); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } case StringType: - { - var b = new StringArray.Builder(); - for (int i = 0; i < length; i++) b.AppendNull(); - return b.Build(); - } + { + var b = new StringArray.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } case BinaryType: - { - var b = new BinaryArray.Builder(); - for (int i = 0; i < length; i++) b.AppendNull(); - return b.Build(); - } + { + var b = new BinaryArray.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } case StringViewType: - { - var b = new StringViewArray.Builder(); - for (int i = 0; i < length; i++) b.AppendNull(); - return b.Build(); - } + { + var b = new StringViewArray.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } case BinaryViewType: - { - var b = new BinaryViewArray.Builder(); - for (int i = 0; i < length; i++) b.AppendNull(); - return b.Build(); - } + { + var b = new BinaryViewArray.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } case GuidType: - { - var b = new GuidArray.Builder(); - for (int i = 0; i < length; i++) b.AppendNull(); - return b.Build(); - } + { + var b = new GuidArray.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } case FixedSizeBinaryType fbt: - { - return BuildNullFixedSizeBinaryArray(fbt, length); - } + { + return BuildNullFixedSizeBinaryArray(fbt, length); + } case Date32Type: - { - var b = new Date32Array.Builder(); - for (int i = 0; i < length; i++) b.AppendNull(); - return b.Build(); - } + { + var b = new Date32Array.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } case Date64Type: - { - var b = new Date64Array.Builder(); - for (int i = 0; i < length; i++) b.AppendNull(); - return b.Build(); - } + { + var b = new Date64Array.Builder(); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } case TimestampType tsType: - { - var b = new TimestampArray.Builder(tsType); - for (int i = 0; i < length; i++) b.AppendNull(); - return b.Build(); - } + { + var b = new TimestampArray.Builder(tsType); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } case Time32Type t32: - { - var b = new Time32Array.Builder(t32); - for (int i = 0; i < length; i++) b.AppendNull(); - return b.Build(); - } + { + var b = new Time32Array.Builder(t32); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } case Time64Type t64: - { - var b = new Time64Array.Builder(t64); - for (int i = 0; i < length; i++) b.AppendNull(); - return b.Build(); - } + { + var b = new Time64Array.Builder(t64); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } case DurationType dur: - { - var b = new DurationArray.Builder(dur); - for (int i = 0; i < length; i++) b.AppendNull(); - return b.Build(); - } + { + var b = new DurationArray.Builder(dur); + for (int i = 0; i < length; i++) b.AppendNull(); + return b.Build(); + } case DictionaryType dt: - { - var idx = new Int16Array.Builder(); - for (int i = 0; i < length; i++) idx.AppendNull(); - var dict = new StringArray.Builder().Build(); - return new DictionaryArray(dt, idx.Build(), dict); - } + { + var idx = new Int16Array.Builder(); + for (int i = 0; i < length; i++) idx.AppendNull(); + var dict = new StringArray.Builder().Build(); + return new DictionaryArray(dt, idx.Build(), dict); + } case StructType st: - { - var children = new IArrowArray[st.Fields.Count]; - for (int i = 0; i < children.Length; i++) - children[i] = BuildNullArray(st.Fields[i].DataType, length); - var bitmapBuilder = new ArrowBuffer.BitmapBuilder(); - for (int i = 0; i < length; i++) bitmapBuilder.Append(false); - return new StructArray(st, length, children, bitmapBuilder.Build(), length); - } + { + var children = new IArrowArray[st.Fields.Count]; + for (int i = 0; i < children.Length; i++) + children[i] = BuildNullArray(st.Fields[i].DataType, length); + var bitmapBuilder = new ArrowBuffer.BitmapBuilder(); + for (int i = 0; i < length; i++) bitmapBuilder.Append(false); + return new StructArray(st, length, children, bitmapBuilder.Build(), length); + } case ListType lt: - { - var lb = new ListArray.Builder(lt.ValueDataType); - for (int i = 0; i < length; i++) lb.AppendNull(); - return lb.Build(); - } + { + var lb = new ListArray.Builder(lt.ValueDataType); + for (int i = 0; i < length; i++) lb.AppendNull(); + return lb.Build(); + } case MapType mt: - { - var mb = new MapArray.Builder(mt); - for (int i = 0; i < length; i++) mb.AppendNull(); - return mb.Build(); - } + { + var mb = new MapArray.Builder(mt); + for (int i = 0; i < length; i++) mb.AppendNull(); + return mb.Build(); + } default: return new NullArray(length); } @@ -406,4 +406,13 @@ public static DateTimeOffset ToWallClockDateTimeOffset(DateTimeOffset value) { return new DateTimeOffset(value.DateTime.Ticks, TimeSpan.Zero); } + + /// + /// Always throws . Used by generated code in + /// expression position for property types that are not supported in a given context. + /// + public static T ThrowNotSupported(string message) + { + throw new NotSupportedException(message); + } } diff --git a/src/Apache.Arrow.Serialization/README.md b/src/Apache.Arrow.Serialization/README.md index 3a0eb041..85554749 100644 --- a/src/Apache.Arrow.Serialization/README.md +++ b/src/Apache.Arrow.Serialization/README.md @@ -159,6 +159,8 @@ Records use `{ get; init; }` properties. Classes and structs use `{ get; set; }` | `ReadOnlyMemory` | `Binary` | Override to `BinaryView` via `[ArrowType("binary_view")]` | | `enum` | `Dictionary(Int16, Utf8)` | Name-based encoding | +> **Note on decimals:** CLR `decimal` and Arrow decimal types do not match exactly. A CLR `decimal` holds 96 bits of mantissa with a floating scale of 0–28, while `Decimal128(38, 18)` is a fixed-point type with 38 significant digits and a fixed scale of 18 fractional digits. Values with more than 18 fractional digits lose precision on write, and values with more than 20 integer digits do not fit. Use `[ArrowType("decimal128(p, s)")]` to pick a precision/scale that matches your data. + ### Collections | C# Type | Arrow Type | diff --git a/src/Apache.Arrow.Serialization/RecordBatchBuilder.cs b/src/Apache.Arrow.Serialization/RecordBatchBuilder.cs index f5825b34..5f96f30b 100644 --- a/src/Apache.Arrow.Serialization/RecordBatchBuilder.cs +++ b/src/Apache.Arrow.Serialization/RecordBatchBuilder.cs @@ -37,27 +37,22 @@ public static class RecordBatchBuilder public static RecordBatch FromObjects(IEnumerable items) { var list = items as IReadOnlyList ?? items.ToList(); - if (list.Count == 0) - throw new ArgumentException("Cannot infer schema from empty collection.", nameof(items)); var properties = typeof(T).GetProperties(BindingFlags.Public | BindingFlags.Instance) .Where(p => p.CanRead) .ToArray(); - var fields = new List(); + var schema = new Schema.Builder(); var builders = new List(); foreach (var prop in properties) { var propType = prop.PropertyType; var (arrowType, nullable) = InferArrowType(propType); - fields.Add(new Field(prop.Name, arrowType, nullable)); + schema.Field(new Field(prop.Name, arrowType, nullable)); builders.Add(CreateColumnBuilder(propType, arrowType)); } - var schema = new Schema.Builder(); - foreach (var f in fields) schema.Field(f); - // Populate builders for (int row = 0; row < list.Count; row++) { @@ -337,7 +332,7 @@ private sealed class DateTimeColumnBuilder : IColumnBuilder public void Append(object? value) { if (value is null) _b.AppendNull(); - else _b.Append(new DateTimeOffset((DateTime)value, TimeSpan.Zero)); + else _b.Append(ArrowArrayHelper.ToUtcDateTimeOffset((DateTime)value)); } public IArrowArray Build() => _b.Build(); } @@ -617,8 +612,21 @@ public IArrowArray Build() var listType = typeof(List<>).MakeGenericType(_clrType); var typedList = (System.Collections.IList)Activator.CreateInstance(listType, length)!; - // For null slots, we need a stand-in value (first non-null item) + // For null slots, we need a stand-in value (first non-null item, + // or a default instance when every item is null) object? standIn = _items.FirstOrDefault(v => v is not null); + if (standIn is null && nullCount > 0) + { + try + { + standIn = Activator.CreateInstance(_clrType); + } + catch (MissingMethodException e) + { + throw new NotSupportedException( + $"Cannot build an all-null struct column for {_clrType.FullName}: the type has no parameterless constructor to use as a null stand-in.", e); + } + } foreach (var item in _items) typedList.Add(item ?? standIn!); diff --git a/test/Apache.Arrow.Serialization.Tests/DiagnosticTests.cs b/test/Apache.Arrow.Serialization.Tests/DiagnosticTests.cs index 1b414b05..98f249cb 100644 --- a/test/Apache.Arrow.Serialization.Tests/DiagnosticTests.cs +++ b/test/Apache.Arrow.Serialization.Tests/DiagnosticTests.cs @@ -14,9 +14,9 @@ // limitations under the License. using System.Collections.Immutable; +using Apache.Arrow.Serialization.Generator; using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp; -using Apache.Arrow.Serialization.Generator; using Xunit; namespace Apache.Arrow.Serialization.Tests; diff --git a/test/Apache.Arrow.Serialization.Tests/RecordBatchBuilderTests.cs b/test/Apache.Arrow.Serialization.Tests/RecordBatchBuilderTests.cs index 5ef14003..9488e2d0 100644 --- a/test/Apache.Arrow.Serialization.Tests/RecordBatchBuilderTests.cs +++ b/test/Apache.Arrow.Serialization.Tests/RecordBatchBuilderTests.cs @@ -14,8 +14,8 @@ // limitations under the License. using Apache.Arrow; -using Apache.Arrow.Types; using Apache.Arrow.Serialization; +using Apache.Arrow.Types; using Xunit; namespace Apache.Arrow.Serialization.Tests; @@ -174,10 +174,16 @@ public void DateTimeAndGuid() } [Fact] - public void EmptyCollection_Throws() + public void EmptyCollection_ProducesEmptyBatch() { - Assert.Throws(() => - RecordBatchBuilder.FromObjects(System.Array.Empty())); + // Schema is inferred from the type, not the data, so an empty + // collection produces a zero-row batch with the full schema. + var data = new[] { new { Name = "x", Age = 1 } }.Take(0); + + var batch = RecordBatchBuilder.FromObjects(data); + + Assert.Equal(0, batch.Length); + Assert.Equal(2, batch.Schema.FieldsList.Count); } [Fact] diff --git a/test/Apache.Arrow.Serialization.Tests/SerializationTests.cs b/test/Apache.Arrow.Serialization.Tests/SerializationTests.cs index c36c7184..4904485a 100644 --- a/test/Apache.Arrow.Serialization.Tests/SerializationTests.cs +++ b/test/Apache.Arrow.Serialization.Tests/SerializationTests.cs @@ -14,8 +14,8 @@ // limitations under the License. using Apache.Arrow; -using Apache.Arrow.Types; using Apache.Arrow.Serialization; +using Apache.Arrow.Types; using Xunit; namespace Apache.Arrow.Serialization.Tests; From 424d64e0dc05b2cdc41ce5ad183b281e417bf5b2 Mon Sep 17 00:00:00 2001 From: Christoph Mettler Date: Tue, 7 Jul 2026 18:45:17 +0200 Subject: [PATCH 7/9] Support .NET Framework and netstandard2.0 targets Multi-target Apache.Arrow.Serialization as netstandard2.0;net462;net8.0 (matching Apache.Arrow) so the serializer works on .NET Framework 4.6.2+ including 4.7.2. The static-abstract IArrowSerializer interface required .NET 7+ runtime support and could not be polyfilled. It is redefined as an instance interface (the resolver pattern used by MessagePack-CSharp and System.Text.Json): the generator now emits, per type, a serializer class delegating to the generated statics and registers it with the new ArrowSerializerRegistry via [ModuleInitializer]. Resolution is one static-generic field read; no reflection, still AOT-safe. The generated statics on each type are unchanged and remain directly usable. - Add IArrowSerializable marker, implemented by generated partials; the generic extension methods keep their exact signatures on all TFMs with a `where T : IArrowSerializable` constraint and registry dispatch - Generator emits an internal ModuleInitializerAttribute polyfill when the consumer compilation lacks an accessible one (referenced assemblies such as Microsoft.CodeAnalysis carry inaccessible internal copies, so the probe checks accessibility, not mere existence) - Replace emitted Enum.Parse (missing on .NET Framework) with an ArrowArrayHelper.ParseEnum helper - Guard DateOnly/TimeOnly/Half code paths with NET6/NET5 conditionals, following the same pattern as Apache.Arrow itself; polyfill RequiresUnreferencedCode on downlevel TFMs - Tests multi-target net8.0;net472 on Windows: 197 pass on net8.0, 182 on net472 (DateOnly/TimeOnly/Half tests are #if-gated) - README: document target frameworks, .NET Framework prerequisites, and the registry-based dispatch Co-Authored-By: Claude Fable 5 --- .../ArrowSerializerGenerator.cs | 33 ++++++++++ .../CodeEmitter.cs | 22 ++++--- .../PolymorphicCodeEmitter.cs | 36 +++++++++-- .../Apache.Arrow.Serialization.csproj | 2 +- .../ArrowArrayHelper.cs | 13 ++++ .../ArrowSerializerExtensions.cs | 30 ++++----- .../IArrowSerializer.cs | 62 ++++++++++++++++--- src/Apache.Arrow.Serialization/Polyfills.cs | 36 +++++++++++ src/Apache.Arrow.Serialization/README.md | 19 ++++-- .../RecordBatchBuilder.cs | 19 ++++-- .../Apache.Arrow.Serialization.Tests.csproj | 9 ++- .../SerializationTests.cs | 10 +++ .../TestPolyfills.cs | 27 ++++++++ .../TestTypes.cs | 20 +++++- 14 files changed, 287 insertions(+), 51 deletions(-) create mode 100644 src/Apache.Arrow.Serialization/Polyfills.cs create mode 100644 test/Apache.Arrow.Serialization.Tests/TestPolyfills.cs diff --git a/src/Apache.Arrow.Serialization.Generator/ArrowSerializerGenerator.cs b/src/Apache.Arrow.Serialization.Generator/ArrowSerializerGenerator.cs index 95cf42c7..1d108064 100644 --- a/src/Apache.Arrow.Serialization.Generator/ArrowSerializerGenerator.cs +++ b/src/Apache.Arrow.Serialization.Generator/ArrowSerializerGenerator.cs @@ -61,6 +61,39 @@ public class ArrowSerializerGenerator : IIncrementalGenerator public void Initialize(IncrementalGeneratorInitializationContext context) { + // Generated registrations use [ModuleInitializer]; the attribute is missing + // from the BCL below .NET 5, so emit an internal polyfill when needed. + // Referenced assemblies may carry their own internal polyfill (e.g. + // Microsoft.CodeAnalysis), which GetTypeByMetadataName would find even though + // the consumer compilation cannot use it — require an accessible declaration. + var needsModuleInitializerPolyfill = context.CompilationProvider.Select(static (compilation, _) => + { + foreach (var type in compilation.GetTypesByMetadataName("System.Runtime.CompilerServices.ModuleInitializerAttribute")) + { + if (type.DeclaredAccessibility == Accessibility.Public || + SymbolEqualityComparer.Default.Equals(type.ContainingAssembly, compilation.Assembly)) + { + return false; + } + } + return true; + }); + + context.RegisterSourceOutput(needsModuleInitializerPolyfill, static (spc, needed) => + { + if (!needed) + return; + spc.AddSource("ModuleInitializerAttribute.Polyfill.g.cs", + "// \n" + + "namespace System.Runtime.CompilerServices\n" + + "{\n" + + " [global::System.AttributeUsage(global::System.AttributeTargets.Method, Inherited = false)]\n" + + " internal sealed class ModuleInitializerAttribute : global::System.Attribute\n" + + " {\n" + + " }\n" + + "}\n"); + }); + var emitSchemaJson = context.AnalyzerConfigOptionsProvider.Select(static (options, _) => { options.GlobalOptions.TryGetValue("build_property.ArrowSerializerEmitSchemaJson", out var value); diff --git a/src/Apache.Arrow.Serialization.Generator/CodeEmitter.cs b/src/Apache.Arrow.Serialization.Generator/CodeEmitter.cs index 85e23376..7f25205a 100644 --- a/src/Apache.Arrow.Serialization.Generator/CodeEmitter.cs +++ b/src/Apache.Arrow.Serialization.Generator/CodeEmitter.cs @@ -55,7 +55,7 @@ public void Emit() } var typeKeyword = GetTypeKeyword(); - Line($"partial {typeKeyword} {_model.TypeName} : IArrowSerializer<{_model.TypeName}>"); + Line($"partial {typeKeyword} {_model.TypeName} : IArrowSerializable"); Line("{"); _indent++; @@ -71,6 +71,8 @@ public void Emit() _indent--; Line("}"); + Line(); + CodeEmitterHelpers.EmitSerializerRegistration(Line, _model.TypeName); } private void EmitSchemaField() @@ -1117,12 +1119,12 @@ private void EmitEnumDeserialize(PropertyModel prop, int index, string colAccess if (prop.IsNullable) { Line($"var dictArr_{index} = (DictionaryArray){colAccess};"); - Line($"{enumType}? prop_{index} = dictArr_{index}.IsNull(0) ? null : System.Enum.Parse<{enumType}>(((StringArray)dictArr_{index}.Dictionary).GetString((int)((Int16Array)dictArr_{index}.Indices).GetValue(0).Value)!);"); + Line($"{enumType}? prop_{index} = dictArr_{index}.IsNull(0) ? null : ArrowArrayHelper.ParseEnum<{enumType}>(((StringArray)dictArr_{index}.Dictionary).GetString((int)((Int16Array)dictArr_{index}.Indices).GetValue(0).Value)!);"); } else { Line($"var dictArr_{index} = (DictionaryArray){colAccess};"); - Line($"var prop_{index} = System.Enum.Parse<{enumType}>(((StringArray)dictArr_{index}.Dictionary).GetString((int)((Int16Array)dictArr_{index}.Indices).GetValue(0).Value)!);"); + Line($"var prop_{index} = ArrowArrayHelper.ParseEnum<{enumType}>(((StringArray)dictArr_{index}.Dictionary).GetString((int)((Int16Array)dictArr_{index}.Indices).GetValue(0).Value)!);"); } } @@ -1233,7 +1235,7 @@ private void EmitManualListDeserialize(PropertyModel prop, TypeInfo elemType, in // Enum list elements are dictionary-encoded: Dictionary(Int16, Utf8) Line($"var {listArr}_dictArr = (DictionaryArray){listArr}.Values;"); Line($"var {listArr}_dictValues = (StringArray){listArr}_dictArr.Dictionary;"); - var readExpr = $"Enumerable.Range({listArr}_start, {listArr}_len).Select(i => System.Enum.Parse<{elemType.FullTypeName}>({listArr}_dictValues.GetString(((Int16Array){listArr}_dictArr.Indices).GetValue(i)!.Value)!))"; + var readExpr = $"Enumerable.Range({listArr}_start, {listArr}_len).Select(i => ArrowArrayHelper.ParseEnum<{elemType.FullTypeName}>({listArr}_dictValues.GetString(((Int16Array){listArr}_dictArr.Indices).GetValue(i)!.Value)!))"; if (isSet) Line(PropDecl($"new System.Collections.Generic.HashSet<{elemType.FullTypeName}>({readExpr})")); else if (prop.Type.Kind == ArrowTypeKind.Array) @@ -1390,7 +1392,7 @@ private string GetElementReader(TypeInfo elemType, string valuesVar) ArrowTypeKind.TimeSpan => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => Apache.Arrow.Serialization.ArrowArrayHelper.ReadDuration((DurationArray){valuesVar}, i))", ArrowTypeKind.Guid => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((GuidArray){valuesVar}).GetGuid(i)!.Value)", ArrowTypeKind.Binary => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ((BinaryArray){valuesVar}).GetBytes(i).ToArray())", - ArrowTypeKind.Enum => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => System.Enum.Parse<{elemType.FullTypeName}>(((StringArray){valuesVar}).GetString(i)!))", + ArrowTypeKind.Enum => $"Enumerable.Range(0, {valuesVar}.Length).Select(i => ArrowArrayHelper.ParseEnum<{elemType.FullTypeName}>(((StringArray){valuesVar}).GetString(i)!))", _ => $"System.Array.Empty<{elemType.FullTypeName}>()", }; } @@ -1482,7 +1484,7 @@ private void EmitDictValueRead(TypeInfo valueType, string dictVar, string keyExp var enumType = valueType.FullTypeName; var dvVar = $"dv_{propIndex}{d}"; Line($"var {dvVar} = (DictionaryArray){valsArrayVar};"); - Line($"{dictVar}[{keyExpr}] = System.Enum.Parse<{enumType}>(((StringArray){dvVar}.Dictionary).GetString((int)((Int16Array){dvVar}.Indices).GetValue({indexVar}).Value)!);"); + Line($"{dictVar}[{keyExpr}] = ArrowArrayHelper.ParseEnum<{enumType}>(((StringArray){dvVar}.Dictionary).GetString((int)((Int16Array){dvVar}.Indices).GetValue({indexVar}).Value)!);"); break; } case ArrowTypeKind.NestedRecord: @@ -1536,7 +1538,7 @@ private string GetScalarReader(TypeInfo type, string arrayVar, string indexVar) ArrowTypeKind.TimeSpan => $"Apache.Arrow.Serialization.ArrowArrayHelper.ReadDuration((DurationArray){arrayVar}, {indexVar})", ArrowTypeKind.Guid => $"((GuidArray){arrayVar}).GetGuid({indexVar})!.Value", ArrowTypeKind.Binary => $"((BinaryArray){arrayVar}).GetBytes({indexVar}).ToArray()", - ArrowTypeKind.Enum => $"System.Enum.Parse<{type.FullTypeName}>(((StringArray){arrayVar}).GetString({indexVar})!)", + ArrowTypeKind.Enum => $"ArrowArrayHelper.ParseEnum<{type.FullTypeName}>(((StringArray){arrayVar}).GetString({indexVar})!)", _ => $"default({type.FullTypeName})", }; } @@ -2462,9 +2464,9 @@ private void EmitMultiRowReadProperty(PropertyModel prop, int index) { var enumType = prop.Type.FullTypeName; if (prop.IsNullable) - Line($"var prop_{index} = col_{index}.IsNull(row) ? ({enumType}?)null : System.Enum.Parse<{enumType}>(((StringArray)col_{index}.Dictionary).GetString((int)((Int16Array)col_{index}.Indices).GetValue(row).Value)!);"); + Line($"var prop_{index} = col_{index}.IsNull(row) ? ({enumType}?)null : ArrowArrayHelper.ParseEnum<{enumType}>(((StringArray)col_{index}.Dictionary).GetString((int)((Int16Array)col_{index}.Indices).GetValue(row).Value)!);"); else - Line($"var prop_{index} = System.Enum.Parse<{enumType}>(((StringArray)col_{index}.Dictionary).GetString((int)((Int16Array)col_{index}.Indices).GetValue(row).Value)!);"); + Line($"var prop_{index} = ArrowArrayHelper.ParseEnum<{enumType}>(((StringArray)col_{index}.Dictionary).GetString((int)((Int16Array)col_{index}.Indices).GetValue(row).Value)!);"); break; } case ArrowTypeKind.List: @@ -2513,7 +2515,7 @@ private void EmitMultiRowReadProperty(PropertyModel prop, int index) // Dictionary-encoded enum values: Dictionary(Int16, Utf8) Line($"var enumDictArr_{index} = (DictionaryArray)col_{index}.Values;"); Line($"var enumDictValues_{index} = (StringArray)enumDictArr_{index}.Dictionary;"); - var readExpr2 = $"Enumerable.Range(listStart_{index}, listLen_{index}).Select(i => System.Enum.Parse<{elemType.FullTypeName}>(enumDictValues_{index}.GetString(((Int16Array)enumDictArr_{index}.Indices).GetValue(i)!.Value)!))"; + var readExpr2 = $"Enumerable.Range(listStart_{index}, listLen_{index}).Select(i => ArrowArrayHelper.ParseEnum<{elemType.FullTypeName}>(enumDictValues_{index}.GetString(((Int16Array)enumDictArr_{index}.Indices).GetValue(i)!.Value)!))"; if (prop.Type.Kind == ArrowTypeKind.Array) Line($"{multiDeclPrefix} = {readExpr2}.ToArray();"); else if (prop.Type.Kind == ArrowTypeKind.Set) diff --git a/src/Apache.Arrow.Serialization.Generator/PolymorphicCodeEmitter.cs b/src/Apache.Arrow.Serialization.Generator/PolymorphicCodeEmitter.cs index 3e8b4fd0..708302b0 100644 --- a/src/Apache.Arrow.Serialization.Generator/PolymorphicCodeEmitter.cs +++ b/src/Apache.Arrow.Serialization.Generator/PolymorphicCodeEmitter.cs @@ -56,7 +56,7 @@ public void Emit() } var typeKeyword = _model.IsInterface ? "interface" : (_model.IsRecord ? "record" : "class"); - Line($"partial {typeKeyword} {_model.TypeName} : IArrowSerializer<{_model.TypeName}>"); + Line($"partial {typeKeyword} {_model.TypeName} : IArrowSerializable"); Line("{"); _indent++; @@ -72,6 +72,8 @@ public void Emit() _indent--; Line("}"); + Line(); + CodeEmitterHelpers.EmitSerializerRegistration(Line, _model.TypeName); } private void EmitSchemaField() @@ -552,13 +554,13 @@ private void EmitReadProperty(PropertyModel prop, int propIndex, int colIndex) if (prop.IsNullable) { Line($"{enumType}? prop_{propIndex} = null;"); - Line($"{{ var da = (DictionaryArray){col}; if (!da.IsNull(0)) {{ var idx = ((Int16Array)da.Indices).GetValue(0).Value; var name = ((StringArray)da.Dictionary).GetString(idx); prop_{propIndex} = System.Enum.Parse<{enumType}>(name!); }} }}"); + Line($"{{ var da = (DictionaryArray){col}; if (!da.IsNull(0)) {{ var idx = ((Int16Array)da.Indices).GetValue(0).Value; var name = ((StringArray)da.Dictionary).GetString(idx); prop_{propIndex} = ArrowArrayHelper.ParseEnum<{enumType}>(name!); }} }}"); } else { Line($"var da_{propIndex} = (DictionaryArray){col};"); Line($"var idx_{propIndex} = ((Int16Array)da_{propIndex}.Indices).GetValue(0).Value;"); - Line($"var prop_{propIndex} = System.Enum.Parse<{enumType}>(((StringArray)da_{propIndex}.Dictionary).GetString(idx_{propIndex})!);"); + Line($"var prop_{propIndex} = ArrowArrayHelper.ParseEnum<{enumType}>(((StringArray)da_{propIndex}.Dictionary).GetString(idx_{propIndex})!);"); } break; } @@ -626,12 +628,12 @@ private void EmitMultiRowReadProperty(PropertyModel prop, int propIndex, int uni if (prop.IsNullable) { Line($"{enumType}? prop_{propIndex} = null;"); - Line($"if (!{col}.IsNull(row)) {{ var idx = ((Int16Array){col}.Indices).GetValue(row).Value; var name = ((StringArray){col}.Dictionary).GetString(idx); prop_{propIndex} = System.Enum.Parse<{enumType}>(name!); }}"); + Line($"if (!{col}.IsNull(row)) {{ var idx = ((Int16Array){col}.Indices).GetValue(row).Value; var name = ((StringArray){col}.Dictionary).GetString(idx); prop_{propIndex} = ArrowArrayHelper.ParseEnum<{enumType}>(name!); }}"); } else { Line($"var idx_{propIndex} = ((Int16Array){col}.Indices).GetValue(row).Value;"); - Line($"var prop_{propIndex} = System.Enum.Parse<{enumType}>(((StringArray){col}.Dictionary).GetString(idx_{propIndex})!);"); + Line($"var prop_{propIndex} = ArrowArrayHelper.ParseEnum<{enumType}>(((StringArray){col}.Dictionary).GetString(idx_{propIndex})!);"); } break; } @@ -680,6 +682,30 @@ private void Line(string text = "") /// internal static class CodeEmitterHelpers { + /// + /// Emits a registration class with an IArrowSerializer<T> implementation that + /// delegates to the generated statics, registered via module initializer so the + /// ArrowSerializerRegistry is populated before any user code in the assembly runs. + /// + public static void EmitSerializerRegistration(Action line, string typeName) + { + line($"internal static class {typeName}ArrowSerializerRegistration"); + line("{"); + line($" private sealed class Serializer : IArrowSerializer<{typeName}>"); + line(" {"); + line($" public Schema ArrowSchema => {typeName}.ArrowSchema;"); + line($" public RecordBatch ToRecordBatch({typeName} value) => {typeName}.ToRecordBatch(value);"); + line($" public {typeName} FromRecordBatch(RecordBatch batch) => {typeName}.FromRecordBatch(batch);"); + line($" public RecordBatch ToRecordBatch(IReadOnlyList<{typeName}> values) => {typeName}.ToRecordBatch(values);"); + line($" public IReadOnlyList<{typeName}> ListFromRecordBatch(RecordBatch batch) => {typeName}.ListFromRecordBatch(batch);"); + line(" }"); + line(""); + line(" [global::System.Runtime.CompilerServices.ModuleInitializer]"); + line(" internal static void Register() =>"); + line($" ArrowSerializerRegistry.Register<{typeName}>(new Serializer());"); + line("}"); + } + /// /// Returns the Arrow type expression for a TypeInfo (e.g., "Int32Type.Default"). /// diff --git a/src/Apache.Arrow.Serialization/Apache.Arrow.Serialization.csproj b/src/Apache.Arrow.Serialization/Apache.Arrow.Serialization.csproj index 1a5d3263..01dbd070 100644 --- a/src/Apache.Arrow.Serialization/Apache.Arrow.Serialization.csproj +++ b/src/Apache.Arrow.Serialization/Apache.Arrow.Serialization.csproj @@ -1,7 +1,7 @@ - net8.0 + netstandard2.0;net462;net8.0 enable enable Source-generated Apache Arrow serialization for .NET. Provides [ArrowSerializable] attribute and IArrowSerializer<T> interface for compile-time Arrow schema derivation, serialization, and deserialization. diff --git a/src/Apache.Arrow.Serialization/ArrowArrayHelper.cs b/src/Apache.Arrow.Serialization/ArrowArrayHelper.cs index 74db0db6..b4acf195 100644 --- a/src/Apache.Arrow.Serialization/ArrowArrayHelper.cs +++ b/src/Apache.Arrow.Serialization/ArrowArrayHelper.cs @@ -92,12 +92,14 @@ public static IArrowArray BuildNullArray(IArrowType type, int length) for (int i = 0; i < length; i++) b.AppendNull(); return b.Build(); } +#if NET5_0_OR_GREATER case HalfFloatType: { var b = new HalfFloatArray.Builder(); for (int i = 0; i < length; i++) b.AppendNull(); return b.Build(); } +#endif case FloatType: { var b = new FloatArray.Builder(); @@ -264,6 +266,7 @@ public static IArrowArray BuildGuidArray(IReadOnlyList items) // --- TimeOnly helpers (Time64) --- +#if NET6_0_OR_GREATER public static IArrowArray BuildTimeOnlyArray(TimeOnly value) { var b = new Time64Array.Builder(Time64Type.Default); @@ -298,6 +301,7 @@ public static TimeOnly ReadTimeOnly(Time64Array array, int index) { return array.GetTime(index)!.Value; } +#endif // --- TimeSpan helpers (Duration) --- @@ -415,4 +419,13 @@ public static T ThrowNotSupported(string message) { throw new NotSupportedException(message); } + + /// + /// Parses an enum value by name. Used by generated code instead of + /// Enum.Parse<T>, whose generic overload does not exist on .NET Framework. + /// + public static T ParseEnum(string name) where T : struct + { + return (T)Enum.Parse(typeof(T), name); + } } diff --git a/src/Apache.Arrow.Serialization/ArrowSerializerExtensions.cs b/src/Apache.Arrow.Serialization/ArrowSerializerExtensions.cs index 501e16ce..e2e53067 100644 --- a/src/Apache.Arrow.Serialization/ArrowSerializerExtensions.cs +++ b/src/Apache.Arrow.Serialization/ArrowSerializerExtensions.cs @@ -26,60 +26,60 @@ public static class ArrowSerializerExtensions /// /// Serialize an instance to Arrow IPC stream bytes. /// - public static byte[] SerializeToBytes(this T value) where T : IArrowSerializer + public static byte[] SerializeToBytes(this T value) where T : IArrowSerializable { - var batch = T.ToRecordBatch(value); + var batch = ArrowSerializerRegistry.Get().ToRecordBatch(value); return RecordBatchToBytes(batch); } /// /// Deserialize an instance from Arrow IPC stream bytes. /// - public static T DeserializeFromBytes(byte[] data) where T : IArrowSerializer + public static T DeserializeFromBytes(byte[] data) where T : IArrowSerializable { var batch = BytesToRecordBatch(data); - return T.FromRecordBatch(batch); + return ArrowSerializerRegistry.Get().FromRecordBatch(batch); } /// /// Serialize an instance to an Arrow IPC stream. /// - public static void SerializeToStream(this T value, Stream destination) where T : IArrowSerializer + public static void SerializeToStream(this T value, Stream destination) where T : IArrowSerializable { - var batch = T.ToRecordBatch(value); + var batch = ArrowSerializerRegistry.Get().ToRecordBatch(value); WriteRecordBatch(batch, destination); } /// /// Deserialize an instance from an Arrow IPC stream. /// - public static T DeserializeFromStream(Stream source) where T : IArrowSerializer + public static T DeserializeFromStream(Stream source) where T : IArrowSerializable { var batch = ReadRecordBatch(source); - return T.FromRecordBatch(batch); + return ArrowSerializerRegistry.Get().FromRecordBatch(batch); } /// /// Serialize multiple instances to a multi-row RecordBatch. /// - public static RecordBatch ToRecordBatch(this IEnumerable items) where T : IArrowSerializer + public static RecordBatch ToRecordBatch(this IEnumerable items) where T : IArrowSerializable { var list = items as IReadOnlyList ?? items.ToList(); - return T.ToRecordBatch(list); + return ArrowSerializerRegistry.Get().ToRecordBatch(list); } /// /// Deserialize all rows from a RecordBatch into a list of instances. /// - public static IReadOnlyList ToList(this RecordBatch batch) where T : IArrowSerializer + public static IReadOnlyList ToList(this RecordBatch batch) where T : IArrowSerializable { - return T.ListFromRecordBatch(batch); + return ArrowSerializerRegistry.Get().ListFromRecordBatch(batch); } /// /// Serialize multiple instances to Arrow IPC stream bytes. /// - public static byte[] SerializeListToBytes(this IEnumerable items) where T : IArrowSerializer + public static byte[] SerializeListToBytes(this IEnumerable items) where T : IArrowSerializable { var batch = items.ToRecordBatch(); return RecordBatchToBytes(batch); @@ -88,10 +88,10 @@ public static byte[] SerializeListToBytes(this IEnumerable items) where T /// /// Deserialize multiple instances from Arrow IPC stream bytes. /// - public static IReadOnlyList DeserializeListFromBytes(byte[] data) where T : IArrowSerializer + public static IReadOnlyList DeserializeListFromBytes(byte[] data) where T : IArrowSerializable { var batch = BytesToRecordBatch(data); - return T.ListFromRecordBatch(batch); + return ArrowSerializerRegistry.Get().ListFromRecordBatch(batch); } /// diff --git a/src/Apache.Arrow.Serialization/IArrowSerializer.cs b/src/Apache.Arrow.Serialization/IArrowSerializer.cs index cb5a8f45..496508aa 100644 --- a/src/Apache.Arrow.Serialization/IArrowSerializer.cs +++ b/src/Apache.Arrow.Serialization/IArrowSerializer.cs @@ -38,33 +38,79 @@ public interface IArrowConverter } /// -/// Interface implemented by source-generated code on [ArrowSerializable] records. -/// Provides static Arrow schema, serialization, and deserialization. +/// Marker interface implemented by source-generated code on [ArrowSerializable] types. +/// Constrains the generic helpers in so they can +/// only be called on types the source generator has produced a serializer for. /// -public interface IArrowSerializer where T : IArrowSerializer +public interface IArrowSerializable +{ +} + +/// +/// Serializer for a specific type, implemented by source-generated code and resolved +/// through . The generated implementation delegates +/// to the static members emitted on the [ArrowSerializable] type itself +/// (T.ArrowSchema, T.ToRecordBatch(...), ...), which remain directly usable. +/// +public interface IArrowSerializer { /// /// The Arrow schema for this type, derived from property types. /// - static abstract Schema ArrowSchema { get; } + Schema ArrowSchema { get; } /// /// Serialize an instance to a single-row RecordBatch. /// - static abstract RecordBatch ToRecordBatch(T value); + RecordBatch ToRecordBatch(T value); /// /// Deserialize an instance from a single-row RecordBatch. /// - static abstract T FromRecordBatch(RecordBatch batch); + T FromRecordBatch(RecordBatch batch); /// /// Serialize multiple instances to a multi-row RecordBatch. /// - static abstract RecordBatch ToRecordBatch(IReadOnlyList values); + RecordBatch ToRecordBatch(IReadOnlyList values); /// /// Deserialize all rows from a RecordBatch into a list of instances. /// - static abstract IReadOnlyList ListFromRecordBatch(RecordBatch batch); + IReadOnlyList ListFromRecordBatch(RecordBatch batch); +} + +/// +/// Registry mapping types to their source-generated +/// implementations. Generated module initializers register a serializer per +/// [ArrowSerializable] type before any user code in that assembly runs, so lookups +/// never race with registration. Resolution is a single static-generic field read — +/// no dictionary lookup and no reflection. +/// +public static class ArrowSerializerRegistry +{ + private static class Cache + { + public static IArrowSerializer? Instance; + } + + /// + /// Register the serializer for . Called by generated code; + /// may also be called manually to override or provide a hand-written serializer. + /// + public static void Register(IArrowSerializer serializer) + { + Cache.Instance = serializer; + } + + /// + /// Resolve the serializer for . + /// + /// No serializer is registered for T. + public static IArrowSerializer Get() + { + return Cache.Instance + ?? throw new NotSupportedException( + $"No Arrow serializer registered for {typeof(T)}. Is the type marked [ArrowSerializable] and declared partial?"); + } } diff --git a/src/Apache.Arrow.Serialization/Polyfills.cs b/src/Apache.Arrow.Serialization/Polyfills.cs new file mode 100644 index 00000000..0a72a3f0 --- /dev/null +++ b/src/Apache.Arrow.Serialization/Polyfills.cs @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if !NET5_0_OR_GREATER + +namespace System.Diagnostics.CodeAnalysis; + +/// +/// Internal polyfill for target frameworks that predate the trimming annotations. +/// +[AttributeUsage(AttributeTargets.Method | AttributeTargets.Constructor | AttributeTargets.Class, Inherited = false)] +internal sealed class RequiresUnreferencedCodeAttribute : Attribute +{ + public RequiresUnreferencedCodeAttribute(string message) + { + Message = message; + } + + public string Message { get; } + + public string? Url { get; set; } +} + +#endif diff --git a/src/Apache.Arrow.Serialization/README.md b/src/Apache.Arrow.Serialization/README.md index 85554749..56dff239 100644 --- a/src/Apache.Arrow.Serialization/README.md +++ b/src/Apache.Arrow.Serialization/README.md @@ -76,13 +76,18 @@ var roundTrip = ArrowSerializerExtensions.DeserializeFromBytes(bytes); dotnet add package Apache.Arrow.Serialization ``` -The NuGet package includes both the runtime library and the Roslyn source generator. Targets `net8.0`. +The NuGet package includes both the runtime library and the Roslyn source generator. Targets `netstandard2.0`, `net462`, and `net8.0` — usable from .NET 8+ and .NET Framework 4.6.2+ (including 4.7.2). + +.NET Framework notes: +- Building requires a Roslyn 4 toolchain (VS 2022 or current .NET SDK) with an SDK-style project, since source generators do not run on older toolchains. +- Generated code uses modern C# syntax; set `latest` if your project targets .NET Framework. +- `DateOnly`, `TimeOnly`, and `Half` mappings are unavailable on .NET Framework because those types do not exist there; `DateTime`/`DateTimeOffset`/`TimeSpan` cover timestamps and durations. ## Quick Start 1. Add `[ArrowSerializable]` to your type 2. Make the type `partial` (required for source generation) -3. The generator emits `IArrowSerializer` — giving you `ArrowSchema`, `ToRecordBatch`, `FromRecordBatch`, and `ListFromRecordBatch` +3. The generator emits static members — `ArrowSchema`, `ToRecordBatch`, `FromRecordBatch`, and `ListFromRecordBatch` — plus an `IArrowSerializer` implementation registered with `ArrowSerializerRegistry`, which powers the generic extension methods ```csharp using Apache.Arrow.Serialization; @@ -99,7 +104,7 @@ public partial record SensorReading The source generator produces a `partial` implementation with these static members: ```csharp -partial record SensorReading : IArrowSerializer +partial record SensorReading : IArrowSerializable { public static Schema ArrowSchema { get; } public static RecordBatch ToRecordBatch(SensorReading value); @@ -109,6 +114,12 @@ partial record SensorReading : IArrowSerializer } ``` +It also generates an `IArrowSerializer` implementation that delegates to these statics and registers it with `ArrowSerializerRegistry` via a module initializer. The registry is what lets the generic extension methods (`value.SerializeToBytes()`, `DeserializeFromBytes`, ...) resolve the right serializer on any target framework, and it can also be used directly: + +```csharp +IArrowSerializer serializer = ArrowSerializerRegistry.Get(); +``` + ## Supported Types ### Type Declarations @@ -576,7 +587,7 @@ When `RecordBatchBuilder` encounters a nested `[ArrowSerializable]` type, it del ## Extension Methods -`ArrowSerializerExtensions` provides convenience methods for any `IArrowSerializer` type: +`ArrowSerializerExtensions` provides convenience methods for any `[ArrowSerializable]` type: | Method | Description | |--------|-------------| diff --git a/src/Apache.Arrow.Serialization/RecordBatchBuilder.cs b/src/Apache.Arrow.Serialization/RecordBatchBuilder.cs index 5f96f30b..a02b2732 100644 --- a/src/Apache.Arrow.Serialization/RecordBatchBuilder.cs +++ b/src/Apache.Arrow.Serialization/RecordBatchBuilder.cs @@ -94,14 +94,18 @@ private static (IArrowType Type, bool Nullable) InferArrowType(Type clrType) if (clrType == typeof(uint)) return (UInt32Type.Default, false); if (clrType == typeof(long)) return (Int64Type.Default, false); if (clrType == typeof(ulong)) return (UInt64Type.Default, false); +#if NET5_0_OR_GREATER if (clrType == typeof(Half)) return (HalfFloatType.Default, false); +#endif if (clrType == typeof(float)) return (FloatType.Default, false); if (clrType == typeof(double)) return (DoubleType.Default, false); if (clrType == typeof(decimal)) return (new Decimal128Type(38, 18), false); if (clrType == typeof(DateTime)) return (new TimestampType(TimeUnit.Microsecond, "UTC"), false); if (clrType == typeof(DateTimeOffset)) return (new TimestampType(TimeUnit.Microsecond, "UTC"), false); +#if NET6_0_OR_GREATER if (clrType == typeof(DateOnly)) return (Date32Type.Default, false); if (clrType == typeof(TimeOnly)) return (new Time64Type(TimeUnit.Microsecond), false); +#endif if (clrType == typeof(TimeSpan)) return (DurationType.Microsecond, false); if (clrType == typeof(Guid)) return (new GuidType(), false); if (clrType == typeof(byte[])) return (BinaryType.Default, true); @@ -165,14 +169,12 @@ private static (IArrowType Type, bool Nullable) InferArrowType(Type clrType) } /// - /// Check if a type implements IArrowSerializer<T> (i.e. has [ArrowSerializable] source-generated code) + /// Check if a type implements IArrowSerializable (i.e. has [ArrowSerializable] source-generated code) /// and return its static ArrowSchema if so. /// private static Schema? GetGeneratedArrowSchema(Type clrType) { - var iface = clrType.GetInterfaces() - .FirstOrDefault(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IArrowSerializer<>)); - if (iface is null) return null; + if (!typeof(IArrowSerializable).IsAssignableFrom(clrType)) return null; var schemaProp = clrType.GetProperty("ArrowSchema", BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy); return schemaProp?.GetValue(null) as Schema; @@ -184,7 +186,8 @@ private static (IArrowType Type, bool Nullable) InferArrowType(Type clrType) private static MethodInfo? GetGeneratedToRecordBatchList(Type clrType) { var listType = typeof(IReadOnlyList<>).MakeGenericType(clrType); - return clrType.GetMethod("ToRecordBatch", BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy, [listType]); + return clrType.GetMethod("ToRecordBatch", BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy, + binder: null, [listType], modifiers: null); } private static IColumnBuilder CreateColumnBuilder(Type clrType, IArrowType arrowType) @@ -203,14 +206,18 @@ private static IColumnBuilder CreateColumnBuilder(Type clrType, IArrowType arrow if (clrType == typeof(uint)) return new TypedColumnBuilder(new(), (b, v) => b.Append(v), b => b.AppendNull(), b => b.Build()); if (clrType == typeof(long)) return new TypedColumnBuilder(new(), (b, v) => b.Append(v), b => b.AppendNull(), b => b.Build()); if (clrType == typeof(ulong)) return new TypedColumnBuilder(new(), (b, v) => b.Append(v), b => b.AppendNull(), b => b.Build()); +#if NET5_0_OR_GREATER if (clrType == typeof(Half)) return new TypedColumnBuilder(new(), (b, v) => b.Append(v), b => b.AppendNull(), b => b.Build()); +#endif if (clrType == typeof(float)) return new TypedColumnBuilder(new(), (b, v) => b.Append(v), b => b.AppendNull(), b => b.Build()); if (clrType == typeof(double)) return new TypedColumnBuilder(new(), (b, v) => b.Append(v), b => b.AppendNull(), b => b.Build()); if (clrType == typeof(decimal)) return new DecimalColumnBuilder(); if (clrType == typeof(DateTime)) return new DateTimeColumnBuilder(); if (clrType == typeof(DateTimeOffset)) return new DateTimeOffsetColumnBuilder(); +#if NET6_0_OR_GREATER if (clrType == typeof(DateOnly)) return new TypedColumnBuilder(new(), (b, v) => b.Append(v), b => b.AppendNull(), b => b.Build()); if (clrType == typeof(TimeOnly)) return new TimeOnlyColumnBuilder(); +#endif if (clrType == typeof(TimeSpan)) return new TimeSpanColumnBuilder(); if (clrType == typeof(Guid)) return new GuidColumnBuilder(); if (clrType == typeof(byte[])) return new BinaryColumnBuilder(); @@ -348,6 +355,7 @@ public void Append(object? value) public IArrowArray Build() => _b.Build(); } +#if NET6_0_OR_GREATER private sealed class TimeOnlyColumnBuilder : IColumnBuilder { private readonly List<(TimeOnly Value, bool IsNull)> _values = new(); @@ -364,6 +372,7 @@ public IArrowArray Build() return b.Build(); } } +#endif private sealed class TimeSpanColumnBuilder : IColumnBuilder { diff --git a/test/Apache.Arrow.Serialization.Tests/Apache.Arrow.Serialization.Tests.csproj b/test/Apache.Arrow.Serialization.Tests/Apache.Arrow.Serialization.Tests.csproj index 943f2302..de053675 100644 --- a/test/Apache.Arrow.Serialization.Tests/Apache.Arrow.Serialization.Tests.csproj +++ b/test/Apache.Arrow.Serialization.Tests/Apache.Arrow.Serialization.Tests.csproj @@ -1,7 +1,7 @@ - net8.0 + true enable enable false @@ -10,6 +10,13 @@ true + + net8.0;net472 + + + net8.0 + + diff --git a/test/Apache.Arrow.Serialization.Tests/SerializationTests.cs b/test/Apache.Arrow.Serialization.Tests/SerializationTests.cs index 4904485a..7a8c9e7b 100644 --- a/test/Apache.Arrow.Serialization.Tests/SerializationTests.cs +++ b/test/Apache.Arrow.Serialization.Tests/SerializationTests.cs @@ -530,6 +530,7 @@ public void OptionalInt_MultiRow_RoundTrip() public class DateTimeTypeTests { +#if NET6_0_OR_GREATER [Fact] public void DateTime_RoundTrip() { @@ -598,6 +599,7 @@ public void DateTime_MultiRow_RoundTrip() Assert.Equal(items[1].Time, restored[1].Time); Assert.Equal(items[1].Duration, restored[1].Duration); } +#endif [Fact] public void DecimalAndGuid_RoundTrip() @@ -633,6 +635,7 @@ public void DecimalAndGuid_MultiRow_RoundTrip() } } +#if NET6_0_OR_GREATER [Fact] public void Half_RoundTrip() { @@ -712,6 +715,7 @@ public void NullableDateTimeTypes_MultiRow_RoundTrip() Assert.Null(restored[1].Id); Assert.Equal(items[2].Date, restored[2].Date); } +#endif } public class ArrowTypeOverrideTests @@ -771,6 +775,7 @@ public void DecimalOverride_RoundTrip() Assert.Equal(obj.Value, restored.Value); } +#if NET6_0_OR_GREATER [Fact] public void TimeOverride_Time32MillisAndTime64Nanos() { @@ -851,6 +856,7 @@ public void DateOverride_MultiRow_RoundTrip() Assert.Equal(items[1].Date64Value, restored[1].Date64Value); Assert.Equal(items[1].Date32Value, restored[1].Date32Value); } +#endif [Fact] public void ViewTypes_Schema_UsesViewTypes() @@ -1743,6 +1749,7 @@ public void DateTimeOffsetList_RoundTrip() Assert.Equal(obj.Timestamps, restored.Timestamps); } +#if NET6_0_OR_GREATER [Fact] public void DateOnlyList_RoundTrip() { @@ -1758,6 +1765,7 @@ public void TimeOnlyList_RoundTrip() var restored = WithTimeOnlyList.FromRecordBatch(WithTimeOnlyList.ToRecordBatch(obj)); Assert.Equal(obj.Times, restored.Times); } +#endif [Fact] public void TimeSpanList_RoundTrip() @@ -1767,6 +1775,7 @@ public void TimeSpanList_RoundTrip() Assert.Equal(obj.Durations, restored.Durations); } +#if NET6_0_OR_GREATER [Fact] public void HalfList_RoundTrip() { @@ -1774,6 +1783,7 @@ public void HalfList_RoundTrip() var restored = WithHalfList.FromRecordBatch(WithHalfList.ToRecordBatch(obj)); Assert.Equal(obj.Values, restored.Values); } +#endif [Fact] public void AllPrimitiveLists_RoundTrip() diff --git a/test/Apache.Arrow.Serialization.Tests/TestPolyfills.cs b/test/Apache.Arrow.Serialization.Tests/TestPolyfills.cs new file mode 100644 index 00000000..29702bd7 --- /dev/null +++ b/test/Apache.Arrow.Serialization.Tests/TestPolyfills.cs @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if !NET5_0_OR_GREATER + +namespace System.Runtime.CompilerServices; + +/// +/// Polyfill enabling C# records and init-only setters on .NET Framework targets. +/// +internal static class IsExternalInit +{ +} + +#endif diff --git a/test/Apache.Arrow.Serialization.Tests/TestTypes.cs b/test/Apache.Arrow.Serialization.Tests/TestTypes.cs index d750681f..23e2ef57 100644 --- a/test/Apache.Arrow.Serialization.Tests/TestTypes.cs +++ b/test/Apache.Arrow.Serialization.Tests/TestTypes.cs @@ -163,6 +163,7 @@ public partial record WithArrowTypeOverride public int Value { get; init; } } +#if NET6_0_OR_GREATER [ArrowSerializable] public partial record WithDateTimeTypes { @@ -172,6 +173,7 @@ public partial record WithDateTimeTypes public TimeOnly Time { get; init; } public TimeSpan Duration { get; init; } } +#endif [ArrowSerializable] public partial record WithDecimalAndGuid @@ -180,12 +182,15 @@ public partial record WithDecimalAndGuid public Guid Id { get; init; } } +#if NET6_0_OR_GREATER [ArrowSerializable] public partial record WithHalf { public Half Value { get; init; } } +#endif +#if NET6_0_OR_GREATER [ArrowSerializable] public partial record WithNullableDateTimeTypes { @@ -197,6 +202,7 @@ public partial record WithNullableDateTimeTypes public Guid? Id { get; init; } public Half? HalfVal { get; init; } } +#endif [ArrowSerializable] public partial record WithTimestampOverride @@ -208,6 +214,7 @@ public partial record WithTimestampOverride public DateTime DateTimeValue { get; init; } } +#if NET6_0_OR_GREATER [ArrowSerializable] public partial record WithTimeOverride { @@ -217,6 +224,7 @@ public partial record WithTimeOverride [ArrowType("time64[ns]")] public TimeOnly Nanos { get; init; } } +#endif [ArrowSerializable] public partial record WithDecimalOverride @@ -285,6 +293,7 @@ public partial record WithWallClockTimestamp public DateTimeOffset UtcDateTimeOffset { get; init; } } +#if NET6_0_OR_GREATER [ArrowSerializable] public partial record WithDateOverride { @@ -296,6 +305,7 @@ public partial record WithDateOverride [ArrowType("date32")] public DateOnly Date32Value { get; init; } } +#endif [ArrowSerializable] public partial record WithBool8 @@ -361,13 +371,13 @@ public class Point2DArrowConverter : IArrowConverter public Apache.Arrow.IArrowArray ToArray(Point2D value) { - return new Apache.Arrow.StringArray.Builder().Append(string.Create(System.Globalization.CultureInfo.InvariantCulture, $"{value.X},{value.Y}")).Build(); + return new Apache.Arrow.StringArray.Builder().Append(FormattableString.Invariant($"{value.X},{value.Y}")).Build(); } public Apache.Arrow.IArrowArray ToArray(IReadOnlyList values) { var b = new Apache.Arrow.StringArray.Builder(); - foreach (var v in values) b.Append(string.Create(System.Globalization.CultureInfo.InvariantCulture, $"{v.X},{v.Y}")); + foreach (var v in values) b.Append(FormattableString.Invariant($"{v.X},{v.Y}")); return b.Build(); } @@ -478,17 +488,21 @@ public partial record WithDateTimeOffsetList public List Timestamps { get; init; } = new(); } +#if NET6_0_OR_GREATER [ArrowSerializable] public partial record WithDateOnlyList { public List Dates { get; init; } = new(); } +#endif +#if NET6_0_OR_GREATER [ArrowSerializable] public partial record WithTimeOnlyList { public List Times { get; init; } = new(); } +#endif [ArrowSerializable] public partial record WithTimeSpanList @@ -496,11 +510,13 @@ public partial record WithTimeSpanList public List Durations { get; init; } = new(); } +#if NET6_0_OR_GREATER [ArrowSerializable] public partial record WithHalfList { public List Values { get; init; } = new(); } +#endif [ArrowSerializable] public partial record WithAllPrimitiveLists From 7b3a5c8a2a9533f1f24bbf933aadd0da18428252 Mon Sep 17 00:00:00 2001 From: Christoph Mettler Date: Tue, 7 Jul 2026 19:01:25 +0200 Subject: [PATCH 8/9] Fix RC verification and integrate serialization docs The RC Verify job iterates every directory under src/ and fetches a matching nupkg; Apache.Arrow.Serialization.Generator is not packable (it ships inside the Apache.Arrow.Serialization package as an analyzer), so skip it in the artifact fetch loop. Add Apache.Arrow.Serialization.Tests to Apache.Arrow.Tests.slnf so the CI test job and RC verification actually run the serialization tests. During binary verification the generator project reference is dropped in favor of the packaged analyzer, and the generator unit tests (DiagnosticTests) are removed there because they need the generator assembly as a compile-time reference, which a packaged analyzer cannot provide; they still run against the source tree. Include the Apache.Arrow.Serialization README in the docfx site as a conceptual "Serialization" page (docs/_site/serialization/) with a toc entry and a link from the landing page. Verified locally with "docfx metadata/build --warningsAsErrors": 0 warnings, 0 errors. Co-Authored-By: Claude Fable 5 --- Apache.Arrow.Tests.slnf | 3 ++- dev/release/verify_rc.sh | 12 +++++++++++- docs/docfx.json | 7 +++++++ docs/index.md | 3 +++ docs/toc.yml | 2 ++ 5 files changed, 25 insertions(+), 2 deletions(-) diff --git a/Apache.Arrow.Tests.slnf b/Apache.Arrow.Tests.slnf index 90bf9bda..10495fbc 100644 --- a/Apache.Arrow.Tests.slnf +++ b/Apache.Arrow.Tests.slnf @@ -8,7 +8,8 @@ "test\\Apache.Arrow.Flight.TestWeb\\Apache.Arrow.Flight.TestWeb.csproj", "test\\Apache.Arrow.Tests\\Apache.Arrow.Tests.csproj", "test\\Apache.Arrow.Scalars.Tests\\Apache.Arrow.Scalars.Tests.csproj", - "test\\Apache.Arrow.Operations.Tests\\Apache.Arrow.Operations.Tests.csproj" + "test\\Apache.Arrow.Operations.Tests\\Apache.Arrow.Operations.Tests.csproj", + "test\\Apache.Arrow.Serialization.Tests\\Apache.Arrow.Serialization.Tests.csproj" ] } } diff --git a/dev/release/verify_rc.sh b/dev/release/verify_rc.sh index 8f592b2f..89854400 100755 --- a/dev/release/verify_rc.sh +++ b/dev/release/verify_rc.sh @@ -173,6 +173,13 @@ test_binary_distribution() { pushd nuget for package in $(cd ../src && echo *); do + case "${package}" in + Apache.Arrow.Serialization.Generator) + # Not packable: the source generator ships inside the + # Apache.Arrow.Serialization package as a Roslyn analyzer. + continue + ;; + esac for package_type in nupkg snupkg; do fetch_artifact "${package}.${VERSION}.${package_type}" done @@ -189,9 +196,12 @@ test_binary_distribution() { reference_package "Apache.Arrow.Serialization" "Apache.Arrow.Serialization.Tests" # The source generator ships inside the Apache.Arrow.Serialization package # as a Roslyn analyzer; drop the direct project reference so the packaged - # generator is exercised instead. + # generator is exercised instead. The generator unit tests need the + # generator assembly as a compile-time reference, which the package's + # analyzer does not provide, so they only run against the source tree. dotnet remove "test/Apache.Arrow.Serialization.Tests" \ reference "src/Apache.Arrow.Serialization.Generator/Apache.Arrow.Serialization.Generator.csproj" + rm test/Apache.Arrow.Serialization.Tests/DiagnosticTests.cs # Move src directory to ensure we are only testing against built packages mv src src.backup diff --git a/docs/docfx.json b/docs/docfx.json index 3f920331..7e963b99 100644 --- a/docs/docfx.json +++ b/docs/docfx.json @@ -27,6 +27,13 @@ "_site/**", "images/**" ] + }, + { + "files": [ + "README.md" + ], + "src": "../src/Apache.Arrow.Serialization", + "dest": "serialization" } ], "resource": [ diff --git a/docs/index.md b/docs/index.md index cbd0fb04..98c1e53d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -27,6 +27,9 @@ An implementation of Arrow targeting .NET. See our current [feature matrix](https://github.com/apache/arrow/blob/main/docs/source/status.rst) for currently available features. +For source-generated POCO serialization (`[ArrowSerializable]`), see +[Apache.Arrow.Serialization](../src/Apache.Arrow.Serialization/README.md). + ## Implementation - Arrow specification 1.0.0. (Support for reading 0.11+.) diff --git a/docs/toc.yml b/docs/toc.yml index 1f897e40..4628d417 100644 --- a/docs/toc.yml +++ b/docs/toc.yml @@ -18,6 +18,8 @@ # under the License. items: +- name: Serialization + href: ../src/Apache.Arrow.Serialization/README.md - name: API Reference type: Namespace href: api/ From 406ed84a842acbb640b36a4288a30ec5d25eaf19 Mon Sep 17 00:00:00 2001 From: Christoph Mettler Date: Tue, 7 Jul 2026 21:06:27 +0200 Subject: [PATCH 9/9] Build docs prerequisite in Release configuration docfx metadata loads projects with Configuration=Release, so the source generator referenced as an analyzer only resolves from the Release output path. The plain (Debug) build left the analyzer unresolved, producing a FailedToResolveAnalyzer warning that --warningsAsErrors turned into the Documentation job failure (docfx exit code 255). Reproduced in a fresh clone: cold Debug build fails metadata with the warning; cold Release build passes. Co-Authored-By: Claude Fable 5 --- ci/scripts/docs.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ci/scripts/docs.sh b/ci/scripts/docs.sh index 61a36395..586ae163 100755 --- a/ci/scripts/docs.sh +++ b/ci/scripts/docs.sh @@ -21,9 +21,10 @@ set -eux source_dir=${1} -# Build the serialization library first so its source generator output is -# available when docfx extracts metadata. -dotnet build "${source_dir}/src/Apache.Arrow.Serialization/Apache.Arrow.Serialization.csproj" +# Build the serialization library first so its source generator is available +# when docfx extracts metadata. docfx loads projects with Configuration=Release, +# so the analyzer project reference only resolves from the Release output. +dotnet build -c Release "${source_dir}/src/Apache.Arrow.Serialization/Apache.Arrow.Serialization.csproj" pushd "${source_dir}/docs"