From 7ce0bfe29e8909f274b2c592fa92ed5742682095 Mon Sep 17 00:00:00 2001 From: jackylee Date: Mon, 10 Aug 2026 18:06:07 +0800 Subject: [PATCH] fix(spark): report the written file size in the commit message bytesWritten accumulated FieldVector.getBufferSize() per batch, which is the uncompressed in-memory Arrow size, and handed that to VortexWriterCommitMessage as "the number of bytes written". VortexWriter.finish() already returns a summary carrying the file's physical size, and close() was only calling finish() and discarding it. Take the size from there and drop the per-batch accumulation, which also removes work from the write path. Signed-off-by: jackylee --- .../main/java/dev/vortex/spark/write/VortexDataWriter.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/write/VortexDataWriter.java b/java/vortex-spark/src/main/java/dev/vortex/spark/write/VortexDataWriter.java index dfadd320a43..48f47284550 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/write/VortexDataWriter.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/write/VortexDataWriter.java @@ -192,9 +192,6 @@ private void writeBatch() throws IOException { vectorSchemaRoot.setRowCount(batchRows.size()); // Export via Arrow C Data Interface and write to Vortex - for (FieldVector vector : vectorSchemaRoot.getFieldVectors()) { - bytesWritten += vector.getBufferSize(); - } try (ArrowArray arrowArray = ArrowArray.allocateNew(allocator); ArrowSchema arrowSchema = ArrowSchema.allocateNew(allocator)) { Data.exportVectorSchemaRoot(allocator, vectorSchemaRoot, null, arrowArray, arrowSchema); @@ -327,10 +324,10 @@ public WriterCommitMessage commit() throws IOException { writeBatch(); } - // Close the Vortex writer to finalize the file + // Finalize the file; the summary carries its physical size if (vortexWriter != null) { try { - vortexWriter.close(); + bytesWritten = vortexWriter.finish().fileSize(); } finally { vortexWriter = null; // Always null out the reference }