diff --git a/native/Cargo.lock b/native/Cargo.lock index d3a4b897df2..918fbbcd4c3 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -1118,11 +1118,11 @@ checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] name = "blake2" -version = "0.10.6" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +checksum = "5b5d4d889834ee8ecfc0f8426ad30faf7cdcb10f741a8e6d7224d95325479f6f" dependencies = [ - "digest 0.10.7", + "digest 0.11.3", ] [[package]] @@ -1893,8 +1893,7 @@ dependencies = [ [[package]] name = "datafusion" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96f76f0167ed0842b29a3d1e41be3c034c0a46409a3a703cc4cc84ee8c24abf4" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "arrow-schema", @@ -1942,8 +1941,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d79ec3460f6ed5c58f9b3f2d873fbc77748b82653bff1b4cdaf06de33bb4e05f" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "async-trait", @@ -1967,8 +1965,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b48cef241e2efcfd496fe05ae4d0d5de20793451862faefe406c397a467e12d4" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "async-trait", @@ -2141,8 +2138,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f72810485975c258f1b4d00baab31728470676c60c5546f366ebd0d99f05ab6" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "arrow-ipc", @@ -2168,8 +2164,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "533c28e75dba52f41bde187d23a1cb24ab91c7c097966824fa471e67b60320ea" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "futures", "log", @@ -2179,8 +2174,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b00a1fa0da26f6087136a82fea7f13c76a672cbab452d4086952a7cf770a19b" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "async-compression", @@ -2215,8 +2209,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ad17ec881bff2ed7768b4bfe971d3efbf3473f2fd1f9d365447bccbdf908678" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "arrow-ipc", @@ -2239,8 +2232,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5345285b0c3eaab412e7539b706973c083bd7e5bce575de5e0a3da488d08d1d" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "async-trait", @@ -2262,8 +2254,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da02fb9324f56bd8c53f1ee2e949547425cb66f76adc6832b10d44f80a1221d2" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "async-trait", @@ -2285,8 +2276,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c0b0dc1453952952fd5c69ad1c7f6042176e69ed233011d47e07cf74ed0949e" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "arrow-schema", @@ -2317,14 +2307,12 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88fd985bc0550c36f557db69543cc9d6393b1509783520b30e902f23c555da6" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" [[package]] name = "datafusion-execution" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a98f1052f91b4991f0bf2ce1e4e36dfbdcda454a956b8c8d562c7c845e8fce1d" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "arrow-buffer", @@ -2350,8 +2338,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464625a1f0e4b9df552d894fafcc8aac953ebbc8b0fa0acdaf20975fd615040e" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "arrow-schema", @@ -2372,8 +2359,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2604994999d5aeca1d1df645ffc98bc787447aaff05dde27aad0342b48fc1fe0" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "datafusion-common", @@ -2384,8 +2370,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "051e97533e6af53e4aa0a0667cadc886abcaf36c4a5925019c55c0aa4c218fde" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "arrow-buffer", @@ -2416,8 +2401,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d0f1bb166d3572b6ed40e1afb2faaacade962abc08c2fcf04babee74681c56b" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "datafusion-common", @@ -2437,8 +2421,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ed756770f5f98369e181d692fd5ee6b1127ffd7322caba92f3730f9f5c92333" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "datafusion-common", @@ -2449,8 +2432,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91173fdb5c0ff2a41169a8ffa1b385b8844f18728747bb0a37e35ad7d5772a4f" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "arrow-ord", @@ -2474,8 +2456,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1bcdfb286a745461b126719c32700777e83df4f17cc44db5d71ebce5731e840" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "async-trait", @@ -2490,8 +2471,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ec4b508f1f93f00038ba3e737e894ec6c775528b4369413386655ae6125f0fc" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "datafusion-common", @@ -2507,8 +2487,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b352020834140073fbf5b46ee0ceb926e5074a9d0bcae1dbd91d0586d999cde" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2517,8 +2496,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15192effab05d38cce10e92a6fb48c967b5f166b27b7195a165a72b232569c58" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "datafusion-doc", "quote", @@ -2528,8 +2506,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854445d9f7847e1e46089cf61b8d341a64382f14484e912c83a0f23b31216896" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "chrono", @@ -2547,10 +2524,10 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "671558dad1d2aa253c39c0a4c52515958b99eb91abf649f4b88d5e69cc55282f" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", + "arrow-schema", "datafusion-common", "datafusion-expr", "datafusion-expr-common", @@ -2562,14 +2539,14 @@ dependencies = [ "itertools 0.15.0", "parking_lot", "petgraph", + "regex", "tokio", ] [[package]] name = "datafusion-physical-expr-adapter" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffae3d78c2da80ecc829cb58536cc5aca2e99cf1365eda694fc75bfe288861e0" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "datafusion-common", @@ -2583,8 +2560,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d9092ed15e7203fbd0903215172f7c9d18f10d94cba35137f3b3836f7c46f16" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "chrono", @@ -2600,8 +2576,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9005b6cf50b57b72d476c6ed4662b04be7ca6be5320ba9127c6d0b7e4218095b" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "datafusion-common", @@ -2619,8 +2594,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5787e4fcff4adc4fce8948441103a99705018b49c8dff0720b650bd7a15da112" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "arrow-data", @@ -2633,6 +2607,7 @@ dependencies = [ "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", + "datafusion-expr-common", "datafusion-functions", "datafusion-functions-aggregate-common", "datafusion-functions-window-common", @@ -2654,8 +2629,7 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e651c8df0b90daed6a7be5921ec0ee379e6909705f063eeff70fd4e35010e4c" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "datafusion-common", @@ -2670,8 +2644,7 @@ dependencies = [ [[package]] name = "datafusion-session" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb56667ee38217efab19b895d9a936052cfb47ed438a19663351bdc42a6214a1" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow-schema", "async-trait", @@ -2679,28 +2652,28 @@ dependencies = [ "datafusion-execution", "datafusion-expr", "datafusion-physical-plan", + "futures", "parking_lot", ] [[package]] name = "datafusion-spark" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "992dd0b954f24cb576cbeee3555c3083b9cf7a5a5f2448ea25f7a437d444163f" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", + "base64 0.23.1", "bigdecimal", "chrono", "crc32fast", "datafusion", - "datafusion-catalog", "datafusion-common", - "datafusion-execution", "datafusion-expr", "datafusion-functions", "datafusion-functions-aggregate", "datafusion-functions-aggregate-common", "datafusion-functions-nested", + "datafusion-session", "log", "num-traits", "percent-encoding", @@ -2715,8 +2688,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c29067cb9d32f8e603c45e15d61ea18f1069f96ceafeceb4e18466b8e5b31d9" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "bigdecimal", diff --git a/native/Cargo.toml b/native/Cargo.toml index ac83bec7844..8cfb9e03a70 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -82,3 +82,38 @@ codegen-units = 16 # Parallel codegen (faster compile, slightly larger binary) debug-assertions = true panic = "unwind" # Allow panics to be caught and logged across FFI boundary # overflow-checks inherited as false from release + +# Struct-field capability and row-group statistics pruning (peterxcli/datafusion#2). +[patch.crates-io] +datafusion = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-catalog = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-catalog-listing = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-common = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-common-runtime = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-datasource = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-datasource-arrow = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-datasource-csv = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-datasource-json = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-datasource-parquet = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-doc = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-execution = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-expr = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-expr-common = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-functions = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-functions-aggregate = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-functions-aggregate-common = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-functions-nested = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-functions-table = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-functions-window = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-functions-window-common = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-macros = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-optimizer = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-physical-expr = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-physical-expr-adapter = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-physical-expr-common = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-physical-optimizer = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-physical-plan = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-pruning = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-session = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-spark = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-sql = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index 8a0972d3977..4e37305c11e 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -833,7 +833,11 @@ impl PhysicalPlanner { ExprStruct::GetStructField(expr) => { let child = self.create_expr(expr.child.as_ref().unwrap(), Arc::clone(&input_schema))?; - Ok(Arc::new(GetStructField::new(child, expr.ordinal as usize))) + Ok(GetStructField::with_field_access( + child, + expr.ordinal as usize, + &input_schema, + )?) } ExprStruct::ToJson(expr) => { let child = self.create_expr(expr.child.as_ref().unwrap(), input_schema)?; diff --git a/native/core/src/parquet/parquet_support.rs b/native/core/src/parquet/parquet_support.rs index b601f984d60..240706751bb 100644 --- a/native/core/src/parquet/parquet_support.rs +++ b/native/core/src/parquet/parquet_support.rs @@ -226,12 +226,10 @@ fn parquet_convert_array_impl( // `millisToMicros` uses `Math.multiplyExact`: // https://github.com/apache/spark/blob/v4.2.0/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/SparkDateTimeUtils.scala#L103-L108 // - // The checked conversion is limited to TOP-LEVEL columns. Spark only avoids the - // error for filtered-out values through row-group statistics pruning, and - // DataFusion's PruningPredicate does not support nested fields yet, so a checked - // conversion on a nested field would fail queries whose predicates Spark prunes - // (e.g. `WHERE s.ts < X` over an all-overflowing file). Nested fields keep the - // pre-existing safe-cast behavior below (overflow -> NULL). + // The checked conversion remains limited to top-level columns. Nested statistics + // pruning alone does not establish Spark parity for conversion-aware predicates, + // dictionary filters, and row filters (see #5739). Nested fields retain the + // existing safe cast below (overflow -> NULL). let micros = array .as_primitive::() .try_unary::<_, TimestampMicrosecondType, _>(|value| value.mul_checked(1_000))? @@ -764,10 +762,8 @@ mod tests { assert!(converted.is_null(0), "overflow must become NULL"); assert!(converted.is_null(1)); - // Nested: DataFusion's PruningPredicate cannot prune nested fields, so a - // checked conversion would fail queries whose predicates Spark satisfies via - // row-group statistics pruning. The nested field keeps the safe-cast behavior: - // overflow becomes NULL. + // Nested statistics pruning does not by itself establish checked-conversion parity + // with Spark (#5739). Preserve the existing nested overflow-to-NULL behavior. let child_field = Arc::new(Field::new( "ts", DataType::Timestamp(TimeUnit::Millisecond, None), diff --git a/native/spark-expr/src/struct_funcs/get_struct_field.rs b/native/spark-expr/src/struct_funcs/get_struct_field.rs index 62447624f5b..a9ccbf96cd9 100644 --- a/native/spark-expr/src/struct_funcs/get_struct_field.rs +++ b/native/spark-expr/src/struct_funcs/get_struct_field.rs @@ -20,8 +20,13 @@ use arrow::buffer::NullBuffer; use arrow::datatypes::{DataType, Field, Schema}; use arrow::record_batch::RecordBatch; use datafusion::common::{DataFusionError, Result as DataFusionResult, ScalarValue}; -use datafusion::logical_expr::ColumnarValue; +use datafusion::config::ConfigOptions; +use datafusion::logical_expr::{ + ColumnarValue, ExpressionPlacement, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF, + ScalarUDFImpl, Signature, StructFieldAccess, Volatility, +}; use datafusion::physical_expr::PhysicalExpr; +use datafusion::physical_expr::ScalarFunctionExpr; use std::{ fmt::{Display, Formatter}, hash::Hash, @@ -51,6 +56,32 @@ impl GetStructField { Self { child, ordinal } } + /// Expose an unambiguous field name to DataFusion's nested projection and pruning. + pub fn with_field_access( + child: Arc, + ordinal: usize, + schema: &Schema, + ) -> DataFusionResult> { + let expr = Self::new(child, ordinal); + let field = expr.child_field(schema)?; + let DataType::Struct(fields) = expr.child.data_type(schema)? else { + unreachable!() + }; + // Spark accesses by ordinal, including structs with duplicate field names. + if fields.iter().filter(|f| f.name() == field.name()).count() != 1 { + return Ok(Arc::new(expr)); + } + Ok(Arc::new(ScalarFunctionExpr::try_new( + Arc::new(ScalarUDF::new_from_impl(StructFieldUdf { + name: field.name().clone(), + signature: Signature::any(1, Volatility::Immutable), + })), + vec![expr.child], + schema, + Arc::new(ConfigOptions::default()), + )?)) + } + fn child_field(&self, input_schema: &Schema) -> DataFusionResult> { match self.child.data_type(input_schema)? { DataType::Struct(fields) => Ok(Arc::clone(&fields[self.ordinal])), @@ -152,6 +183,92 @@ impl Display for GetStructField { } } +/// Name-based evaluation survives the schema adapter reordering or narrowing the struct. +#[derive(Debug, PartialEq, Eq, Hash)] +struct StructFieldUdf { + name: String, + signature: Signature, +} + +impl ScalarUDFImpl for StructFieldUdf { + fn name(&self) -> &str { + "spark_get_struct_field" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, types: &[DataType]) -> DataFusionResult { + match &types[0] { + DataType::Struct(fields) => fields + .iter() + .find(|f| f.name() == &self.name) + .map(|f| f.data_type().clone()) + .ok_or_else(|| { + DataFusionError::Plan(format!("Missing struct field {}", self.name)) + }), + _ => datafusion::common::exec_err!("Expected a struct"), + } + } + + fn return_field_from_args(&self, args: ReturnFieldArgs) -> DataFusionResult> { + let parent = &args.arg_fields[0]; + let DataType::Struct(fields) = parent.data_type() else { + return datafusion::common::exec_err!("Expected a struct"); + }; + let field = fields + .iter() + .find(|f| f.name() == &self.name) + .ok_or_else(|| DataFusionError::Plan(format!("Missing struct field {}", self.name)))?; + Ok(Arc::new(field.as_ref().clone().with_nullable( + parent.is_nullable() || field.is_nullable(), + ))) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { + let scalar = matches!(args.args[0], ColumnarValue::Scalar(_)); + let array = args.args[0] + .clone() + .into_array(if scalar { 1 } else { args.number_rows })?; + let array = array + .as_any() + .downcast_ref::() + .ok_or_else(|| DataFusionError::Execution("Expected a struct".into()))?; + let ordinal = array + .fields() + .iter() + .position(|f| f.name() == &self.name) + .ok_or_else(|| { + DataFusionError::Execution(format!("Missing struct field {}", self.name)) + })?; + let result = GetStructField::project_field(array, ordinal)?; + if scalar { + Ok(ColumnarValue::Scalar(ScalarValue::try_from_array( + &result, 0, + )?)) + } else { + Ok(ColumnarValue::Array(result)) + } + } + + fn struct_field_access(&self, _args: &[Option]) -> Option { + Some(StructFieldAccess { + source_arg: 0, + field_path: vec![self.name.clone()], + }) + } + + fn placement(&self, args: &[ExpressionPlacement]) -> ExpressionPlacement { + match args.first() { + Some(ExpressionPlacement::Column | ExpressionPlacement::MoveTowardsLeafNodes) => { + ExpressionPlacement::MoveTowardsLeafNodes + } + _ => ExpressionPlacement::KeepInPlace, + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -159,6 +276,57 @@ mod tests { use arrow::datatypes::Fields; use datafusion::physical_expr::expressions::Column; + #[test] + fn field_access_handles_reordered_fields_scalars_and_duplicate_names() { + use datafusion::physical_expr::expressions::Literal; + let fields: Fields = vec![ + Field::new("other", DataType::Int64, false), + Field::new("k.dot", DataType::Int64, false), + ] + .into(); + let schema = Schema::new(vec![Field::new( + "s", + DataType::Struct(fields.clone()), + false, + )]); + let expr = + GetStructField::with_field_access(Arc::new(Column::new("s", 0)), 1, &schema).unwrap(); + let reordered = StructArray::new( + vec![fields[1].clone(), fields[0].clone()].into(), + vec![ + Arc::new(Int64Array::from(vec![42])), + Arc::new(Int64Array::from(vec![99])), + ], + None, + ); + let literal: Arc = + Arc::new(Literal::new(ScalarValue::Struct(Arc::new(reordered)))); + let expr = expr.with_new_children(vec![literal]).unwrap(); + let batch = RecordBatch::new_empty(Arc::new(schema)); + assert!(matches!( + expr.evaluate(&batch).unwrap(), + ColumnarValue::Scalar(ScalarValue::Int64(Some(42))) + )); + + let duplicate = StructArray::new( + vec![fields[1].clone(), fields[1].clone()].into(), + vec![ + Arc::new(Int64Array::from(vec![42])), + Arc::new(Int64Array::from(vec![99])), + ], + None, + ); + let literal: Arc = + Arc::new(Literal::new(ScalarValue::Struct(Arc::new(duplicate)))); + let expr = GetStructField::with_field_access(literal, 1, &Schema::empty()).unwrap(); + assert!(expr.downcast_ref::().is_some()); + let output = expr.evaluate(&batch).unwrap().into_array(1).unwrap(); + assert_eq!( + ScalarValue::try_from_array(&output, 0).unwrap(), + ScalarValue::Int64(Some(99)) + ); + } + // A field of a NULL struct must be NULL (Spark semantics) even when the child buffer holds a // non-null value at that row -- Arrow stores child validity independently of the parent // struct's null mask, so a logically-null struct column read from parquet can still carry a @@ -174,7 +342,9 @@ mod tests { let schema = Schema::new(vec![Field::new("cm", DataType::Struct(fields), true)]); let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(struct_array)]).unwrap(); - let expr = GetStructField::new(Arc::new(Column::new("cm", 0)), 0); + let expr = + GetStructField::with_field_access(Arc::new(Column::new("cm", 0)), 0, &batch.schema()) + .unwrap(); let out = expr .evaluate(&batch) .unwrap() @@ -203,7 +373,8 @@ mod tests { /* struct nullable */ true, )]); - let expr = GetStructField::new(Arc::new(Column::new("add", 0)), 0); + let expr = + GetStructField::with_field_access(Arc::new(Column::new("add", 0)), 0, &schema).unwrap(); assert!( expr.nullable(&schema).unwrap(), "a field of a nullable struct must be nullable even if the field itself is non-nullable" @@ -220,7 +391,8 @@ mod tests { /* struct nullable */ false, )]); - let expr = GetStructField::new(Arc::new(Column::new("add", 0)), 0); + let expr = + GetStructField::with_field_access(Arc::new(Column::new("add", 0)), 0, &schema).unwrap(); assert!( !expr.nullable(&schema).unwrap(), "a non-nullable field of a non-nullable struct must remain non-nullable" diff --git a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala index 799197bf33d..be7972bbef4 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala @@ -2090,6 +2090,56 @@ class ParquetReadV1Suite extends ParquetReadSuite with AdaptiveSparkPlanHelper { } } + test("nested struct filters prune row groups using leaf statistics") { + withTempDir { dir => + val path = new Path(dir.toURI.toString, "nested.parquet") + val schema = MessageTypeParser.parseMessageType("""message root { + | optional group s { + | optional group inner { optional int32 k; } + | optional int32 payload; + | } + |}""".stripMargin) + val writer = ExampleParquetWriter + .builder(path) + .withType(schema) + .withDictionaryEncoding(false) + .withRowGroupSize(4096L) + .withConf(spark.sessionState.newHadoopConf()) + .build() + try { + (0 until 8192).foreach { i => + val row = new SimpleGroup(schema) + if (i % 16 != 0) { + val s = row.addGroup(0) + val inner = s.addGroup(0) + if (i % 17 != 0) inner.add(0, i) + s.add(1, i) + } + writer.write(row) + } + } finally { + writer.close() + } + Seq(false, true).foreach { pruning => + withSQLConf( + CometConf.COMET_RESPECT_DATAFUSION_CONFIGS.key -> "true", + "spark.comet.datafusion.execution.parquet.pruning" -> pruning.toString, + "spark.comet.datafusion.execution.parquet.enable_page_index" -> "false", + CometConf.COMET_PARQUET_ROW_FILTER_PUSHDOWN_ENABLED.key -> "false") { + val query = spark.read + .parquet(path.toString) + .where("s.inner.k >= 7000") + .select("s.inner.k", "s.payload") + val (_, plan) = checkSparkAnswer(query) + val scans = collect(plan) { case scan: CometNativeScanExec => scan } + assert(scans.size == 1) + val pruned = scans.head.metrics("row_groups_pruned_statistics").value + assert(if (pruning) pruned > 0 else pruned == 0) + } + } + } + } + test("test V1 parquet scan filter pushdown of primitive types") { withTempPath { dir => val path = new Path(dir.toURI.toString, "test1.parquet")