From d54ad1b1c1c3babe4fea3470aebaeb2b541efa54 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Tue, 8 Sep 2026 22:38:09 +0800 Subject: [PATCH 1/3] perf: demonstrate nested struct statistics pruning --- benchmarks/nested-pruning/README.md | 135 + benchmarks/nested-pruning/benchmark.py | 149 + benchmarks/nested-pruning/hashes.sha256 | 5 + benchmarks/nested-pruning/results.json | 5986 +++++++++++++++++ native/Cargo.lock | 110 +- native/Cargo.toml | 35 + native/core/src/execution/planner.rs | 6 +- native/core/src/parquet/parquet_support.rs | 16 +- .../src/struct_funcs/get_struct_field.rs | 180 +- .../comet/parquet/ParquetReadSuite.scala | 50 + 10 files changed, 6588 insertions(+), 84 deletions(-) create mode 100644 benchmarks/nested-pruning/README.md create mode 100644 benchmarks/nested-pruning/benchmark.py create mode 100644 benchmarks/nested-pruning/hashes.sha256 create mode 100644 benchmarks/nested-pruning/results.json diff --git a/benchmarks/nested-pruning/README.md b/benchmarks/nested-pruning/README.md new file mode 100644 index 00000000000..489a66c9b89 --- /dev/null +++ b/benchmarks/nested-pruning/README.md @@ -0,0 +1,135 @@ + + +# Nested struct statistics pruning + +Comet needs native expression plumbing to use +[peterxcli/datafusion#2](https://github.com/peterxcli/datafusion/pull/2). +Spark already serializes the nested predicates, but Comet's custom `GetStructField` +physical expression does not expose DataFusion's scalar-UDF field-access capability. +The adapter declares the selected field while retaining Comet's parent-null handling. +It resolves the field by name after schema adaptation and retains ordinal access when +field names are duplicated. + +The dependency is pinned to `0b0506a9acab9d5892ecf7e89243c3b34664bcc6`, which includes +the field-access capability, nested input requirements, and row-group pruning. +The baseline is unchanged Comet `bb9e74020adc228e486f6f4d0fa68292b30bff31`, with its +checked-in DataFusion 55.0.0 dependencies. The comparison therefore includes the +DataFusion fork update and the Comet adapter. A pruning-disabled candidate control +isolates the effect of statistics pruning. + +This change does not resolve [#5739](https://github.com/apache/datafusion-comet/issues/5739). +Timestamp conversion and Spark's other filtering paths still need separate parity work. +The optimization applies to supported struct-leaf predicates with useful statistics; +it does not imply equivalent improvements for arrays, maps, Variant, or unfiltered scans. + +## Results (2026-09-08) + +On `chia-ping-aws1` (AMD EPYC 7282, 16 vCPUs, 31 GiB RAM), using Rust 1.98.0, +OpenJDK 17.0.20, Spark 4.1.3, NumPy 2.5.3, PyArrow 25.0.1, and the settings below: + +| Query | Baseline median | Candidate median | Baseline / candidate | +| --- | ---: | ---: | ---: | +| Nested filter, sorted keys | 642.3 ms | 182.6 ms | 3.52x | +| Top-level filter, sorted keys | 138.8 ms | 138.3 ms | 1.00x | +| Projection only | 530.2 ms | 537.4 ms | 0.99x | +| Nested filter, shuffled keys | 585.4 ms | 577.1 ms | 1.01x | + +The selective nested query skips **63 of 64 row groups**. Reader bytes decrease from +**182,481,422 to 2,851,301 (98.4% less)**, and scan output decreases from 4,194,304 to +65,536 rows. The final aggregate checksums are identical. Disabling pruning in the +candidate restores 0 pruned groups, 182,481,422 reader bytes, and a **638.0 ms** median +(five control measurements), close to the baseline. This supports attributing the +selective-query improvement to statistics pruning. + +The three other comparisons differ by less than 2%, with overlapping interquartile +ranges. They show no consistent speedup. The selective query has interquartile ranges +of 629.3–647.1 ms for baseline and 168.8–199.3 ms for candidate. + +All measured queries used `CometNativeScan`; the plans, metrics, checksums, individual +timings, and summary statistics are retained in [results.json](results.json). +Additional candidate controls with decoder filtering enabled passed checksum and +pruning-counter checks. These controls are not a matched comparison of decoder-filter +performance. Pilot timings were excluded from the reported medians. + +Validation passed: four Rust expression tests; five Spark tests covering the new +pruning regression, nested schema evolution, field-ID mapping, and nullable struct +access (one pre-existing Spark test remains ignored); native release builds; Maven +packaging; Rust formatting; and the Spark style/format checks. The Spark regression +exercises the real native scan with statistics pruning both enabled and disabled. + +Remote checkouts are `~/oss/comet-nested-pruning` and `~/oss/comet-nested-baseline`. +Data is in `~/oss/comet-nested-data`; libraries, raw logs, samples, the implementation +patch, and [binary/data hashes](hashes.sha256) are retained in +`~/oss/comet-nested-results`. The Python environment is `~/oss/comet-nested-venv`. +The host-side `~/oss/comet-nested-run-rounds.py` records the alternating run order. + +## Reproduce + +Install `numpy==2.5.3`, `pyarrow==25.0.1`, and `pyspark==4.1.3` in a dedicated Python environment. +Generate the files once and reuse them for every revision: + +```sh +python benchmark.py --generate --data /path/to/data +``` + +Each file contains 4,194,304 rows and 64 row groups of 65,536 rows. The schema has +`k: int64` and `s: struct, p0: int64, ..., p7: int64>`. +The payload values are seeded random integers. One file has sorted keys; the other +has shuffled keys. Both use Snappy compression, no dictionary encoding, and row-group +statistics. The selective predicate retains 65,536 rows (1.5625%). Every query sums +all eight payload fields and checks the answer against sums computed during generation. + +Build both native revisions with the same settings, from their respective `native` +directories, and retain each resulting `libcomet.so` in a separate directory: + +```sh +CARGO_BUILD_JOBS=8 CARGO_PROFILE_RELEASE_LTO=false \ + CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16 CARGO_PROFILE_RELEASE_DEBUG=0 \ + cargo build --release --no-default-features --locked +``` + +Build the Spark jar from the Comet root after the native build: + +```sh +./mvnw package -DskipTests -Djni.dir=/path/to/candidate-library-directory +``` + +Use the same jar, data, JVM, and CPU affinity for both native libraries: + +```sh +SPARK_LOCAL_IP=127.0.0.1 COMET_WORKER_THREADS=8 taskset -c 0-7 \ + python benchmark.py --data /path/to/data --jar /path/to/comet.jar \ + --library-dir /path/to/candidate-library-directory --label candidate-0 \ + --runs 5 --warmup 5 --expect-nested-pruned 63 --output candidate-0.json +``` + +Use `--expect-nested-pruned 0` for the baseline or the candidate with `--pruning-off`. +The harness explicitly selects Comet's default of disabled decoder row filtering. +`--row-filter` exercises decoder filtering as an additional control. Page-index and +Bloom-filter pruning are disabled. Spark uses `local[1]`, one shuffle partition, +and disabled AQE. Timings include SQL planning, execution, and collecting the aggregate +result; they exclude Spark startup, schema inference, and data generation. + +The measured comparison uses four rounds, alternating baseline/candidate order, +with five warmups followed by five measured executions per query in each process. +Report medians over the 20 measurements per revision, alongside the scan metrics. +`bytes_scanned` excludes metadata reads, as documented by Comet's reader factory. +These are warm-cache measurements on synthetic local files, using matching release +builds with LTO disabled; they do not establish production or object-storage speedups. diff --git a/benchmarks/nested-pruning/benchmark.py b/benchmarks/nested-pruning/benchmark.py new file mode 100644 index 00000000000..95b50444378 --- /dev/null +++ b/benchmarks/nested-pruning/benchmark.py @@ -0,0 +1,149 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Generate deterministic nested Parquet data, then time checked Spark queries. + +Dependencies: numpy, pyarrow, pyspark matching the Comet build. +Run --generate once; reuse its files for every native library revision. +""" +import argparse +import json +import time +from pathlib import Path + + +def generate(directory, rows): + import numpy as np + import pyarrow as pa + import pyarrow.parquet as pq + + directory.mkdir(parents=True, exist_ok=True) + expected = {} + for layout in ("sorted", "shuffled"): + rng = np.random.default_rng(5739) + keys = np.arange(rows, dtype=np.int64) + if layout == "shuffled": + rng.shuffle(keys) + payload = [rng.integers(0, 1000000, rows, dtype=np.int64) for _ in range(8)] + inner = pa.StructArray.from_arrays([pa.array(keys)], names=["k"]) + nested = pa.StructArray.from_arrays( + [inner] + [pa.array(p) for p in payload], + names=["inner"] + [f"p{i}" for i in range(8)], + ) + table = pa.table({"k": keys, "s": nested}) + pq.write_table(table, directory / f"{layout}.parquet", row_group_size=65536, + compression="snappy", use_dictionary=False, write_statistics=True) + mask = keys >= rows - 65536 + expected[layout] = { + "all": [int(p.sum()) for p in payload], + "selective": [int(p[mask].sum()) for p in payload], + } + (directory / "expected.json").write_text(json.dumps({"rows": rows, "sums": expected})) + + +def metrics(node): + result = [] + it = node.metrics().iterator() + values = {} + while it.hasNext(): + entry = it.next() + values[entry._1()] = entry._2().value() + result.append({"node": node.nodeName(), "metrics": values}) + children = node.children().iterator() + while children.hasNext(): + result.extend(metrics(children.next())) + return result + + +def run(args): + from pyspark.sql import SparkSession + + spark = (SparkSession.builder.master("local[1]").appName("Comet nested pruning") + .config("spark.jars", str(args.jar.resolve())) + .config("spark.driver.extraJavaOptions", f"-Djava.library.path={args.library_dir.resolve()}") + .config("spark.sql.extensions", "org.apache.comet.CometSparkSessionExtensions") + .config("spark.shuffle.manager", "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") + .config("spark.sql.adaptive.enabled", "false") + .config("spark.sql.shuffle.partitions", "1") + .config("spark.sql.files.maxPartitionBytes", str(1024**3)) + .config("spark.comet.enabled", "true") + .config("spark.comet.exec.enabled", "true") + .config("spark.comet.exec.respectDataFusionConfigs", "true") + .config("spark.comet.datafusion.execution.parquet.pruning", str(not args.pruning_off).lower()) + .config("spark.comet.datafusion.execution.parquet.enable_page_index", "false") + .config("spark.comet.datafusion.execution.parquet.bloom_filter_on_read", "false") + .config("spark.comet.parquet.rowFilterPushdown.enabled", str(args.row_filter).lower()) + .getOrCreate()) + spark.sparkContext.setLogLevel("ERROR") + expected = json.loads((args.data / "expected.json").read_text()) + for layout in ("sorted", "shuffled"): + spark.read.parquet(str((args.data / f"{layout}.parquet").resolve())).createOrReplaceTempView(layout) + sums = ", ".join(f"sum(s.p{i})" for i in range(8)) + threshold = expected["rows"] - 65536 + cases = [ + ("nested_selective", "sorted", f"s.inner.k >= {threshold}", "selective"), + ("top_level_selective", "sorted", f"k >= {threshold}", "selective"), + ("projection_only", "sorted", "true", "all"), + ("nested_unprunable", "shuffled", f"s.inner.k >= {threshold}", "selective"), + ] + output = {"label": args.label, "spark": spark.version, "pruning_off": args.pruning_off, + "row_filter": args.row_filter, "warmup": args.warmup, "cases": {}} + for name, layout, predicate, expected_key in cases: + query = f"SELECT {sums} FROM {layout} WHERE {predicate}" + samples = [] + for iteration in range(args.runs + args.warmup): + start = time.perf_counter() + df = spark.sql(query) + answer = list(df.collect()[0]) + elapsed = time.perf_counter() - start + assert answer == expected["sums"][layout][expected_key], (name, answer) + plan = df._jdf.queryExecution().executedPlan() + plan_text = plan.toString() + assert "CometNativeScan" in plan_text, plan_text + if iteration >= args.warmup: + samples.append(elapsed) + scan_metrics = metrics(plan) + if args.expect_nested_pruned is not None and name == "nested_selective": + pruned = sum(n["metrics"].get("row_groups_pruned_statistics", 0) + for n in scan_metrics if "CometNativeScan" in n["node"]) + assert pruned == args.expect_nested_pruned, scan_metrics + output["cases"][name] = {"seconds": samples, "query": query, "answer": answer, + "plan": plan_text, "metrics": scan_metrics} + print(json.dumps({"label": args.label, "case": name, "seconds": samples}), flush=True) + args.output.write_text(json.dumps(output, indent=2)) + spark.stop() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--data", type=Path, required=True) + parser.add_argument("--generate", action="store_true") + parser.add_argument("--rows", type=int, default=4194304) + parser.add_argument("--jar", type=Path) + parser.add_argument("--library-dir", type=Path) + parser.add_argument("--label", default="candidate") + parser.add_argument("--output", type=Path, default=Path("results.json")) + parser.add_argument("--runs", type=int, default=5) + parser.add_argument("--warmup", type=int, default=5) + parser.add_argument("--pruning-off", action="store_true") + parser.add_argument("--row-filter", action="store_true") + parser.add_argument("--expect-nested-pruned", type=int) + args = parser.parse_args() + if args.generate: + generate(args.data, args.rows) + else: + run(args) diff --git a/benchmarks/nested-pruning/hashes.sha256 b/benchmarks/nested-pruning/hashes.sha256 new file mode 100644 index 00000000000..4c928543d37 --- /dev/null +++ b/benchmarks/nested-pruning/hashes.sha256 @@ -0,0 +1,5 @@ +7c127e5ab7945c10b679d9c2665718adc040ad082e0d4fbf773f2d22ee438c30 /home/ubuntu/oss/comet-nested-results/baseline/libcomet.so +e8c2a1ef7db65836dfe935008083e518fdb2d6258498554854b79b666eee40f9 /home/ubuntu/oss/comet-nested-results/candidate/libcomet.so +ace52025f280edeb5f9723235392054ae45b8e0a914120f260e71ec701909f8d /home/ubuntu/oss/comet-nested-data/shuffled.parquet +c10d2c19f517e62f496a8a02da4c8491b9c4fff712ea7d893c986821727ca0af /home/ubuntu/oss/comet-nested-data/sorted.parquet +12877fbd1f236c8755f2209aafb87961e0f39333f6345ee1b93f00c59d404e39 /home/ubuntu/oss/comet-nested-pruning/spark/target/comet-spark-spark4.1_2.13-1.1.0-SNAPSHOT.jar diff --git a/benchmarks/nested-pruning/results.json b/benchmarks/nested-pruning/results.json new file mode 100644 index 00000000000..a35778bee17 --- /dev/null +++ b/benchmarks/nested-pruning/results.json @@ -0,0 +1,5986 @@ +{ + "summary": { + "nested_selective": { + "baseline": { + "median_ms": 642.2919891774654, + "iqr_ms": [ + 629.2613609693944, + 647.1081066410989 + ], + "samples": 20, + "bytes_scanned": 182481422, + "output_rows": 4194304, + "row_groups_pruned_statistics": 0, + "row_groups_matched_statistics": 64 + }, + "candidate": { + "median_ms": 182.61525640264153, + "iqr_ms": [ + 168.84455806575716, + 199.33200534433126 + ], + "samples": 20, + "bytes_scanned": 2851301, + "output_rows": 65536, + "row_groups_pruned_statistics": 63, + "row_groups_matched_statistics": 1 + }, + "pruning-off": { + "median_ms": 638.0214085802436, + "iqr_ms": [ + 615.5594941228628, + 638.3354254066944 + ], + "samples": 5, + "bytes_scanned": 182481422, + "output_rows": 4194304, + "row_groups_pruned_statistics": 0, + "row_groups_matched_statistics": 64 + }, + "pruning-off-row-filter": { + "median_ms": 299.7523508965969, + "iqr_ms": [ + 283.56191609054804, + 299.83343090862036 + ], + "samples": 5, + "bytes_scanned": 19396952, + "output_rows": 65536, + "row_groups_pruned_statistics": 0, + "row_groups_matched_statistics": 64 + } + }, + "top_level_selective": { + "baseline": { + "median_ms": 138.81242321804166, + "iqr_ms": [ + 128.66710755042732, + 143.74887361191213 + ], + "samples": 20, + "bytes_scanned": 2851301, + "output_rows": 65536, + "row_groups_pruned_statistics": 63, + "row_groups_matched_statistics": 1 + }, + "candidate": { + "median_ms": 138.2634500041604, + "iqr_ms": [ + 131.87488308176398, + 146.0748442914337 + ], + "samples": 20, + "bytes_scanned": 2851301, + "output_rows": 65536, + "row_groups_pruned_statistics": 63, + "row_groups_matched_statistics": 1 + }, + "pruning-off": { + "median_ms": 605.2072821184993, + "iqr_ms": [ + 598.0431707575917, + 610.489190556109 + ], + "samples": 5, + "bytes_scanned": 182481422, + "output_rows": 4194304, + "row_groups_pruned_statistics": 0, + "row_groups_matched_statistics": 64 + }, + "pruning-off-row-filter": { + "median_ms": 188.66712413728237, + "iqr_ms": [ + 176.94623582065105, + 191.35639816522598 + ], + "samples": 5, + "bytes_scanned": 19396952, + "output_rows": 65536, + "row_groups_pruned_statistics": 0, + "row_groups_matched_statistics": 64 + } + }, + "projection_only": { + "baseline": { + "median_ms": 530.1858955062926, + "iqr_ms": [ + 522.7551013231277, + 539.3509692512453 + ], + "samples": 20, + "bytes_scanned": 165673158, + "output_rows": 4194304, + "row_groups_pruned_statistics": 0, + "row_groups_matched_statistics": 64 + }, + "candidate": { + "median_ms": 537.4044002965093, + "iqr_ms": [ + 520.9687126334757, + 550.8011817000806 + ], + "samples": 20, + "bytes_scanned": 165673158, + "output_rows": 4194304, + "row_groups_pruned_statistics": 0, + "row_groups_matched_statistics": 64 + }, + "pruning-off": { + "median_ms": 540.7594768330455, + "iqr_ms": [ + 525.6619229912758, + 550.6223440170288 + ], + "samples": 5, + "bytes_scanned": 165673158, + "output_rows": 4194304, + "row_groups_pruned_statistics": 0, + "row_groups_matched_statistics": 64 + }, + "pruning-off-row-filter": { + "median_ms": 534.7809121012688, + "iqr_ms": [ + 521.8340344727039, + 552.2861639037728 + ], + "samples": 5, + "bytes_scanned": 165673158, + "output_rows": 4194304, + "row_groups_pruned_statistics": 0, + "row_groups_matched_statistics": 64 + } + }, + "nested_unprunable": { + "baseline": { + "median_ms": 585.3609410114586, + "iqr_ms": [ + 571.0740983486176, + 591.5737492032349 + ], + "samples": 20, + "bytes_scanned": 186772555, + "output_rows": 4194304, + "row_groups_pruned_statistics": 0, + "row_groups_matched_statistics": 64 + }, + "candidate": { + "median_ms": 577.0554314367473, + "iqr_ms": [ + 565.5243212822825, + 592.3003694042563 + ], + "samples": 20, + "bytes_scanned": 186772555, + "output_rows": 4194304, + "row_groups_pruned_statistics": 0, + "row_groups_matched_statistics": 64 + }, + "pruning-off": { + "median_ms": 560.1397510617971, + "iqr_ms": [ + 551.8598509952426, + 570.2894749119878 + ], + "samples": 5, + "bytes_scanned": 186772555, + "output_rows": 4194304, + "row_groups_pruned_statistics": 0, + "row_groups_matched_statistics": 64 + }, + "pruning-off-row-filter": { + "median_ms": 751.1665793135762, + "iqr_ms": [ + 743.246354162693, + 769.7837706655264 + ], + "samples": 5, + "bytes_scanned": 186772555, + "output_rows": 65536, + "row_groups_pruned_statistics": 0, + "row_groups_matched_statistics": 64 + } + } + }, + "runs": [ + { + "label": "baseline-0", + "spark": "4.1.3", + "pruning_off": false, + "row_filter": false, + "warmup": 5, + "cases": { + "nested_selective": { + "seconds": [ + 0.6894052512943745, + 0.6490190038457513, + 0.6236759554594755, + 0.6345376027747989, + 0.6294151488691568 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE s.inner.k >= 4128768", + "answer": [ + 32741059719, + 32774396751, + 32772938793, + 32799973223, + 32905236128, + 32807159809, + 32718967833, + 32661111725 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#486L, sum#487L, sum#488L, sum#489L, sum#490L, sum#491L, sum#492L, sum#493L], [Final], [sum(_extract_p0#470L), sum(_extract_p1#471L), sum(_extract_p2#472L), sum(_extract_p3#473L), sum(_extract_p4#474L), sum(_extract_p5#475L), sum(_extract_p6#476L), sum(_extract_p7#477L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=594]\n +- CometHashAggregate [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [Partial], [partial_sum(_extract_p0#470L), partial_sum(_extract_p1#471L), partial_sum(_extract_p2#472L), partial_sum(_extract_p3#473L), partial_sum(_extract_p4#474L), partial_sum(_extract_p5#475L), partial_sum(_extract_p6#476L), partial_sum(_extract_p7#477L)]\n +- CometProject [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [s#1.p0 AS _extract_p0#470L, s#1.p1 AS _extract_p1#471L, s#1.p2 AS _extract_p2#472L, s#1.p3 AS _extract_p3#473L, s#1.p4 AS _extract_p4#474L, s#1.p5 AS _extract_p5#475L, s#1.p6 AS _extract_p6#476L, s#1.p7 AS _extract_p7#477L]\n +- CometFilter [s#1], (isnotnull(s#1.inner.k) AND (s#1.inner.k >= 4128768))\n +- CometNativeScan parquet [s#1] Batched: true, DataFilters: [isnotnull(s#1.inner.k), (s#1.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 3 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 19480 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 526, + "decode_time": 746503, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 151809, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 1064290, + "numPartitions": 1, + "encode_time": 26980, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 526, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 539487 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 47948 + } + }, + { + "node": "CometFilter", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 12320530 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 466543403, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 0, + "predicate_cache_inner_records": 0, + "filesSize": 199358457, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 2694414, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 64, + "row_pushdown_eval_time": 2, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 0, + "page_index_eval_time": 2, + "bytes_scanned": 182481422, + "predicate_evaluation_errors": 0, + "metadata_load_time": 754783, + "output_rows": 4194304, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 101, + "metadataTime": 0, + "scan_efficiency_ratio_total": 199358457, + "row_groups_matched_statistics": 64, + "statistics_eval_time": 2, + "time_elapsed_opening": 835172, + "row_groups_pruned_statistics": 0, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 385843795, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + }, + "top_level_selective": { + "seconds": [ + 0.14080541394650936, + 0.12375261634588242, + 0.11019718460738659, + 0.145455040037632, + 0.12480535730719566 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE k >= 4128768", + "answer": [ + 32741059719, + 32774396751, + 32772938793, + 32799973223, + 32905236128, + 32807159809, + 32718967833, + 32661111725 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#986L, sum#987L, sum#988L, sum#989L, sum#990L, sum#991L, sum#992L, sum#993L], [Final], [sum(_extract_p0#968L), sum(_extract_p1#969L), sum(_extract_p2#970L), sum(_extract_p3#971L), sum(_extract_p4#972L), sum(_extract_p5#973L), sum(_extract_p6#974L), sum(_extract_p7#975L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1204]\n +- CometHashAggregate [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [Partial], [partial_sum(_extract_p0#968L), partial_sum(_extract_p1#969L), partial_sum(_extract_p2#970L), partial_sum(_extract_p3#971L), partial_sum(_extract_p4#972L), partial_sum(_extract_p5#973L), partial_sum(_extract_p6#974L), partial_sum(_extract_p7#975L)]\n +- CometProject [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [s#1.p0 AS _extract_p0#968L, s#1.p1 AS _extract_p1#969L, s#1.p2 AS _extract_p2#970L, s#1.p3 AS _extract_p3#971L, s#1.p4 AS _extract_p4#972L, s#1.p5 AS _extract_p5#973L, s#1.p6 AS _extract_p6#974L, s#1.p7 AS _extract_p7#975L]\n +- CometFilter [k#0L, s#1], (isnotnull(k#0L) AND (k#0L >= 4128768))\n +- CometNativeScan parquet [k#0L,s#1] Batched: true, DataFilters: [isnotnull(k#0L), (k#0L >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(k), GreaterThanOrEqual(k,4128768)], ReadSchema: struct>\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 2 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 19440 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 526, + "decode_time": 414026, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 131429, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 794482, + "numPartitions": 1, + "encode_time": 32149, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 526, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 34312512 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 4194304, + "elapsed_compute": 3722041 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 452814152, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 0, + "predicate_cache_inner_records": 0, + "filesSize": 199358457, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 2521226, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 64, + "row_pushdown_eval_time": 2, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 0, + "page_index_eval_time": 2, + "bytes_scanned": 165673158, + "predicate_evaluation_errors": 0, + "metadata_load_time": 745524, + "output_rows": 4194304, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 121, + "metadataTime": 0, + "scan_efficiency_ratio_total": 199358457, + "row_groups_matched_statistics": 64, + "statistics_eval_time": 2, + "time_elapsed_opening": 955812, + "row_groups_pruned_statistics": 0, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 343454356, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + }, + "nested_unprunable": { + "seconds": [ + 0.5853548347949982, + 0.5719598717987537, + 0.5750899221748114, + 0.5677164318040013, + 0.5670498479157686 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM shuffled WHERE s.inner.k >= 4128768", + "answer": [ + 32658560674, + 32812041019, + 32867632839, + 32498761177, + 32648273638, + 32881993944, + 32750626290, + 32782672280 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#1976L, sum#1977L, sum#1978L, sum#1979L, sum#1980L, sum#1981L, sum#1982L, sum#1983L], [Final], [sum(_extract_p0#1960L), sum(_extract_p1#1961L), sum(_extract_p2#1962L), sum(_extract_p3#1963L), sum(_extract_p4#1964L), sum(_extract_p5#1965L), sum(_extract_p6#1966L), sum(_extract_p7#1967L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2334]\n +- CometHashAggregate [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [Partial], [partial_sum(_extract_p0#1960L), partial_sum(_extract_p1#1961L), partial_sum(_extract_p2#1962L), partial_sum(_extract_p3#1963L), partial_sum(_extract_p4#1964L), partial_sum(_extract_p5#1965L), partial_sum(_extract_p6#1966L), partial_sum(_extract_p7#1967L)]\n +- CometProject [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [s#3.p0 AS _extract_p0#1960L, s#3.p1 AS _extract_p1#1961L, s#3.p2 AS _extract_p2#1962L, s#3.p3 AS _extract_p3#1963L, s#3.p4 AS _extract_p4#1964L, s#3.p5 AS _extract_p5#1965L, s#3.p6 AS _extract_p6#1966L, s#3.p7 AS _extract_p7#1967L]\n +- CometFilter [s#3], (isnotnull(s#3.inner.k) AND (s#3.inner.k >= 4128768))\n +- CometNativeScan parquet [s#3] Batched: true, DataFilters: [isnotnull(s#3.inner.k), (s#3.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/shuffled.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 2 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 20070 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 527, + "decode_time": 455145, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 141189, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 861171, + "numPartitions": 1, + "encode_time": 29249, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 527, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 552116 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 74758 + } + }, + { + "node": "CometFilter", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 18256290 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 466942698, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 0, + "predicate_cache_inner_records": 0, + "filesSize": 207941114, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 2784123, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 64, + "row_pushdown_eval_time": 2, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 0, + "page_index_eval_time": 2, + "bytes_scanned": 186772555, + "predicate_evaluation_errors": 0, + "metadata_load_time": 793873, + "output_rows": 4194304, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 121, + "metadataTime": 0, + "scan_efficiency_ratio_total": 207941114, + "row_groups_matched_statistics": 64, + "statistics_eval_time": 2, + "time_elapsed_opening": 911391, + "row_groups_pruned_statistics": 0, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 383420159, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + } + } + }, + { + "label": "baseline-1", + "spark": "4.1.3", + "pruning_off": false, + "row_filter": false, + "warmup": 5, + "cases": { + "nested_selective": { + "seconds": [ + 0.6316028796136379, + 0.6436244547367096, + 0.6065702280029655, + 0.6287999972701073, + 0.6161443758755922 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE s.inner.k >= 4128768", + "answer": [ + 32741059719, + 32774396751, + 32772938793, + 32799973223, + 32905236128, + 32807159809, + 32718967833, + 32661111725 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#486L, sum#487L, sum#488L, sum#489L, sum#490L, sum#491L, sum#492L, sum#493L], [Final], [sum(_extract_p0#470L), sum(_extract_p1#471L), sum(_extract_p2#472L), sum(_extract_p3#473L), sum(_extract_p4#474L), sum(_extract_p5#475L), sum(_extract_p6#476L), sum(_extract_p7#477L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=594]\n +- CometHashAggregate [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [Partial], [partial_sum(_extract_p0#470L), partial_sum(_extract_p1#471L), partial_sum(_extract_p2#472L), partial_sum(_extract_p3#473L), partial_sum(_extract_p4#474L), partial_sum(_extract_p5#475L), partial_sum(_extract_p6#476L), partial_sum(_extract_p7#477L)]\n +- CometProject [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [s#1.p0 AS _extract_p0#470L, s#1.p1 AS _extract_p1#471L, s#1.p2 AS _extract_p2#472L, s#1.p3 AS _extract_p3#473L, s#1.p4 AS _extract_p4#474L, s#1.p5 AS _extract_p5#475L, s#1.p6 AS _extract_p6#476L, s#1.p7 AS _extract_p7#477L]\n +- CometFilter [s#1], (isnotnull(s#1.inner.k) AND (s#1.inner.k >= 4128768))\n +- CometNativeScan parquet [s#1] Batched: true, DataFilters: [isnotnull(s#1.inner.k), (s#1.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 4 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 24360 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 526, + "decode_time": 1010631, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 131768, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 1252738, + "numPartitions": 1, + "encode_time": 27390, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 526, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 506868 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 46139 + } + }, + { + "node": "CometFilter", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 12313974 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 470639031, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 0, + "predicate_cache_inner_records": 0, + "filesSize": 199358457, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 2616935, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 64, + "row_pushdown_eval_time": 2, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 0, + "page_index_eval_time": 2, + "bytes_scanned": 182481422, + "predicate_evaluation_errors": 0, + "metadata_load_time": 730644, + "output_rows": 4194304, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 131, + "metadataTime": 0, + "scan_efficiency_ratio_total": 199358457, + "row_groups_matched_statistics": 64, + "statistics_eval_time": 2, + "time_elapsed_opening": 858293, + "row_groups_pruned_statistics": 0, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 384754173, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + }, + "top_level_selective": { + "seconds": [ + 0.13424876146018505, + 0.14073058031499386, + 0.11925629619508982, + 0.14138470403850079, + 0.1284897569566965 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE k >= 4128768", + "answer": [ + 32741059719, + 32774396751, + 32772938793, + 32799973223, + 32905236128, + 32807159809, + 32718967833, + 32661111725 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#986L, sum#987L, sum#988L, sum#989L, sum#990L, sum#991L, sum#992L, sum#993L], [Final], [sum(_extract_p0#968L), sum(_extract_p1#969L), sum(_extract_p2#970L), sum(_extract_p3#971L), sum(_extract_p4#972L), sum(_extract_p5#973L), sum(_extract_p6#974L), sum(_extract_p7#975L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1204]\n +- CometHashAggregate [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [Partial], [partial_sum(_extract_p0#968L), partial_sum(_extract_p1#969L), partial_sum(_extract_p2#970L), partial_sum(_extract_p3#971L), partial_sum(_extract_p4#972L), partial_sum(_extract_p5#973L), partial_sum(_extract_p6#974L), partial_sum(_extract_p7#975L)]\n +- CometProject [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [s#1.p0 AS _extract_p0#968L, s#1.p1 AS _extract_p1#969L, s#1.p2 AS _extract_p2#970L, s#1.p3 AS _extract_p3#971L, s#1.p4 AS _extract_p4#972L, s#1.p5 AS _extract_p5#973L, s#1.p6 AS _extract_p6#974L, s#1.p7 AS _extract_p7#975L]\n +- CometFilter [k#0L, s#1], (isnotnull(k#0L) AND (k#0L >= 4128768))\n +- CometNativeScan parquet [k#0L,s#1] Batched: true, DataFilters: [isnotnull(k#0L), (k#0L >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(k), GreaterThanOrEqual(k,4128768)], ReadSchema: struct>\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 2 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 18509 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 526, + "decode_time": 414736, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 139100, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 838573, + "numPartitions": 1, + "encode_time": 33560, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 526, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 29503811 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 4194304, + "elapsed_compute": 3183327 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 415581284, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 0, + "predicate_cache_inner_records": 0, + "filesSize": 199358457, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 2352628, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 64, + "row_pushdown_eval_time": 2, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 0, + "page_index_eval_time": 2, + "bytes_scanned": 165673158, + "predicate_evaluation_errors": 0, + "metadata_load_time": 776343, + "output_rows": 4194304, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 141, + "metadataTime": 0, + "scan_efficiency_ratio_total": 199358457, + "row_groups_matched_statistics": 64, + "statistics_eval_time": 2, + "time_elapsed_opening": 922081, + "row_groups_pruned_statistics": 0, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 326972042, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + }, + "nested_unprunable": { + "seconds": [ + 0.568416777998209, + 0.5853670472279191, + 0.6337875267490745, + 0.5857159039005637, + 0.5950113451108336 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM shuffled WHERE s.inner.k >= 4128768", + "answer": [ + 32658560674, + 32812041019, + 32867632839, + 32498761177, + 32648273638, + 32881993944, + 32750626290, + 32782672280 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#1976L, sum#1977L, sum#1978L, sum#1979L, sum#1980L, sum#1981L, sum#1982L, sum#1983L], [Final], [sum(_extract_p0#1960L), sum(_extract_p1#1961L), sum(_extract_p2#1962L), sum(_extract_p3#1963L), sum(_extract_p4#1964L), sum(_extract_p5#1965L), sum(_extract_p6#1966L), sum(_extract_p7#1967L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2334]\n +- CometHashAggregate [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [Partial], [partial_sum(_extract_p0#1960L), partial_sum(_extract_p1#1961L), partial_sum(_extract_p2#1962L), partial_sum(_extract_p3#1963L), partial_sum(_extract_p4#1964L), partial_sum(_extract_p5#1965L), partial_sum(_extract_p6#1966L), partial_sum(_extract_p7#1967L)]\n +- CometProject [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [s#3.p0 AS _extract_p0#1960L, s#3.p1 AS _extract_p1#1961L, s#3.p2 AS _extract_p2#1962L, s#3.p3 AS _extract_p3#1963L, s#3.p4 AS _extract_p4#1964L, s#3.p5 AS _extract_p5#1965L, s#3.p6 AS _extract_p6#1966L, s#3.p7 AS _extract_p7#1967L]\n +- CometFilter [s#3], (isnotnull(s#3.inner.k) AND (s#3.inner.k >= 4128768))\n +- CometNativeScan parquet [s#3] Batched: true, DataFilters: [isnotnull(s#3.inner.k), (s#3.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/shuffled.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 2 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 19230 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 527, + "decode_time": 531525, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 148458, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 773863, + "numPartitions": 1, + "encode_time": 27970, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 527, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 528616 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 69109 + } + }, + { + "node": "CometFilter", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 18318110 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 472289234, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 0, + "predicate_cache_inner_records": 0, + "filesSize": 207941114, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 2838723, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 64, + "row_pushdown_eval_time": 2, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 0, + "page_index_eval_time": 2, + "bytes_scanned": 186772555, + "predicate_evaluation_errors": 0, + "metadata_load_time": 859733, + "output_rows": 4194304, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 121, + "metadataTime": 0, + "scan_efficiency_ratio_total": 207941114, + "row_groups_matched_statistics": 64, + "statistics_eval_time": 2, + "time_elapsed_opening": 995810, + "row_groups_pruned_statistics": 0, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 386357774, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + } + } + }, + { + "label": "baseline-2", + "spark": "4.1.3", + "pruning_off": false, + "row_filter": false, + "warmup": 5, + "cases": { + "nested_selective": { + "seconds": [ + 0.6409595236182213, + 0.6275981506332755, + 0.6454171715304255, + 0.6464711409062147, + 0.6355891628190875 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE s.inner.k >= 4128768", + "answer": [ + 32741059719, + 32774396751, + 32772938793, + 32799973223, + 32905236128, + 32807159809, + 32718967833, + 32661111725 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#486L, sum#487L, sum#488L, sum#489L, sum#490L, sum#491L, sum#492L, sum#493L], [Final], [sum(_extract_p0#470L), sum(_extract_p1#471L), sum(_extract_p2#472L), sum(_extract_p3#473L), sum(_extract_p4#474L), sum(_extract_p5#475L), sum(_extract_p6#476L), sum(_extract_p7#477L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=594]\n +- CometHashAggregate [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [Partial], [partial_sum(_extract_p0#470L), partial_sum(_extract_p1#471L), partial_sum(_extract_p2#472L), partial_sum(_extract_p3#473L), partial_sum(_extract_p4#474L), partial_sum(_extract_p5#475L), partial_sum(_extract_p6#476L), partial_sum(_extract_p7#477L)]\n +- CometProject [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [s#1.p0 AS _extract_p0#470L, s#1.p1 AS _extract_p1#471L, s#1.p2 AS _extract_p2#472L, s#1.p3 AS _extract_p3#473L, s#1.p4 AS _extract_p4#474L, s#1.p5 AS _extract_p5#475L, s#1.p6 AS _extract_p6#476L, s#1.p7 AS _extract_p7#477L]\n +- CometFilter [s#1], (isnotnull(s#1.inner.k) AND (s#1.inner.k >= 4128768))\n +- CometNativeScan parquet [s#1] Batched: true, DataFilters: [isnotnull(s#1.inner.k), (s#1.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 3 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 23170 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 526, + "decode_time": 763003, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 138378, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 1160478, + "numPartitions": 1, + "encode_time": 28010, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 526, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 408018 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 37948 + } + }, + { + "node": "CometFilter", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 12861779 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 470629325, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 0, + "predicate_cache_inner_records": 0, + "filesSize": 199358457, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 4383559, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 64, + "row_pushdown_eval_time": 2, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 0, + "page_index_eval_time": 2, + "bytes_scanned": 182481422, + "predicate_evaluation_errors": 0, + "metadata_load_time": 942381, + "output_rows": 4194304, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 141, + "metadataTime": 0, + "scan_efficiency_ratio_total": 199358457, + "row_groups_matched_statistics": 64, + "statistics_eval_time": 2, + "time_elapsed_opening": 1060260, + "row_groups_pruned_statistics": 0, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 388815665, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + }, + "top_level_selective": { + "seconds": [ + 0.13689426612108946, + 0.14238543342798948, + 0.12872622441500425, + 0.13351120799779892, + 0.1360515933483839 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE k >= 4128768", + "answer": [ + 32741059719, + 32774396751, + 32772938793, + 32799973223, + 32905236128, + 32807159809, + 32718967833, + 32661111725 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#986L, sum#987L, sum#988L, sum#989L, sum#990L, sum#991L, sum#992L, sum#993L], [Final], [sum(_extract_p0#968L), sum(_extract_p1#969L), sum(_extract_p2#970L), sum(_extract_p3#971L), sum(_extract_p4#972L), sum(_extract_p5#973L), sum(_extract_p6#974L), sum(_extract_p7#975L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1204]\n +- CometHashAggregate [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [Partial], [partial_sum(_extract_p0#968L), partial_sum(_extract_p1#969L), partial_sum(_extract_p2#970L), partial_sum(_extract_p3#971L), partial_sum(_extract_p4#972L), partial_sum(_extract_p5#973L), partial_sum(_extract_p6#974L), partial_sum(_extract_p7#975L)]\n +- CometProject [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [s#1.p0 AS _extract_p0#968L, s#1.p1 AS _extract_p1#969L, s#1.p2 AS _extract_p2#970L, s#1.p3 AS _extract_p3#971L, s#1.p4 AS _extract_p4#972L, s#1.p5 AS _extract_p5#973L, s#1.p6 AS _extract_p6#974L, s#1.p7 AS _extract_p7#975L]\n +- CometFilter [k#0L, s#1], (isnotnull(k#0L) AND (k#0L >= 4128768))\n +- CometNativeScan parquet [k#0L,s#1] Batched: true, DataFilters: [isnotnull(k#0L), (k#0L >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(k), GreaterThanOrEqual(k,4128768)], ReadSchema: struct>\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 3 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 23500 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 526, + "decode_time": 616295, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 125098, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 1003540, + "numPartitions": 1, + "encode_time": 28750, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 526, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 31021971 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 4194304, + "elapsed_compute": 3687545 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 437298712, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 0, + "predicate_cache_inner_records": 0, + "filesSize": 199358457, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 2228229, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 64, + "row_pushdown_eval_time": 2, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 0, + "page_index_eval_time": 2, + "bytes_scanned": 165673158, + "predicate_evaluation_errors": 0, + "metadata_load_time": 603585, + "output_rows": 4194304, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 121, + "metadataTime": 0, + "scan_efficiency_ratio_total": 199358457, + "row_groups_matched_statistics": 64, + "statistics_eval_time": 2, + "time_elapsed_opening": 748802, + "row_groups_pruned_statistics": 0, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 334708499, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + }, + "nested_unprunable": { + "seconds": [ + 0.595560303889215, + 0.590427883900702, + 0.575502255000174, + 0.5648535676300526, + 0.5603784089908004 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM shuffled WHERE s.inner.k >= 4128768", + "answer": [ + 32658560674, + 32812041019, + 32867632839, + 32498761177, + 32648273638, + 32881993944, + 32750626290, + 32782672280 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#1976L, sum#1977L, sum#1978L, sum#1979L, sum#1980L, sum#1981L, sum#1982L, sum#1983L], [Final], [sum(_extract_p0#1960L), sum(_extract_p1#1961L), sum(_extract_p2#1962L), sum(_extract_p3#1963L), sum(_extract_p4#1964L), sum(_extract_p5#1965L), sum(_extract_p6#1966L), sum(_extract_p7#1967L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2334]\n +- CometHashAggregate [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [Partial], [partial_sum(_extract_p0#1960L), partial_sum(_extract_p1#1961L), partial_sum(_extract_p2#1962L), partial_sum(_extract_p3#1963L), partial_sum(_extract_p4#1964L), partial_sum(_extract_p5#1965L), partial_sum(_extract_p6#1966L), partial_sum(_extract_p7#1967L)]\n +- CometProject [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [s#3.p0 AS _extract_p0#1960L, s#3.p1 AS _extract_p1#1961L, s#3.p2 AS _extract_p2#1962L, s#3.p3 AS _extract_p3#1963L, s#3.p4 AS _extract_p4#1964L, s#3.p5 AS _extract_p5#1965L, s#3.p6 AS _extract_p6#1966L, s#3.p7 AS _extract_p7#1967L]\n +- CometFilter [s#3], (isnotnull(s#3.inner.k) AND (s#3.inner.k >= 4128768))\n +- CometNativeScan parquet [s#3] Batched: true, DataFilters: [isnotnull(s#3.inner.k), (s#3.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/shuffled.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 2 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 24990 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 527, + "decode_time": 507995, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 133459, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 901032, + "numPartitions": 1, + "encode_time": 30301, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 527, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 505934 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 66330 + } + }, + { + "node": "CometFilter", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 17572895 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 466456893, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 0, + "predicate_cache_inner_records": 0, + "filesSize": 207941114, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 2937692, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 64, + "row_pushdown_eval_time": 2, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 0, + "page_index_eval_time": 2, + "bytes_scanned": 186772555, + "predicate_evaluation_errors": 0, + "metadata_load_time": 687675, + "output_rows": 4194304, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 91, + "metadataTime": 0, + "scan_efficiency_ratio_total": 207941114, + "row_groups_matched_statistics": 64, + "statistics_eval_time": 2, + "time_elapsed_opening": 820532, + "row_groups_pruned_statistics": 0, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 381924008, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + } + } + }, + { + "label": "baseline-3", + "spark": "4.1.3", + "pruning_off": false, + "row_filter": false, + "warmup": 5, + "cases": { + "nested_selective": { + "seconds": [ + 0.6592776225879788, + 0.6437941892072558, + 0.6677187196910381, + 0.6587780667468905, + 0.6453634444624186 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE s.inner.k >= 4128768", + "answer": [ + 32741059719, + 32774396751, + 32772938793, + 32799973223, + 32905236128, + 32807159809, + 32718967833, + 32661111725 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#486L, sum#487L, sum#488L, sum#489L, sum#490L, sum#491L, sum#492L, sum#493L], [Final], [sum(_extract_p0#470L), sum(_extract_p1#471L), sum(_extract_p2#472L), sum(_extract_p3#473L), sum(_extract_p4#474L), sum(_extract_p5#475L), sum(_extract_p6#476L), sum(_extract_p7#477L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=594]\n +- CometHashAggregate [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [Partial], [partial_sum(_extract_p0#470L), partial_sum(_extract_p1#471L), partial_sum(_extract_p2#472L), partial_sum(_extract_p3#473L), partial_sum(_extract_p4#474L), partial_sum(_extract_p5#475L), partial_sum(_extract_p6#476L), partial_sum(_extract_p7#477L)]\n +- CometProject [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [s#1.p0 AS _extract_p0#470L, s#1.p1 AS _extract_p1#471L, s#1.p2 AS _extract_p2#472L, s#1.p3 AS _extract_p3#473L, s#1.p4 AS _extract_p4#474L, s#1.p5 AS _extract_p5#475L, s#1.p6 AS _extract_p6#476L, s#1.p7 AS _extract_p7#477L]\n +- CometFilter [s#1], (isnotnull(s#1.inner.k) AND (s#1.inner.k >= 4128768))\n +- CometNativeScan parquet [s#1] Batched: true, DataFilters: [isnotnull(s#1.inner.k), (s#1.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 3 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 19510 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 526, + "decode_time": 735554, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 145849, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 1099190, + "numPartitions": 1, + "encode_time": 28100, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 526, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 526914 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 43539 + } + }, + { + "node": "CometFilter", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 14507373 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 495112241, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 0, + "predicate_cache_inner_records": 0, + "filesSize": 199358457, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 3515616, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 64, + "row_pushdown_eval_time": 2, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 0, + "page_index_eval_time": 2, + "bytes_scanned": 182481422, + "predicate_evaluation_errors": 0, + "metadata_load_time": 914442, + "output_rows": 4194304, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 111, + "metadataTime": 0, + "scan_efficiency_ratio_total": 199358457, + "row_groups_matched_statistics": 64, + "statistics_eval_time": 2, + "time_elapsed_opening": 1007160, + "row_groups_pruned_statistics": 0, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 404711922, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + }, + "top_level_selective": { + "seconds": [ + 0.15336241573095322, + 0.21535575296729803, + 0.1927366890013218, + 0.16786616574972868, + 0.1431801514700055 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE k >= 4128768", + "answer": [ + 32741059719, + 32774396751, + 32772938793, + 32799973223, + 32905236128, + 32807159809, + 32718967833, + 32661111725 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#986L, sum#987L, sum#988L, sum#989L, sum#990L, sum#991L, sum#992L, sum#993L], [Final], [sum(_extract_p0#968L), sum(_extract_p1#969L), sum(_extract_p2#970L), sum(_extract_p3#971L), sum(_extract_p4#972L), sum(_extract_p5#973L), sum(_extract_p6#974L), sum(_extract_p7#975L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1204]\n +- CometHashAggregate [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [Partial], [partial_sum(_extract_p0#968L), partial_sum(_extract_p1#969L), partial_sum(_extract_p2#970L), partial_sum(_extract_p3#971L), partial_sum(_extract_p4#972L), partial_sum(_extract_p5#973L), partial_sum(_extract_p6#974L), partial_sum(_extract_p7#975L)]\n +- CometProject [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [s#1.p0 AS _extract_p0#968L, s#1.p1 AS _extract_p1#969L, s#1.p2 AS _extract_p2#970L, s#1.p3 AS _extract_p3#971L, s#1.p4 AS _extract_p4#972L, s#1.p5 AS _extract_p5#973L, s#1.p6 AS _extract_p6#974L, s#1.p7 AS _extract_p7#975L]\n +- CometFilter [k#0L, s#1], (isnotnull(k#0L) AND (k#0L >= 4128768))\n +- CometNativeScan parquet [k#0L,s#1] Batched: true, DataFilters: [isnotnull(k#0L), (k#0L >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(k), GreaterThanOrEqual(k,4128768)], ReadSchema: struct>\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 2 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 18769 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 526, + "decode_time": 454405, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 131129, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 726884, + "numPartitions": 1, + "encode_time": 27160, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 526, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 30755644 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 4194304, + "elapsed_compute": 3470066 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 428848163, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 0, + "predicate_cache_inner_records": 0, + "filesSize": 199358457, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 2374218, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 64, + "row_pushdown_eval_time": 2, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 0, + "page_index_eval_time": 2, + "bytes_scanned": 165673158, + "predicate_evaluation_errors": 0, + "metadata_load_time": 649384, + "output_rows": 4194304, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 81, + "metadataTime": 0, + "scan_efficiency_ratio_total": 199358457, + "row_groups_matched_statistics": 64, + "statistics_eval_time": 2, + "time_elapsed_opening": 772642, + "row_groups_pruned_statistics": 0, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 329854295, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + }, + "nested_unprunable": { + "seconds": [ + 0.6037412909790874, + 0.588707254268229, + 0.6129186730831861, + 0.5801800154149532, + 0.5867422120645642 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM shuffled WHERE s.inner.k >= 4128768", + "answer": [ + 32658560674, + 32812041019, + 32867632839, + 32498761177, + 32648273638, + 32881993944, + 32750626290, + 32782672280 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#1976L, sum#1977L, sum#1978L, sum#1979L, sum#1980L, sum#1981L, sum#1982L, sum#1983L], [Final], [sum(_extract_p0#1960L), sum(_extract_p1#1961L), sum(_extract_p2#1962L), sum(_extract_p3#1963L), sum(_extract_p4#1964L), sum(_extract_p5#1965L), sum(_extract_p6#1966L), sum(_extract_p7#1967L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2334]\n +- CometHashAggregate [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [Partial], [partial_sum(_extract_p0#1960L), partial_sum(_extract_p1#1961L), partial_sum(_extract_p2#1962L), partial_sum(_extract_p3#1963L), partial_sum(_extract_p4#1964L), partial_sum(_extract_p5#1965L), partial_sum(_extract_p6#1966L), partial_sum(_extract_p7#1967L)]\n +- CometProject [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [s#3.p0 AS _extract_p0#1960L, s#3.p1 AS _extract_p1#1961L, s#3.p2 AS _extract_p2#1962L, s#3.p3 AS _extract_p3#1963L, s#3.p4 AS _extract_p4#1964L, s#3.p5 AS _extract_p5#1965L, s#3.p6 AS _extract_p6#1966L, s#3.p7 AS _extract_p7#1967L]\n +- CometFilter [s#3], (isnotnull(s#3.inner.k) AND (s#3.inner.k >= 4128768))\n +- CometNativeScan parquet [s#3] Batched: true, DataFilters: [isnotnull(s#3.inner.k), (s#3.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/shuffled.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 2 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 22099 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 527, + "decode_time": 471595, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 146169, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 811843, + "numPartitions": 1, + "encode_time": 32070, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 527, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 592173 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 65260 + } + }, + { + "node": "CometFilter", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 18570095 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 477367026, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 0, + "predicate_cache_inner_records": 0, + "filesSize": 207941114, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 2710304, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 64, + "row_pushdown_eval_time": 2, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 0, + "page_index_eval_time": 2, + "bytes_scanned": 186772555, + "predicate_evaluation_errors": 0, + "metadata_load_time": 730174, + "output_rows": 4194304, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 101, + "metadataTime": 0, + "scan_efficiency_ratio_total": 207941114, + "row_groups_matched_statistics": 64, + "statistics_eval_time": 2, + "time_elapsed_opening": 818543, + "row_groups_pruned_statistics": 0, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 393760680, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + } + } + }, + { + "label": "candidate-0", + "spark": "4.1.3", + "pruning_off": false, + "row_filter": false, + "warmup": 5, + "cases": { + "nested_selective": { + "seconds": [ + 0.2042244616895914, + 0.18543519917875528, + 0.16565270721912384, + 0.1565515324473381, + 0.15750608406960964 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE s.inner.k >= 4128768", + "answer": [ + 32741059719, + 32774396751, + 32772938793, + 32799973223, + 32905236128, + 32807159809, + 32718967833, + 32661111725 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#486L, sum#487L, sum#488L, sum#489L, sum#490L, sum#491L, sum#492L, sum#493L], [Final], [sum(_extract_p0#470L), sum(_extract_p1#471L), sum(_extract_p2#472L), sum(_extract_p3#473L), sum(_extract_p4#474L), sum(_extract_p5#475L), sum(_extract_p6#476L), sum(_extract_p7#477L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=594]\n +- CometHashAggregate [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [Partial], [partial_sum(_extract_p0#470L), partial_sum(_extract_p1#471L), partial_sum(_extract_p2#472L), partial_sum(_extract_p3#473L), partial_sum(_extract_p4#474L), partial_sum(_extract_p5#475L), partial_sum(_extract_p6#476L), partial_sum(_extract_p7#477L)]\n +- CometProject [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [s#1.p0 AS _extract_p0#470L, s#1.p1 AS _extract_p1#471L, s#1.p2 AS _extract_p2#472L, s#1.p3 AS _extract_p3#473L, s#1.p4 AS _extract_p4#474L, s#1.p5 AS _extract_p5#475L, s#1.p6 AS _extract_p6#476L, s#1.p7 AS _extract_p7#477L]\n +- CometFilter [s#1], (isnotnull(s#1.inner.k) AND (s#1.inner.k >= 4128768))\n +- CometNativeScan parquet [s#1] Batched: true, DataFilters: [isnotnull(s#1.inner.k), (s#1.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 3 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 19640 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 526, + "decode_time": 721214, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 133359, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 1454205, + "numPartitions": 1, + "encode_time": 30339, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 526, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 582483 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 77019 + } + }, + { + "node": "CometFilter", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 247780 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 8220012, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 0, + "predicate_cache_inner_records": 0, + "filesSize": 199358457, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 2743914, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 1, + "row_pushdown_eval_time": 2, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 0, + "page_index_eval_time": 2, + "bytes_scanned": 2851301, + "predicate_evaluation_errors": 0, + "metadata_load_time": 873414, + "output_rows": 65536, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 111, + "metadataTime": 0, + "scan_efficiency_ratio_total": 199358457, + "row_groups_matched_statistics": 1, + "statistics_eval_time": 161710, + "time_elapsed_opening": 1247678, + "row_groups_pruned_statistics": 63, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 7230549, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + }, + "top_level_selective": { + "seconds": [ + 0.144645806401968, + 0.14989290665835142, + 0.14998808596283197, + 0.1314064422622323, + 0.1390496101230383 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE k >= 4128768", + "answer": [ + 32741059719, + 32774396751, + 32772938793, + 32799973223, + 32905236128, + 32807159809, + 32718967833, + 32661111725 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#986L, sum#987L, sum#988L, sum#989L, sum#990L, sum#991L, sum#992L, sum#993L], [Final], [sum(_extract_p0#968L), sum(_extract_p1#969L), sum(_extract_p2#970L), sum(_extract_p3#971L), sum(_extract_p4#972L), sum(_extract_p5#973L), sum(_extract_p6#974L), sum(_extract_p7#975L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1204]\n +- CometHashAggregate [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [Partial], [partial_sum(_extract_p0#968L), partial_sum(_extract_p1#969L), partial_sum(_extract_p2#970L), partial_sum(_extract_p3#971L), partial_sum(_extract_p4#972L), partial_sum(_extract_p5#973L), partial_sum(_extract_p6#974L), partial_sum(_extract_p7#975L)]\n +- CometProject [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [s#1.p0 AS _extract_p0#968L, s#1.p1 AS _extract_p1#969L, s#1.p2 AS _extract_p2#970L, s#1.p3 AS _extract_p3#971L, s#1.p4 AS _extract_p4#972L, s#1.p5 AS _extract_p5#973L, s#1.p6 AS _extract_p6#974L, s#1.p7 AS _extract_p7#975L]\n +- CometFilter [k#0L, s#1], (isnotnull(k#0L) AND (k#0L >= 4128768))\n +- CometNativeScan parquet [k#0L,s#1] Batched: true, DataFilters: [isnotnull(k#0L), (k#0L >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(k), GreaterThanOrEqual(k,4128768)], ReadSchema: struct>\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 2 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 16140 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 526, + "decode_time": 506835, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 155388, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 761703, + "numPartitions": 1, + "encode_time": 36300, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 526, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 30316709 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 4194304, + "elapsed_compute": 5984147 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 419252397, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 0, + "predicate_cache_inner_records": 0, + "filesSize": 199358457, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 2213759, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 64, + "row_pushdown_eval_time": 2, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 0, + "page_index_eval_time": 2, + "bytes_scanned": 165673158, + "predicate_evaluation_errors": 0, + "metadata_load_time": 683165, + "output_rows": 4194304, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 91, + "metadataTime": 0, + "scan_efficiency_ratio_total": 199358457, + "row_groups_matched_statistics": 64, + "statistics_eval_time": 2, + "time_elapsed_opening": 847602, + "row_groups_pruned_statistics": 0, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 328324266, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + }, + "nested_unprunable": { + "seconds": [ + 0.5705090211704373, + 0.5631797285750508, + 0.5575059540569782, + 0.5655996361747384, + 0.58751414809376 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM shuffled WHERE s.inner.k >= 4128768", + "answer": [ + 32658560674, + 32812041019, + 32867632839, + 32498761177, + 32648273638, + 32881993944, + 32750626290, + 32782672280 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#1976L, sum#1977L, sum#1978L, sum#1979L, sum#1980L, sum#1981L, sum#1982L, sum#1983L], [Final], [sum(_extract_p0#1960L), sum(_extract_p1#1961L), sum(_extract_p2#1962L), sum(_extract_p3#1963L), sum(_extract_p4#1964L), sum(_extract_p5#1965L), sum(_extract_p6#1966L), sum(_extract_p7#1967L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2334]\n +- CometHashAggregate [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [Partial], [partial_sum(_extract_p0#1960L), partial_sum(_extract_p1#1961L), partial_sum(_extract_p2#1962L), partial_sum(_extract_p3#1963L), partial_sum(_extract_p4#1964L), partial_sum(_extract_p5#1965L), partial_sum(_extract_p6#1966L), partial_sum(_extract_p7#1967L)]\n +- CometProject [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [s#3.p0 AS _extract_p0#1960L, s#3.p1 AS _extract_p1#1961L, s#3.p2 AS _extract_p2#1962L, s#3.p3 AS _extract_p3#1963L, s#3.p4 AS _extract_p4#1964L, s#3.p5 AS _extract_p5#1965L, s#3.p6 AS _extract_p6#1966L, s#3.p7 AS _extract_p7#1967L]\n +- CometFilter [s#3], (isnotnull(s#3.inner.k) AND (s#3.inner.k >= 4128768))\n +- CometNativeScan parquet [s#3] Batched: true, DataFilters: [isnotnull(s#3.inner.k), (s#3.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/shuffled.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 2 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 16840 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 527, + "decode_time": 419237, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 136768, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 801402, + "numPartitions": 1, + "encode_time": 28240, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 527, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 529916 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 125728 + } + }, + { + "node": "CometFilter", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 22522085 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 482903791, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 0, + "predicate_cache_inner_records": 0, + "filesSize": 207941114, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 2613066, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 64, + "row_pushdown_eval_time": 2, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 0, + "page_index_eval_time": 2, + "bytes_scanned": 186772555, + "predicate_evaluation_errors": 0, + "metadata_load_time": 621005, + "output_rows": 4194304, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 81, + "metadataTime": 0, + "scan_efficiency_ratio_total": 207941114, + "row_groups_matched_statistics": 64, + "statistics_eval_time": 105420, + "time_elapsed_opening": 951250, + "row_groups_pruned_statistics": 0, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 394684340, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + } + } + }, + { + "label": "candidate-1", + "spark": "4.1.3", + "pruning_off": false, + "row_filter": false, + "warmup": 5, + "cases": { + "nested_selective": { + "seconds": [ + 0.18005445785820484, + 0.1846651053056121, + 0.2008777316659689, + 0.19776866026222706, + 0.1689277533441782 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE s.inner.k >= 4128768", + "answer": [ + 32741059719, + 32774396751, + 32772938793, + 32799973223, + 32905236128, + 32807159809, + 32718967833, + 32661111725 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#486L, sum#487L, sum#488L, sum#489L, sum#490L, sum#491L, sum#492L, sum#493L], [Final], [sum(_extract_p0#470L), sum(_extract_p1#471L), sum(_extract_p2#472L), sum(_extract_p3#473L), sum(_extract_p4#474L), sum(_extract_p5#475L), sum(_extract_p6#476L), sum(_extract_p7#477L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=594]\n +- CometHashAggregate [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [Partial], [partial_sum(_extract_p0#470L), partial_sum(_extract_p1#471L), partial_sum(_extract_p2#472L), partial_sum(_extract_p3#473L), partial_sum(_extract_p4#474L), partial_sum(_extract_p5#475L), partial_sum(_extract_p6#476L), partial_sum(_extract_p7#477L)]\n +- CometProject [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [s#1.p0 AS _extract_p0#470L, s#1.p1 AS _extract_p1#471L, s#1.p2 AS _extract_p2#472L, s#1.p3 AS _extract_p3#473L, s#1.p4 AS _extract_p4#474L, s#1.p5 AS _extract_p5#475L, s#1.p6 AS _extract_p6#476L, s#1.p7 AS _extract_p7#477L]\n +- CometFilter [s#1], (isnotnull(s#1.inner.k) AND (s#1.inner.k >= 4128768))\n +- CometNativeScan parquet [s#1] Batched: true, DataFilters: [isnotnull(s#1.inner.k), (s#1.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 5 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 25490 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 526, + "decode_time": 709473, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 127649, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 2567907, + "numPartitions": 1, + "encode_time": 35670, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 526, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 455267 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 64959 + } + }, + { + "node": "CometFilter", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 204199 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 7833416, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 0, + "predicate_cache_inner_records": 0, + "filesSize": 199358457, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 2729324, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 1, + "row_pushdown_eval_time": 2, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 0, + "page_index_eval_time": 2, + "bytes_scanned": 2851301, + "predicate_evaluation_errors": 0, + "metadata_load_time": 940631, + "output_rows": 65536, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 111, + "metadataTime": 0, + "scan_efficiency_ratio_total": 199358457, + "row_groups_matched_statistics": 1, + "statistics_eval_time": 122540, + "time_elapsed_opening": 1236698, + "row_groups_pruned_statistics": 63, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 7021741, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + }, + "top_level_selective": { + "seconds": [ + 0.1509197847917676, + 0.1415041647851467, + 0.15639991406351328, + 0.11335313320159912, + 0.12271884363144636 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE k >= 4128768", + "answer": [ + 32741059719, + 32774396751, + 32772938793, + 32799973223, + 32905236128, + 32807159809, + 32718967833, + 32661111725 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#986L, sum#987L, sum#988L, sum#989L, sum#990L, sum#991L, sum#992L, sum#993L], [Final], [sum(_extract_p0#968L), sum(_extract_p1#969L), sum(_extract_p2#970L), sum(_extract_p3#971L), sum(_extract_p4#972L), sum(_extract_p5#973L), sum(_extract_p6#974L), sum(_extract_p7#975L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1204]\n +- CometHashAggregate [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [Partial], [partial_sum(_extract_p0#968L), partial_sum(_extract_p1#969L), partial_sum(_extract_p2#970L), partial_sum(_extract_p3#971L), partial_sum(_extract_p4#972L), partial_sum(_extract_p5#973L), partial_sum(_extract_p6#974L), partial_sum(_extract_p7#975L)]\n +- CometProject [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [s#1.p0 AS _extract_p0#968L, s#1.p1 AS _extract_p1#969L, s#1.p2 AS _extract_p2#970L, s#1.p3 AS _extract_p3#971L, s#1.p4 AS _extract_p4#972L, s#1.p5 AS _extract_p5#973L, s#1.p6 AS _extract_p6#974L, s#1.p7 AS _extract_p7#975L]\n +- CometFilter [k#0L, s#1], (isnotnull(k#0L) AND (k#0L >= 4128768))\n +- CometNativeScan parquet [k#0L,s#1] Batched: true, DataFilters: [isnotnull(k#0L), (k#0L >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(k), GreaterThanOrEqual(k,4128768)], ReadSchema: struct>\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 2 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 16520 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 526, + "decode_time": 483245, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 131669, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 828622, + "numPartitions": 1, + "encode_time": 33180, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 526, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 35990879 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 4194304, + "elapsed_compute": 6089895 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 448517056, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 0, + "predicate_cache_inner_records": 0, + "filesSize": 199358457, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 2653374, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 64, + "row_pushdown_eval_time": 2, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 0, + "page_index_eval_time": 2, + "bytes_scanned": 165673158, + "predicate_evaluation_errors": 0, + "metadata_load_time": 836503, + "output_rows": 4194304, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 111, + "metadataTime": 0, + "scan_efficiency_ratio_total": 199358457, + "row_groups_matched_statistics": 64, + "statistics_eval_time": 2, + "time_elapsed_opening": 1017221, + "row_groups_pruned_statistics": 0, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 345028311, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + }, + "nested_unprunable": { + "seconds": [ + 0.5922109065577388, + 0.6122729973867536, + 0.6169724743813276, + 0.5767420660704374, + 0.5864941030740738 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM shuffled WHERE s.inner.k >= 4128768", + "answer": [ + 32658560674, + 32812041019, + 32867632839, + 32498761177, + 32648273638, + 32881993944, + 32750626290, + 32782672280 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#1976L, sum#1977L, sum#1978L, sum#1979L, sum#1980L, sum#1981L, sum#1982L, sum#1983L], [Final], [sum(_extract_p0#1960L), sum(_extract_p1#1961L), sum(_extract_p2#1962L), sum(_extract_p3#1963L), sum(_extract_p4#1964L), sum(_extract_p5#1965L), sum(_extract_p6#1966L), sum(_extract_p7#1967L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2334]\n +- CometHashAggregate [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [Partial], [partial_sum(_extract_p0#1960L), partial_sum(_extract_p1#1961L), partial_sum(_extract_p2#1962L), partial_sum(_extract_p3#1963L), partial_sum(_extract_p4#1964L), partial_sum(_extract_p5#1965L), partial_sum(_extract_p6#1966L), partial_sum(_extract_p7#1967L)]\n +- CometProject [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [s#3.p0 AS _extract_p0#1960L, s#3.p1 AS _extract_p1#1961L, s#3.p2 AS _extract_p2#1962L, s#3.p3 AS _extract_p3#1963L, s#3.p4 AS _extract_p4#1964L, s#3.p5 AS _extract_p5#1965L, s#3.p6 AS _extract_p6#1966L, s#3.p7 AS _extract_p7#1967L]\n +- CometFilter [s#3], (isnotnull(s#3.inner.k) AND (s#3.inner.k >= 4128768))\n +- CometNativeScan parquet [s#3] Batched: true, DataFilters: [isnotnull(s#3.inner.k), (s#3.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/shuffled.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 2 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 16040 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 527, + "decode_time": 388976, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 143020, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 764594, + "numPartitions": 1, + "encode_time": 35479, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 527, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 601804 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 120608 + } + }, + { + "node": "CometFilter", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 21920600 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 494768133, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 0, + "predicate_cache_inner_records": 0, + "filesSize": 207941114, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 3986671, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 64, + "row_pushdown_eval_time": 2, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 0, + "page_index_eval_time": 2, + "bytes_scanned": 186772555, + "predicate_evaluation_errors": 0, + "metadata_load_time": 745275, + "output_rows": 4194304, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 91, + "metadataTime": 0, + "scan_efficiency_ratio_total": 207941114, + "row_groups_matched_statistics": 64, + "statistics_eval_time": 100610, + "time_elapsed_opening": 1060151, + "row_groups_pruned_statistics": 0, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 406038990, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + } + } + }, + { + "label": "candidate-2", + "spark": "4.1.3", + "pruning_off": false, + "row_filter": false, + "warmup": 5, + "cases": { + "nested_selective": { + "seconds": [ + 0.19452941324561834, + 0.17460494581609964, + 0.18056540749967098, + 0.16720692533999681, + 0.16859497223049402 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE s.inner.k >= 4128768", + "answer": [ + 32741059719, + 32774396751, + 32772938793, + 32799973223, + 32905236128, + 32807159809, + 32718967833, + 32661111725 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#486L, sum#487L, sum#488L, sum#489L, sum#490L, sum#491L, sum#492L, sum#493L], [Final], [sum(_extract_p0#470L), sum(_extract_p1#471L), sum(_extract_p2#472L), sum(_extract_p3#473L), sum(_extract_p4#474L), sum(_extract_p5#475L), sum(_extract_p6#476L), sum(_extract_p7#477L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=594]\n +- CometHashAggregate [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [Partial], [partial_sum(_extract_p0#470L), partial_sum(_extract_p1#471L), partial_sum(_extract_p2#472L), partial_sum(_extract_p3#473L), partial_sum(_extract_p4#474L), partial_sum(_extract_p5#475L), partial_sum(_extract_p6#476L), partial_sum(_extract_p7#477L)]\n +- CometProject [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [s#1.p0 AS _extract_p0#470L, s#1.p1 AS _extract_p1#471L, s#1.p2 AS _extract_p2#472L, s#1.p3 AS _extract_p3#473L, s#1.p4 AS _extract_p4#474L, s#1.p5 AS _extract_p5#475L, s#1.p6 AS _extract_p6#476L, s#1.p7 AS _extract_p7#477L]\n +- CometFilter [s#1], (isnotnull(s#1.inner.k) AND (s#1.inner.k >= 4128768))\n +- CometNativeScan parquet [s#1] Batched: true, DataFilters: [isnotnull(s#1.inner.k), (s#1.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 3 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 16580 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 526, + "decode_time": 660194, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 141798, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 1243107, + "numPartitions": 1, + "encode_time": 35350, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 526, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 452257 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 58556 + } + }, + { + "node": "CometFilter", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 170330 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 7269680, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 0, + "predicate_cache_inner_records": 0, + "filesSize": 199358457, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 2442397, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 1, + "row_pushdown_eval_time": 2, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 0, + "page_index_eval_time": 2, + "bytes_scanned": 2851301, + "predicate_evaluation_errors": 0, + "metadata_load_time": 740214, + "output_rows": 65536, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 91, + "metadataTime": 0, + "scan_efficiency_ratio_total": 199358457, + "row_groups_matched_statistics": 1, + "statistics_eval_time": 94821, + "time_elapsed_opening": 983571, + "row_groups_pruned_statistics": 63, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 6501088, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + }, + "top_level_selective": { + "seconds": [ + 0.15506690181791782, + 0.12573741003870964, + 0.14402167592197657, + 0.1330686118453741, + 0.13747728988528252 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE k >= 4128768", + "answer": [ + 32741059719, + 32774396751, + 32772938793, + 32799973223, + 32905236128, + 32807159809, + 32718967833, + 32661111725 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#986L, sum#987L, sum#988L, sum#989L, sum#990L, sum#991L, sum#992L, sum#993L], [Final], [sum(_extract_p0#968L), sum(_extract_p1#969L), sum(_extract_p2#970L), sum(_extract_p3#971L), sum(_extract_p4#972L), sum(_extract_p5#973L), sum(_extract_p6#974L), sum(_extract_p7#975L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1204]\n +- CometHashAggregate [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [Partial], [partial_sum(_extract_p0#968L), partial_sum(_extract_p1#969L), partial_sum(_extract_p2#970L), partial_sum(_extract_p3#971L), partial_sum(_extract_p4#972L), partial_sum(_extract_p5#973L), partial_sum(_extract_p6#974L), partial_sum(_extract_p7#975L)]\n +- CometProject [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [s#1.p0 AS _extract_p0#968L, s#1.p1 AS _extract_p1#969L, s#1.p2 AS _extract_p2#970L, s#1.p3 AS _extract_p3#971L, s#1.p4 AS _extract_p4#972L, s#1.p5 AS _extract_p5#973L, s#1.p6 AS _extract_p6#974L, s#1.p7 AS _extract_p7#975L]\n +- CometFilter [k#0L, s#1], (isnotnull(k#0L) AND (k#0L >= 4128768))\n +- CometNativeScan parquet [k#0L,s#1] Batched: true, DataFilters: [isnotnull(k#0L), (k#0L >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(k), GreaterThanOrEqual(k,4128768)], ReadSchema: struct>\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 3 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 19280 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 526, + "decode_time": 502416, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 132858, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 849513, + "numPartitions": 1, + "encode_time": 28440, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 526, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 34173480 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 4194304, + "elapsed_compute": 6662563 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 457767984, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 0, + "predicate_cache_inner_records": 0, + "filesSize": 199358457, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 2357397, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 64, + "row_pushdown_eval_time": 2, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 0, + "page_index_eval_time": 2, + "bytes_scanned": 165673158, + "predicate_evaluation_errors": 0, + "metadata_load_time": 757584, + "output_rows": 4194304, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 111, + "metadataTime": 0, + "scan_efficiency_ratio_total": 199358457, + "row_groups_matched_statistics": 64, + "statistics_eval_time": 2, + "time_elapsed_opening": 916321, + "row_groups_pruned_statistics": 0, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 349320757, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + }, + "nested_unprunable": { + "seconds": [ + 0.5652983766049147, + 0.5627129822969437, + 0.5944593185558915, + 0.592568757943809, + 0.6130315512418747 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM shuffled WHERE s.inner.k >= 4128768", + "answer": [ + 32658560674, + 32812041019, + 32867632839, + 32498761177, + 32648273638, + 32881993944, + 32750626290, + 32782672280 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#1976L, sum#1977L, sum#1978L, sum#1979L, sum#1980L, sum#1981L, sum#1982L, sum#1983L], [Final], [sum(_extract_p0#1960L), sum(_extract_p1#1961L), sum(_extract_p2#1962L), sum(_extract_p3#1963L), sum(_extract_p4#1964L), sum(_extract_p5#1965L), sum(_extract_p6#1966L), sum(_extract_p7#1967L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2334]\n +- CometHashAggregate [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [Partial], [partial_sum(_extract_p0#1960L), partial_sum(_extract_p1#1961L), partial_sum(_extract_p2#1962L), partial_sum(_extract_p3#1963L), partial_sum(_extract_p4#1964L), partial_sum(_extract_p5#1965L), partial_sum(_extract_p6#1966L), partial_sum(_extract_p7#1967L)]\n +- CometProject [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [s#3.p0 AS _extract_p0#1960L, s#3.p1 AS _extract_p1#1961L, s#3.p2 AS _extract_p2#1962L, s#3.p3 AS _extract_p3#1963L, s#3.p4 AS _extract_p4#1964L, s#3.p5 AS _extract_p5#1965L, s#3.p6 AS _extract_p6#1966L, s#3.p7 AS _extract_p7#1967L]\n +- CometFilter [s#3], (isnotnull(s#3.inner.k) AND (s#3.inner.k >= 4128768))\n +- CometNativeScan parquet [s#3] Batched: true, DataFilters: [isnotnull(s#3.inner.k), (s#3.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/shuffled.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 2 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 19840 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 527, + "decode_time": 577975, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 134939, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 928573, + "numPartitions": 1, + "encode_time": 31879, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 527, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 651932 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 130430 + } + }, + { + "node": "CometFilter", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 21826400 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 500684622, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 0, + "predicate_cache_inner_records": 0, + "filesSize": 207941114, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 2513986, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 64, + "row_pushdown_eval_time": 2, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 0, + "page_index_eval_time": 2, + "bytes_scanned": 186772555, + "predicate_evaluation_errors": 0, + "metadata_load_time": 639414, + "output_rows": 4194304, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 111, + "metadataTime": 0, + "scan_efficiency_ratio_total": 207941114, + "row_groups_matched_statistics": 64, + "statistics_eval_time": 111380, + "time_elapsed_opening": 926681, + "row_groups_pruned_statistics": 0, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 406449858, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + } + } + }, + { + "label": "candidate-3", + "spark": "4.1.3", + "pruning_off": false, + "row_filter": false, + "warmup": 5, + "cases": { + "nested_selective": { + "seconds": [ + 0.2269756831228733, + 0.20965376868844032, + 0.22189291194081306, + 0.19881676323711872, + 0.17740720696747303 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE s.inner.k >= 4128768", + "answer": [ + 32741059719, + 32774396751, + 32772938793, + 32799973223, + 32905236128, + 32807159809, + 32718967833, + 32661111725 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#486L, sum#487L, sum#488L, sum#489L, sum#490L, sum#491L, sum#492L, sum#493L], [Final], [sum(_extract_p0#470L), sum(_extract_p1#471L), sum(_extract_p2#472L), sum(_extract_p3#473L), sum(_extract_p4#474L), sum(_extract_p5#475L), sum(_extract_p6#476L), sum(_extract_p7#477L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=594]\n +- CometHashAggregate [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [Partial], [partial_sum(_extract_p0#470L), partial_sum(_extract_p1#471L), partial_sum(_extract_p2#472L), partial_sum(_extract_p3#473L), partial_sum(_extract_p4#474L), partial_sum(_extract_p5#475L), partial_sum(_extract_p6#476L), partial_sum(_extract_p7#477L)]\n +- CometProject [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [s#1.p0 AS _extract_p0#470L, s#1.p1 AS _extract_p1#471L, s#1.p2 AS _extract_p2#472L, s#1.p3 AS _extract_p3#473L, s#1.p4 AS _extract_p4#474L, s#1.p5 AS _extract_p5#475L, s#1.p6 AS _extract_p6#476L, s#1.p7 AS _extract_p7#477L]\n +- CometFilter [s#1], (isnotnull(s#1.inner.k) AND (s#1.inner.k >= 4128768))\n +- CometNativeScan parquet [s#1] Batched: true, DataFilters: [isnotnull(s#1.inner.k), (s#1.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 4 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 16800 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 526, + "decode_time": 651635, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 130000, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 1220159, + "numPartitions": 1, + "encode_time": 30460, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 526, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 516816 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 69168 + } + }, + { + "node": "CometFilter", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 190860 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 7765226, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 0, + "predicate_cache_inner_records": 0, + "filesSize": 199358457, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 2598115, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 1, + "row_pushdown_eval_time": 2, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 0, + "page_index_eval_time": 2, + "bytes_scanned": 2851301, + "predicate_evaluation_errors": 0, + "metadata_load_time": 764723, + "output_rows": 65536, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 111, + "metadataTime": 0, + "scan_efficiency_ratio_total": 199358457, + "row_groups_matched_statistics": 1, + "statistics_eval_time": 120310, + "time_elapsed_opening": 1105879, + "row_groups_pruned_statistics": 63, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 6849684, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + }, + "top_level_selective": { + "seconds": [ + 0.14480215683579445, + 0.13203103002160788, + 0.1324948649853468, + 0.1340858293697238, + 0.12480908911675215 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE k >= 4128768", + "answer": [ + 32741059719, + 32774396751, + 32772938793, + 32799973223, + 32905236128, + 32807159809, + 32718967833, + 32661111725 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#986L, sum#987L, sum#988L, sum#989L, sum#990L, sum#991L, sum#992L, sum#993L], [Final], [sum(_extract_p0#968L), sum(_extract_p1#969L), sum(_extract_p2#970L), sum(_extract_p3#971L), sum(_extract_p4#972L), sum(_extract_p5#973L), sum(_extract_p6#974L), sum(_extract_p7#975L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1204]\n +- CometHashAggregate [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [Partial], [partial_sum(_extract_p0#968L), partial_sum(_extract_p1#969L), partial_sum(_extract_p2#970L), partial_sum(_extract_p3#971L), partial_sum(_extract_p4#972L), partial_sum(_extract_p5#973L), partial_sum(_extract_p6#974L), partial_sum(_extract_p7#975L)]\n +- CometProject [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [s#1.p0 AS _extract_p0#968L, s#1.p1 AS _extract_p1#969L, s#1.p2 AS _extract_p2#970L, s#1.p3 AS _extract_p3#971L, s#1.p4 AS _extract_p4#972L, s#1.p5 AS _extract_p5#973L, s#1.p6 AS _extract_p6#974L, s#1.p7 AS _extract_p7#975L]\n +- CometFilter [k#0L, s#1], (isnotnull(k#0L) AND (k#0L >= 4128768))\n +- CometNativeScan parquet [k#0L,s#1] Batched: true, DataFilters: [isnotnull(k#0L), (k#0L >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(k), GreaterThanOrEqual(k,4128768)], ReadSchema: struct>\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 2 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 17109 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 526, + "decode_time": 416336, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 114758, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 790602, + "numPartitions": 1, + "encode_time": 28780, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 526, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 33829637 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 4194304, + "elapsed_compute": 6278255 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 430781919, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 0, + "predicate_cache_inner_records": 0, + "filesSize": 199358457, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 2239719, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 64, + "row_pushdown_eval_time": 2, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 0, + "page_index_eval_time": 2, + "bytes_scanned": 165673158, + "predicate_evaluation_errors": 0, + "metadata_load_time": 667365, + "output_rows": 4194304, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 81, + "metadataTime": 0, + "scan_efficiency_ratio_total": 199358457, + "row_groups_matched_statistics": 64, + "statistics_eval_time": 2, + "time_elapsed_opening": 840452, + "row_groups_pruned_statistics": 0, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 335087732, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + }, + "nested_unprunable": { + "seconds": [ + 0.5824264092370868, + 0.5773687968030572, + 0.5656023295596242, + 0.5545061454176903, + 0.5747098913416266 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM shuffled WHERE s.inner.k >= 4128768", + "answer": [ + 32658560674, + 32812041019, + 32867632839, + 32498761177, + 32648273638, + 32881993944, + 32750626290, + 32782672280 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#1976L, sum#1977L, sum#1978L, sum#1979L, sum#1980L, sum#1981L, sum#1982L, sum#1983L], [Final], [sum(_extract_p0#1960L), sum(_extract_p1#1961L), sum(_extract_p2#1962L), sum(_extract_p3#1963L), sum(_extract_p4#1964L), sum(_extract_p5#1965L), sum(_extract_p6#1966L), sum(_extract_p7#1967L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2334]\n +- CometHashAggregate [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [Partial], [partial_sum(_extract_p0#1960L), partial_sum(_extract_p1#1961L), partial_sum(_extract_p2#1962L), partial_sum(_extract_p3#1963L), partial_sum(_extract_p4#1964L), partial_sum(_extract_p5#1965L), partial_sum(_extract_p6#1966L), partial_sum(_extract_p7#1967L)]\n +- CometProject [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [s#3.p0 AS _extract_p0#1960L, s#3.p1 AS _extract_p1#1961L, s#3.p2 AS _extract_p2#1962L, s#3.p3 AS _extract_p3#1963L, s#3.p4 AS _extract_p4#1964L, s#3.p5 AS _extract_p5#1965L, s#3.p6 AS _extract_p6#1966L, s#3.p7 AS _extract_p7#1967L]\n +- CometFilter [s#3], (isnotnull(s#3.inner.k) AND (s#3.inner.k >= 4128768))\n +- CometNativeScan parquet [s#3] Batched: true, DataFilters: [isnotnull(s#3.inner.k), (s#3.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/shuffled.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 2 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 17099 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 527, + "decode_time": 430956, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 133750, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 817622, + "numPartitions": 1, + "encode_time": 29140, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 527, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 759554 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 131507 + } + }, + { + "node": "CometFilter", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 22934138 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 482883949, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 0, + "predicate_cache_inner_records": 0, + "filesSize": 207941114, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 2619785, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 64, + "row_pushdown_eval_time": 2, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 0, + "page_index_eval_time": 2, + "bytes_scanned": 186772555, + "predicate_evaluation_errors": 0, + "metadata_load_time": 745495, + "output_rows": 4194304, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 81, + "metadataTime": 0, + "scan_efficiency_ratio_total": 207941114, + "row_groups_matched_statistics": 64, + "statistics_eval_time": 95980, + "time_elapsed_opening": 990912, + "row_groups_pruned_statistics": 0, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 398224205, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + } + } + }, + { + "label": "pruning-off-row-filter", + "spark": "4.1.3", + "pruning_off": true, + "row_filter": true, + "warmup": 5, + "cases": { + "nested_selective": { + "seconds": [ + 0.29983343090862036, + 0.31047218944877386, + 0.2997523508965969, + 0.28356191609054804, + 0.2667716462165117 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE s.inner.k >= 4128768", + "answer": [ + 32741059719, + 32774396751, + 32772938793, + 32799973223, + 32905236128, + 32807159809, + 32718967833, + 32661111725 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#486L, sum#487L, sum#488L, sum#489L, sum#490L, sum#491L, sum#492L, sum#493L], [Final], [sum(_extract_p0#470L), sum(_extract_p1#471L), sum(_extract_p2#472L), sum(_extract_p3#473L), sum(_extract_p4#474L), sum(_extract_p5#475L), sum(_extract_p6#476L), sum(_extract_p7#477L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=594]\n +- CometHashAggregate [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [Partial], [partial_sum(_extract_p0#470L), partial_sum(_extract_p1#471L), partial_sum(_extract_p2#472L), partial_sum(_extract_p3#473L), partial_sum(_extract_p4#474L), partial_sum(_extract_p5#475L), partial_sum(_extract_p6#476L), partial_sum(_extract_p7#477L)]\n +- CometProject [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [s#1.p0 AS _extract_p0#470L, s#1.p1 AS _extract_p1#471L, s#1.p2 AS _extract_p2#472L, s#1.p3 AS _extract_p3#473L, s#1.p4 AS _extract_p4#474L, s#1.p5 AS _extract_p5#475L, s#1.p6 AS _extract_p6#476L, s#1.p7 AS _extract_p7#477L]\n +- CometFilter [s#1], (isnotnull(s#1.inner.k) AND (s#1.inner.k >= 4128768))\n +- CometNativeScan parquet [s#1] Batched: true, DataFilters: [isnotnull(s#1.inner.k), (s#1.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 3 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 18750 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 526, + "decode_time": 712513, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 148899, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 1110219, + "numPartitions": 1, + "encode_time": 32699, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 526, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 561406 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 71000 + } + }, + { + "node": "CometFilter", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 179287 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 126723837, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 65536, + "predicate_cache_inner_records": 0, + "filesSize": 199358457, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 121052961, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 64, + "row_pushdown_eval_time": 6995880, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 4128768, + "page_index_eval_time": 2, + "bytes_scanned": 19396952, + "predicate_evaluation_errors": 0, + "metadata_load_time": 651616, + "output_rows": 65536, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 91, + "metadataTime": 0, + "scan_efficiency_ratio_total": 199358457, + "row_groups_matched_statistics": 64, + "statistics_eval_time": 2, + "time_elapsed_opening": 877952, + "row_groups_pruned_statistics": 0, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 117336764, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + }, + "top_level_selective": { + "seconds": [ + 0.19135639816522598, + 0.18866712413728237, + 0.19497105479240417, + 0.17380876746028662, + 0.17694623582065105 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE k >= 4128768", + "answer": [ + 32741059719, + 32774396751, + 32772938793, + 32799973223, + 32905236128, + 32807159809, + 32718967833, + 32661111725 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#986L, sum#987L, sum#988L, sum#989L, sum#990L, sum#991L, sum#992L, sum#993L], [Final], [sum(_extract_p0#968L), sum(_extract_p1#969L), sum(_extract_p2#970L), sum(_extract_p3#971L), sum(_extract_p4#972L), sum(_extract_p5#973L), sum(_extract_p6#974L), sum(_extract_p7#975L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1204]\n +- CometHashAggregate [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [Partial], [partial_sum(_extract_p0#968L), partial_sum(_extract_p1#969L), partial_sum(_extract_p2#970L), partial_sum(_extract_p3#971L), partial_sum(_extract_p4#972L), partial_sum(_extract_p5#973L), partial_sum(_extract_p6#974L), partial_sum(_extract_p7#975L)]\n +- CometProject [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [s#1.p0 AS _extract_p0#968L, s#1.p1 AS _extract_p1#969L, s#1.p2 AS _extract_p2#970L, s#1.p3 AS _extract_p3#971L, s#1.p4 AS _extract_p4#972L, s#1.p5 AS _extract_p5#973L, s#1.p6 AS _extract_p6#974L, s#1.p7 AS _extract_p7#975L]\n +- CometFilter [k#0L, s#1], (isnotnull(k#0L) AND (k#0L >= 4128768))\n +- CometNativeScan parquet [k#0L,s#1] Batched: true, DataFilters: [isnotnull(k#0L), (k#0L >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(k), GreaterThanOrEqual(k,4128768)], ReadSchema: struct>\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 2 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 15989 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 526, + "decode_time": 408797, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 135238, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 723212, + "numPartitions": 1, + "encode_time": 28930, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 526, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 33986400 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 4194304, + "elapsed_compute": 6452212 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 443018252, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 0, + "predicate_cache_inner_records": 0, + "filesSize": 199358457, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 2586726, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 64, + "row_pushdown_eval_time": 2, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 0, + "page_index_eval_time": 2, + "bytes_scanned": 165673158, + "predicate_evaluation_errors": 0, + "metadata_load_time": 817833, + "output_rows": 4194304, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 121, + "metadataTime": 0, + "scan_efficiency_ratio_total": 199358457, + "row_groups_matched_statistics": 64, + "statistics_eval_time": 2, + "time_elapsed_opening": 1043659, + "row_groups_pruned_statistics": 0, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 339187186, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + }, + "nested_unprunable": { + "seconds": [ + 0.802385420538485, + 0.7697837706655264, + 0.7511665793135762, + 0.743246354162693, + 0.7415530709549785 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM shuffled WHERE s.inner.k >= 4128768", + "answer": [ + 32658560674, + 32812041019, + 32867632839, + 32498761177, + 32648273638, + 32881993944, + 32750626290, + 32782672280 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#1976L, sum#1977L, sum#1978L, sum#1979L, sum#1980L, sum#1981L, sum#1982L, sum#1983L], [Final], [sum(_extract_p0#1960L), sum(_extract_p1#1961L), sum(_extract_p2#1962L), sum(_extract_p3#1963L), sum(_extract_p4#1964L), sum(_extract_p5#1965L), sum(_extract_p6#1966L), sum(_extract_p7#1967L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2334]\n +- CometHashAggregate [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [Partial], [partial_sum(_extract_p0#1960L), partial_sum(_extract_p1#1961L), partial_sum(_extract_p2#1962L), partial_sum(_extract_p3#1963L), partial_sum(_extract_p4#1964L), partial_sum(_extract_p5#1965L), partial_sum(_extract_p6#1966L), partial_sum(_extract_p7#1967L)]\n +- CometProject [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [s#3.p0 AS _extract_p0#1960L, s#3.p1 AS _extract_p1#1961L, s#3.p2 AS _extract_p2#1962L, s#3.p3 AS _extract_p3#1963L, s#3.p4 AS _extract_p4#1964L, s#3.p5 AS _extract_p5#1965L, s#3.p6 AS _extract_p6#1966L, s#3.p7 AS _extract_p7#1967L]\n +- CometFilter [s#3], (isnotnull(s#3.inner.k) AND (s#3.inner.k >= 4128768))\n +- CometNativeScan parquet [s#3] Batched: true, DataFilters: [isnotnull(s#3.inner.k), (s#3.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/shuffled.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 2 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 37750 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 527, + "decode_time": 516295, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 138420, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 779512, + "numPartitions": 1, + "encode_time": 33369, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 527, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 592795 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 109828 + } + }, + { + "node": "CometFilter", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 4746354 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 649387752, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 65536, + "predicate_cache_inner_records": 0, + "filesSize": 207941114, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 8882425, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 64, + "row_pushdown_eval_time": 7648663, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 4128768, + "page_index_eval_time": 2, + "bytes_scanned": 186772555, + "predicate_evaluation_errors": 0, + "metadata_load_time": 800924, + "output_rows": 65536, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 111, + "metadataTime": 0, + "scan_efficiency_ratio_total": 207941114, + "row_groups_matched_statistics": 64, + "statistics_eval_time": 2, + "time_elapsed_opening": 1081740, + "row_groups_pruned_statistics": 0, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 579288015, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + } + } + }, + { + "label": "pruning-off", + "spark": "4.1.3", + "pruning_off": true, + "row_filter": false, + "warmup": 5, + "cases": { + "nested_selective": { + "seconds": [ + 0.6383354254066944, + 0.6380214085802436, + 0.6571799078956246, + 0.6155594941228628, + 0.600575627759099 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE s.inner.k >= 4128768", + "answer": [ + 32741059719, + 32774396751, + 32772938793, + 32799973223, + 32905236128, + 32807159809, + 32718967833, + 32661111725 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#486L, sum#487L, sum#488L, sum#489L, sum#490L, sum#491L, sum#492L, sum#493L], [Final], [sum(_extract_p0#470L), sum(_extract_p1#471L), sum(_extract_p2#472L), sum(_extract_p3#473L), sum(_extract_p4#474L), sum(_extract_p5#475L), sum(_extract_p6#476L), sum(_extract_p7#477L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=594]\n +- CometHashAggregate [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [Partial], [partial_sum(_extract_p0#470L), partial_sum(_extract_p1#471L), partial_sum(_extract_p2#472L), partial_sum(_extract_p3#473L), partial_sum(_extract_p4#474L), partial_sum(_extract_p5#475L), partial_sum(_extract_p6#476L), partial_sum(_extract_p7#477L)]\n +- CometProject [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [s#1.p0 AS _extract_p0#470L, s#1.p1 AS _extract_p1#471L, s#1.p2 AS _extract_p2#472L, s#1.p3 AS _extract_p3#473L, s#1.p4 AS _extract_p4#474L, s#1.p5 AS _extract_p5#475L, s#1.p6 AS _extract_p6#476L, s#1.p7 AS _extract_p7#477L]\n +- CometFilter [s#1], (isnotnull(s#1.inner.k) AND (s#1.inner.k >= 4128768))\n +- CometNativeScan parquet [s#1] Batched: true, DataFilters: [isnotnull(s#1.inner.k), (s#1.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 4 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 22330 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 526, + "decode_time": 708473, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 126429, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 1266088, + "numPartitions": 1, + "encode_time": 30110, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 526, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 528445 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 75548 + } + }, + { + "node": "CometFilter", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 13918695 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 454680822, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 0, + "predicate_cache_inner_records": 0, + "filesSize": 199358457, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 2553836, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 64, + "row_pushdown_eval_time": 2, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 0, + "page_index_eval_time": 2, + "bytes_scanned": 182481422, + "predicate_evaluation_errors": 0, + "metadata_load_time": 937502, + "output_rows": 4194304, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 111, + "metadataTime": 0, + "scan_efficiency_ratio_total": 199358457, + "row_groups_matched_statistics": 64, + "statistics_eval_time": 2, + "time_elapsed_opening": 1107139, + "row_groups_pruned_statistics": 0, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 378507122, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + }, + "top_level_selective": { + "seconds": [ + 0.6123585933819413, + 0.610489190556109, + 0.5980431707575917, + 0.6052072821184993, + 0.5822479920461774 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE k >= 4128768", + "answer": [ + 32741059719, + 32774396751, + 32772938793, + 32799973223, + 32905236128, + 32807159809, + 32718967833, + 32661111725 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#986L, sum#987L, sum#988L, sum#989L, sum#990L, sum#991L, sum#992L, sum#993L], [Final], [sum(_extract_p0#968L), sum(_extract_p1#969L), sum(_extract_p2#970L), sum(_extract_p3#971L), sum(_extract_p4#972L), sum(_extract_p5#973L), sum(_extract_p6#974L), sum(_extract_p7#975L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1204]\n +- CometHashAggregate [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [Partial], [partial_sum(_extract_p0#968L), partial_sum(_extract_p1#969L), partial_sum(_extract_p2#970L), partial_sum(_extract_p3#971L), partial_sum(_extract_p4#972L), partial_sum(_extract_p5#973L), partial_sum(_extract_p6#974L), partial_sum(_extract_p7#975L)]\n +- CometProject [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [s#1.p0 AS _extract_p0#968L, s#1.p1 AS _extract_p1#969L, s#1.p2 AS _extract_p2#970L, s#1.p3 AS _extract_p3#971L, s#1.p4 AS _extract_p4#972L, s#1.p5 AS _extract_p5#973L, s#1.p6 AS _extract_p6#974L, s#1.p7 AS _extract_p7#975L]\n +- CometFilter [k#0L, s#1], (isnotnull(k#0L) AND (k#0L >= 4128768))\n +- CometNativeScan parquet [k#0L,s#1] Batched: true, DataFilters: [isnotnull(k#0L), (k#0L >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(k), GreaterThanOrEqual(k,4128768)], ReadSchema: struct>\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 3 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 26110 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 526, + "decode_time": 487555, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 139549, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 942530, + "numPartitions": 1, + "encode_time": 29679, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 526, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 34212055 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 4194304, + "elapsed_compute": 6810633 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 448676357, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 0, + "predicate_cache_inner_records": 0, + "filesSize": 199358457, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 2319278, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 64, + "row_pushdown_eval_time": 2, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 0, + "page_index_eval_time": 2, + "bytes_scanned": 165673158, + "predicate_evaluation_errors": 0, + "metadata_load_time": 650476, + "output_rows": 4194304, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 91, + "metadataTime": 0, + "scan_efficiency_ratio_total": 199358457, + "row_groups_matched_statistics": 64, + "statistics_eval_time": 2, + "time_elapsed_opening": 777864, + "row_groups_pruned_statistics": 0, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 344153367, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + }, + "nested_unprunable": { + "seconds": [ + 0.5745589630678296, + 0.5518598509952426, + 0.551765551790595, + 0.5702894749119878, + 0.5601397510617971 + ], + "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM shuffled WHERE s.inner.k >= 4128768", + "answer": [ + 32658560674, + 32812041019, + 32867632839, + 32498761177, + 32648273638, + 32881993944, + 32750626290, + 32782672280 + ], + "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#1976L, sum#1977L, sum#1978L, sum#1979L, sum#1980L, sum#1981L, sum#1982L, sum#1983L], [Final], [sum(_extract_p0#1960L), sum(_extract_p1#1961L), sum(_extract_p2#1962L), sum(_extract_p3#1963L), sum(_extract_p4#1964L), sum(_extract_p5#1965L), sum(_extract_p6#1966L), sum(_extract_p7#1967L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2334]\n +- CometHashAggregate [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [Partial], [partial_sum(_extract_p0#1960L), partial_sum(_extract_p1#1961L), partial_sum(_extract_p2#1962L), partial_sum(_extract_p3#1963L), partial_sum(_extract_p4#1964L), partial_sum(_extract_p5#1965L), partial_sum(_extract_p6#1966L), partial_sum(_extract_p7#1967L)]\n +- CometProject [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [s#3.p0 AS _extract_p0#1960L, s#3.p1 AS _extract_p1#1961L, s#3.p2 AS _extract_p2#1962L, s#3.p3 AS _extract_p3#1963L, s#3.p4 AS _extract_p4#1964L, s#3.p5 AS _extract_p5#1965L, s#3.p6 AS _extract_p6#1966L, s#3.p7 AS _extract_p7#1967L]\n +- CometFilter [s#3], (isnotnull(s#3.inner.k) AND (s#3.inner.k >= 4128768))\n +- CometNativeScan parquet [s#3] Batched: true, DataFilters: [isnotnull(s#3.inner.k), (s#3.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/shuffled.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", + "metrics": [ + { + "node": "WholeStageCodegen (1)", + "metrics": { + "pipelineTime": 2 + } + }, + { + "node": "CometColumnarToRow", + "metrics": { + "numOutputRows": 1, + "numInputBatches": 1 + } + }, + { + "node": "InputAdapter", + "metrics": {} + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 19599 + } + }, + { + "node": "CometExchange", + "metrics": { + "shuffleRecordsWritten": 1, + "spill_count": 0, + "remoteMergedBlocksFetched": 0, + "recordsRead": 1, + "localBytesRead": 527, + "decode_time": 436956, + "remoteBlocksFetched": 0, + "localMergedBytesRead": 0, + "localMergedChunksFetched": 0, + "shuffleWriteTime": 114760, + "remoteMergedBytesRead": 0, + "localMergedBlocksFetched": 0, + "corruptMergedBlockChunks": 0, + "input_batches": 1, + "remoteMergedReqsDuration": 0, + "fetchWaitTime": 0, + "remoteBytesRead": 0, + "mergedFetchFallbackCount": 0, + "repart_time": 1, + "memory_spilled_bytes": 0, + "localBlocksFetched": 1, + "remoteMergedChunksFetched": 0, + "spilled_bytes": 0, + "dataSize": 832, + "elapsed_compute": 736584, + "numPartitions": 1, + "encode_time": 26600, + "remoteReqsDuration": 0, + "remoteBytesReadToDisk": 0, + "shuffleBytesWritten": 527, + "interleave_time": 1 + } + }, + { + "node": "CometHashAggregate", + "metrics": { + "output_rows": 1, + "elapsed_compute": 548733 + } + }, + { + "node": "CometProject", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 113850 + } + }, + { + "node": "CometFilter", + "metrics": { + "output_rows": 65536, + "elapsed_compute": 19650433 + } + }, + { + "node": "CometNativeScan parquet ", + "metrics": { + "time_elapsed_scanning_total": 457130777, + "limit_matched_row_groups": 0, + "page_index_rows_pruned": 0, + "pushdown_rows_matched": 0, + "predicate_cache_inner_records": 0, + "filesSize": 207941114, + "files_ranges_pruned_statistics": 0, + "time_elapsed_scanning_until_data": 2619095, + "page_index_pages_matched": 0, + "row_groups_matched_bloom_filter": 64, + "row_pushdown_eval_time": 2, + "page_index_rows_matched": 0, + "row_groups_pruned_bloom_filter": 0, + "limit_pruned_row_groups": 0, + "numFiles": 1, + "pushdown_rows_pruned": 0, + "page_index_eval_time": 2, + "bytes_scanned": 186772555, + "predicate_evaluation_errors": 0, + "metadata_load_time": 600915, + "output_rows": 4194304, + "predicate_cache_records": 0, + "bloom_filter_eval_time": 111, + "metadataTime": 0, + "scan_efficiency_ratio_total": 207941114, + "row_groups_matched_statistics": 64, + "statistics_eval_time": 2, + "time_elapsed_opening": 846571, + "row_groups_pruned_statistics": 0, + "files_ranges_matched_statistics": 1, + "time_elapsed_processing": 377532689, + "file_scan_errors": 0, + "num_predicate_creation_errors": 0, + "file_open_errors": 0, + "page_index_pages_pruned": 0 + } + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/native/Cargo.lock b/native/Cargo.lock index d3a4b897df2..918fbbcd4c3 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -1118,11 +1118,11 @@ checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] name = "blake2" -version = "0.10.6" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +checksum = "5b5d4d889834ee8ecfc0f8426ad30faf7cdcb10f741a8e6d7224d95325479f6f" dependencies = [ - "digest 0.10.7", + "digest 0.11.3", ] [[package]] @@ -1893,8 +1893,7 @@ dependencies = [ [[package]] name = "datafusion" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96f76f0167ed0842b29a3d1e41be3c034c0a46409a3a703cc4cc84ee8c24abf4" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "arrow-schema", @@ -1942,8 +1941,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d79ec3460f6ed5c58f9b3f2d873fbc77748b82653bff1b4cdaf06de33bb4e05f" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "async-trait", @@ -1967,8 +1965,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b48cef241e2efcfd496fe05ae4d0d5de20793451862faefe406c397a467e12d4" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "async-trait", @@ -2141,8 +2138,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f72810485975c258f1b4d00baab31728470676c60c5546f366ebd0d99f05ab6" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "arrow-ipc", @@ -2168,8 +2164,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "533c28e75dba52f41bde187d23a1cb24ab91c7c097966824fa471e67b60320ea" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "futures", "log", @@ -2179,8 +2174,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b00a1fa0da26f6087136a82fea7f13c76a672cbab452d4086952a7cf770a19b" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "async-compression", @@ -2215,8 +2209,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ad17ec881bff2ed7768b4bfe971d3efbf3473f2fd1f9d365447bccbdf908678" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "arrow-ipc", @@ -2239,8 +2232,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5345285b0c3eaab412e7539b706973c083bd7e5bce575de5e0a3da488d08d1d" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "async-trait", @@ -2262,8 +2254,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da02fb9324f56bd8c53f1ee2e949547425cb66f76adc6832b10d44f80a1221d2" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "async-trait", @@ -2285,8 +2276,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c0b0dc1453952952fd5c69ad1c7f6042176e69ed233011d47e07cf74ed0949e" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "arrow-schema", @@ -2317,14 +2307,12 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88fd985bc0550c36f557db69543cc9d6393b1509783520b30e902f23c555da6" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" [[package]] name = "datafusion-execution" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a98f1052f91b4991f0bf2ce1e4e36dfbdcda454a956b8c8d562c7c845e8fce1d" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "arrow-buffer", @@ -2350,8 +2338,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464625a1f0e4b9df552d894fafcc8aac953ebbc8b0fa0acdaf20975fd615040e" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "arrow-schema", @@ -2372,8 +2359,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2604994999d5aeca1d1df645ffc98bc787447aaff05dde27aad0342b48fc1fe0" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "datafusion-common", @@ -2384,8 +2370,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "051e97533e6af53e4aa0a0667cadc886abcaf36c4a5925019c55c0aa4c218fde" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "arrow-buffer", @@ -2416,8 +2401,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d0f1bb166d3572b6ed40e1afb2faaacade962abc08c2fcf04babee74681c56b" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "datafusion-common", @@ -2437,8 +2421,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ed756770f5f98369e181d692fd5ee6b1127ffd7322caba92f3730f9f5c92333" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "datafusion-common", @@ -2449,8 +2432,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91173fdb5c0ff2a41169a8ffa1b385b8844f18728747bb0a37e35ad7d5772a4f" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "arrow-ord", @@ -2474,8 +2456,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1bcdfb286a745461b126719c32700777e83df4f17cc44db5d71ebce5731e840" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "async-trait", @@ -2490,8 +2471,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ec4b508f1f93f00038ba3e737e894ec6c775528b4369413386655ae6125f0fc" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "datafusion-common", @@ -2507,8 +2487,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b352020834140073fbf5b46ee0ceb926e5074a9d0bcae1dbd91d0586d999cde" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2517,8 +2496,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15192effab05d38cce10e92a6fb48c967b5f166b27b7195a165a72b232569c58" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "datafusion-doc", "quote", @@ -2528,8 +2506,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854445d9f7847e1e46089cf61b8d341a64382f14484e912c83a0f23b31216896" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "chrono", @@ -2547,10 +2524,10 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "671558dad1d2aa253c39c0a4c52515958b99eb91abf649f4b88d5e69cc55282f" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", + "arrow-schema", "datafusion-common", "datafusion-expr", "datafusion-expr-common", @@ -2562,14 +2539,14 @@ dependencies = [ "itertools 0.15.0", "parking_lot", "petgraph", + "regex", "tokio", ] [[package]] name = "datafusion-physical-expr-adapter" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffae3d78c2da80ecc829cb58536cc5aca2e99cf1365eda694fc75bfe288861e0" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "datafusion-common", @@ -2583,8 +2560,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d9092ed15e7203fbd0903215172f7c9d18f10d94cba35137f3b3836f7c46f16" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "chrono", @@ -2600,8 +2576,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9005b6cf50b57b72d476c6ed4662b04be7ca6be5320ba9127c6d0b7e4218095b" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "datafusion-common", @@ -2619,8 +2594,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5787e4fcff4adc4fce8948441103a99705018b49c8dff0720b650bd7a15da112" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "arrow-data", @@ -2633,6 +2607,7 @@ dependencies = [ "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", + "datafusion-expr-common", "datafusion-functions", "datafusion-functions-aggregate-common", "datafusion-functions-window-common", @@ -2654,8 +2629,7 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e651c8df0b90daed6a7be5921ec0ee379e6909705f063eeff70fd4e35010e4c" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "datafusion-common", @@ -2670,8 +2644,7 @@ dependencies = [ [[package]] name = "datafusion-session" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb56667ee38217efab19b895d9a936052cfb47ed438a19663351bdc42a6214a1" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow-schema", "async-trait", @@ -2679,28 +2652,28 @@ dependencies = [ "datafusion-execution", "datafusion-expr", "datafusion-physical-plan", + "futures", "parking_lot", ] [[package]] name = "datafusion-spark" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "992dd0b954f24cb576cbeee3555c3083b9cf7a5a5f2448ea25f7a437d444163f" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", + "base64 0.23.1", "bigdecimal", "chrono", "crc32fast", "datafusion", - "datafusion-catalog", "datafusion-common", - "datafusion-execution", "datafusion-expr", "datafusion-functions", "datafusion-functions-aggregate", "datafusion-functions-aggregate-common", "datafusion-functions-nested", + "datafusion-session", "log", "num-traits", "percent-encoding", @@ -2715,8 +2688,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c29067cb9d32f8e603c45e15d61ea18f1069f96ceafeceb4e18466b8e5b31d9" +source = "git+https://github.com/peterxcli/datafusion.git?rev=0b0506a9acab9d5892ecf7e89243c3b34664bcc6#0b0506a9acab9d5892ecf7e89243c3b34664bcc6" dependencies = [ "arrow", "bigdecimal", diff --git a/native/Cargo.toml b/native/Cargo.toml index ac83bec7844..8cfb9e03a70 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -82,3 +82,38 @@ codegen-units = 16 # Parallel codegen (faster compile, slightly larger binary) debug-assertions = true panic = "unwind" # Allow panics to be caught and logged across FFI boundary # overflow-checks inherited as false from release + +# Struct-field capability and row-group statistics pruning (peterxcli/datafusion#2). +[patch.crates-io] +datafusion = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-catalog = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-catalog-listing = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-common = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-common-runtime = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-datasource = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-datasource-arrow = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-datasource-csv = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-datasource-json = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-datasource-parquet = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-doc = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-execution = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-expr = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-expr-common = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-functions = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-functions-aggregate = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-functions-aggregate-common = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-functions-nested = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-functions-table = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-functions-window = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-functions-window-common = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-macros = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-optimizer = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-physical-expr = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-physical-expr-adapter = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-physical-expr-common = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-physical-optimizer = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-physical-plan = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-pruning = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-session = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-spark = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } +datafusion-sql = { git = "https://github.com/peterxcli/datafusion.git", rev = "0b0506a9acab9d5892ecf7e89243c3b34664bcc6" } diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index 8a0972d3977..4e37305c11e 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -833,7 +833,11 @@ impl PhysicalPlanner { ExprStruct::GetStructField(expr) => { let child = self.create_expr(expr.child.as_ref().unwrap(), Arc::clone(&input_schema))?; - Ok(Arc::new(GetStructField::new(child, expr.ordinal as usize))) + Ok(GetStructField::with_field_access( + child, + expr.ordinal as usize, + &input_schema, + )?) } ExprStruct::ToJson(expr) => { let child = self.create_expr(expr.child.as_ref().unwrap(), input_schema)?; diff --git a/native/core/src/parquet/parquet_support.rs b/native/core/src/parquet/parquet_support.rs index b601f984d60..240706751bb 100644 --- a/native/core/src/parquet/parquet_support.rs +++ b/native/core/src/parquet/parquet_support.rs @@ -226,12 +226,10 @@ fn parquet_convert_array_impl( // `millisToMicros` uses `Math.multiplyExact`: // https://github.com/apache/spark/blob/v4.2.0/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/SparkDateTimeUtils.scala#L103-L108 // - // The checked conversion is limited to TOP-LEVEL columns. Spark only avoids the - // error for filtered-out values through row-group statistics pruning, and - // DataFusion's PruningPredicate does not support nested fields yet, so a checked - // conversion on a nested field would fail queries whose predicates Spark prunes - // (e.g. `WHERE s.ts < X` over an all-overflowing file). Nested fields keep the - // pre-existing safe-cast behavior below (overflow -> NULL). + // The checked conversion remains limited to top-level columns. Nested statistics + // pruning alone does not establish Spark parity for conversion-aware predicates, + // dictionary filters, and row filters (see #5739). Nested fields retain the + // existing safe cast below (overflow -> NULL). let micros = array .as_primitive::() .try_unary::<_, TimestampMicrosecondType, _>(|value| value.mul_checked(1_000))? @@ -764,10 +762,8 @@ mod tests { assert!(converted.is_null(0), "overflow must become NULL"); assert!(converted.is_null(1)); - // Nested: DataFusion's PruningPredicate cannot prune nested fields, so a - // checked conversion would fail queries whose predicates Spark satisfies via - // row-group statistics pruning. The nested field keeps the safe-cast behavior: - // overflow becomes NULL. + // Nested statistics pruning does not by itself establish checked-conversion parity + // with Spark (#5739). Preserve the existing nested overflow-to-NULL behavior. let child_field = Arc::new(Field::new( "ts", DataType::Timestamp(TimeUnit::Millisecond, None), diff --git a/native/spark-expr/src/struct_funcs/get_struct_field.rs b/native/spark-expr/src/struct_funcs/get_struct_field.rs index 62447624f5b..a9ccbf96cd9 100644 --- a/native/spark-expr/src/struct_funcs/get_struct_field.rs +++ b/native/spark-expr/src/struct_funcs/get_struct_field.rs @@ -20,8 +20,13 @@ use arrow::buffer::NullBuffer; use arrow::datatypes::{DataType, Field, Schema}; use arrow::record_batch::RecordBatch; use datafusion::common::{DataFusionError, Result as DataFusionResult, ScalarValue}; -use datafusion::logical_expr::ColumnarValue; +use datafusion::config::ConfigOptions; +use datafusion::logical_expr::{ + ColumnarValue, ExpressionPlacement, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF, + ScalarUDFImpl, Signature, StructFieldAccess, Volatility, +}; use datafusion::physical_expr::PhysicalExpr; +use datafusion::physical_expr::ScalarFunctionExpr; use std::{ fmt::{Display, Formatter}, hash::Hash, @@ -51,6 +56,32 @@ impl GetStructField { Self { child, ordinal } } + /// Expose an unambiguous field name to DataFusion's nested projection and pruning. + pub fn with_field_access( + child: Arc, + ordinal: usize, + schema: &Schema, + ) -> DataFusionResult> { + let expr = Self::new(child, ordinal); + let field = expr.child_field(schema)?; + let DataType::Struct(fields) = expr.child.data_type(schema)? else { + unreachable!() + }; + // Spark accesses by ordinal, including structs with duplicate field names. + if fields.iter().filter(|f| f.name() == field.name()).count() != 1 { + return Ok(Arc::new(expr)); + } + Ok(Arc::new(ScalarFunctionExpr::try_new( + Arc::new(ScalarUDF::new_from_impl(StructFieldUdf { + name: field.name().clone(), + signature: Signature::any(1, Volatility::Immutable), + })), + vec![expr.child], + schema, + Arc::new(ConfigOptions::default()), + )?)) + } + fn child_field(&self, input_schema: &Schema) -> DataFusionResult> { match self.child.data_type(input_schema)? { DataType::Struct(fields) => Ok(Arc::clone(&fields[self.ordinal])), @@ -152,6 +183,92 @@ impl Display for GetStructField { } } +/// Name-based evaluation survives the schema adapter reordering or narrowing the struct. +#[derive(Debug, PartialEq, Eq, Hash)] +struct StructFieldUdf { + name: String, + signature: Signature, +} + +impl ScalarUDFImpl for StructFieldUdf { + fn name(&self) -> &str { + "spark_get_struct_field" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, types: &[DataType]) -> DataFusionResult { + match &types[0] { + DataType::Struct(fields) => fields + .iter() + .find(|f| f.name() == &self.name) + .map(|f| f.data_type().clone()) + .ok_or_else(|| { + DataFusionError::Plan(format!("Missing struct field {}", self.name)) + }), + _ => datafusion::common::exec_err!("Expected a struct"), + } + } + + fn return_field_from_args(&self, args: ReturnFieldArgs) -> DataFusionResult> { + let parent = &args.arg_fields[0]; + let DataType::Struct(fields) = parent.data_type() else { + return datafusion::common::exec_err!("Expected a struct"); + }; + let field = fields + .iter() + .find(|f| f.name() == &self.name) + .ok_or_else(|| DataFusionError::Plan(format!("Missing struct field {}", self.name)))?; + Ok(Arc::new(field.as_ref().clone().with_nullable( + parent.is_nullable() || field.is_nullable(), + ))) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { + let scalar = matches!(args.args[0], ColumnarValue::Scalar(_)); + let array = args.args[0] + .clone() + .into_array(if scalar { 1 } else { args.number_rows })?; + let array = array + .as_any() + .downcast_ref::() + .ok_or_else(|| DataFusionError::Execution("Expected a struct".into()))?; + let ordinal = array + .fields() + .iter() + .position(|f| f.name() == &self.name) + .ok_or_else(|| { + DataFusionError::Execution(format!("Missing struct field {}", self.name)) + })?; + let result = GetStructField::project_field(array, ordinal)?; + if scalar { + Ok(ColumnarValue::Scalar(ScalarValue::try_from_array( + &result, 0, + )?)) + } else { + Ok(ColumnarValue::Array(result)) + } + } + + fn struct_field_access(&self, _args: &[Option]) -> Option { + Some(StructFieldAccess { + source_arg: 0, + field_path: vec![self.name.clone()], + }) + } + + fn placement(&self, args: &[ExpressionPlacement]) -> ExpressionPlacement { + match args.first() { + Some(ExpressionPlacement::Column | ExpressionPlacement::MoveTowardsLeafNodes) => { + ExpressionPlacement::MoveTowardsLeafNodes + } + _ => ExpressionPlacement::KeepInPlace, + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -159,6 +276,57 @@ mod tests { use arrow::datatypes::Fields; use datafusion::physical_expr::expressions::Column; + #[test] + fn field_access_handles_reordered_fields_scalars_and_duplicate_names() { + use datafusion::physical_expr::expressions::Literal; + let fields: Fields = vec![ + Field::new("other", DataType::Int64, false), + Field::new("k.dot", DataType::Int64, false), + ] + .into(); + let schema = Schema::new(vec![Field::new( + "s", + DataType::Struct(fields.clone()), + false, + )]); + let expr = + GetStructField::with_field_access(Arc::new(Column::new("s", 0)), 1, &schema).unwrap(); + let reordered = StructArray::new( + vec![fields[1].clone(), fields[0].clone()].into(), + vec![ + Arc::new(Int64Array::from(vec![42])), + Arc::new(Int64Array::from(vec![99])), + ], + None, + ); + let literal: Arc = + Arc::new(Literal::new(ScalarValue::Struct(Arc::new(reordered)))); + let expr = expr.with_new_children(vec![literal]).unwrap(); + let batch = RecordBatch::new_empty(Arc::new(schema)); + assert!(matches!( + expr.evaluate(&batch).unwrap(), + ColumnarValue::Scalar(ScalarValue::Int64(Some(42))) + )); + + let duplicate = StructArray::new( + vec![fields[1].clone(), fields[1].clone()].into(), + vec![ + Arc::new(Int64Array::from(vec![42])), + Arc::new(Int64Array::from(vec![99])), + ], + None, + ); + let literal: Arc = + Arc::new(Literal::new(ScalarValue::Struct(Arc::new(duplicate)))); + let expr = GetStructField::with_field_access(literal, 1, &Schema::empty()).unwrap(); + assert!(expr.downcast_ref::().is_some()); + let output = expr.evaluate(&batch).unwrap().into_array(1).unwrap(); + assert_eq!( + ScalarValue::try_from_array(&output, 0).unwrap(), + ScalarValue::Int64(Some(99)) + ); + } + // A field of a NULL struct must be NULL (Spark semantics) even when the child buffer holds a // non-null value at that row -- Arrow stores child validity independently of the parent // struct's null mask, so a logically-null struct column read from parquet can still carry a @@ -174,7 +342,9 @@ mod tests { let schema = Schema::new(vec![Field::new("cm", DataType::Struct(fields), true)]); let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(struct_array)]).unwrap(); - let expr = GetStructField::new(Arc::new(Column::new("cm", 0)), 0); + let expr = + GetStructField::with_field_access(Arc::new(Column::new("cm", 0)), 0, &batch.schema()) + .unwrap(); let out = expr .evaluate(&batch) .unwrap() @@ -203,7 +373,8 @@ mod tests { /* struct nullable */ true, )]); - let expr = GetStructField::new(Arc::new(Column::new("add", 0)), 0); + let expr = + GetStructField::with_field_access(Arc::new(Column::new("add", 0)), 0, &schema).unwrap(); assert!( expr.nullable(&schema).unwrap(), "a field of a nullable struct must be nullable even if the field itself is non-nullable" @@ -220,7 +391,8 @@ mod tests { /* struct nullable */ false, )]); - let expr = GetStructField::new(Arc::new(Column::new("add", 0)), 0); + let expr = + GetStructField::with_field_access(Arc::new(Column::new("add", 0)), 0, &schema).unwrap(); assert!( !expr.nullable(&schema).unwrap(), "a non-nullable field of a non-nullable struct must remain non-nullable" diff --git a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala index 799197bf33d..be7972bbef4 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala @@ -2090,6 +2090,56 @@ class ParquetReadV1Suite extends ParquetReadSuite with AdaptiveSparkPlanHelper { } } + test("nested struct filters prune row groups using leaf statistics") { + withTempDir { dir => + val path = new Path(dir.toURI.toString, "nested.parquet") + val schema = MessageTypeParser.parseMessageType("""message root { + | optional group s { + | optional group inner { optional int32 k; } + | optional int32 payload; + | } + |}""".stripMargin) + val writer = ExampleParquetWriter + .builder(path) + .withType(schema) + .withDictionaryEncoding(false) + .withRowGroupSize(4096L) + .withConf(spark.sessionState.newHadoopConf()) + .build() + try { + (0 until 8192).foreach { i => + val row = new SimpleGroup(schema) + if (i % 16 != 0) { + val s = row.addGroup(0) + val inner = s.addGroup(0) + if (i % 17 != 0) inner.add(0, i) + s.add(1, i) + } + writer.write(row) + } + } finally { + writer.close() + } + Seq(false, true).foreach { pruning => + withSQLConf( + CometConf.COMET_RESPECT_DATAFUSION_CONFIGS.key -> "true", + "spark.comet.datafusion.execution.parquet.pruning" -> pruning.toString, + "spark.comet.datafusion.execution.parquet.enable_page_index" -> "false", + CometConf.COMET_PARQUET_ROW_FILTER_PUSHDOWN_ENABLED.key -> "false") { + val query = spark.read + .parquet(path.toString) + .where("s.inner.k >= 7000") + .select("s.inner.k", "s.payload") + val (_, plan) = checkSparkAnswer(query) + val scans = collect(plan) { case scan: CometNativeScanExec => scan } + assert(scans.size == 1) + val pruned = scans.head.metrics("row_groups_pruned_statistics").value + assert(if (pruning) pruned > 0 else pruned == 0) + } + } + } + } + test("test V1 parquet scan filter pushdown of primitive types") { withTempPath { dir => val path = new Path(dir.toURI.toString, "test1.parquet") From 96499b1ed5c1eac79330776b009e5c592a8f569e Mon Sep 17 00:00:00 2001 From: peterxcli Date: Tue, 8 Sep 2026 23:55:27 +0800 Subject: [PATCH 2/3] Fix benchmark artifact license and Markdown formatting --- benchmarks/nested-pruning/README.md | 12 ++++++------ benchmarks/nested-pruning/hashes.sha256 | 17 +++++++++++++++++ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/benchmarks/nested-pruning/README.md b/benchmarks/nested-pruning/README.md index 489a66c9b89..09cd9623087 100644 --- a/benchmarks/nested-pruning/README.md +++ b/benchmarks/nested-pruning/README.md @@ -44,12 +44,12 @@ it does not imply equivalent improvements for arrays, maps, Variant, or unfilter On `chia-ping-aws1` (AMD EPYC 7282, 16 vCPUs, 31 GiB RAM), using Rust 1.98.0, OpenJDK 17.0.20, Spark 4.1.3, NumPy 2.5.3, PyArrow 25.0.1, and the settings below: -| Query | Baseline median | Candidate median | Baseline / candidate | -| --- | ---: | ---: | ---: | -| Nested filter, sorted keys | 642.3 ms | 182.6 ms | 3.52x | -| Top-level filter, sorted keys | 138.8 ms | 138.3 ms | 1.00x | -| Projection only | 530.2 ms | 537.4 ms | 0.99x | -| Nested filter, shuffled keys | 585.4 ms | 577.1 ms | 1.01x | +| Query | Baseline median | Candidate median | Baseline / candidate | +| ----------------------------- | --------------: | ---------------: | -------------------: | +| Nested filter, sorted keys | 642.3 ms | 182.6 ms | 3.52x | +| Top-level filter, sorted keys | 138.8 ms | 138.3 ms | 1.00x | +| Projection only | 530.2 ms | 537.4 ms | 0.99x | +| Nested filter, shuffled keys | 585.4 ms | 577.1 ms | 1.01x | The selective nested query skips **63 of 64 row groups**. Reader bytes decrease from **182,481,422 to 2,851,301 (98.4% less)**, and scan output decreases from 4,194,304 to diff --git a/benchmarks/nested-pruning/hashes.sha256 b/benchmarks/nested-pruning/hashes.sha256 index 4c928543d37..a55381feb79 100644 --- a/benchmarks/nested-pruning/hashes.sha256 +++ b/benchmarks/nested-pruning/hashes.sha256 @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + 7c127e5ab7945c10b679d9c2665718adc040ad082e0d4fbf773f2d22ee438c30 /home/ubuntu/oss/comet-nested-results/baseline/libcomet.so e8c2a1ef7db65836dfe935008083e518fdb2d6258498554854b79b666eee40f9 /home/ubuntu/oss/comet-nested-results/candidate/libcomet.so ace52025f280edeb5f9723235392054ae45b8e0a914120f260e71ec701909f8d /home/ubuntu/oss/comet-nested-data/shuffled.parquet From ae98020eca0210734bb2fe3a56f16ea53a1de907 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 9 Sep 2026 01:44:05 +0800 Subject: [PATCH 3/3] Remove benchmark artifacts from nested pruning PR --- benchmarks/nested-pruning/README.md | 135 - benchmarks/nested-pruning/benchmark.py | 149 - benchmarks/nested-pruning/hashes.sha256 | 22 - benchmarks/nested-pruning/results.json | 5986 ----------------------- 4 files changed, 6292 deletions(-) delete mode 100644 benchmarks/nested-pruning/README.md delete mode 100644 benchmarks/nested-pruning/benchmark.py delete mode 100644 benchmarks/nested-pruning/hashes.sha256 delete mode 100644 benchmarks/nested-pruning/results.json diff --git a/benchmarks/nested-pruning/README.md b/benchmarks/nested-pruning/README.md deleted file mode 100644 index 09cd9623087..00000000000 --- a/benchmarks/nested-pruning/README.md +++ /dev/null @@ -1,135 +0,0 @@ - - -# Nested struct statistics pruning - -Comet needs native expression plumbing to use -[peterxcli/datafusion#2](https://github.com/peterxcli/datafusion/pull/2). -Spark already serializes the nested predicates, but Comet's custom `GetStructField` -physical expression does not expose DataFusion's scalar-UDF field-access capability. -The adapter declares the selected field while retaining Comet's parent-null handling. -It resolves the field by name after schema adaptation and retains ordinal access when -field names are duplicated. - -The dependency is pinned to `0b0506a9acab9d5892ecf7e89243c3b34664bcc6`, which includes -the field-access capability, nested input requirements, and row-group pruning. -The baseline is unchanged Comet `bb9e74020adc228e486f6f4d0fa68292b30bff31`, with its -checked-in DataFusion 55.0.0 dependencies. The comparison therefore includes the -DataFusion fork update and the Comet adapter. A pruning-disabled candidate control -isolates the effect of statistics pruning. - -This change does not resolve [#5739](https://github.com/apache/datafusion-comet/issues/5739). -Timestamp conversion and Spark's other filtering paths still need separate parity work. -The optimization applies to supported struct-leaf predicates with useful statistics; -it does not imply equivalent improvements for arrays, maps, Variant, or unfiltered scans. - -## Results (2026-09-08) - -On `chia-ping-aws1` (AMD EPYC 7282, 16 vCPUs, 31 GiB RAM), using Rust 1.98.0, -OpenJDK 17.0.20, Spark 4.1.3, NumPy 2.5.3, PyArrow 25.0.1, and the settings below: - -| Query | Baseline median | Candidate median | Baseline / candidate | -| ----------------------------- | --------------: | ---------------: | -------------------: | -| Nested filter, sorted keys | 642.3 ms | 182.6 ms | 3.52x | -| Top-level filter, sorted keys | 138.8 ms | 138.3 ms | 1.00x | -| Projection only | 530.2 ms | 537.4 ms | 0.99x | -| Nested filter, shuffled keys | 585.4 ms | 577.1 ms | 1.01x | - -The selective nested query skips **63 of 64 row groups**. Reader bytes decrease from -**182,481,422 to 2,851,301 (98.4% less)**, and scan output decreases from 4,194,304 to -65,536 rows. The final aggregate checksums are identical. Disabling pruning in the -candidate restores 0 pruned groups, 182,481,422 reader bytes, and a **638.0 ms** median -(five control measurements), close to the baseline. This supports attributing the -selective-query improvement to statistics pruning. - -The three other comparisons differ by less than 2%, with overlapping interquartile -ranges. They show no consistent speedup. The selective query has interquartile ranges -of 629.3–647.1 ms for baseline and 168.8–199.3 ms for candidate. - -All measured queries used `CometNativeScan`; the plans, metrics, checksums, individual -timings, and summary statistics are retained in [results.json](results.json). -Additional candidate controls with decoder filtering enabled passed checksum and -pruning-counter checks. These controls are not a matched comparison of decoder-filter -performance. Pilot timings were excluded from the reported medians. - -Validation passed: four Rust expression tests; five Spark tests covering the new -pruning regression, nested schema evolution, field-ID mapping, and nullable struct -access (one pre-existing Spark test remains ignored); native release builds; Maven -packaging; Rust formatting; and the Spark style/format checks. The Spark regression -exercises the real native scan with statistics pruning both enabled and disabled. - -Remote checkouts are `~/oss/comet-nested-pruning` and `~/oss/comet-nested-baseline`. -Data is in `~/oss/comet-nested-data`; libraries, raw logs, samples, the implementation -patch, and [binary/data hashes](hashes.sha256) are retained in -`~/oss/comet-nested-results`. The Python environment is `~/oss/comet-nested-venv`. -The host-side `~/oss/comet-nested-run-rounds.py` records the alternating run order. - -## Reproduce - -Install `numpy==2.5.3`, `pyarrow==25.0.1`, and `pyspark==4.1.3` in a dedicated Python environment. -Generate the files once and reuse them for every revision: - -```sh -python benchmark.py --generate --data /path/to/data -``` - -Each file contains 4,194,304 rows and 64 row groups of 65,536 rows. The schema has -`k: int64` and `s: struct, p0: int64, ..., p7: int64>`. -The payload values are seeded random integers. One file has sorted keys; the other -has shuffled keys. Both use Snappy compression, no dictionary encoding, and row-group -statistics. The selective predicate retains 65,536 rows (1.5625%). Every query sums -all eight payload fields and checks the answer against sums computed during generation. - -Build both native revisions with the same settings, from their respective `native` -directories, and retain each resulting `libcomet.so` in a separate directory: - -```sh -CARGO_BUILD_JOBS=8 CARGO_PROFILE_RELEASE_LTO=false \ - CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16 CARGO_PROFILE_RELEASE_DEBUG=0 \ - cargo build --release --no-default-features --locked -``` - -Build the Spark jar from the Comet root after the native build: - -```sh -./mvnw package -DskipTests -Djni.dir=/path/to/candidate-library-directory -``` - -Use the same jar, data, JVM, and CPU affinity for both native libraries: - -```sh -SPARK_LOCAL_IP=127.0.0.1 COMET_WORKER_THREADS=8 taskset -c 0-7 \ - python benchmark.py --data /path/to/data --jar /path/to/comet.jar \ - --library-dir /path/to/candidate-library-directory --label candidate-0 \ - --runs 5 --warmup 5 --expect-nested-pruned 63 --output candidate-0.json -``` - -Use `--expect-nested-pruned 0` for the baseline or the candidate with `--pruning-off`. -The harness explicitly selects Comet's default of disabled decoder row filtering. -`--row-filter` exercises decoder filtering as an additional control. Page-index and -Bloom-filter pruning are disabled. Spark uses `local[1]`, one shuffle partition, -and disabled AQE. Timings include SQL planning, execution, and collecting the aggregate -result; they exclude Spark startup, schema inference, and data generation. - -The measured comparison uses four rounds, alternating baseline/candidate order, -with five warmups followed by five measured executions per query in each process. -Report medians over the 20 measurements per revision, alongside the scan metrics. -`bytes_scanned` excludes metadata reads, as documented by Comet's reader factory. -These are warm-cache measurements on synthetic local files, using matching release -builds with LTO disabled; they do not establish production or object-storage speedups. diff --git a/benchmarks/nested-pruning/benchmark.py b/benchmarks/nested-pruning/benchmark.py deleted file mode 100644 index 95b50444378..00000000000 --- a/benchmarks/nested-pruning/benchmark.py +++ /dev/null @@ -1,149 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -"""Generate deterministic nested Parquet data, then time checked Spark queries. - -Dependencies: numpy, pyarrow, pyspark matching the Comet build. -Run --generate once; reuse its files for every native library revision. -""" -import argparse -import json -import time -from pathlib import Path - - -def generate(directory, rows): - import numpy as np - import pyarrow as pa - import pyarrow.parquet as pq - - directory.mkdir(parents=True, exist_ok=True) - expected = {} - for layout in ("sorted", "shuffled"): - rng = np.random.default_rng(5739) - keys = np.arange(rows, dtype=np.int64) - if layout == "shuffled": - rng.shuffle(keys) - payload = [rng.integers(0, 1000000, rows, dtype=np.int64) for _ in range(8)] - inner = pa.StructArray.from_arrays([pa.array(keys)], names=["k"]) - nested = pa.StructArray.from_arrays( - [inner] + [pa.array(p) for p in payload], - names=["inner"] + [f"p{i}" for i in range(8)], - ) - table = pa.table({"k": keys, "s": nested}) - pq.write_table(table, directory / f"{layout}.parquet", row_group_size=65536, - compression="snappy", use_dictionary=False, write_statistics=True) - mask = keys >= rows - 65536 - expected[layout] = { - "all": [int(p.sum()) for p in payload], - "selective": [int(p[mask].sum()) for p in payload], - } - (directory / "expected.json").write_text(json.dumps({"rows": rows, "sums": expected})) - - -def metrics(node): - result = [] - it = node.metrics().iterator() - values = {} - while it.hasNext(): - entry = it.next() - values[entry._1()] = entry._2().value() - result.append({"node": node.nodeName(), "metrics": values}) - children = node.children().iterator() - while children.hasNext(): - result.extend(metrics(children.next())) - return result - - -def run(args): - from pyspark.sql import SparkSession - - spark = (SparkSession.builder.master("local[1]").appName("Comet nested pruning") - .config("spark.jars", str(args.jar.resolve())) - .config("spark.driver.extraJavaOptions", f"-Djava.library.path={args.library_dir.resolve()}") - .config("spark.sql.extensions", "org.apache.comet.CometSparkSessionExtensions") - .config("spark.shuffle.manager", "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") - .config("spark.sql.adaptive.enabled", "false") - .config("spark.sql.shuffle.partitions", "1") - .config("spark.sql.files.maxPartitionBytes", str(1024**3)) - .config("spark.comet.enabled", "true") - .config("spark.comet.exec.enabled", "true") - .config("spark.comet.exec.respectDataFusionConfigs", "true") - .config("spark.comet.datafusion.execution.parquet.pruning", str(not args.pruning_off).lower()) - .config("spark.comet.datafusion.execution.parquet.enable_page_index", "false") - .config("spark.comet.datafusion.execution.parquet.bloom_filter_on_read", "false") - .config("spark.comet.parquet.rowFilterPushdown.enabled", str(args.row_filter).lower()) - .getOrCreate()) - spark.sparkContext.setLogLevel("ERROR") - expected = json.loads((args.data / "expected.json").read_text()) - for layout in ("sorted", "shuffled"): - spark.read.parquet(str((args.data / f"{layout}.parquet").resolve())).createOrReplaceTempView(layout) - sums = ", ".join(f"sum(s.p{i})" for i in range(8)) - threshold = expected["rows"] - 65536 - cases = [ - ("nested_selective", "sorted", f"s.inner.k >= {threshold}", "selective"), - ("top_level_selective", "sorted", f"k >= {threshold}", "selective"), - ("projection_only", "sorted", "true", "all"), - ("nested_unprunable", "shuffled", f"s.inner.k >= {threshold}", "selective"), - ] - output = {"label": args.label, "spark": spark.version, "pruning_off": args.pruning_off, - "row_filter": args.row_filter, "warmup": args.warmup, "cases": {}} - for name, layout, predicate, expected_key in cases: - query = f"SELECT {sums} FROM {layout} WHERE {predicate}" - samples = [] - for iteration in range(args.runs + args.warmup): - start = time.perf_counter() - df = spark.sql(query) - answer = list(df.collect()[0]) - elapsed = time.perf_counter() - start - assert answer == expected["sums"][layout][expected_key], (name, answer) - plan = df._jdf.queryExecution().executedPlan() - plan_text = plan.toString() - assert "CometNativeScan" in plan_text, plan_text - if iteration >= args.warmup: - samples.append(elapsed) - scan_metrics = metrics(plan) - if args.expect_nested_pruned is not None and name == "nested_selective": - pruned = sum(n["metrics"].get("row_groups_pruned_statistics", 0) - for n in scan_metrics if "CometNativeScan" in n["node"]) - assert pruned == args.expect_nested_pruned, scan_metrics - output["cases"][name] = {"seconds": samples, "query": query, "answer": answer, - "plan": plan_text, "metrics": scan_metrics} - print(json.dumps({"label": args.label, "case": name, "seconds": samples}), flush=True) - args.output.write_text(json.dumps(output, indent=2)) - spark.stop() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--data", type=Path, required=True) - parser.add_argument("--generate", action="store_true") - parser.add_argument("--rows", type=int, default=4194304) - parser.add_argument("--jar", type=Path) - parser.add_argument("--library-dir", type=Path) - parser.add_argument("--label", default="candidate") - parser.add_argument("--output", type=Path, default=Path("results.json")) - parser.add_argument("--runs", type=int, default=5) - parser.add_argument("--warmup", type=int, default=5) - parser.add_argument("--pruning-off", action="store_true") - parser.add_argument("--row-filter", action="store_true") - parser.add_argument("--expect-nested-pruned", type=int) - args = parser.parse_args() - if args.generate: - generate(args.data, args.rows) - else: - run(args) diff --git a/benchmarks/nested-pruning/hashes.sha256 b/benchmarks/nested-pruning/hashes.sha256 deleted file mode 100644 index a55381feb79..00000000000 --- a/benchmarks/nested-pruning/hashes.sha256 +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -7c127e5ab7945c10b679d9c2665718adc040ad082e0d4fbf773f2d22ee438c30 /home/ubuntu/oss/comet-nested-results/baseline/libcomet.so -e8c2a1ef7db65836dfe935008083e518fdb2d6258498554854b79b666eee40f9 /home/ubuntu/oss/comet-nested-results/candidate/libcomet.so -ace52025f280edeb5f9723235392054ae45b8e0a914120f260e71ec701909f8d /home/ubuntu/oss/comet-nested-data/shuffled.parquet -c10d2c19f517e62f496a8a02da4c8491b9c4fff712ea7d893c986821727ca0af /home/ubuntu/oss/comet-nested-data/sorted.parquet -12877fbd1f236c8755f2209aafb87961e0f39333f6345ee1b93f00c59d404e39 /home/ubuntu/oss/comet-nested-pruning/spark/target/comet-spark-spark4.1_2.13-1.1.0-SNAPSHOT.jar diff --git a/benchmarks/nested-pruning/results.json b/benchmarks/nested-pruning/results.json deleted file mode 100644 index a35778bee17..00000000000 --- a/benchmarks/nested-pruning/results.json +++ /dev/null @@ -1,5986 +0,0 @@ -{ - "summary": { - "nested_selective": { - "baseline": { - "median_ms": 642.2919891774654, - "iqr_ms": [ - 629.2613609693944, - 647.1081066410989 - ], - "samples": 20, - "bytes_scanned": 182481422, - "output_rows": 4194304, - "row_groups_pruned_statistics": 0, - "row_groups_matched_statistics": 64 - }, - "candidate": { - "median_ms": 182.61525640264153, - "iqr_ms": [ - 168.84455806575716, - 199.33200534433126 - ], - "samples": 20, - "bytes_scanned": 2851301, - "output_rows": 65536, - "row_groups_pruned_statistics": 63, - "row_groups_matched_statistics": 1 - }, - "pruning-off": { - "median_ms": 638.0214085802436, - "iqr_ms": [ - 615.5594941228628, - 638.3354254066944 - ], - "samples": 5, - "bytes_scanned": 182481422, - "output_rows": 4194304, - "row_groups_pruned_statistics": 0, - "row_groups_matched_statistics": 64 - }, - "pruning-off-row-filter": { - "median_ms": 299.7523508965969, - "iqr_ms": [ - 283.56191609054804, - 299.83343090862036 - ], - "samples": 5, - "bytes_scanned": 19396952, - "output_rows": 65536, - "row_groups_pruned_statistics": 0, - "row_groups_matched_statistics": 64 - } - }, - "top_level_selective": { - "baseline": { - "median_ms": 138.81242321804166, - "iqr_ms": [ - 128.66710755042732, - 143.74887361191213 - ], - "samples": 20, - "bytes_scanned": 2851301, - "output_rows": 65536, - "row_groups_pruned_statistics": 63, - "row_groups_matched_statistics": 1 - }, - "candidate": { - "median_ms": 138.2634500041604, - "iqr_ms": [ - 131.87488308176398, - 146.0748442914337 - ], - "samples": 20, - "bytes_scanned": 2851301, - "output_rows": 65536, - "row_groups_pruned_statistics": 63, - "row_groups_matched_statistics": 1 - }, - "pruning-off": { - "median_ms": 605.2072821184993, - "iqr_ms": [ - 598.0431707575917, - 610.489190556109 - ], - "samples": 5, - "bytes_scanned": 182481422, - "output_rows": 4194304, - "row_groups_pruned_statistics": 0, - "row_groups_matched_statistics": 64 - }, - "pruning-off-row-filter": { - "median_ms": 188.66712413728237, - "iqr_ms": [ - 176.94623582065105, - 191.35639816522598 - ], - "samples": 5, - "bytes_scanned": 19396952, - "output_rows": 65536, - "row_groups_pruned_statistics": 0, - "row_groups_matched_statistics": 64 - } - }, - "projection_only": { - "baseline": { - "median_ms": 530.1858955062926, - "iqr_ms": [ - 522.7551013231277, - 539.3509692512453 - ], - "samples": 20, - "bytes_scanned": 165673158, - "output_rows": 4194304, - "row_groups_pruned_statistics": 0, - "row_groups_matched_statistics": 64 - }, - "candidate": { - "median_ms": 537.4044002965093, - "iqr_ms": [ - 520.9687126334757, - 550.8011817000806 - ], - "samples": 20, - "bytes_scanned": 165673158, - "output_rows": 4194304, - "row_groups_pruned_statistics": 0, - "row_groups_matched_statistics": 64 - }, - "pruning-off": { - "median_ms": 540.7594768330455, - "iqr_ms": [ - 525.6619229912758, - 550.6223440170288 - ], - "samples": 5, - "bytes_scanned": 165673158, - "output_rows": 4194304, - "row_groups_pruned_statistics": 0, - "row_groups_matched_statistics": 64 - }, - "pruning-off-row-filter": { - "median_ms": 534.7809121012688, - "iqr_ms": [ - 521.8340344727039, - 552.2861639037728 - ], - "samples": 5, - "bytes_scanned": 165673158, - "output_rows": 4194304, - "row_groups_pruned_statistics": 0, - "row_groups_matched_statistics": 64 - } - }, - "nested_unprunable": { - "baseline": { - "median_ms": 585.3609410114586, - "iqr_ms": [ - 571.0740983486176, - 591.5737492032349 - ], - "samples": 20, - "bytes_scanned": 186772555, - "output_rows": 4194304, - "row_groups_pruned_statistics": 0, - "row_groups_matched_statistics": 64 - }, - "candidate": { - "median_ms": 577.0554314367473, - "iqr_ms": [ - 565.5243212822825, - 592.3003694042563 - ], - "samples": 20, - "bytes_scanned": 186772555, - "output_rows": 4194304, - "row_groups_pruned_statistics": 0, - "row_groups_matched_statistics": 64 - }, - "pruning-off": { - "median_ms": 560.1397510617971, - "iqr_ms": [ - 551.8598509952426, - 570.2894749119878 - ], - "samples": 5, - "bytes_scanned": 186772555, - "output_rows": 4194304, - "row_groups_pruned_statistics": 0, - "row_groups_matched_statistics": 64 - }, - "pruning-off-row-filter": { - "median_ms": 751.1665793135762, - "iqr_ms": [ - 743.246354162693, - 769.7837706655264 - ], - "samples": 5, - "bytes_scanned": 186772555, - "output_rows": 65536, - "row_groups_pruned_statistics": 0, - "row_groups_matched_statistics": 64 - } - } - }, - "runs": [ - { - "label": "baseline-0", - "spark": "4.1.3", - "pruning_off": false, - "row_filter": false, - "warmup": 5, - "cases": { - "nested_selective": { - "seconds": [ - 0.6894052512943745, - 0.6490190038457513, - 0.6236759554594755, - 0.6345376027747989, - 0.6294151488691568 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE s.inner.k >= 4128768", - "answer": [ - 32741059719, - 32774396751, - 32772938793, - 32799973223, - 32905236128, - 32807159809, - 32718967833, - 32661111725 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#486L, sum#487L, sum#488L, sum#489L, sum#490L, sum#491L, sum#492L, sum#493L], [Final], [sum(_extract_p0#470L), sum(_extract_p1#471L), sum(_extract_p2#472L), sum(_extract_p3#473L), sum(_extract_p4#474L), sum(_extract_p5#475L), sum(_extract_p6#476L), sum(_extract_p7#477L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=594]\n +- CometHashAggregate [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [Partial], [partial_sum(_extract_p0#470L), partial_sum(_extract_p1#471L), partial_sum(_extract_p2#472L), partial_sum(_extract_p3#473L), partial_sum(_extract_p4#474L), partial_sum(_extract_p5#475L), partial_sum(_extract_p6#476L), partial_sum(_extract_p7#477L)]\n +- CometProject [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [s#1.p0 AS _extract_p0#470L, s#1.p1 AS _extract_p1#471L, s#1.p2 AS _extract_p2#472L, s#1.p3 AS _extract_p3#473L, s#1.p4 AS _extract_p4#474L, s#1.p5 AS _extract_p5#475L, s#1.p6 AS _extract_p6#476L, s#1.p7 AS _extract_p7#477L]\n +- CometFilter [s#1], (isnotnull(s#1.inner.k) AND (s#1.inner.k >= 4128768))\n +- CometNativeScan parquet [s#1] Batched: true, DataFilters: [isnotnull(s#1.inner.k), (s#1.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 3 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 19480 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 526, - "decode_time": 746503, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 151809, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 1064290, - "numPartitions": 1, - "encode_time": 26980, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 526, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 539487 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 47948 - } - }, - { - "node": "CometFilter", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 12320530 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 466543403, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 0, - "predicate_cache_inner_records": 0, - "filesSize": 199358457, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 2694414, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 64, - "row_pushdown_eval_time": 2, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 0, - "page_index_eval_time": 2, - "bytes_scanned": 182481422, - "predicate_evaluation_errors": 0, - "metadata_load_time": 754783, - "output_rows": 4194304, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 101, - "metadataTime": 0, - "scan_efficiency_ratio_total": 199358457, - "row_groups_matched_statistics": 64, - "statistics_eval_time": 2, - "time_elapsed_opening": 835172, - "row_groups_pruned_statistics": 0, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 385843795, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - }, - "top_level_selective": { - "seconds": [ - 0.14080541394650936, - 0.12375261634588242, - 0.11019718460738659, - 0.145455040037632, - 0.12480535730719566 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE k >= 4128768", - "answer": [ - 32741059719, - 32774396751, - 32772938793, - 32799973223, - 32905236128, - 32807159809, - 32718967833, - 32661111725 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#986L, sum#987L, sum#988L, sum#989L, sum#990L, sum#991L, sum#992L, sum#993L], [Final], [sum(_extract_p0#968L), sum(_extract_p1#969L), sum(_extract_p2#970L), sum(_extract_p3#971L), sum(_extract_p4#972L), sum(_extract_p5#973L), sum(_extract_p6#974L), sum(_extract_p7#975L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1204]\n +- CometHashAggregate [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [Partial], [partial_sum(_extract_p0#968L), partial_sum(_extract_p1#969L), partial_sum(_extract_p2#970L), partial_sum(_extract_p3#971L), partial_sum(_extract_p4#972L), partial_sum(_extract_p5#973L), partial_sum(_extract_p6#974L), partial_sum(_extract_p7#975L)]\n +- CometProject [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [s#1.p0 AS _extract_p0#968L, s#1.p1 AS _extract_p1#969L, s#1.p2 AS _extract_p2#970L, s#1.p3 AS _extract_p3#971L, s#1.p4 AS _extract_p4#972L, s#1.p5 AS _extract_p5#973L, s#1.p6 AS _extract_p6#974L, s#1.p7 AS _extract_p7#975L]\n +- CometFilter [k#0L, s#1], (isnotnull(k#0L) AND (k#0L >= 4128768))\n +- CometNativeScan parquet [k#0L,s#1] Batched: true, DataFilters: [isnotnull(k#0L), (k#0L >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(k), GreaterThanOrEqual(k,4128768)], ReadSchema: struct>\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 2 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 19440 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 526, - "decode_time": 414026, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 131429, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 794482, - "numPartitions": 1, - "encode_time": 32149, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 526, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 34312512 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 4194304, - "elapsed_compute": 3722041 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 452814152, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 0, - "predicate_cache_inner_records": 0, - "filesSize": 199358457, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 2521226, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 64, - "row_pushdown_eval_time": 2, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 0, - "page_index_eval_time": 2, - "bytes_scanned": 165673158, - "predicate_evaluation_errors": 0, - "metadata_load_time": 745524, - "output_rows": 4194304, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 121, - "metadataTime": 0, - "scan_efficiency_ratio_total": 199358457, - "row_groups_matched_statistics": 64, - "statistics_eval_time": 2, - "time_elapsed_opening": 955812, - "row_groups_pruned_statistics": 0, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 343454356, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - }, - "nested_unprunable": { - "seconds": [ - 0.5853548347949982, - 0.5719598717987537, - 0.5750899221748114, - 0.5677164318040013, - 0.5670498479157686 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM shuffled WHERE s.inner.k >= 4128768", - "answer": [ - 32658560674, - 32812041019, - 32867632839, - 32498761177, - 32648273638, - 32881993944, - 32750626290, - 32782672280 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#1976L, sum#1977L, sum#1978L, sum#1979L, sum#1980L, sum#1981L, sum#1982L, sum#1983L], [Final], [sum(_extract_p0#1960L), sum(_extract_p1#1961L), sum(_extract_p2#1962L), sum(_extract_p3#1963L), sum(_extract_p4#1964L), sum(_extract_p5#1965L), sum(_extract_p6#1966L), sum(_extract_p7#1967L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2334]\n +- CometHashAggregate [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [Partial], [partial_sum(_extract_p0#1960L), partial_sum(_extract_p1#1961L), partial_sum(_extract_p2#1962L), partial_sum(_extract_p3#1963L), partial_sum(_extract_p4#1964L), partial_sum(_extract_p5#1965L), partial_sum(_extract_p6#1966L), partial_sum(_extract_p7#1967L)]\n +- CometProject [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [s#3.p0 AS _extract_p0#1960L, s#3.p1 AS _extract_p1#1961L, s#3.p2 AS _extract_p2#1962L, s#3.p3 AS _extract_p3#1963L, s#3.p4 AS _extract_p4#1964L, s#3.p5 AS _extract_p5#1965L, s#3.p6 AS _extract_p6#1966L, s#3.p7 AS _extract_p7#1967L]\n +- CometFilter [s#3], (isnotnull(s#3.inner.k) AND (s#3.inner.k >= 4128768))\n +- CometNativeScan parquet [s#3] Batched: true, DataFilters: [isnotnull(s#3.inner.k), (s#3.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/shuffled.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 2 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 20070 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 527, - "decode_time": 455145, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 141189, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 861171, - "numPartitions": 1, - "encode_time": 29249, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 527, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 552116 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 74758 - } - }, - { - "node": "CometFilter", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 18256290 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 466942698, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 0, - "predicate_cache_inner_records": 0, - "filesSize": 207941114, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 2784123, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 64, - "row_pushdown_eval_time": 2, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 0, - "page_index_eval_time": 2, - "bytes_scanned": 186772555, - "predicate_evaluation_errors": 0, - "metadata_load_time": 793873, - "output_rows": 4194304, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 121, - "metadataTime": 0, - "scan_efficiency_ratio_total": 207941114, - "row_groups_matched_statistics": 64, - "statistics_eval_time": 2, - "time_elapsed_opening": 911391, - "row_groups_pruned_statistics": 0, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 383420159, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - } - } - }, - { - "label": "baseline-1", - "spark": "4.1.3", - "pruning_off": false, - "row_filter": false, - "warmup": 5, - "cases": { - "nested_selective": { - "seconds": [ - 0.6316028796136379, - 0.6436244547367096, - 0.6065702280029655, - 0.6287999972701073, - 0.6161443758755922 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE s.inner.k >= 4128768", - "answer": [ - 32741059719, - 32774396751, - 32772938793, - 32799973223, - 32905236128, - 32807159809, - 32718967833, - 32661111725 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#486L, sum#487L, sum#488L, sum#489L, sum#490L, sum#491L, sum#492L, sum#493L], [Final], [sum(_extract_p0#470L), sum(_extract_p1#471L), sum(_extract_p2#472L), sum(_extract_p3#473L), sum(_extract_p4#474L), sum(_extract_p5#475L), sum(_extract_p6#476L), sum(_extract_p7#477L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=594]\n +- CometHashAggregate [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [Partial], [partial_sum(_extract_p0#470L), partial_sum(_extract_p1#471L), partial_sum(_extract_p2#472L), partial_sum(_extract_p3#473L), partial_sum(_extract_p4#474L), partial_sum(_extract_p5#475L), partial_sum(_extract_p6#476L), partial_sum(_extract_p7#477L)]\n +- CometProject [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [s#1.p0 AS _extract_p0#470L, s#1.p1 AS _extract_p1#471L, s#1.p2 AS _extract_p2#472L, s#1.p3 AS _extract_p3#473L, s#1.p4 AS _extract_p4#474L, s#1.p5 AS _extract_p5#475L, s#1.p6 AS _extract_p6#476L, s#1.p7 AS _extract_p7#477L]\n +- CometFilter [s#1], (isnotnull(s#1.inner.k) AND (s#1.inner.k >= 4128768))\n +- CometNativeScan parquet [s#1] Batched: true, DataFilters: [isnotnull(s#1.inner.k), (s#1.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 4 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 24360 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 526, - "decode_time": 1010631, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 131768, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 1252738, - "numPartitions": 1, - "encode_time": 27390, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 526, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 506868 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 46139 - } - }, - { - "node": "CometFilter", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 12313974 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 470639031, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 0, - "predicate_cache_inner_records": 0, - "filesSize": 199358457, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 2616935, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 64, - "row_pushdown_eval_time": 2, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 0, - "page_index_eval_time": 2, - "bytes_scanned": 182481422, - "predicate_evaluation_errors": 0, - "metadata_load_time": 730644, - "output_rows": 4194304, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 131, - "metadataTime": 0, - "scan_efficiency_ratio_total": 199358457, - "row_groups_matched_statistics": 64, - "statistics_eval_time": 2, - "time_elapsed_opening": 858293, - "row_groups_pruned_statistics": 0, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 384754173, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - }, - "top_level_selective": { - "seconds": [ - 0.13424876146018505, - 0.14073058031499386, - 0.11925629619508982, - 0.14138470403850079, - 0.1284897569566965 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE k >= 4128768", - "answer": [ - 32741059719, - 32774396751, - 32772938793, - 32799973223, - 32905236128, - 32807159809, - 32718967833, - 32661111725 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#986L, sum#987L, sum#988L, sum#989L, sum#990L, sum#991L, sum#992L, sum#993L], [Final], [sum(_extract_p0#968L), sum(_extract_p1#969L), sum(_extract_p2#970L), sum(_extract_p3#971L), sum(_extract_p4#972L), sum(_extract_p5#973L), sum(_extract_p6#974L), sum(_extract_p7#975L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1204]\n +- CometHashAggregate [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [Partial], [partial_sum(_extract_p0#968L), partial_sum(_extract_p1#969L), partial_sum(_extract_p2#970L), partial_sum(_extract_p3#971L), partial_sum(_extract_p4#972L), partial_sum(_extract_p5#973L), partial_sum(_extract_p6#974L), partial_sum(_extract_p7#975L)]\n +- CometProject [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [s#1.p0 AS _extract_p0#968L, s#1.p1 AS _extract_p1#969L, s#1.p2 AS _extract_p2#970L, s#1.p3 AS _extract_p3#971L, s#1.p4 AS _extract_p4#972L, s#1.p5 AS _extract_p5#973L, s#1.p6 AS _extract_p6#974L, s#1.p7 AS _extract_p7#975L]\n +- CometFilter [k#0L, s#1], (isnotnull(k#0L) AND (k#0L >= 4128768))\n +- CometNativeScan parquet [k#0L,s#1] Batched: true, DataFilters: [isnotnull(k#0L), (k#0L >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(k), GreaterThanOrEqual(k,4128768)], ReadSchema: struct>\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 2 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 18509 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 526, - "decode_time": 414736, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 139100, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 838573, - "numPartitions": 1, - "encode_time": 33560, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 526, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 29503811 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 4194304, - "elapsed_compute": 3183327 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 415581284, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 0, - "predicate_cache_inner_records": 0, - "filesSize": 199358457, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 2352628, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 64, - "row_pushdown_eval_time": 2, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 0, - "page_index_eval_time": 2, - "bytes_scanned": 165673158, - "predicate_evaluation_errors": 0, - "metadata_load_time": 776343, - "output_rows": 4194304, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 141, - "metadataTime": 0, - "scan_efficiency_ratio_total": 199358457, - "row_groups_matched_statistics": 64, - "statistics_eval_time": 2, - "time_elapsed_opening": 922081, - "row_groups_pruned_statistics": 0, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 326972042, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - }, - "nested_unprunable": { - "seconds": [ - 0.568416777998209, - 0.5853670472279191, - 0.6337875267490745, - 0.5857159039005637, - 0.5950113451108336 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM shuffled WHERE s.inner.k >= 4128768", - "answer": [ - 32658560674, - 32812041019, - 32867632839, - 32498761177, - 32648273638, - 32881993944, - 32750626290, - 32782672280 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#1976L, sum#1977L, sum#1978L, sum#1979L, sum#1980L, sum#1981L, sum#1982L, sum#1983L], [Final], [sum(_extract_p0#1960L), sum(_extract_p1#1961L), sum(_extract_p2#1962L), sum(_extract_p3#1963L), sum(_extract_p4#1964L), sum(_extract_p5#1965L), sum(_extract_p6#1966L), sum(_extract_p7#1967L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2334]\n +- CometHashAggregate [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [Partial], [partial_sum(_extract_p0#1960L), partial_sum(_extract_p1#1961L), partial_sum(_extract_p2#1962L), partial_sum(_extract_p3#1963L), partial_sum(_extract_p4#1964L), partial_sum(_extract_p5#1965L), partial_sum(_extract_p6#1966L), partial_sum(_extract_p7#1967L)]\n +- CometProject [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [s#3.p0 AS _extract_p0#1960L, s#3.p1 AS _extract_p1#1961L, s#3.p2 AS _extract_p2#1962L, s#3.p3 AS _extract_p3#1963L, s#3.p4 AS _extract_p4#1964L, s#3.p5 AS _extract_p5#1965L, s#3.p6 AS _extract_p6#1966L, s#3.p7 AS _extract_p7#1967L]\n +- CometFilter [s#3], (isnotnull(s#3.inner.k) AND (s#3.inner.k >= 4128768))\n +- CometNativeScan parquet [s#3] Batched: true, DataFilters: [isnotnull(s#3.inner.k), (s#3.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/shuffled.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 2 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 19230 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 527, - "decode_time": 531525, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 148458, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 773863, - "numPartitions": 1, - "encode_time": 27970, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 527, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 528616 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 69109 - } - }, - { - "node": "CometFilter", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 18318110 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 472289234, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 0, - "predicate_cache_inner_records": 0, - "filesSize": 207941114, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 2838723, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 64, - "row_pushdown_eval_time": 2, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 0, - "page_index_eval_time": 2, - "bytes_scanned": 186772555, - "predicate_evaluation_errors": 0, - "metadata_load_time": 859733, - "output_rows": 4194304, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 121, - "metadataTime": 0, - "scan_efficiency_ratio_total": 207941114, - "row_groups_matched_statistics": 64, - "statistics_eval_time": 2, - "time_elapsed_opening": 995810, - "row_groups_pruned_statistics": 0, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 386357774, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - } - } - }, - { - "label": "baseline-2", - "spark": "4.1.3", - "pruning_off": false, - "row_filter": false, - "warmup": 5, - "cases": { - "nested_selective": { - "seconds": [ - 0.6409595236182213, - 0.6275981506332755, - 0.6454171715304255, - 0.6464711409062147, - 0.6355891628190875 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE s.inner.k >= 4128768", - "answer": [ - 32741059719, - 32774396751, - 32772938793, - 32799973223, - 32905236128, - 32807159809, - 32718967833, - 32661111725 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#486L, sum#487L, sum#488L, sum#489L, sum#490L, sum#491L, sum#492L, sum#493L], [Final], [sum(_extract_p0#470L), sum(_extract_p1#471L), sum(_extract_p2#472L), sum(_extract_p3#473L), sum(_extract_p4#474L), sum(_extract_p5#475L), sum(_extract_p6#476L), sum(_extract_p7#477L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=594]\n +- CometHashAggregate [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [Partial], [partial_sum(_extract_p0#470L), partial_sum(_extract_p1#471L), partial_sum(_extract_p2#472L), partial_sum(_extract_p3#473L), partial_sum(_extract_p4#474L), partial_sum(_extract_p5#475L), partial_sum(_extract_p6#476L), partial_sum(_extract_p7#477L)]\n +- CometProject [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [s#1.p0 AS _extract_p0#470L, s#1.p1 AS _extract_p1#471L, s#1.p2 AS _extract_p2#472L, s#1.p3 AS _extract_p3#473L, s#1.p4 AS _extract_p4#474L, s#1.p5 AS _extract_p5#475L, s#1.p6 AS _extract_p6#476L, s#1.p7 AS _extract_p7#477L]\n +- CometFilter [s#1], (isnotnull(s#1.inner.k) AND (s#1.inner.k >= 4128768))\n +- CometNativeScan parquet [s#1] Batched: true, DataFilters: [isnotnull(s#1.inner.k), (s#1.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 3 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 23170 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 526, - "decode_time": 763003, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 138378, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 1160478, - "numPartitions": 1, - "encode_time": 28010, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 526, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 408018 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 37948 - } - }, - { - "node": "CometFilter", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 12861779 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 470629325, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 0, - "predicate_cache_inner_records": 0, - "filesSize": 199358457, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 4383559, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 64, - "row_pushdown_eval_time": 2, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 0, - "page_index_eval_time": 2, - "bytes_scanned": 182481422, - "predicate_evaluation_errors": 0, - "metadata_load_time": 942381, - "output_rows": 4194304, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 141, - "metadataTime": 0, - "scan_efficiency_ratio_total": 199358457, - "row_groups_matched_statistics": 64, - "statistics_eval_time": 2, - "time_elapsed_opening": 1060260, - "row_groups_pruned_statistics": 0, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 388815665, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - }, - "top_level_selective": { - "seconds": [ - 0.13689426612108946, - 0.14238543342798948, - 0.12872622441500425, - 0.13351120799779892, - 0.1360515933483839 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE k >= 4128768", - "answer": [ - 32741059719, - 32774396751, - 32772938793, - 32799973223, - 32905236128, - 32807159809, - 32718967833, - 32661111725 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#986L, sum#987L, sum#988L, sum#989L, sum#990L, sum#991L, sum#992L, sum#993L], [Final], [sum(_extract_p0#968L), sum(_extract_p1#969L), sum(_extract_p2#970L), sum(_extract_p3#971L), sum(_extract_p4#972L), sum(_extract_p5#973L), sum(_extract_p6#974L), sum(_extract_p7#975L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1204]\n +- CometHashAggregate [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [Partial], [partial_sum(_extract_p0#968L), partial_sum(_extract_p1#969L), partial_sum(_extract_p2#970L), partial_sum(_extract_p3#971L), partial_sum(_extract_p4#972L), partial_sum(_extract_p5#973L), partial_sum(_extract_p6#974L), partial_sum(_extract_p7#975L)]\n +- CometProject [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [s#1.p0 AS _extract_p0#968L, s#1.p1 AS _extract_p1#969L, s#1.p2 AS _extract_p2#970L, s#1.p3 AS _extract_p3#971L, s#1.p4 AS _extract_p4#972L, s#1.p5 AS _extract_p5#973L, s#1.p6 AS _extract_p6#974L, s#1.p7 AS _extract_p7#975L]\n +- CometFilter [k#0L, s#1], (isnotnull(k#0L) AND (k#0L >= 4128768))\n +- CometNativeScan parquet [k#0L,s#1] Batched: true, DataFilters: [isnotnull(k#0L), (k#0L >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(k), GreaterThanOrEqual(k,4128768)], ReadSchema: struct>\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 3 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 23500 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 526, - "decode_time": 616295, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 125098, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 1003540, - "numPartitions": 1, - "encode_time": 28750, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 526, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 31021971 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 4194304, - "elapsed_compute": 3687545 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 437298712, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 0, - "predicate_cache_inner_records": 0, - "filesSize": 199358457, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 2228229, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 64, - "row_pushdown_eval_time": 2, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 0, - "page_index_eval_time": 2, - "bytes_scanned": 165673158, - "predicate_evaluation_errors": 0, - "metadata_load_time": 603585, - "output_rows": 4194304, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 121, - "metadataTime": 0, - "scan_efficiency_ratio_total": 199358457, - "row_groups_matched_statistics": 64, - "statistics_eval_time": 2, - "time_elapsed_opening": 748802, - "row_groups_pruned_statistics": 0, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 334708499, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - }, - "nested_unprunable": { - "seconds": [ - 0.595560303889215, - 0.590427883900702, - 0.575502255000174, - 0.5648535676300526, - 0.5603784089908004 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM shuffled WHERE s.inner.k >= 4128768", - "answer": [ - 32658560674, - 32812041019, - 32867632839, - 32498761177, - 32648273638, - 32881993944, - 32750626290, - 32782672280 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#1976L, sum#1977L, sum#1978L, sum#1979L, sum#1980L, sum#1981L, sum#1982L, sum#1983L], [Final], [sum(_extract_p0#1960L), sum(_extract_p1#1961L), sum(_extract_p2#1962L), sum(_extract_p3#1963L), sum(_extract_p4#1964L), sum(_extract_p5#1965L), sum(_extract_p6#1966L), sum(_extract_p7#1967L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2334]\n +- CometHashAggregate [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [Partial], [partial_sum(_extract_p0#1960L), partial_sum(_extract_p1#1961L), partial_sum(_extract_p2#1962L), partial_sum(_extract_p3#1963L), partial_sum(_extract_p4#1964L), partial_sum(_extract_p5#1965L), partial_sum(_extract_p6#1966L), partial_sum(_extract_p7#1967L)]\n +- CometProject [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [s#3.p0 AS _extract_p0#1960L, s#3.p1 AS _extract_p1#1961L, s#3.p2 AS _extract_p2#1962L, s#3.p3 AS _extract_p3#1963L, s#3.p4 AS _extract_p4#1964L, s#3.p5 AS _extract_p5#1965L, s#3.p6 AS _extract_p6#1966L, s#3.p7 AS _extract_p7#1967L]\n +- CometFilter [s#3], (isnotnull(s#3.inner.k) AND (s#3.inner.k >= 4128768))\n +- CometNativeScan parquet [s#3] Batched: true, DataFilters: [isnotnull(s#3.inner.k), (s#3.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/shuffled.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 2 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 24990 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 527, - "decode_time": 507995, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 133459, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 901032, - "numPartitions": 1, - "encode_time": 30301, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 527, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 505934 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 66330 - } - }, - { - "node": "CometFilter", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 17572895 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 466456893, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 0, - "predicate_cache_inner_records": 0, - "filesSize": 207941114, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 2937692, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 64, - "row_pushdown_eval_time": 2, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 0, - "page_index_eval_time": 2, - "bytes_scanned": 186772555, - "predicate_evaluation_errors": 0, - "metadata_load_time": 687675, - "output_rows": 4194304, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 91, - "metadataTime": 0, - "scan_efficiency_ratio_total": 207941114, - "row_groups_matched_statistics": 64, - "statistics_eval_time": 2, - "time_elapsed_opening": 820532, - "row_groups_pruned_statistics": 0, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 381924008, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - } - } - }, - { - "label": "baseline-3", - "spark": "4.1.3", - "pruning_off": false, - "row_filter": false, - "warmup": 5, - "cases": { - "nested_selective": { - "seconds": [ - 0.6592776225879788, - 0.6437941892072558, - 0.6677187196910381, - 0.6587780667468905, - 0.6453634444624186 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE s.inner.k >= 4128768", - "answer": [ - 32741059719, - 32774396751, - 32772938793, - 32799973223, - 32905236128, - 32807159809, - 32718967833, - 32661111725 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#486L, sum#487L, sum#488L, sum#489L, sum#490L, sum#491L, sum#492L, sum#493L], [Final], [sum(_extract_p0#470L), sum(_extract_p1#471L), sum(_extract_p2#472L), sum(_extract_p3#473L), sum(_extract_p4#474L), sum(_extract_p5#475L), sum(_extract_p6#476L), sum(_extract_p7#477L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=594]\n +- CometHashAggregate [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [Partial], [partial_sum(_extract_p0#470L), partial_sum(_extract_p1#471L), partial_sum(_extract_p2#472L), partial_sum(_extract_p3#473L), partial_sum(_extract_p4#474L), partial_sum(_extract_p5#475L), partial_sum(_extract_p6#476L), partial_sum(_extract_p7#477L)]\n +- CometProject [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [s#1.p0 AS _extract_p0#470L, s#1.p1 AS _extract_p1#471L, s#1.p2 AS _extract_p2#472L, s#1.p3 AS _extract_p3#473L, s#1.p4 AS _extract_p4#474L, s#1.p5 AS _extract_p5#475L, s#1.p6 AS _extract_p6#476L, s#1.p7 AS _extract_p7#477L]\n +- CometFilter [s#1], (isnotnull(s#1.inner.k) AND (s#1.inner.k >= 4128768))\n +- CometNativeScan parquet [s#1] Batched: true, DataFilters: [isnotnull(s#1.inner.k), (s#1.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 3 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 19510 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 526, - "decode_time": 735554, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 145849, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 1099190, - "numPartitions": 1, - "encode_time": 28100, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 526, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 526914 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 43539 - } - }, - { - "node": "CometFilter", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 14507373 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 495112241, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 0, - "predicate_cache_inner_records": 0, - "filesSize": 199358457, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 3515616, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 64, - "row_pushdown_eval_time": 2, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 0, - "page_index_eval_time": 2, - "bytes_scanned": 182481422, - "predicate_evaluation_errors": 0, - "metadata_load_time": 914442, - "output_rows": 4194304, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 111, - "metadataTime": 0, - "scan_efficiency_ratio_total": 199358457, - "row_groups_matched_statistics": 64, - "statistics_eval_time": 2, - "time_elapsed_opening": 1007160, - "row_groups_pruned_statistics": 0, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 404711922, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - }, - "top_level_selective": { - "seconds": [ - 0.15336241573095322, - 0.21535575296729803, - 0.1927366890013218, - 0.16786616574972868, - 0.1431801514700055 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE k >= 4128768", - "answer": [ - 32741059719, - 32774396751, - 32772938793, - 32799973223, - 32905236128, - 32807159809, - 32718967833, - 32661111725 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#986L, sum#987L, sum#988L, sum#989L, sum#990L, sum#991L, sum#992L, sum#993L], [Final], [sum(_extract_p0#968L), sum(_extract_p1#969L), sum(_extract_p2#970L), sum(_extract_p3#971L), sum(_extract_p4#972L), sum(_extract_p5#973L), sum(_extract_p6#974L), sum(_extract_p7#975L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1204]\n +- CometHashAggregate [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [Partial], [partial_sum(_extract_p0#968L), partial_sum(_extract_p1#969L), partial_sum(_extract_p2#970L), partial_sum(_extract_p3#971L), partial_sum(_extract_p4#972L), partial_sum(_extract_p5#973L), partial_sum(_extract_p6#974L), partial_sum(_extract_p7#975L)]\n +- CometProject [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [s#1.p0 AS _extract_p0#968L, s#1.p1 AS _extract_p1#969L, s#1.p2 AS _extract_p2#970L, s#1.p3 AS _extract_p3#971L, s#1.p4 AS _extract_p4#972L, s#1.p5 AS _extract_p5#973L, s#1.p6 AS _extract_p6#974L, s#1.p7 AS _extract_p7#975L]\n +- CometFilter [k#0L, s#1], (isnotnull(k#0L) AND (k#0L >= 4128768))\n +- CometNativeScan parquet [k#0L,s#1] Batched: true, DataFilters: [isnotnull(k#0L), (k#0L >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(k), GreaterThanOrEqual(k,4128768)], ReadSchema: struct>\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 2 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 18769 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 526, - "decode_time": 454405, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 131129, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 726884, - "numPartitions": 1, - "encode_time": 27160, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 526, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 30755644 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 4194304, - "elapsed_compute": 3470066 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 428848163, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 0, - "predicate_cache_inner_records": 0, - "filesSize": 199358457, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 2374218, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 64, - "row_pushdown_eval_time": 2, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 0, - "page_index_eval_time": 2, - "bytes_scanned": 165673158, - "predicate_evaluation_errors": 0, - "metadata_load_time": 649384, - "output_rows": 4194304, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 81, - "metadataTime": 0, - "scan_efficiency_ratio_total": 199358457, - "row_groups_matched_statistics": 64, - "statistics_eval_time": 2, - "time_elapsed_opening": 772642, - "row_groups_pruned_statistics": 0, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 329854295, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - }, - "nested_unprunable": { - "seconds": [ - 0.6037412909790874, - 0.588707254268229, - 0.6129186730831861, - 0.5801800154149532, - 0.5867422120645642 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM shuffled WHERE s.inner.k >= 4128768", - "answer": [ - 32658560674, - 32812041019, - 32867632839, - 32498761177, - 32648273638, - 32881993944, - 32750626290, - 32782672280 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#1976L, sum#1977L, sum#1978L, sum#1979L, sum#1980L, sum#1981L, sum#1982L, sum#1983L], [Final], [sum(_extract_p0#1960L), sum(_extract_p1#1961L), sum(_extract_p2#1962L), sum(_extract_p3#1963L), sum(_extract_p4#1964L), sum(_extract_p5#1965L), sum(_extract_p6#1966L), sum(_extract_p7#1967L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2334]\n +- CometHashAggregate [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [Partial], [partial_sum(_extract_p0#1960L), partial_sum(_extract_p1#1961L), partial_sum(_extract_p2#1962L), partial_sum(_extract_p3#1963L), partial_sum(_extract_p4#1964L), partial_sum(_extract_p5#1965L), partial_sum(_extract_p6#1966L), partial_sum(_extract_p7#1967L)]\n +- CometProject [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [s#3.p0 AS _extract_p0#1960L, s#3.p1 AS _extract_p1#1961L, s#3.p2 AS _extract_p2#1962L, s#3.p3 AS _extract_p3#1963L, s#3.p4 AS _extract_p4#1964L, s#3.p5 AS _extract_p5#1965L, s#3.p6 AS _extract_p6#1966L, s#3.p7 AS _extract_p7#1967L]\n +- CometFilter [s#3], (isnotnull(s#3.inner.k) AND (s#3.inner.k >= 4128768))\n +- CometNativeScan parquet [s#3] Batched: true, DataFilters: [isnotnull(s#3.inner.k), (s#3.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/shuffled.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 2 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 22099 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 527, - "decode_time": 471595, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 146169, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 811843, - "numPartitions": 1, - "encode_time": 32070, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 527, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 592173 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 65260 - } - }, - { - "node": "CometFilter", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 18570095 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 477367026, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 0, - "predicate_cache_inner_records": 0, - "filesSize": 207941114, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 2710304, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 64, - "row_pushdown_eval_time": 2, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 0, - "page_index_eval_time": 2, - "bytes_scanned": 186772555, - "predicate_evaluation_errors": 0, - "metadata_load_time": 730174, - "output_rows": 4194304, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 101, - "metadataTime": 0, - "scan_efficiency_ratio_total": 207941114, - "row_groups_matched_statistics": 64, - "statistics_eval_time": 2, - "time_elapsed_opening": 818543, - "row_groups_pruned_statistics": 0, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 393760680, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - } - } - }, - { - "label": "candidate-0", - "spark": "4.1.3", - "pruning_off": false, - "row_filter": false, - "warmup": 5, - "cases": { - "nested_selective": { - "seconds": [ - 0.2042244616895914, - 0.18543519917875528, - 0.16565270721912384, - 0.1565515324473381, - 0.15750608406960964 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE s.inner.k >= 4128768", - "answer": [ - 32741059719, - 32774396751, - 32772938793, - 32799973223, - 32905236128, - 32807159809, - 32718967833, - 32661111725 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#486L, sum#487L, sum#488L, sum#489L, sum#490L, sum#491L, sum#492L, sum#493L], [Final], [sum(_extract_p0#470L), sum(_extract_p1#471L), sum(_extract_p2#472L), sum(_extract_p3#473L), sum(_extract_p4#474L), sum(_extract_p5#475L), sum(_extract_p6#476L), sum(_extract_p7#477L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=594]\n +- CometHashAggregate [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [Partial], [partial_sum(_extract_p0#470L), partial_sum(_extract_p1#471L), partial_sum(_extract_p2#472L), partial_sum(_extract_p3#473L), partial_sum(_extract_p4#474L), partial_sum(_extract_p5#475L), partial_sum(_extract_p6#476L), partial_sum(_extract_p7#477L)]\n +- CometProject [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [s#1.p0 AS _extract_p0#470L, s#1.p1 AS _extract_p1#471L, s#1.p2 AS _extract_p2#472L, s#1.p3 AS _extract_p3#473L, s#1.p4 AS _extract_p4#474L, s#1.p5 AS _extract_p5#475L, s#1.p6 AS _extract_p6#476L, s#1.p7 AS _extract_p7#477L]\n +- CometFilter [s#1], (isnotnull(s#1.inner.k) AND (s#1.inner.k >= 4128768))\n +- CometNativeScan parquet [s#1] Batched: true, DataFilters: [isnotnull(s#1.inner.k), (s#1.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 3 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 19640 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 526, - "decode_time": 721214, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 133359, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 1454205, - "numPartitions": 1, - "encode_time": 30339, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 526, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 582483 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 77019 - } - }, - { - "node": "CometFilter", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 247780 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 8220012, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 0, - "predicate_cache_inner_records": 0, - "filesSize": 199358457, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 2743914, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 1, - "row_pushdown_eval_time": 2, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 0, - "page_index_eval_time": 2, - "bytes_scanned": 2851301, - "predicate_evaluation_errors": 0, - "metadata_load_time": 873414, - "output_rows": 65536, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 111, - "metadataTime": 0, - "scan_efficiency_ratio_total": 199358457, - "row_groups_matched_statistics": 1, - "statistics_eval_time": 161710, - "time_elapsed_opening": 1247678, - "row_groups_pruned_statistics": 63, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 7230549, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - }, - "top_level_selective": { - "seconds": [ - 0.144645806401968, - 0.14989290665835142, - 0.14998808596283197, - 0.1314064422622323, - 0.1390496101230383 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE k >= 4128768", - "answer": [ - 32741059719, - 32774396751, - 32772938793, - 32799973223, - 32905236128, - 32807159809, - 32718967833, - 32661111725 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#986L, sum#987L, sum#988L, sum#989L, sum#990L, sum#991L, sum#992L, sum#993L], [Final], [sum(_extract_p0#968L), sum(_extract_p1#969L), sum(_extract_p2#970L), sum(_extract_p3#971L), sum(_extract_p4#972L), sum(_extract_p5#973L), sum(_extract_p6#974L), sum(_extract_p7#975L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1204]\n +- CometHashAggregate [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [Partial], [partial_sum(_extract_p0#968L), partial_sum(_extract_p1#969L), partial_sum(_extract_p2#970L), partial_sum(_extract_p3#971L), partial_sum(_extract_p4#972L), partial_sum(_extract_p5#973L), partial_sum(_extract_p6#974L), partial_sum(_extract_p7#975L)]\n +- CometProject [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [s#1.p0 AS _extract_p0#968L, s#1.p1 AS _extract_p1#969L, s#1.p2 AS _extract_p2#970L, s#1.p3 AS _extract_p3#971L, s#1.p4 AS _extract_p4#972L, s#1.p5 AS _extract_p5#973L, s#1.p6 AS _extract_p6#974L, s#1.p7 AS _extract_p7#975L]\n +- CometFilter [k#0L, s#1], (isnotnull(k#0L) AND (k#0L >= 4128768))\n +- CometNativeScan parquet [k#0L,s#1] Batched: true, DataFilters: [isnotnull(k#0L), (k#0L >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(k), GreaterThanOrEqual(k,4128768)], ReadSchema: struct>\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 2 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 16140 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 526, - "decode_time": 506835, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 155388, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 761703, - "numPartitions": 1, - "encode_time": 36300, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 526, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 30316709 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 4194304, - "elapsed_compute": 5984147 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 419252397, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 0, - "predicate_cache_inner_records": 0, - "filesSize": 199358457, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 2213759, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 64, - "row_pushdown_eval_time": 2, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 0, - "page_index_eval_time": 2, - "bytes_scanned": 165673158, - "predicate_evaluation_errors": 0, - "metadata_load_time": 683165, - "output_rows": 4194304, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 91, - "metadataTime": 0, - "scan_efficiency_ratio_total": 199358457, - "row_groups_matched_statistics": 64, - "statistics_eval_time": 2, - "time_elapsed_opening": 847602, - "row_groups_pruned_statistics": 0, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 328324266, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - }, - "nested_unprunable": { - "seconds": [ - 0.5705090211704373, - 0.5631797285750508, - 0.5575059540569782, - 0.5655996361747384, - 0.58751414809376 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM shuffled WHERE s.inner.k >= 4128768", - "answer": [ - 32658560674, - 32812041019, - 32867632839, - 32498761177, - 32648273638, - 32881993944, - 32750626290, - 32782672280 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#1976L, sum#1977L, sum#1978L, sum#1979L, sum#1980L, sum#1981L, sum#1982L, sum#1983L], [Final], [sum(_extract_p0#1960L), sum(_extract_p1#1961L), sum(_extract_p2#1962L), sum(_extract_p3#1963L), sum(_extract_p4#1964L), sum(_extract_p5#1965L), sum(_extract_p6#1966L), sum(_extract_p7#1967L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2334]\n +- CometHashAggregate [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [Partial], [partial_sum(_extract_p0#1960L), partial_sum(_extract_p1#1961L), partial_sum(_extract_p2#1962L), partial_sum(_extract_p3#1963L), partial_sum(_extract_p4#1964L), partial_sum(_extract_p5#1965L), partial_sum(_extract_p6#1966L), partial_sum(_extract_p7#1967L)]\n +- CometProject [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [s#3.p0 AS _extract_p0#1960L, s#3.p1 AS _extract_p1#1961L, s#3.p2 AS _extract_p2#1962L, s#3.p3 AS _extract_p3#1963L, s#3.p4 AS _extract_p4#1964L, s#3.p5 AS _extract_p5#1965L, s#3.p6 AS _extract_p6#1966L, s#3.p7 AS _extract_p7#1967L]\n +- CometFilter [s#3], (isnotnull(s#3.inner.k) AND (s#3.inner.k >= 4128768))\n +- CometNativeScan parquet [s#3] Batched: true, DataFilters: [isnotnull(s#3.inner.k), (s#3.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/shuffled.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 2 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 16840 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 527, - "decode_time": 419237, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 136768, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 801402, - "numPartitions": 1, - "encode_time": 28240, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 527, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 529916 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 125728 - } - }, - { - "node": "CometFilter", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 22522085 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 482903791, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 0, - "predicate_cache_inner_records": 0, - "filesSize": 207941114, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 2613066, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 64, - "row_pushdown_eval_time": 2, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 0, - "page_index_eval_time": 2, - "bytes_scanned": 186772555, - "predicate_evaluation_errors": 0, - "metadata_load_time": 621005, - "output_rows": 4194304, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 81, - "metadataTime": 0, - "scan_efficiency_ratio_total": 207941114, - "row_groups_matched_statistics": 64, - "statistics_eval_time": 105420, - "time_elapsed_opening": 951250, - "row_groups_pruned_statistics": 0, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 394684340, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - } - } - }, - { - "label": "candidate-1", - "spark": "4.1.3", - "pruning_off": false, - "row_filter": false, - "warmup": 5, - "cases": { - "nested_selective": { - "seconds": [ - 0.18005445785820484, - 0.1846651053056121, - 0.2008777316659689, - 0.19776866026222706, - 0.1689277533441782 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE s.inner.k >= 4128768", - "answer": [ - 32741059719, - 32774396751, - 32772938793, - 32799973223, - 32905236128, - 32807159809, - 32718967833, - 32661111725 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#486L, sum#487L, sum#488L, sum#489L, sum#490L, sum#491L, sum#492L, sum#493L], [Final], [sum(_extract_p0#470L), sum(_extract_p1#471L), sum(_extract_p2#472L), sum(_extract_p3#473L), sum(_extract_p4#474L), sum(_extract_p5#475L), sum(_extract_p6#476L), sum(_extract_p7#477L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=594]\n +- CometHashAggregate [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [Partial], [partial_sum(_extract_p0#470L), partial_sum(_extract_p1#471L), partial_sum(_extract_p2#472L), partial_sum(_extract_p3#473L), partial_sum(_extract_p4#474L), partial_sum(_extract_p5#475L), partial_sum(_extract_p6#476L), partial_sum(_extract_p7#477L)]\n +- CometProject [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [s#1.p0 AS _extract_p0#470L, s#1.p1 AS _extract_p1#471L, s#1.p2 AS _extract_p2#472L, s#1.p3 AS _extract_p3#473L, s#1.p4 AS _extract_p4#474L, s#1.p5 AS _extract_p5#475L, s#1.p6 AS _extract_p6#476L, s#1.p7 AS _extract_p7#477L]\n +- CometFilter [s#1], (isnotnull(s#1.inner.k) AND (s#1.inner.k >= 4128768))\n +- CometNativeScan parquet [s#1] Batched: true, DataFilters: [isnotnull(s#1.inner.k), (s#1.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 5 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 25490 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 526, - "decode_time": 709473, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 127649, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 2567907, - "numPartitions": 1, - "encode_time": 35670, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 526, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 455267 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 64959 - } - }, - { - "node": "CometFilter", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 204199 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 7833416, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 0, - "predicate_cache_inner_records": 0, - "filesSize": 199358457, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 2729324, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 1, - "row_pushdown_eval_time": 2, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 0, - "page_index_eval_time": 2, - "bytes_scanned": 2851301, - "predicate_evaluation_errors": 0, - "metadata_load_time": 940631, - "output_rows": 65536, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 111, - "metadataTime": 0, - "scan_efficiency_ratio_total": 199358457, - "row_groups_matched_statistics": 1, - "statistics_eval_time": 122540, - "time_elapsed_opening": 1236698, - "row_groups_pruned_statistics": 63, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 7021741, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - }, - "top_level_selective": { - "seconds": [ - 0.1509197847917676, - 0.1415041647851467, - 0.15639991406351328, - 0.11335313320159912, - 0.12271884363144636 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE k >= 4128768", - "answer": [ - 32741059719, - 32774396751, - 32772938793, - 32799973223, - 32905236128, - 32807159809, - 32718967833, - 32661111725 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#986L, sum#987L, sum#988L, sum#989L, sum#990L, sum#991L, sum#992L, sum#993L], [Final], [sum(_extract_p0#968L), sum(_extract_p1#969L), sum(_extract_p2#970L), sum(_extract_p3#971L), sum(_extract_p4#972L), sum(_extract_p5#973L), sum(_extract_p6#974L), sum(_extract_p7#975L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1204]\n +- CometHashAggregate [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [Partial], [partial_sum(_extract_p0#968L), partial_sum(_extract_p1#969L), partial_sum(_extract_p2#970L), partial_sum(_extract_p3#971L), partial_sum(_extract_p4#972L), partial_sum(_extract_p5#973L), partial_sum(_extract_p6#974L), partial_sum(_extract_p7#975L)]\n +- CometProject [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [s#1.p0 AS _extract_p0#968L, s#1.p1 AS _extract_p1#969L, s#1.p2 AS _extract_p2#970L, s#1.p3 AS _extract_p3#971L, s#1.p4 AS _extract_p4#972L, s#1.p5 AS _extract_p5#973L, s#1.p6 AS _extract_p6#974L, s#1.p7 AS _extract_p7#975L]\n +- CometFilter [k#0L, s#1], (isnotnull(k#0L) AND (k#0L >= 4128768))\n +- CometNativeScan parquet [k#0L,s#1] Batched: true, DataFilters: [isnotnull(k#0L), (k#0L >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(k), GreaterThanOrEqual(k,4128768)], ReadSchema: struct>\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 2 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 16520 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 526, - "decode_time": 483245, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 131669, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 828622, - "numPartitions": 1, - "encode_time": 33180, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 526, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 35990879 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 4194304, - "elapsed_compute": 6089895 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 448517056, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 0, - "predicate_cache_inner_records": 0, - "filesSize": 199358457, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 2653374, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 64, - "row_pushdown_eval_time": 2, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 0, - "page_index_eval_time": 2, - "bytes_scanned": 165673158, - "predicate_evaluation_errors": 0, - "metadata_load_time": 836503, - "output_rows": 4194304, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 111, - "metadataTime": 0, - "scan_efficiency_ratio_total": 199358457, - "row_groups_matched_statistics": 64, - "statistics_eval_time": 2, - "time_elapsed_opening": 1017221, - "row_groups_pruned_statistics": 0, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 345028311, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - }, - "nested_unprunable": { - "seconds": [ - 0.5922109065577388, - 0.6122729973867536, - 0.6169724743813276, - 0.5767420660704374, - 0.5864941030740738 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM shuffled WHERE s.inner.k >= 4128768", - "answer": [ - 32658560674, - 32812041019, - 32867632839, - 32498761177, - 32648273638, - 32881993944, - 32750626290, - 32782672280 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#1976L, sum#1977L, sum#1978L, sum#1979L, sum#1980L, sum#1981L, sum#1982L, sum#1983L], [Final], [sum(_extract_p0#1960L), sum(_extract_p1#1961L), sum(_extract_p2#1962L), sum(_extract_p3#1963L), sum(_extract_p4#1964L), sum(_extract_p5#1965L), sum(_extract_p6#1966L), sum(_extract_p7#1967L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2334]\n +- CometHashAggregate [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [Partial], [partial_sum(_extract_p0#1960L), partial_sum(_extract_p1#1961L), partial_sum(_extract_p2#1962L), partial_sum(_extract_p3#1963L), partial_sum(_extract_p4#1964L), partial_sum(_extract_p5#1965L), partial_sum(_extract_p6#1966L), partial_sum(_extract_p7#1967L)]\n +- CometProject [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [s#3.p0 AS _extract_p0#1960L, s#3.p1 AS _extract_p1#1961L, s#3.p2 AS _extract_p2#1962L, s#3.p3 AS _extract_p3#1963L, s#3.p4 AS _extract_p4#1964L, s#3.p5 AS _extract_p5#1965L, s#3.p6 AS _extract_p6#1966L, s#3.p7 AS _extract_p7#1967L]\n +- CometFilter [s#3], (isnotnull(s#3.inner.k) AND (s#3.inner.k >= 4128768))\n +- CometNativeScan parquet [s#3] Batched: true, DataFilters: [isnotnull(s#3.inner.k), (s#3.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/shuffled.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 2 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 16040 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 527, - "decode_time": 388976, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 143020, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 764594, - "numPartitions": 1, - "encode_time": 35479, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 527, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 601804 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 120608 - } - }, - { - "node": "CometFilter", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 21920600 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 494768133, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 0, - "predicate_cache_inner_records": 0, - "filesSize": 207941114, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 3986671, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 64, - "row_pushdown_eval_time": 2, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 0, - "page_index_eval_time": 2, - "bytes_scanned": 186772555, - "predicate_evaluation_errors": 0, - "metadata_load_time": 745275, - "output_rows": 4194304, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 91, - "metadataTime": 0, - "scan_efficiency_ratio_total": 207941114, - "row_groups_matched_statistics": 64, - "statistics_eval_time": 100610, - "time_elapsed_opening": 1060151, - "row_groups_pruned_statistics": 0, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 406038990, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - } - } - }, - { - "label": "candidate-2", - "spark": "4.1.3", - "pruning_off": false, - "row_filter": false, - "warmup": 5, - "cases": { - "nested_selective": { - "seconds": [ - 0.19452941324561834, - 0.17460494581609964, - 0.18056540749967098, - 0.16720692533999681, - 0.16859497223049402 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE s.inner.k >= 4128768", - "answer": [ - 32741059719, - 32774396751, - 32772938793, - 32799973223, - 32905236128, - 32807159809, - 32718967833, - 32661111725 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#486L, sum#487L, sum#488L, sum#489L, sum#490L, sum#491L, sum#492L, sum#493L], [Final], [sum(_extract_p0#470L), sum(_extract_p1#471L), sum(_extract_p2#472L), sum(_extract_p3#473L), sum(_extract_p4#474L), sum(_extract_p5#475L), sum(_extract_p6#476L), sum(_extract_p7#477L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=594]\n +- CometHashAggregate [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [Partial], [partial_sum(_extract_p0#470L), partial_sum(_extract_p1#471L), partial_sum(_extract_p2#472L), partial_sum(_extract_p3#473L), partial_sum(_extract_p4#474L), partial_sum(_extract_p5#475L), partial_sum(_extract_p6#476L), partial_sum(_extract_p7#477L)]\n +- CometProject [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [s#1.p0 AS _extract_p0#470L, s#1.p1 AS _extract_p1#471L, s#1.p2 AS _extract_p2#472L, s#1.p3 AS _extract_p3#473L, s#1.p4 AS _extract_p4#474L, s#1.p5 AS _extract_p5#475L, s#1.p6 AS _extract_p6#476L, s#1.p7 AS _extract_p7#477L]\n +- CometFilter [s#1], (isnotnull(s#1.inner.k) AND (s#1.inner.k >= 4128768))\n +- CometNativeScan parquet [s#1] Batched: true, DataFilters: [isnotnull(s#1.inner.k), (s#1.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 3 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 16580 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 526, - "decode_time": 660194, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 141798, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 1243107, - "numPartitions": 1, - "encode_time": 35350, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 526, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 452257 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 58556 - } - }, - { - "node": "CometFilter", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 170330 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 7269680, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 0, - "predicate_cache_inner_records": 0, - "filesSize": 199358457, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 2442397, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 1, - "row_pushdown_eval_time": 2, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 0, - "page_index_eval_time": 2, - "bytes_scanned": 2851301, - "predicate_evaluation_errors": 0, - "metadata_load_time": 740214, - "output_rows": 65536, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 91, - "metadataTime": 0, - "scan_efficiency_ratio_total": 199358457, - "row_groups_matched_statistics": 1, - "statistics_eval_time": 94821, - "time_elapsed_opening": 983571, - "row_groups_pruned_statistics": 63, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 6501088, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - }, - "top_level_selective": { - "seconds": [ - 0.15506690181791782, - 0.12573741003870964, - 0.14402167592197657, - 0.1330686118453741, - 0.13747728988528252 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE k >= 4128768", - "answer": [ - 32741059719, - 32774396751, - 32772938793, - 32799973223, - 32905236128, - 32807159809, - 32718967833, - 32661111725 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#986L, sum#987L, sum#988L, sum#989L, sum#990L, sum#991L, sum#992L, sum#993L], [Final], [sum(_extract_p0#968L), sum(_extract_p1#969L), sum(_extract_p2#970L), sum(_extract_p3#971L), sum(_extract_p4#972L), sum(_extract_p5#973L), sum(_extract_p6#974L), sum(_extract_p7#975L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1204]\n +- CometHashAggregate [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [Partial], [partial_sum(_extract_p0#968L), partial_sum(_extract_p1#969L), partial_sum(_extract_p2#970L), partial_sum(_extract_p3#971L), partial_sum(_extract_p4#972L), partial_sum(_extract_p5#973L), partial_sum(_extract_p6#974L), partial_sum(_extract_p7#975L)]\n +- CometProject [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [s#1.p0 AS _extract_p0#968L, s#1.p1 AS _extract_p1#969L, s#1.p2 AS _extract_p2#970L, s#1.p3 AS _extract_p3#971L, s#1.p4 AS _extract_p4#972L, s#1.p5 AS _extract_p5#973L, s#1.p6 AS _extract_p6#974L, s#1.p7 AS _extract_p7#975L]\n +- CometFilter [k#0L, s#1], (isnotnull(k#0L) AND (k#0L >= 4128768))\n +- CometNativeScan parquet [k#0L,s#1] Batched: true, DataFilters: [isnotnull(k#0L), (k#0L >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(k), GreaterThanOrEqual(k,4128768)], ReadSchema: struct>\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 3 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 19280 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 526, - "decode_time": 502416, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 132858, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 849513, - "numPartitions": 1, - "encode_time": 28440, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 526, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 34173480 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 4194304, - "elapsed_compute": 6662563 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 457767984, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 0, - "predicate_cache_inner_records": 0, - "filesSize": 199358457, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 2357397, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 64, - "row_pushdown_eval_time": 2, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 0, - "page_index_eval_time": 2, - "bytes_scanned": 165673158, - "predicate_evaluation_errors": 0, - "metadata_load_time": 757584, - "output_rows": 4194304, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 111, - "metadataTime": 0, - "scan_efficiency_ratio_total": 199358457, - "row_groups_matched_statistics": 64, - "statistics_eval_time": 2, - "time_elapsed_opening": 916321, - "row_groups_pruned_statistics": 0, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 349320757, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - }, - "nested_unprunable": { - "seconds": [ - 0.5652983766049147, - 0.5627129822969437, - 0.5944593185558915, - 0.592568757943809, - 0.6130315512418747 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM shuffled WHERE s.inner.k >= 4128768", - "answer": [ - 32658560674, - 32812041019, - 32867632839, - 32498761177, - 32648273638, - 32881993944, - 32750626290, - 32782672280 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#1976L, sum#1977L, sum#1978L, sum#1979L, sum#1980L, sum#1981L, sum#1982L, sum#1983L], [Final], [sum(_extract_p0#1960L), sum(_extract_p1#1961L), sum(_extract_p2#1962L), sum(_extract_p3#1963L), sum(_extract_p4#1964L), sum(_extract_p5#1965L), sum(_extract_p6#1966L), sum(_extract_p7#1967L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2334]\n +- CometHashAggregate [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [Partial], [partial_sum(_extract_p0#1960L), partial_sum(_extract_p1#1961L), partial_sum(_extract_p2#1962L), partial_sum(_extract_p3#1963L), partial_sum(_extract_p4#1964L), partial_sum(_extract_p5#1965L), partial_sum(_extract_p6#1966L), partial_sum(_extract_p7#1967L)]\n +- CometProject [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [s#3.p0 AS _extract_p0#1960L, s#3.p1 AS _extract_p1#1961L, s#3.p2 AS _extract_p2#1962L, s#3.p3 AS _extract_p3#1963L, s#3.p4 AS _extract_p4#1964L, s#3.p5 AS _extract_p5#1965L, s#3.p6 AS _extract_p6#1966L, s#3.p7 AS _extract_p7#1967L]\n +- CometFilter [s#3], (isnotnull(s#3.inner.k) AND (s#3.inner.k >= 4128768))\n +- CometNativeScan parquet [s#3] Batched: true, DataFilters: [isnotnull(s#3.inner.k), (s#3.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/shuffled.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 2 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 19840 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 527, - "decode_time": 577975, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 134939, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 928573, - "numPartitions": 1, - "encode_time": 31879, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 527, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 651932 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 130430 - } - }, - { - "node": "CometFilter", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 21826400 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 500684622, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 0, - "predicate_cache_inner_records": 0, - "filesSize": 207941114, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 2513986, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 64, - "row_pushdown_eval_time": 2, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 0, - "page_index_eval_time": 2, - "bytes_scanned": 186772555, - "predicate_evaluation_errors": 0, - "metadata_load_time": 639414, - "output_rows": 4194304, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 111, - "metadataTime": 0, - "scan_efficiency_ratio_total": 207941114, - "row_groups_matched_statistics": 64, - "statistics_eval_time": 111380, - "time_elapsed_opening": 926681, - "row_groups_pruned_statistics": 0, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 406449858, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - } - } - }, - { - "label": "candidate-3", - "spark": "4.1.3", - "pruning_off": false, - "row_filter": false, - "warmup": 5, - "cases": { - "nested_selective": { - "seconds": [ - 0.2269756831228733, - 0.20965376868844032, - 0.22189291194081306, - 0.19881676323711872, - 0.17740720696747303 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE s.inner.k >= 4128768", - "answer": [ - 32741059719, - 32774396751, - 32772938793, - 32799973223, - 32905236128, - 32807159809, - 32718967833, - 32661111725 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#486L, sum#487L, sum#488L, sum#489L, sum#490L, sum#491L, sum#492L, sum#493L], [Final], [sum(_extract_p0#470L), sum(_extract_p1#471L), sum(_extract_p2#472L), sum(_extract_p3#473L), sum(_extract_p4#474L), sum(_extract_p5#475L), sum(_extract_p6#476L), sum(_extract_p7#477L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=594]\n +- CometHashAggregate [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [Partial], [partial_sum(_extract_p0#470L), partial_sum(_extract_p1#471L), partial_sum(_extract_p2#472L), partial_sum(_extract_p3#473L), partial_sum(_extract_p4#474L), partial_sum(_extract_p5#475L), partial_sum(_extract_p6#476L), partial_sum(_extract_p7#477L)]\n +- CometProject [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [s#1.p0 AS _extract_p0#470L, s#1.p1 AS _extract_p1#471L, s#1.p2 AS _extract_p2#472L, s#1.p3 AS _extract_p3#473L, s#1.p4 AS _extract_p4#474L, s#1.p5 AS _extract_p5#475L, s#1.p6 AS _extract_p6#476L, s#1.p7 AS _extract_p7#477L]\n +- CometFilter [s#1], (isnotnull(s#1.inner.k) AND (s#1.inner.k >= 4128768))\n +- CometNativeScan parquet [s#1] Batched: true, DataFilters: [isnotnull(s#1.inner.k), (s#1.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 4 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 16800 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 526, - "decode_time": 651635, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 130000, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 1220159, - "numPartitions": 1, - "encode_time": 30460, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 526, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 516816 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 69168 - } - }, - { - "node": "CometFilter", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 190860 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 7765226, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 0, - "predicate_cache_inner_records": 0, - "filesSize": 199358457, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 2598115, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 1, - "row_pushdown_eval_time": 2, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 0, - "page_index_eval_time": 2, - "bytes_scanned": 2851301, - "predicate_evaluation_errors": 0, - "metadata_load_time": 764723, - "output_rows": 65536, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 111, - "metadataTime": 0, - "scan_efficiency_ratio_total": 199358457, - "row_groups_matched_statistics": 1, - "statistics_eval_time": 120310, - "time_elapsed_opening": 1105879, - "row_groups_pruned_statistics": 63, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 6849684, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - }, - "top_level_selective": { - "seconds": [ - 0.14480215683579445, - 0.13203103002160788, - 0.1324948649853468, - 0.1340858293697238, - 0.12480908911675215 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE k >= 4128768", - "answer": [ - 32741059719, - 32774396751, - 32772938793, - 32799973223, - 32905236128, - 32807159809, - 32718967833, - 32661111725 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#986L, sum#987L, sum#988L, sum#989L, sum#990L, sum#991L, sum#992L, sum#993L], [Final], [sum(_extract_p0#968L), sum(_extract_p1#969L), sum(_extract_p2#970L), sum(_extract_p3#971L), sum(_extract_p4#972L), sum(_extract_p5#973L), sum(_extract_p6#974L), sum(_extract_p7#975L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1204]\n +- CometHashAggregate [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [Partial], [partial_sum(_extract_p0#968L), partial_sum(_extract_p1#969L), partial_sum(_extract_p2#970L), partial_sum(_extract_p3#971L), partial_sum(_extract_p4#972L), partial_sum(_extract_p5#973L), partial_sum(_extract_p6#974L), partial_sum(_extract_p7#975L)]\n +- CometProject [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [s#1.p0 AS _extract_p0#968L, s#1.p1 AS _extract_p1#969L, s#1.p2 AS _extract_p2#970L, s#1.p3 AS _extract_p3#971L, s#1.p4 AS _extract_p4#972L, s#1.p5 AS _extract_p5#973L, s#1.p6 AS _extract_p6#974L, s#1.p7 AS _extract_p7#975L]\n +- CometFilter [k#0L, s#1], (isnotnull(k#0L) AND (k#0L >= 4128768))\n +- CometNativeScan parquet [k#0L,s#1] Batched: true, DataFilters: [isnotnull(k#0L), (k#0L >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(k), GreaterThanOrEqual(k,4128768)], ReadSchema: struct>\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 2 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 17109 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 526, - "decode_time": 416336, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 114758, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 790602, - "numPartitions": 1, - "encode_time": 28780, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 526, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 33829637 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 4194304, - "elapsed_compute": 6278255 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 430781919, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 0, - "predicate_cache_inner_records": 0, - "filesSize": 199358457, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 2239719, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 64, - "row_pushdown_eval_time": 2, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 0, - "page_index_eval_time": 2, - "bytes_scanned": 165673158, - "predicate_evaluation_errors": 0, - "metadata_load_time": 667365, - "output_rows": 4194304, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 81, - "metadataTime": 0, - "scan_efficiency_ratio_total": 199358457, - "row_groups_matched_statistics": 64, - "statistics_eval_time": 2, - "time_elapsed_opening": 840452, - "row_groups_pruned_statistics": 0, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 335087732, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - }, - "nested_unprunable": { - "seconds": [ - 0.5824264092370868, - 0.5773687968030572, - 0.5656023295596242, - 0.5545061454176903, - 0.5747098913416266 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM shuffled WHERE s.inner.k >= 4128768", - "answer": [ - 32658560674, - 32812041019, - 32867632839, - 32498761177, - 32648273638, - 32881993944, - 32750626290, - 32782672280 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#1976L, sum#1977L, sum#1978L, sum#1979L, sum#1980L, sum#1981L, sum#1982L, sum#1983L], [Final], [sum(_extract_p0#1960L), sum(_extract_p1#1961L), sum(_extract_p2#1962L), sum(_extract_p3#1963L), sum(_extract_p4#1964L), sum(_extract_p5#1965L), sum(_extract_p6#1966L), sum(_extract_p7#1967L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2334]\n +- CometHashAggregate [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [Partial], [partial_sum(_extract_p0#1960L), partial_sum(_extract_p1#1961L), partial_sum(_extract_p2#1962L), partial_sum(_extract_p3#1963L), partial_sum(_extract_p4#1964L), partial_sum(_extract_p5#1965L), partial_sum(_extract_p6#1966L), partial_sum(_extract_p7#1967L)]\n +- CometProject [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [s#3.p0 AS _extract_p0#1960L, s#3.p1 AS _extract_p1#1961L, s#3.p2 AS _extract_p2#1962L, s#3.p3 AS _extract_p3#1963L, s#3.p4 AS _extract_p4#1964L, s#3.p5 AS _extract_p5#1965L, s#3.p6 AS _extract_p6#1966L, s#3.p7 AS _extract_p7#1967L]\n +- CometFilter [s#3], (isnotnull(s#3.inner.k) AND (s#3.inner.k >= 4128768))\n +- CometNativeScan parquet [s#3] Batched: true, DataFilters: [isnotnull(s#3.inner.k), (s#3.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/shuffled.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 2 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 17099 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 527, - "decode_time": 430956, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 133750, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 817622, - "numPartitions": 1, - "encode_time": 29140, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 527, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 759554 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 131507 - } - }, - { - "node": "CometFilter", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 22934138 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 482883949, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 0, - "predicate_cache_inner_records": 0, - "filesSize": 207941114, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 2619785, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 64, - "row_pushdown_eval_time": 2, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 0, - "page_index_eval_time": 2, - "bytes_scanned": 186772555, - "predicate_evaluation_errors": 0, - "metadata_load_time": 745495, - "output_rows": 4194304, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 81, - "metadataTime": 0, - "scan_efficiency_ratio_total": 207941114, - "row_groups_matched_statistics": 64, - "statistics_eval_time": 95980, - "time_elapsed_opening": 990912, - "row_groups_pruned_statistics": 0, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 398224205, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - } - } - }, - { - "label": "pruning-off-row-filter", - "spark": "4.1.3", - "pruning_off": true, - "row_filter": true, - "warmup": 5, - "cases": { - "nested_selective": { - "seconds": [ - 0.29983343090862036, - 0.31047218944877386, - 0.2997523508965969, - 0.28356191609054804, - 0.2667716462165117 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE s.inner.k >= 4128768", - "answer": [ - 32741059719, - 32774396751, - 32772938793, - 32799973223, - 32905236128, - 32807159809, - 32718967833, - 32661111725 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#486L, sum#487L, sum#488L, sum#489L, sum#490L, sum#491L, sum#492L, sum#493L], [Final], [sum(_extract_p0#470L), sum(_extract_p1#471L), sum(_extract_p2#472L), sum(_extract_p3#473L), sum(_extract_p4#474L), sum(_extract_p5#475L), sum(_extract_p6#476L), sum(_extract_p7#477L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=594]\n +- CometHashAggregate [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [Partial], [partial_sum(_extract_p0#470L), partial_sum(_extract_p1#471L), partial_sum(_extract_p2#472L), partial_sum(_extract_p3#473L), partial_sum(_extract_p4#474L), partial_sum(_extract_p5#475L), partial_sum(_extract_p6#476L), partial_sum(_extract_p7#477L)]\n +- CometProject [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [s#1.p0 AS _extract_p0#470L, s#1.p1 AS _extract_p1#471L, s#1.p2 AS _extract_p2#472L, s#1.p3 AS _extract_p3#473L, s#1.p4 AS _extract_p4#474L, s#1.p5 AS _extract_p5#475L, s#1.p6 AS _extract_p6#476L, s#1.p7 AS _extract_p7#477L]\n +- CometFilter [s#1], (isnotnull(s#1.inner.k) AND (s#1.inner.k >= 4128768))\n +- CometNativeScan parquet [s#1] Batched: true, DataFilters: [isnotnull(s#1.inner.k), (s#1.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 3 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 18750 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 526, - "decode_time": 712513, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 148899, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 1110219, - "numPartitions": 1, - "encode_time": 32699, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 526, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 561406 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 71000 - } - }, - { - "node": "CometFilter", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 179287 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 126723837, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 65536, - "predicate_cache_inner_records": 0, - "filesSize": 199358457, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 121052961, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 64, - "row_pushdown_eval_time": 6995880, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 4128768, - "page_index_eval_time": 2, - "bytes_scanned": 19396952, - "predicate_evaluation_errors": 0, - "metadata_load_time": 651616, - "output_rows": 65536, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 91, - "metadataTime": 0, - "scan_efficiency_ratio_total": 199358457, - "row_groups_matched_statistics": 64, - "statistics_eval_time": 2, - "time_elapsed_opening": 877952, - "row_groups_pruned_statistics": 0, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 117336764, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - }, - "top_level_selective": { - "seconds": [ - 0.19135639816522598, - 0.18866712413728237, - 0.19497105479240417, - 0.17380876746028662, - 0.17694623582065105 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE k >= 4128768", - "answer": [ - 32741059719, - 32774396751, - 32772938793, - 32799973223, - 32905236128, - 32807159809, - 32718967833, - 32661111725 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#986L, sum#987L, sum#988L, sum#989L, sum#990L, sum#991L, sum#992L, sum#993L], [Final], [sum(_extract_p0#968L), sum(_extract_p1#969L), sum(_extract_p2#970L), sum(_extract_p3#971L), sum(_extract_p4#972L), sum(_extract_p5#973L), sum(_extract_p6#974L), sum(_extract_p7#975L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1204]\n +- CometHashAggregate [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [Partial], [partial_sum(_extract_p0#968L), partial_sum(_extract_p1#969L), partial_sum(_extract_p2#970L), partial_sum(_extract_p3#971L), partial_sum(_extract_p4#972L), partial_sum(_extract_p5#973L), partial_sum(_extract_p6#974L), partial_sum(_extract_p7#975L)]\n +- CometProject [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [s#1.p0 AS _extract_p0#968L, s#1.p1 AS _extract_p1#969L, s#1.p2 AS _extract_p2#970L, s#1.p3 AS _extract_p3#971L, s#1.p4 AS _extract_p4#972L, s#1.p5 AS _extract_p5#973L, s#1.p6 AS _extract_p6#974L, s#1.p7 AS _extract_p7#975L]\n +- CometFilter [k#0L, s#1], (isnotnull(k#0L) AND (k#0L >= 4128768))\n +- CometNativeScan parquet [k#0L,s#1] Batched: true, DataFilters: [isnotnull(k#0L), (k#0L >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(k), GreaterThanOrEqual(k,4128768)], ReadSchema: struct>\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 2 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 15989 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 526, - "decode_time": 408797, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 135238, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 723212, - "numPartitions": 1, - "encode_time": 28930, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 526, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 33986400 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 4194304, - "elapsed_compute": 6452212 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 443018252, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 0, - "predicate_cache_inner_records": 0, - "filesSize": 199358457, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 2586726, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 64, - "row_pushdown_eval_time": 2, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 0, - "page_index_eval_time": 2, - "bytes_scanned": 165673158, - "predicate_evaluation_errors": 0, - "metadata_load_time": 817833, - "output_rows": 4194304, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 121, - "metadataTime": 0, - "scan_efficiency_ratio_total": 199358457, - "row_groups_matched_statistics": 64, - "statistics_eval_time": 2, - "time_elapsed_opening": 1043659, - "row_groups_pruned_statistics": 0, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 339187186, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - }, - "nested_unprunable": { - "seconds": [ - 0.802385420538485, - 0.7697837706655264, - 0.7511665793135762, - 0.743246354162693, - 0.7415530709549785 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM shuffled WHERE s.inner.k >= 4128768", - "answer": [ - 32658560674, - 32812041019, - 32867632839, - 32498761177, - 32648273638, - 32881993944, - 32750626290, - 32782672280 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#1976L, sum#1977L, sum#1978L, sum#1979L, sum#1980L, sum#1981L, sum#1982L, sum#1983L], [Final], [sum(_extract_p0#1960L), sum(_extract_p1#1961L), sum(_extract_p2#1962L), sum(_extract_p3#1963L), sum(_extract_p4#1964L), sum(_extract_p5#1965L), sum(_extract_p6#1966L), sum(_extract_p7#1967L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2334]\n +- CometHashAggregate [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [Partial], [partial_sum(_extract_p0#1960L), partial_sum(_extract_p1#1961L), partial_sum(_extract_p2#1962L), partial_sum(_extract_p3#1963L), partial_sum(_extract_p4#1964L), partial_sum(_extract_p5#1965L), partial_sum(_extract_p6#1966L), partial_sum(_extract_p7#1967L)]\n +- CometProject [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [s#3.p0 AS _extract_p0#1960L, s#3.p1 AS _extract_p1#1961L, s#3.p2 AS _extract_p2#1962L, s#3.p3 AS _extract_p3#1963L, s#3.p4 AS _extract_p4#1964L, s#3.p5 AS _extract_p5#1965L, s#3.p6 AS _extract_p6#1966L, s#3.p7 AS _extract_p7#1967L]\n +- CometFilter [s#3], (isnotnull(s#3.inner.k) AND (s#3.inner.k >= 4128768))\n +- CometNativeScan parquet [s#3] Batched: true, DataFilters: [isnotnull(s#3.inner.k), (s#3.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/shuffled.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 2 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 37750 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 527, - "decode_time": 516295, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 138420, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 779512, - "numPartitions": 1, - "encode_time": 33369, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 527, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 592795 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 109828 - } - }, - { - "node": "CometFilter", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 4746354 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 649387752, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 65536, - "predicate_cache_inner_records": 0, - "filesSize": 207941114, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 8882425, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 64, - "row_pushdown_eval_time": 7648663, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 4128768, - "page_index_eval_time": 2, - "bytes_scanned": 186772555, - "predicate_evaluation_errors": 0, - "metadata_load_time": 800924, - "output_rows": 65536, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 111, - "metadataTime": 0, - "scan_efficiency_ratio_total": 207941114, - "row_groups_matched_statistics": 64, - "statistics_eval_time": 2, - "time_elapsed_opening": 1081740, - "row_groups_pruned_statistics": 0, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 579288015, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - } - } - }, - { - "label": "pruning-off", - "spark": "4.1.3", - "pruning_off": true, - "row_filter": false, - "warmup": 5, - "cases": { - "nested_selective": { - "seconds": [ - 0.6383354254066944, - 0.6380214085802436, - 0.6571799078956246, - 0.6155594941228628, - 0.600575627759099 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE s.inner.k >= 4128768", - "answer": [ - 32741059719, - 32774396751, - 32772938793, - 32799973223, - 32905236128, - 32807159809, - 32718967833, - 32661111725 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#486L, sum#487L, sum#488L, sum#489L, sum#490L, sum#491L, sum#492L, sum#493L], [Final], [sum(_extract_p0#470L), sum(_extract_p1#471L), sum(_extract_p2#472L), sum(_extract_p3#473L), sum(_extract_p4#474L), sum(_extract_p5#475L), sum(_extract_p6#476L), sum(_extract_p7#477L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=594]\n +- CometHashAggregate [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [Partial], [partial_sum(_extract_p0#470L), partial_sum(_extract_p1#471L), partial_sum(_extract_p2#472L), partial_sum(_extract_p3#473L), partial_sum(_extract_p4#474L), partial_sum(_extract_p5#475L), partial_sum(_extract_p6#476L), partial_sum(_extract_p7#477L)]\n +- CometProject [_extract_p0#470L, _extract_p1#471L, _extract_p2#472L, _extract_p3#473L, _extract_p4#474L, _extract_p5#475L, _extract_p6#476L, _extract_p7#477L], [s#1.p0 AS _extract_p0#470L, s#1.p1 AS _extract_p1#471L, s#1.p2 AS _extract_p2#472L, s#1.p3 AS _extract_p3#473L, s#1.p4 AS _extract_p4#474L, s#1.p5 AS _extract_p5#475L, s#1.p6 AS _extract_p6#476L, s#1.p7 AS _extract_p7#477L]\n +- CometFilter [s#1], (isnotnull(s#1.inner.k) AND (s#1.inner.k >= 4128768))\n +- CometNativeScan parquet [s#1] Batched: true, DataFilters: [isnotnull(s#1.inner.k), (s#1.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 4 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 22330 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 526, - "decode_time": 708473, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 126429, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 1266088, - "numPartitions": 1, - "encode_time": 30110, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 526, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 528445 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 75548 - } - }, - { - "node": "CometFilter", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 13918695 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 454680822, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 0, - "predicate_cache_inner_records": 0, - "filesSize": 199358457, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 2553836, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 64, - "row_pushdown_eval_time": 2, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 0, - "page_index_eval_time": 2, - "bytes_scanned": 182481422, - "predicate_evaluation_errors": 0, - "metadata_load_time": 937502, - "output_rows": 4194304, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 111, - "metadataTime": 0, - "scan_efficiency_ratio_total": 199358457, - "row_groups_matched_statistics": 64, - "statistics_eval_time": 2, - "time_elapsed_opening": 1107139, - "row_groups_pruned_statistics": 0, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 378507122, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - }, - "top_level_selective": { - "seconds": [ - 0.6123585933819413, - 0.610489190556109, - 0.5980431707575917, - 0.6052072821184993, - 0.5822479920461774 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM sorted WHERE k >= 4128768", - "answer": [ - 32741059719, - 32774396751, - 32772938793, - 32799973223, - 32905236128, - 32807159809, - 32718967833, - 32661111725 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#986L, sum#987L, sum#988L, sum#989L, sum#990L, sum#991L, sum#992L, sum#993L], [Final], [sum(_extract_p0#968L), sum(_extract_p1#969L), sum(_extract_p2#970L), sum(_extract_p3#971L), sum(_extract_p4#972L), sum(_extract_p5#973L), sum(_extract_p6#974L), sum(_extract_p7#975L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1204]\n +- CometHashAggregate [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [Partial], [partial_sum(_extract_p0#968L), partial_sum(_extract_p1#969L), partial_sum(_extract_p2#970L), partial_sum(_extract_p3#971L), partial_sum(_extract_p4#972L), partial_sum(_extract_p5#973L), partial_sum(_extract_p6#974L), partial_sum(_extract_p7#975L)]\n +- CometProject [_extract_p0#968L, _extract_p1#969L, _extract_p2#970L, _extract_p3#971L, _extract_p4#972L, _extract_p5#973L, _extract_p6#974L, _extract_p7#975L], [s#1.p0 AS _extract_p0#968L, s#1.p1 AS _extract_p1#969L, s#1.p2 AS _extract_p2#970L, s#1.p3 AS _extract_p3#971L, s#1.p4 AS _extract_p4#972L, s#1.p5 AS _extract_p5#973L, s#1.p6 AS _extract_p6#974L, s#1.p7 AS _extract_p7#975L]\n +- CometFilter [k#0L, s#1], (isnotnull(k#0L) AND (k#0L >= 4128768))\n +- CometNativeScan parquet [k#0L,s#1] Batched: true, DataFilters: [isnotnull(k#0L), (k#0L >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/sorted.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(k), GreaterThanOrEqual(k,4128768)], ReadSchema: struct>\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 3 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 26110 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 526, - "decode_time": 487555, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 139549, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 942530, - "numPartitions": 1, - "encode_time": 29679, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 526, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 34212055 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 4194304, - "elapsed_compute": 6810633 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 448676357, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 0, - "predicate_cache_inner_records": 0, - "filesSize": 199358457, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 2319278, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 64, - "row_pushdown_eval_time": 2, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 0, - "page_index_eval_time": 2, - "bytes_scanned": 165673158, - "predicate_evaluation_errors": 0, - "metadata_load_time": 650476, - "output_rows": 4194304, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 91, - "metadataTime": 0, - "scan_efficiency_ratio_total": 199358457, - "row_groups_matched_statistics": 64, - "statistics_eval_time": 2, - "time_elapsed_opening": 777864, - "row_groups_pruned_statistics": 0, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 344153367, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - }, - "nested_unprunable": { - "seconds": [ - 0.5745589630678296, - 0.5518598509952426, - 0.551765551790595, - 0.5702894749119878, - 0.5601397510617971 - ], - "query": "SELECT sum(s.p0), sum(s.p1), sum(s.p2), sum(s.p3), sum(s.p4), sum(s.p5), sum(s.p6), sum(s.p7) FROM shuffled WHERE s.inner.k >= 4128768", - "answer": [ - 32658560674, - 32812041019, - 32867632839, - 32498761177, - 32648273638, - 32881993944, - 32750626290, - 32782672280 - ], - "plan": "*(1) CometColumnarToRow\n+- CometHashAggregate [sum#1976L, sum#1977L, sum#1978L, sum#1979L, sum#1980L, sum#1981L, sum#1982L, sum#1983L], [Final], [sum(_extract_p0#1960L), sum(_extract_p1#1961L), sum(_extract_p2#1962L), sum(_extract_p3#1963L), sum(_extract_p4#1964L), sum(_extract_p5#1965L), sum(_extract_p6#1966L), sum(_extract_p7#1967L)]\n +- CometExchange SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2334]\n +- CometHashAggregate [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [Partial], [partial_sum(_extract_p0#1960L), partial_sum(_extract_p1#1961L), partial_sum(_extract_p2#1962L), partial_sum(_extract_p3#1963L), partial_sum(_extract_p4#1964L), partial_sum(_extract_p5#1965L), partial_sum(_extract_p6#1966L), partial_sum(_extract_p7#1967L)]\n +- CometProject [_extract_p0#1960L, _extract_p1#1961L, _extract_p2#1962L, _extract_p3#1963L, _extract_p4#1964L, _extract_p5#1965L, _extract_p6#1966L, _extract_p7#1967L], [s#3.p0 AS _extract_p0#1960L, s#3.p1 AS _extract_p1#1961L, s#3.p2 AS _extract_p2#1962L, s#3.p3 AS _extract_p3#1963L, s#3.p4 AS _extract_p4#1964L, s#3.p5 AS _extract_p5#1965L, s#3.p6 AS _extract_p6#1966L, s#3.p7 AS _extract_p7#1967L]\n +- CometFilter [s#3], (isnotnull(s#3.inner.k) AND (s#3.inner.k >= 4128768))\n +- CometNativeScan parquet [s#3] Batched: true, DataFilters: [isnotnull(s#3.inner.k), (s#3.inner.k >= 4128768)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/ubuntu/oss/comet-nested-data/shuffled.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(s.inner.k), GreaterThanOrEqual(s.inner.k,4128768)], ReadSchema: struct,p0:bigint,p1:bigint,p2:bigint,p3:bigint,p4:bigint,p5:bigin...\n", - "metrics": [ - { - "node": "WholeStageCodegen (1)", - "metrics": { - "pipelineTime": 2 - } - }, - { - "node": "CometColumnarToRow", - "metrics": { - "numOutputRows": 1, - "numInputBatches": 1 - } - }, - { - "node": "InputAdapter", - "metrics": {} - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 19599 - } - }, - { - "node": "CometExchange", - "metrics": { - "shuffleRecordsWritten": 1, - "spill_count": 0, - "remoteMergedBlocksFetched": 0, - "recordsRead": 1, - "localBytesRead": 527, - "decode_time": 436956, - "remoteBlocksFetched": 0, - "localMergedBytesRead": 0, - "localMergedChunksFetched": 0, - "shuffleWriteTime": 114760, - "remoteMergedBytesRead": 0, - "localMergedBlocksFetched": 0, - "corruptMergedBlockChunks": 0, - "input_batches": 1, - "remoteMergedReqsDuration": 0, - "fetchWaitTime": 0, - "remoteBytesRead": 0, - "mergedFetchFallbackCount": 0, - "repart_time": 1, - "memory_spilled_bytes": 0, - "localBlocksFetched": 1, - "remoteMergedChunksFetched": 0, - "spilled_bytes": 0, - "dataSize": 832, - "elapsed_compute": 736584, - "numPartitions": 1, - "encode_time": 26600, - "remoteReqsDuration": 0, - "remoteBytesReadToDisk": 0, - "shuffleBytesWritten": 527, - "interleave_time": 1 - } - }, - { - "node": "CometHashAggregate", - "metrics": { - "output_rows": 1, - "elapsed_compute": 548733 - } - }, - { - "node": "CometProject", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 113850 - } - }, - { - "node": "CometFilter", - "metrics": { - "output_rows": 65536, - "elapsed_compute": 19650433 - } - }, - { - "node": "CometNativeScan parquet ", - "metrics": { - "time_elapsed_scanning_total": 457130777, - "limit_matched_row_groups": 0, - "page_index_rows_pruned": 0, - "pushdown_rows_matched": 0, - "predicate_cache_inner_records": 0, - "filesSize": 207941114, - "files_ranges_pruned_statistics": 0, - "time_elapsed_scanning_until_data": 2619095, - "page_index_pages_matched": 0, - "row_groups_matched_bloom_filter": 64, - "row_pushdown_eval_time": 2, - "page_index_rows_matched": 0, - "row_groups_pruned_bloom_filter": 0, - "limit_pruned_row_groups": 0, - "numFiles": 1, - "pushdown_rows_pruned": 0, - "page_index_eval_time": 2, - "bytes_scanned": 186772555, - "predicate_evaluation_errors": 0, - "metadata_load_time": 600915, - "output_rows": 4194304, - "predicate_cache_records": 0, - "bloom_filter_eval_time": 111, - "metadataTime": 0, - "scan_efficiency_ratio_total": 207941114, - "row_groups_matched_statistics": 64, - "statistics_eval_time": 2, - "time_elapsed_opening": 846571, - "row_groups_pruned_statistics": 0, - "files_ranges_matched_statistics": 1, - "time_elapsed_processing": 377532689, - "file_scan_errors": 0, - "num_predicate_creation_errors": 0, - "file_open_errors": 0, - "page_index_pages_pruned": 0 - } - } - ] - } - } - } - ] -} \ No newline at end of file