From bec5faa80e2ec5f2966288569bdd59ce996f2a84 Mon Sep 17 00:00:00 2001 From: Miles Cole Date: Wed, 7 Oct 2026 13:55:02 -0600 Subject: [PATCH 1/2] improve docs and simplify databricks constructors --- README.md | 236 ++---------------- docs/engines/README.md | 41 +++ docs/engines/daft/README.md | 36 +++ docs/engines/databricks-spark/README.md | 68 +++++ .../databricks-sql-warehouse/README.md | 60 +++++ docs/engines/duckdb/README.md | 37 +++ docs/engines/fabric-data-warehouse/README.md | 52 ++++ docs/engines/fabric-spark/README.md | 54 ++++ docs/engines/hdi-spark/README.md | 39 +++ docs/engines/polars/README.md | 37 +++ docs/engines/sail/README.md | 41 +++ docs/engines/spark/README.md | 58 +++++ docs/engines/synapse-spark/README.md | 45 ++++ pyproject.toml | 1 - scripts/refresh_databricks_azure_dbu_map.py | 4 +- src/lakebench/engines/base.py | 13 +- src/lakebench/engines/daft.py | 19 +- src/lakebench/engines/databricks_spark.py | 59 +++-- .../engines/databricks_sql_warehouse.py | 36 ++- src/lakebench/engines/delta_rs.py | 10 +- src/lakebench/engines/duckdb.py | 21 +- .../engines/fabric_data_warehouse.py | 16 +- src/lakebench/engines/fabric_spark.py | 29 ++- src/lakebench/engines/hdi_spark.py | 23 +- src/lakebench/engines/polars.py | 21 +- src/lakebench/engines/resources/__init__.py | 1 + .../databricks_azure_dbu_map.py | 2 +- .../{ => resources}/databricks_pricing.py | 23 +- src/lakebench/engines/sail.py | 23 +- src/lakebench/engines/spark.py | 35 ++- src/lakebench/engines/synapse_spark.py | 25 +- tests/test_databricks_engines.py | 27 +- tests/test_engine.py | 46 ++++ uv.lock | 9 +- 34 files changed, 919 insertions(+), 328 deletions(-) create mode 100644 docs/engines/README.md create mode 100644 docs/engines/daft/README.md create mode 100644 docs/engines/databricks-spark/README.md create mode 100644 docs/engines/databricks-sql-warehouse/README.md create mode 100644 docs/engines/duckdb/README.md create mode 100644 docs/engines/fabric-data-warehouse/README.md create mode 100644 docs/engines/fabric-spark/README.md create mode 100644 docs/engines/hdi-spark/README.md create mode 100644 docs/engines/polars/README.md create mode 100644 docs/engines/sail/README.md create mode 100644 docs/engines/spark/README.md create mode 100644 docs/engines/synapse-spark/README.md create mode 100644 src/lakebench/engines/resources/__init__.py rename src/lakebench/engines/{ => resources}/databricks_azure_dbu_map.py (99%) rename src/lakebench/engines/{ => resources}/databricks_pricing.py (91%) diff --git a/README.md b/README.md index e501d6c..163a39b 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ LakeBench exists to bring clarity, trust, accessibility, and relevance to engine ## ✅ Why LakeBench? -- **Multi-Engine**: Benchmark Spark, DuckDB, Polars, Daft, Sail and others, side-by-side +- **Multi-Engine**: Benchmark various lakehouse compute engines side-by-side - **Lifecycle Coverage**: Ingest, transform, maintain, and query—just like real workloads - **Diverse Workloads**: Test performance across varied data shapes and operations - **Consistent Execution**: One framework, many engines @@ -65,41 +65,27 @@ LakeBench supports multiple lakehouse compute engines. Each benchmark scenario d | Engine | ELTBench | TPC-DS | TPC-H | ClickBench | |-----------------|:--------:|:------:|:-------:|:----------:| -| Spark (Generic) | ✅ | ✅ | ✅ | ✅ | -| Fabric Spark | ✅ | ✅ | ✅ | ✅ | -| Synapse Spark | ✅ | ✅ | ✅ | ✅ | -| HDInsight Spark | ✅ | ✅ | ✅ | ✅ | -| Databricks Spark (AWS/Azure/GCP) | ✅ | ✅ | ✅ | ✅ | -| Fabric Data Warehouse| ✅ | ✅ | ✅ | ✅ | -| Databricks SQL Warehouse (AWS/Azure/GCP) | ✅ | ✅ | ✅ | ✅ | -| DuckDB | ✅ | ✅ | ✅ | ✅ | -| Polars | ✅ | ⚠️ | ⚠️ | ⚠️ | -| Daft | ❌ | ❌ | ❌ | ⚠️ | -| Sail | ✅ | ✅ | ✅ | ✅ | +| [Spark (Generic)](docs/engines/spark/README.md) | ✅ | ✅ | ✅ | ✅ | +| [Fabric Spark](docs/engines/fabric-spark/README.md) | ✅ | ✅ | ✅ | ✅ | +| [Synapse Spark](docs/engines/synapse-spark/README.md) | ✅ | ✅ | ✅ | ✅ | +| [HDInsight Spark](docs/engines/hdi-spark/README.md) | ✅ | ✅ | ✅ | ✅ | +| [Databricks Spark](docs/engines/databricks-spark/README.md) | ✅ | ✅ | ✅ | ✅ | +| [Fabric Data Warehouse](docs/engines/fabric-data-warehouse/README.md) | ✅ | ✅ | ✅ | ✅ | +| [Databricks SQL Warehouse](docs/engines/databricks-sql-warehouse/README.md) | ✅ | ✅ | ✅ | ✅ | +| [DuckDB](docs/engines/duckdb/README.md) | ✅ | ✅ | ✅ | ✅ | +| [Polars](docs/engines/polars/README.md) | ✅ | ⚠️ | ⚠️ | ⚠️ | +| [Daft](docs/engines/daft/README.md) | ❌ | ❌ | ❌ | ⚠️ | +| [Sail](docs/engines/sail/README.md) | ✅ | ✅ | ✅ | ✅ | > **Legend:** > > - ✅ = Supported -> - ⚠️ = Some queries fail due to syntax issues (i.e. Polars doesn't support SQL non-equi joins, Daft is missing a lot of standard SQL contructs, i.e. DATE_ADD, CROSS JOIN, Subqueries, non-equi joins, CASE with operand, etc.). +> - ⚠️ = Partial support; see the linked engine page for current limitations. > - ❌ = Not supported > - 🔜 = Coming Soon > - (Blank) = Not currently supported -> **Daft is unsupported as of LakeBench 2.0.0.** Daft cannot read Parquet whose embedded Arrow schema uses `Utf8View`, and the bundled TPC generators emit that type, so every TPC-H, TPC-DS, and ELTBench table fails to load with `DaftError::ValueError Unsupported Arrow DataType: Utf8View`. The generators deliberately keep the type rather than rewriting output for one engine. The engine, its `daft` extra, and its integration tests all remain in the repository so support can be restored as soon as Daft handles `Utf8View` — only ClickBench, which loads from a plain Parquet sample, still runs. See [reports/coverage/daft.md](reports/coverage/daft.md). - -For detailed pass rates and per-query failure analysis, see the [coverage reports](reports/coverage/). - -## 📊 Engine Coverage Reports - -Per-engine coverage reports are auto-generated by the integration test suite and show pass rates with individual query failure details. To refresh: run the integration tests for your engine of choice (see [`tests/integration/README.md`](tests/integration/README.md)). - -| Engine | Report | -|--------|--------| -| DuckDB | [reports/coverage/duckdb.md](reports/coverage/duckdb.md) | -| Polars | [reports/coverage/polars.md](reports/coverage/polars.md) | -| Daft | [reports/coverage/daft.md](reports/coverage/daft.md) (unsupported as of 2.0.0) | -| Spark | [reports/coverage/spark.md](reports/coverage/spark.md) | -| Sail | [reports/coverage/sail.md](reports/coverage/sail.md) | +Installation, configuration, authentication, pricing, storage support, limitations, and coverage links live in the [engine documentation](docs/engines/README.md). ## Where Can I Run LakeBench? Multiple modalities doesn't end at just benchmarks and engines, LakeBench also supports different runtimes and storage backends: @@ -130,7 +116,7 @@ LakeBench is designed to be _extensible_, both for additional engines and benchm - You can register **new engines** without modifying core benchmark logic. - You can add **new benchmarks** that reuse existing engines and shared engine methods. -- LakeBench extension libraries can be created to extend core LakeBench capabilities with additional custom benchmarks and engines (i.e. `MyCustomSynapseSpark(Spark)`, `MyOrgsELT(BaseBenchmark)`). +- LakeBench extension libraries can extend core capabilities with custom benchmarks and engines. New engines can be added via subclassing an existing engine class. Existing benchmarks can then register support for additional engines via the below: @@ -174,22 +160,13 @@ class MyCustomEngine(BaseEngine): ## 📦 Installation -Install from PyPi: +Install the core package from PyPI: ```bash -pip install lakebench[duckdb,polars,fabric_data_warehouse,databricks_spark,databricks_sql_warehouse,sail,sparkmeasure] -``` - -Engines import their heavy dependencies lazily, so a forgotten extra used to surface partway into a run as an error that named neither the engine nor the package to install. Every engine now checks its dependencies when it is constructed and reports all of the missing ones at once: - -```python ->>> from lakebench.engines import DuckDB ->>> DuckDB(schema_or_working_directory_uri="file:///tmp/lakebench") -lakebench.engines.base.MissingDependenciesError: DuckDB requires `duckdb`, which is not -installed. Install with `pip install lakebench[duckdb]`. +pip install lakebench ``` -`MissingDependenciesError` subclasses `ImportError`, so existing `except ImportError` handling keeps working. Availability is resolved through the import system rather than through installed distribution names, so modules supplied by a managed runtime — such as `pyspark` on Fabric, Synapse, or HDInsight — count as present and do not trigger a false failure. `FabricDataWarehouse` additionally checks for the Microsoft ODBC Driver 18 for SQL Server, which pip cannot install. +Select the optional dependency group and any non-Python prerequisites from the [engine documentation](docs/engines/README.md). Engines validate their required modules when constructed and raise `MissingDependenciesError` with an installation hint when dependencies are unavailable. ## Example Usage To run any LakeBench benchmark, first do a one time generation of the data required for the benchmark and scale of interest. LakeBench provides datagen classes to quickly generate parquet datasets required by the benchmarks. @@ -245,7 +222,6 @@ TPC-H and TPC-DS can be generated and loaded in the TPC tools' native pipe-delim ```python from lakebench.datagen import TPCHDataGenerator from lakebench.benchmarks import TPCH -from lakebench.engines import Polars TPCHDataGenerator( scale_factor=1, @@ -254,7 +230,7 @@ TPCHDataGenerator( ).run() benchmark = TPCH( - engine=Polars(schema_or_working_directory_uri='...'), + engine=engine, # Construct an engine using docs/engines/ scenario_name='native-load', scale_factor=1, input_folder_uri='/lakehouse/default/Files/tpch_sf1_native', @@ -270,7 +246,7 @@ _Notes:_ - Every generated line ends with a trailing delimiter. LakeBench reads one extra trailing column and drops it, so no reader needs a lenient mode. - Empty fields are read as `NULL`, and quoting is disabled so `"` is treated as ordinary data. - Parquet-only options (`target_row_group_size_mb`, `compression`, and `compression_factor`) are rejected with `output_format="native"`. Automatic part counts use per-table native-to-uncompressed-Parquet size ratios measured at SF1. -- Supported on the DuckDB, Polars, Daft, Sail, and Spark engines. Engines with a benchmark-specific Parquet loader (such as Fabric Data Warehouse) reject `input_format="native"` rather than silently loading Parquet. +- Native-input support varies by engine; see the selected [engine page](docs/engines/README.md). - `output_format="native"` requires `backend="rust"`. With the currently bundled generator, `tpcgen-cli tpcds dat` has no thread option, so TPC-DS native generation runs single-threaded regardless of the available core count and rejects `num_threads`. Part count controls output splitting but does not make the current TPC-DS native run parallel. `tpcgen-cli tpch tbl` does accept `--num-threads`, so TPC-H native honours it exactly like Parquet (defaulting to all available CPU cores). #### Naming the Input Location @@ -308,171 +284,11 @@ The `analyze` option supports: - `"full"`: Ask the engine to generate statistics for every table column. - `"selective"`: Generate statistics only for the benchmark-maintained columns used by the workload. -For backward compatibility, `analyze=True` is equivalent to `"full"` and `analyze=False` is equivalent to `"none"`. - -Fabric Spark separately enables Delta extended statistics during writes by default. Set `collect_stats_on_write=False` only when isolating explicit analyze costs: - -```python -engine = FabricSpark( - lakehouse_name="lakehouse", - lakehouse_schema_name="schema", - collect_stats_on_write=False, -) -``` - -### Fabric Spark -```python -from lakebench.engines import FabricSpark -from lakebench.benchmarks import ELTBench - -engine = FabricSpark( - lakehouse_workspace_name="workspace", - lakehouse_name="lakehouse", - lakehouse_schema_name="schema", - spark_measure_telemetry=True -) - -benchmark = ELTBench( - engine=engine, - scenario_name="sf10", - mode="light", - input_folder_uri="abfss://...", - save_results=True, - result_table_uri="abfss://..." -) - -benchmark.run() -``` - -> _Note: The `spark_measure_telemetry` flag can be enabled to capture stage metrics in the results. The `sparkmeasure` install option must be used when `spark_measure_telemetry` is enabled (`%pip install lakebench[sparkmeasure]`). Additionally, the Spark-Measure JAR must be installed from Maven: https://mvnrepository.com/artifact/ch.cern.sparkmeasure/spark-measure_2.13/0.24_ - -### Databricks Spark -```python -from lakebench.benchmarks import TPCDS -from lakebench.engines import DatabricksSpark - -engine = DatabricksSpark( - catalog_name="main", - schema_name="lakebench", - schema_uri="s3://bucket/lakebench", # abfss://... on Azure or gs://... on GCP - cost_per_hour=12.00, # Or cost_per_vcore_hour; optional manual estimate on AWS and GCP - workload_type="jobs-compute", # Use "all-purpose-compute" for interactive clusters -) - -benchmark = TPCDS( - engine=engine, - scenario_name="sf1000", - scale_factor=1000, - input_folder_uri="s3://bucket/tpcds/sf1000", -) -benchmark.run() -``` - -Install Spark support with `pip install lakebench[databricks_spark]`. The engine detects AWS, Azure, or -GCP from Databricks runtime metadata and writes `cloud_provider`, `compute_region`, workload, and pricing -components to `engine_properties`. For fixed-size Azure clusters, automatic estimates combine the -published workload-and-VM-specific DBU consumption mapping with regional DBU and VM rates from the Azure -Retail Prices API. The estimate uses public pay-as-you-go list prices and does not include private -discounts, commitments, or negotiated pricing. - -On AWS and GCP, provide either an all-inclusive manual `cost_per_hour` or `cost_per_vcore_hour` to report -estimated job cost; otherwise LakeBench emits a warning and leaves the estimate unset. LakeBench does not -call AWS or GCP cloud pricing catalogs. Manual rates should include both Databricks DBU and underlying -cloud infrastructure costs. The two manual options are mutually exclusive. Refresh the generated Azure -DBU mapping from the public Azure Databricks pricing page with: - -```bash -uv run python scripts/refresh_databricks_azure_dbu_map.py -``` +For backward compatibility, `analyze=True` is equivalent to `"full"` and `analyze=False` is equivalent to `"none"`. Engine-specific statistics behavior is documented on each [engine page](docs/engines/README.md). -### Databricks SQL Warehouse -```python -import os - -from lakebench.benchmarks import TPCDS -from lakebench.engines import DatabricksSQLWarehouse - -engine = DatabricksSQLWarehouse( - server_hostname="dbc-....cloud.databricks.com", - warehouse_name="Serverless Starter Warehouse", - catalog_name="main", - schema_name="lakebench", - access_token=os.environ["DATABRICKS_TOKEN"], - cost_per_hour=12.00, # Optional fallback; serverless can use list_prices on any cloud -) +## Configure an engine -benchmark = TPCDS( - engine=engine, - scenario_name="sf1000", - scale_factor=1000, - input_folder_uri="s3://bucket/tpcds/sf1000", -) -benchmark.run() -``` - -Install SQL Warehouse support with `pip install lakebench[databricks_sql_warehouse]`. LakeBench always -detects the SQL Warehouse cloud and region from `current_metastore()`. On AWS, Azure, and GCP, -serverless warehouse estimates use the published warehouse-size DBU consumption and the current -cloud-specific USD list price from `system.billing.list_prices`; the serverless DBU price includes -infrastructure. -LakeBench first resolves the warehouse's exact billed SKU from recent `system.billing.usage`. When no -usage exists yet, AWS falls back to the regional serverless SQL SKU; pricing tiers are accepted only -when they resolve to the same rate. On Azure, unavailable or inaccessible billing tables fall back to -the exact regional `Premium Serverless SQL DBU` meter from the public Azure Retail Prices API. On AWS -and GCP, if the billing tables are unavailable or the matching SKUs do not yield one unambiguous active -rate, LakeBench logs a warning and leaves the estimate unset. In that case, provide an all-inclusive -manual `cost_per_hour`. Classic and pro warehouses also use manual `cost_per_hour`; without it, -LakeBench runs the benchmark but does not report estimated cost. SQL Warehouse intentionally does not -accept a per-vCore rate. The access token is used for both the Warehouse REST API and SQL connector and -is never added to result metadata. - -### Fabric Data Warehouse -```python -from lakebench.engines import FabricDataWarehouse -from lakebench.benchmarks import TPCDS - -engine = FabricDataWarehouse( - warehouse_name="warehouse", - warehouse_server="xxxxx.datawarehouse.fabric.microsoft.com", - schema_name="dbo", -) - -benchmark = TPCDS( - engine=engine, - scenario_name="sf1000", - scale_factor=1000, - input_folder_uri="abfss://...", - save_results=True, - result_table_uri="abfss://..." -) - -benchmark.run() -``` - -> _Note: use Python 3.10+ and install with `pip install lakebench[fabric_data_warehouse]`. Make sure Microsoft ODBC Driver 18 for SQL Server is available on the host. The engine authenticates with the notebook identity's Fabric token, so it must run inside a Fabric notebook._ -> -> Delta (`deltalake==1.5.1`) and PyArrow are used only for result-log appends. The extra relies on the core `pyarrow>=14.0.0` floor rather than forcing an upgrade of Fabric Runtime 1.3's Arrow installation, and can share Daft's Delta 1.5.x dependency. SQL reads require pandas 1.4+ with SQLAlchemy 2; pandas 2 is not required. - -### Polars -```python -from lakebench.engines import Polars -from lakebench.benchmarks import ELTBench - -engine = Polars( - schema_or_working_directory_uri = 'abfss://...' -) - -benchmark = ELTBench( - engine=engine, - scenario_name="sf10", - mode="light", - input_folder_uri="abfss://...", - save_results=True, - result_table_uri="abfss://..." -) - -benchmark.run() -``` +Construct the engine appropriate for your runtime, storage, and pricing requirements, then pass it to any supported benchmark. See the [engine documentation index](docs/engines/README.md) for installation and configuration. --- ## Managing Queries Over Various Dialects @@ -509,9 +325,7 @@ Static TPC query sets currently cover SF1000 and SF10000. Other data scales log SQLGlot is pinned to **30.18.0 on Python 3.9+**. Python 3.8 retains **26.30.0** because newer SQLGlot releases require Python 3.9+. The AST adapters support both versions, including FROM/WITH argument names, DROP VIEW target lists, and GROUPING function nodes. -`tests/test_tpc_sqlglot_compatibility.py` checks reviewed output fingerprints in `tests/fixtures/tpc_query_rendering.json` for all **500** TPC-H/TPC-DS renderings (both static scales, Spark and Fabric, on both pinned SQLGlot versions). Each dialect is rendered through the engine that emits it, so engine-registered rules are part of what the fingerprints pin. Review actual SQL differences before refreshing these fingerprints; do not regenerate them just to clear a failure. The 26.30.0-to-30.18.0 comparison found 472 identical outputs and 28 differences limited to equivalent NOT LIKE spelling and generated subquery alias names. Generated source hashes remain independently checked against their manifests. - -The upgrade does not make the existing compatibility rules redundant. The built-in Fabric dialect is a suitable Warehouse target, but does not replace them — see the per-benchmark pages linked above for what each rule still covers. +`tests/test_tpc_sqlglot_compatibility.py` checks reviewed output fingerprints in `tests/fixtures/tpc_query_rendering.json` for all **500** TPC-H/TPC-DS renderings across both static scales, target dialects, and pinned SQLGlot versions. Engine-registered rules are part of what the fingerprints pin. Review actual SQL differences before refreshing these fingerprints; do not regenerate them just to clear a failure. Generated source hashes remain independently checked against their manifests. Case-sensitive binding checks intentionally bypass identifier normalization so they catch mismatches like SR_FEE versus the declared sr_fee column. Successful grammar parsing alone does not establish Warehouse execution support. @@ -530,7 +344,7 @@ TPCH.ENGINE_QUERY_NORMALIZERS = { } ``` -Preserve generated literals and make AST rules idempotent. Current engine accommodations include Daft DOUBLE arithmetic casts in TPC-H q1/q8/q9/q14, Sail's NULLIF denominator in TPC-DS q12, and the Fabric Data Warehouse T-SQL accommodations for TPC-DS q17/q22/q29/q35/q39 and ClickBench grouping, aggregate widening, and q29 host extraction. These are **semantic accommodations** (numeric precision and division-by-zero behavior), not merely syntax fixes. Applied rule identifiers are recorded per query in `execution_telemetry["query_normalization_rules"]`, alongside the benchmark's normalizer version in engine metadata. Successful transpilation alone does not establish specification equivalence or engine execution support. +Preserve generated literals and make AST rules idempotent. Engine accommodations can be semantic changes, such as numeric-precision or division-by-zero safeguards, rather than merely syntax fixes. Applied rule identifiers are recorded per query in `execution_telemetry["query_normalization_rules"]`, alongside the benchmark's normalizer version in engine metadata. Successful transpilation alone does not establish specification equivalence or engine execution support. The per-benchmark normalization pages inventory the current rules. Full guidance on writing and registering a rule — including bumping `NORMALIZER_VERSION` and refreshing rendering fingerprints — is in [docs/query-normalization/](docs/query-normalization/README.md#adding-a-rule). diff --git a/docs/engines/README.md b/docs/engines/README.md new file mode 100644 index 0000000..5645031 --- /dev/null +++ b/docs/engines/README.md @@ -0,0 +1,41 @@ +# Engine documentation + +LakeBench engines provide a consistent benchmark interface while preserving each runtime's native storage, authentication, telemetry, and pricing behavior. + +| Engine | Install | Primary runtime | Documentation | Coverage | +|---|---|---|---|---| +| Generic Spark | `lakebench[spark]` | Local or Spark runtime | [Spark](spark/README.md) | [Report](../../reports/coverage/spark.md) | +| Fabric Spark | Runtime-provided Spark | Microsoft Fabric | [Fabric Spark](fabric-spark/README.md) | — | +| Synapse Spark | Runtime-provided Spark | Azure Synapse | [Synapse Spark](synapse-spark/README.md) | — | +| HDInsight Spark | Runtime-provided Spark | Azure HDInsight | [HDInsight Spark](hdi-spark/README.md) | — | +| Databricks Spark | `lakebench[databricks_spark]` | Databricks on AWS, Azure, or GCP | [Databricks Spark](databricks-spark/README.md) | — | +| Databricks SQL Warehouse | `lakebench[databricks_sql_warehouse]` | Databricks SQL | [Databricks SQL Warehouse](databricks-sql-warehouse/README.md) | — | +| Fabric Data Warehouse | `lakebench[fabric_data_warehouse]` | Microsoft Fabric | [Fabric Data Warehouse](fabric-data-warehouse/README.md) | — | +| DuckDB | `lakebench[duckdb]` | Local or notebook runtime | [DuckDB](duckdb/README.md) | [Report](../../reports/coverage/duckdb.md) | +| Polars | `lakebench[polars]` | Local or notebook runtime | [Polars](polars/README.md) | [Report](../../reports/coverage/polars.md) | +| Daft | `lakebench[daft]` | Local or notebook runtime | [Daft](daft/README.md) | [Report](../../reports/coverage/daft.md) | +| Sail | `lakebench[sail]` | Local or notebook runtime | [Sail](sail/README.md) | [Report](../../reports/coverage/sail.md) | + +All engines write benchmark results through the same shared result contract. A benchmark accepts the engine instance, so the benchmark workflow remains the same after engine construction: + +```python +from lakebench.benchmarks import TPCH + +benchmark = TPCH( + engine=engine, + scenario_name="sf10", + scale_factor=10, + input_folder_uri="path-or-cloud-uri/tpch/sf10", +) +benchmark.run() +``` + +## Shared pricing contract + +Engines backed by vCore compute can accept either `cost_per_vcore_hour` or `cost_per_hour`; the inputs are mutually exclusive. A total hourly rate should include every cost component the user wants represented in `estimated_retail_job_cost`. + +Automatic estimates use public list prices and are not invoice calculations. They do not account for negotiated discounts, commitments, credits, taxes, or other private billing adjustments. + +## Adding an engine + +Contributor guidance for custom engines remains in the root [README](../../README.md). Internal helpers such as `DeltaRs` do not have standalone engine pages. diff --git a/docs/engines/daft/README.md b/docs/engines/daft/README.md new file mode 100644 index 0000000..c0b551d --- /dev/null +++ b/docs/engines/daft/README.md @@ -0,0 +1,36 @@ +# Daft + +The `Daft` engine integrates Daft DataFrames with LakeBench and Delta Lake. + +## Current support status + +Daft is unsupported for ELTBench, TPC-H, and TPC-DS as of LakeBench 2.0.0. The bundled TPC generators emit Parquet containing Arrow `Utf8View`, which the pinned Daft reader cannot load. ClickBench remains partially usable because its sample does not use that generated schema. + +See the [Daft coverage report](../../../reports/coverage/daft.md). + +## Installation + +```bash +pip install "lakebench[daft]" +``` + +## Example + +```python +from lakebench.engines import Daft + +engine = Daft( + schema_or_working_directory_uri="file:///tmp/lakebench", +) +``` + +## Configuration + +- `schema_or_working_directory_uri`: Delta table root. +- `cost_per_vcore_hour` or `cost_per_hour`: mutually exclusive manual pricing inputs. + +For ADLS Gen2, set `AZURE_STORAGE_TOKEN`. OneLake paths are rejected; use an ADLS Gen2 path instead. + +## Engine behavior + +Daft reads and writes Delta tables through its native Delta integration and the shared DeltaRs helper where appropriate. Native TPC `.tbl` and `.dat` input is implemented, but the generated Parquet limitation currently prevents supported end-to-end TPC/ELT execution. diff --git a/docs/engines/databricks-spark/README.md b/docs/engines/databricks-spark/README.md new file mode 100644 index 0000000..660de0d --- /dev/null +++ b/docs/engines/databricks-spark/README.md @@ -0,0 +1,68 @@ +# Databricks Spark + +`DatabricksSpark` runs LakeBench on Databricks clusters across AWS, Azure, and Google Cloud. + +## Support + +- Benchmarks: ELTBench, TPC-DS, TPC-H, and ClickBench. +- Compute: Jobs Compute and All-Purpose Compute. +- Runtime: Databricks Runtime with PySpark in **Dedicated** access mode. +- Storage: S3 on AWS, ADLS Gen2 on Azure, and Google Cloud Storage on GCP. +- Catalogs: Hive metastore or Unity Catalog. + +## Installation + +```python +%pip install lakebench +``` + +## Example + +```python +from lakebench.engines import DatabricksSpark + +engine = DatabricksSpark( + catalog_name="main", + schema_name="lakebench", + schema_uri="abfss://container@account.dfs.core.windows.net/lakebench" #optional +) +``` + +Use `s3://...` on AWS or `gs://...` on GCP. + +## Compute and access mode + +LakeBench supports both Databricks Jobs Compute and All-Purpose Compute. The compute type is automatically detected and logged in the results. + +The compute must use **Dedicated** access mode. LakeBench uses the driver `SparkContext` to set a job description for every timed phase and test item. Those descriptions include labels such as benchmark phase and query ID, making individual operations easy to identify in the Spark UI. Databricks compute modes that do not expose `SparkContext` cannot provide this labeling or the other Spark runtime metadata LakeBench requires. + +## Configuration + +- `catalog_name`: defaults to `hive_metastore`; use a Unity Catalog name when applicable. +- `schema_name` and `schema_uri`: benchmark table namespace and Delta root. +- `cost_per_hour` or `cost_per_vcore_hour`: mutually exclusive manual rates. +- `spark_measure_telemetry` and `tblproperties`: shared Spark options. + +## Pricing + +### Azure + +For fixed-size clusters, LakeBench combines: + +1. Driver and worker DBUs/hour from the published workload-, Photon-, and VM-specific mapping. +2. The regional DBU price from Azure Retail Prices. +3. Regional driver and worker VM infrastructure prices from Azure Retail Prices. + +Autoscaling or missing cluster metadata causes a warning and leaves cost unset unless a manual rate is supplied. Public prices do not include negotiated discounts or commitments. + +Refresh the generated mapping with: + +```bash +uv run python scripts/refresh_databricks_azure_dbu_map.py +``` + +### AWS and GCP + +Supply an all-inclusive `cost_per_hour` or `cost_per_vcore_hour`. The rate should include both Databricks DBUs and cloud infrastructure. Without a manual rate, the benchmark continues and estimated cost remains unset. + +Native TPC `.tbl` and `.dat` input is supported through the shared Spark loader. diff --git a/docs/engines/databricks-sql-warehouse/README.md b/docs/engines/databricks-sql-warehouse/README.md new file mode 100644 index 0000000..9a2afcc --- /dev/null +++ b/docs/engines/databricks-sql-warehouse/README.md @@ -0,0 +1,60 @@ +# Databricks SQL Warehouse + +`DatabricksSQLWarehouse` runs LakeBench through the Databricks SQL connector and Warehouses REST API on AWS, Azure, or GCP. + +## Support + +- Benchmarks: ELTBench, TPC-DS, TPC-H, and ClickBench. +- Warehouse types: serverless, classic, and pro. +- Storage: cloud object storage accessible to the warehouse. +- Result persistence: SQL executed by the remote warehouse, including result-table appends. + +## Installation + +```bash +pip install "lakebench[databricks_sql_warehouse]" +``` + +## Example + +```python +import os + +from lakebench.engines import DatabricksSQLWarehouse + +engine = DatabricksSQLWarehouse( + server_hostname="dbc-....cloud.databricks.com", + warehouse_name="Serverless Starter Warehouse", + catalog_name="main", + schema_name="lakebench", + access_token=os.environ["DATABRICKS_TOKEN"], +) +``` + +## Configuration + +- `server_hostname`: workspace hostname without `https://`. +- `warehouse_name`: Warehouses API display name. +- `catalog_name` and `schema_name`: target namespace. +- `access_token`: used by the REST API and SQL connector; never written to result metadata. +- `schema_uri`: optional managed/external schema location. +- `enable_result_caching`: defaults to `False` for reproducible benchmark execution. +- `cost_per_hour`: all-inclusive manual hourly rate. +- `dbu_rate`: optional serverless DBU-rate override. + +SQL Warehouse intentionally does not accept `cost_per_vcore_hour`. + +## Serverless pricing + +Warehouse-size DBUs/hour are multiplied by a DBU list rate. Resolution order is: + +1. Explicit `dbu_rate`. +2. Exact recent warehouse SKU from `system.billing.usage`, then its active USD DBU rate from `system.billing.list_prices`. +3. When no recent usage exists, AWS regional SKU matching if all matching pricing tiers have one rate. +4. On Azure, the regional `Premium Serverless SQL DBU` meter from the public Azure Retail Prices API. + +Insufficient billing-table permissions are logged at warning level. Azure can still use its public catalog; AWS and GCP continue without an estimate unless a manual rate is supplied. Classic and pro warehouses require `cost_per_hour` for estimated cost. + +List prices are estimates and do not include negotiated discounts, commitments, or credits. + +Native TPC `.tbl` and `.dat` input is loaded with `read_files` and the benchmark DDL. diff --git a/docs/engines/duckdb/README.md b/docs/engines/duckdb/README.md new file mode 100644 index 0000000..b6fe64c --- /dev/null +++ b/docs/engines/duckdb/README.md @@ -0,0 +1,37 @@ +# DuckDB + +The `DuckDB` engine provides an embedded local analytical engine with Delta Lake persistence. + +## Support + +- Benchmarks: ELTBench, TPC-DS, TPC-H, and ClickBench. +- Storage: local filesystem, mount-style paths, ADLS Gen2, and OneLake with appropriate credentials. +- Coverage: [DuckDB coverage report](../../../reports/coverage/duckdb.md). + +## Installation + +```bash +pip install "lakebench[duckdb]" +``` + +## Example + +```python +from lakebench.engines import DuckDB + +engine = DuckDB( + schema_or_working_directory_uri="file:///tmp/lakebench", +) +``` + +## Configuration + +- `schema_or_working_directory_uri`: Delta table root. +- `storage_options`: options forwarded to DeltaRs and filesystem clients. +- `cost_per_vcore_hour` or `cost_per_hour`: mutually exclusive manual pricing inputs. + +For OneLake `abfss://` paths, set `AZURE_STORAGE_TOKEN`; LakeBench creates a DuckDB Azure secret using that bearer token. + +## Engine behavior + +DuckDB creates typed empty tables in memory from benchmark DDL and persists them as Delta tables. Delta writes use the shared DeltaRs helper. Native TPC delimited input is supported. diff --git a/docs/engines/fabric-data-warehouse/README.md b/docs/engines/fabric-data-warehouse/README.md new file mode 100644 index 0000000..fc2b794 --- /dev/null +++ b/docs/engines/fabric-data-warehouse/README.md @@ -0,0 +1,52 @@ +# Fabric Data Warehouse + +`FabricDataWarehouse` runs LakeBench through the Microsoft Fabric Warehouse SQL endpoint. + +## Support + +- Benchmarks: ELTBench, TPC-DS, TPC-H, and ClickBench. +- Runtime: Microsoft Fabric notebook. +- Storage: OneLake, ADLS Gen2 sources supported by Warehouse SQL, Parquet, and native TPC delimited files. +- SQL dialect: Fabric Warehouse T-SQL. + +## Installation + +Use Python 3.10 or later and install: + +```python +%pip install "lakebench[fabric_data_warehouse]" +``` + +The host must also have Microsoft ODBC Driver 18 for SQL Server. Pip cannot install this system driver. + +## Example + +```python +from lakebench.engines import FabricDataWarehouse + +engine = FabricDataWarehouse( + warehouse_name="warehouse", + warehouse_server="xxxxx.datawarehouse.fabric.microsoft.com", + schema_name="dbo", +) +``` + +Warehouse and schema names cannot contain spaces. + +## Authentication + +The engine uses the current Fabric notebook identity to obtain a Fabric access token. The identity must be able to connect to the Warehouse, create and modify benchmark tables, and read source storage. + +## Engine behavior + +- ODBC connections use encryption and token authentication. +- Parquet loads use Warehouse-native SQL paths. +- TPC native delimited files are loaded with the generator's trailing delimiter handled explicitly. +- Result-log appends use DeltaRs when required by the result destination. +- Capacity SKU, Warehouse version, server, Warehouse name, and V-Order state are recorded when visible. + +## Pricing + +`estimated_retail_job_cost` remains unset. Fabric capacity cost cannot be reliably attributed to an individual Warehouse query or load, so LakeBench does not report a misleading per-job estimate. + +`FabricWarehouse` is retained as a compatibility alias for `FabricDataWarehouse`. diff --git a/docs/engines/fabric-spark/README.md b/docs/engines/fabric-spark/README.md new file mode 100644 index 0000000..9527666 --- /dev/null +++ b/docs/engines/fabric-spark/README.md @@ -0,0 +1,54 @@ +# Fabric Spark + +`FabricSpark` specializes the generic Spark engine for Microsoft Fabric notebooks and Lakehouses. + +## Support + +- Benchmarks: ELTBench, TPC-DS, TPC-H, and ClickBench. +- Runtime: Microsoft Fabric Spark. +- Storage: OneLake and Lakehouse schemas. +- Authentication: the Fabric notebook identity and runtime-provided `notebookutils`. + +## Installation + +Fabric supplies PySpark. Install LakeBench in the notebook: + +```python +%pip install lakebench +``` + +For stage telemetry: + +```python +%pip install "lakebench[sparkmeasure]" +``` + +## Example + +```python +from lakebench.engines import FabricSpark + +engine = FabricSpark( + lakehouse_name="lakehouse", + lakehouse_schema_name="lakebench", +) +``` + +## Configuration + +- `lakehouse_name`: Fabric Lakehouse name. +- `lakehouse_schema_name`: schema used for benchmark tables. +- `spark_measure_telemetry`: enables Spark Measure stage metrics. +- `collect_stats_on_write`: enables Fabric Delta extended statistics during writes; defaults to `True`. +- `cost_per_vcore_hour` or `cost_per_hour`: mutually exclusive manual overrides. +- `tblproperties`: Delta properties applied at table creation. + +`compute_stats_all_cols` is a deprecated alias for `collect_stats_on_write`. Explicit benchmark statistics should use the benchmark's `analyze` option. + +## Pricing and metadata + +When no manual rate is supplied, LakeBench derives a Fabric vCore rate from visible capacity metadata. Results include the capacity ID, computed hourly cost, Spark history URL, runtime version, and relevant Fabric Spark configuration. + +Fast optimize is temporarily disabled while LakeBench runs explicit compaction so the measured operation uses the intended Delta optimization path. + +Native TPC `.tbl` and `.dat` input is supported through the shared Spark loader. diff --git a/docs/engines/hdi-spark/README.md b/docs/engines/hdi-spark/README.md new file mode 100644 index 0000000..90f7961 --- /dev/null +++ b/docs/engines/hdi-spark/README.md @@ -0,0 +1,39 @@ +# HDInsight Spark + +`HDISpark` adapts the generic Spark engine for Azure HDInsight Spark clusters. + +## Support + +- Benchmarks: ELTBench, TPC-DS, TPC-H, and ClickBench. +- Runtime: HDInsight Spark. +- Storage: storage configured for the HDInsight cluster. +- Authentication: cluster/runtime credentials and Hadoop configuration. + +## Installation + +HDInsight supplies PySpark. Install LakeBench on the cluster or notebook: + +```python +%pip install lakebench +``` + +## Example + +```python +from lakebench.engines import HDISpark + +engine = HDISpark(schema_name="lakebench") +``` + +## Configuration + +- `schema_name`: benchmark schema/database. +- `spark_measure_telemetry`: optional Spark Measure stage metrics. +- `cost_per_vcore_hour` or `cost_per_hour`: mutually exclusive manual pricing inputs. +- `tblproperties`: Delta table properties. + +## Pricing + +HDInsight does not automatically assemble cluster pricing. Supply an all-inclusive `cost_per_hour` or a `cost_per_vcore_hour` appropriate for the cluster when estimated job cost is required. + +Native TPC `.tbl` and `.dat` input is supported through the shared Spark loader. diff --git a/docs/engines/polars/README.md b/docs/engines/polars/README.md new file mode 100644 index 0000000..47f276a --- /dev/null +++ b/docs/engines/polars/README.md @@ -0,0 +1,37 @@ +# Polars + +The `Polars` engine executes LakeBench with Polars lazy DataFrames and SQL context while persisting tables in Delta Lake. + +## Support + +- Benchmarks: ELTBench; partial TPC-DS, TPC-H, and ClickBench support. +- Storage: local filesystem, mount paths, ADLS Gen2, and OneLake through storage options. +- Coverage: [Polars coverage report](../../../reports/coverage/polars.md). + +## Installation + +```bash +pip install "lakebench[polars]" +``` + +## Example + +```python +from lakebench.engines import Polars + +engine = Polars( + schema_or_working_directory_uri="file:///tmp/lakebench", +) +``` + +## Configuration + +- `schema_or_working_directory_uri`: Delta table root. +- `storage_options`: cloud filesystem and DeltaRs authentication options. +- `cost_per_vcore_hour` or `cost_per_hour`: mutually exclusive manual pricing inputs. + +## Engine behavior and limitations + +Polars uses lazy Parquet scans and a Polars SQL context. Delta writes use DeltaRs. Native TPC delimited input is supported. + +Some benchmark queries remain unsupported because Polars SQL does not implement every required construct, including some non-equi joins. Use the coverage report for current query-level results. diff --git a/docs/engines/sail/README.md b/docs/engines/sail/README.md new file mode 100644 index 0000000..96ea1c3 --- /dev/null +++ b/docs/engines/sail/README.md @@ -0,0 +1,41 @@ +# Sail + +The `Sail` engine runs LakeBench through a local Sail Spark Connect server. + +## Support + +- Benchmarks: ELTBench, TPC-DS, TPC-H, and ClickBench. +- Storage: local files, mount-style paths, and OneLake/cloud paths supported by Sail and DeltaRs. +- Coverage: [Sail coverage report](../../../reports/coverage/sail.md). + +## Installation + +```bash +pip install "lakebench[sail]" +``` + +Sail and the generic Spark extra conflict because they require different Spark distributions. Install only the engine needed in that environment. + +## Example + +```python +from lakebench.engines import Sail + +engine = Sail( + schema_or_working_directory_uri="file:///tmp/lakebench", +) +``` + +## Configuration + +- `schema_or_working_directory_uri`: Delta table root and Spark warehouse directory. +- `storage_options`: options passed to DeltaRs/filesystem access. +- `cost_per_vcore_hour` or `cost_per_hour`: mutually exclusive manual pricing inputs. + +## Runtime behavior + +LakeBench starts one background Sail Spark Connect server per Python process and reuses its Spark session across engine instances. Join reordering is enabled for benchmark execution. + +After upgrading Sail packages in a live notebook kernel, a kernel restart may be required before engine initialization. + +Native TPC `.tbl` and `.dat` input is supported. diff --git a/docs/engines/spark/README.md b/docs/engines/spark/README.md new file mode 100644 index 0000000..abd42a5 --- /dev/null +++ b/docs/engines/spark/README.md @@ -0,0 +1,58 @@ +# Generic Spark + +The generic `Spark` engine runs LakeBench against a local Spark session or an existing Spark runtime. + +## Support + +- Benchmarks: ELTBench, TPC-DS, TPC-H, and ClickBench. +- Storage: local files, mount paths, and storage exposed through the runtime's Hadoop configuration. +- Table format: Delta Lake. +- Coverage: [Spark coverage report](../../../reports/coverage/spark.md). + +## Installation + +```bash +pip install "lakebench[spark]" +``` + +Install `lakebench[sparkmeasure]` as well when using stage telemetry. + +## Example + +```python +from lakebench.engines import Spark + +engine = Spark( + schema_name="lakebench", + schema_uri="file:///tmp/lakebench", +) +``` + +For an existing managed Spark runtime, use that runtime's schema path and optional catalog: + +```python +engine = Spark( + catalog_name="catalog", + schema_name="lakebench", + schema_uri="abfss://container@account.dfs.core.windows.net/lakebench", +) +``` + +## Configuration + +- `schema_name`: schema/database used for benchmark tables. +- `catalog_name`: optional metastore catalog. +- `schema_uri`: Delta table root; required for local Spark. +- `spark_measure_telemetry`: captures stage metrics when Spark Measure is installed and configured. +- `cost_per_vcore_hour` or `cost_per_hour`: mutually exclusive manual pricing inputs. +- `tblproperties`: Delta table properties applied during table creation. + +Use the benchmark's `analyze` option for statistics. `compute_stats_all_cols` is deprecated. + +## Runtime behavior + +When LakeBench detects a local runtime, it creates a local Spark session with Delta extensions and uses all local cores. On a managed runtime it reuses the runtime's Spark session and authentication. + +Spark Measure also requires the matching Spark Measure JAR. See the [sparkmeasure package documentation](https://github.com/LucaCanali/sparkMeasure). + +Native TPC `.tbl` and `.dat` input is supported. diff --git a/docs/engines/synapse-spark/README.md b/docs/engines/synapse-spark/README.md new file mode 100644 index 0000000..39e4250 --- /dev/null +++ b/docs/engines/synapse-spark/README.md @@ -0,0 +1,45 @@ +# Synapse Spark + +`SynapseSpark` runs LakeBench in an Azure Synapse serverless Apache Spark pool. + +## Support + +- Benchmarks: ELTBench, TPC-DS, TPC-H, and ClickBench. +- Runtime: Azure Synapse Spark only. +- Storage: ADLS Gen2 and runtime-accessible mount paths. +- Authentication: the Synapse notebook/runtime identity. + +## Installation + +Synapse supplies PySpark. Install LakeBench in the notebook environment: + +```python +%pip install lakebench +``` + +## Example + +```python +from lakebench.engines import SynapseSpark + +engine = SynapseSpark( + schema_name="lakebench", + schema_uri="abfss://container@account.dfs.core.windows.net/lakebench", +) +``` + +## Configuration + +- `schema_name`: benchmark schema/database. +- `schema_uri`: Delta table root. +- `spark_measure_telemetry`: optional Spark Measure stage metrics. +- `cost_per_vcore_hour` or `cost_per_hour`: mutually exclusive manual pricing overrides. +- `tblproperties`: Delta table properties. + +## Pricing and metadata + +Without a manual rate, LakeBench queries the Azure Retail Prices API for the regional Azure Synapse serverless Spark vCore rate and derives the total hourly rate from active cores. Results also include the Synapse region, Spark history URL, and selected runtime configuration. + +Construction fails outside a detected Synapse runtime. + +Native TPC `.tbl` and `.dat` input is supported through the shared Spark loader. diff --git a/pyproject.toml b/pyproject.toml index b874530..44389a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,6 @@ tpch_datagen = [] tpcds_duckdb_datagen = ["duckdb==1.5.5; python_version >= '3.10'"] sparkmeasure = ["sparkmeasure==0.28.0; python_version >= '3.9'"] spark = ["pyspark>=3.5.0,<4.0.0; python_version >= '3.9'", "delta-spark>=3.2.0,<4.0.0; python_version >= '3.9'"] -databricks_spark = ["pyspark>=3.5.0,<4.0.0; python_version >= '3.9'", "requests>=2.28.0; python_version >= '3.9'"] databricks_sql_warehouse = ["databricks-sql-connector>=3.0.0; python_version >= '3.9'", "pandas>=1.4.0; python_version >= '3.9'", "requests>=2.28.0; python_version >= '3.9'"] # pysail is a floor rather than the newest release: it is a lower bound only, so # the resolver picks the newest version available in whichever index is diff --git a/scripts/refresh_databricks_azure_dbu_map.py b/scripts/refresh_databricks_azure_dbu_map.py index 4032ea0..d065fad 100644 --- a/scripts/refresh_databricks_azure_dbu_map.py +++ b/scripts/refresh_databricks_azure_dbu_map.py @@ -11,7 +11,9 @@ from pathlib import Path SOURCE_URL = "https://azure.microsoft.com/en-us/pricing/details/databricks/" -DEFAULT_OUTPUT = Path(__file__).resolve().parents[1] / "src" / "lakebench" / "engines" / "databricks_azure_dbu_map.py" +DEFAULT_OUTPUT = ( + Path(__file__).resolve().parents[1] / "src" / "lakebench" / "engines" / "resources" / "databricks_azure_dbu_map.py" +) SPARK_TABLES = { "all-purpose-compute-premium": "all-purpose-compute", diff --git a/src/lakebench/engines/base.py b/src/lakebench/engines/base.py index b19f5be..43cc18c 100644 --- a/src/lakebench/engines/base.py +++ b/src/lakebench/engines/base.py @@ -21,15 +21,10 @@ class BaseEngine(ABC): Attributes ---------- - SQLGLOT_DIALECT : str, optional - Specifies the SQL dialect to be used by the engine when SQL transpiling - is required. Default is None. - SUPPORTS_ONELAKE : bool - Indicates if the engine supports OneLake URIs (e.g., abfss://workspace@onelake.dfs.fabric.microsoft.com/...) - SUPPORTS_SCHEMA_PREP : bool - Indicates if the engine supports schema preparation (creation of empty table with defined schema) - SUPPORTS_MOUNT_PATH : bool - Indicates if the engine supports mount URIs (e.g., /mnt/...) + schema_or_working_directory_uri : str, optional + Base URI used to store Delta tables. + storage_options : dict, optional + Filesystem credentials and options passed to storage clients. Methods ------- diff --git a/src/lakebench/engines/daft.py b/src/lakebench/engines/daft.py index 6ca7537..fe88d08 100644 --- a/src/lakebench/engines/daft.py +++ b/src/lakebench/engines/daft.py @@ -9,8 +9,23 @@ class Daft(BaseEngine): - """ - Daft Engine + """Execute LakeBench workloads with the Daft DataFrame engine. + + Attributes + ---------- + schema_or_working_directory_uri : str + Base URI used to store Delta tables. + cost_per_vcore_hour : float, optional + Retail cost per vCore hour used for estimated job cost. + cost_per_hour : float, optional + Total hourly compute cost, mutually exclusive with + ``cost_per_vcore_hour``. + + Notes + ----- + Daft supports ADLS Gen2 paths but not OneLake paths. Decimal columns in + native TPC input are read as text and cast because of Daft CSV precision + handling. """ SQLGLOT_DIALECT = "mysql" diff --git a/src/lakebench/engines/databricks_spark.py b/src/lakebench/engines/databricks_spark.py index ee53445..08bece3 100644 --- a/src/lakebench/engines/databricks_spark.py +++ b/src/lakebench/engines/databricks_spark.py @@ -3,7 +3,7 @@ import logging from typing import Optional -from .databricks_pricing import ( +from .resources.databricks_pricing import ( DatabricksPricingError, estimate_azure_spark_cost, infer_cloud_provider, @@ -15,13 +15,41 @@ class DatabricksSpark(Spark): - """Databricks Spark engine.""" + """Execute LakeBench workloads on Databricks Spark compute. + + Attributes + ---------- + schema_name : str + Name of the Databricks schema used for benchmark tables. + schema_uri : str, optional + Storage location assigned to the schema. + catalog_name : str, optional + Databricks catalog containing the benchmark schema. + spark_measure_telemetry : bool, optional + Whether to collect sparkMeasure execution telemetry. + cost_per_vcore_hour : float, optional + Retail cost per vCore hour used for estimated job cost. + cost_per_hour : float, optional + Total hourly compute cost, mutually exclusive with + ``cost_per_vcore_hour``. + compute_stats_all_cols : bool, optional + Deprecated compatibility option for post-load column statistics. + tblproperties : dict, optional + Delta table properties added to table creation statements. + + Notes + ----- + Cloud provider, region, workload type, Photon state, and Unity Catalog + state are detected automatically from Databricks runtime configuration. + Jobs Compute and All-Purpose Compute are supported. Dedicated access mode + is required because LakeBench uses ``SparkContext`` to label timed phases + and query IDs in the Spark UI. Azure fixed-size clusters can resolve + automatic DBU and infrastructure pricing from public Azure APIs. + """ SUPPORTS_MOUNT_PATH = True SUPPORTS_ONELAKE = False SUPPORTS_SCHEMA_PREP = True - REQUIRED_MODULES = ("pyspark", "requests") - INSTALL_EXTRA = "databricks_spark" def __init__( self, @@ -31,10 +59,6 @@ def __init__( spark_measure_telemetry: bool = False, cost_per_vcore_hour: Optional[float] = None, cost_per_hour: Optional[float] = None, - dbu_rate: Optional[float] = None, - infrastructure_cost_per_hour: Optional[float] = None, - workload_type: str = "jobs-compute", - cloud_provider: Optional[str] = None, compute_stats_all_cols: bool = False, tblproperties: Optional[dict] = None, ): @@ -47,9 +71,6 @@ def __init__( tblproperties=tblproperties, ) - if workload_type not in {"jobs-compute", "all-purpose-compute"}: - raise ValueError("workload_type must be 'jobs-compute' or 'all-purpose-compute'.") - runtime_version = self.spark.conf.get( "spark.databricks.clusterUsageTags.sparkVersion", "unknown", @@ -65,15 +86,19 @@ def __init__( and catalog_name != "hive_metastore" ) self.cloud_provider = infer_cloud_provider( - explicit_cloud_provider=cloud_provider, + explicit_cloud_provider=None, spark_configs=self.spark_configs, ) self.region = self.spark.conf.get( "spark.databricks.clusterUsageTags.region", None, ) - self.workload_type = workload_type - pricing_workload_type = f"{workload_type}-with-photon" if self.photon_enabled else workload_type + self.workload_type = ( + "jobs-compute" + if self.spark_configs.get("spark.databricks.clusterUsageTags.workloadType", "unknown") == "AUTOMATED" + else "all-purpose-compute" + ) + pricing_workload_type = f"{self.workload_type}-with-photon" if self.photon_enabled else self.workload_type workspace_url = self.spark_configs.get("spark.databricks.workspaceUrl") cluster_id = self.spark_configs.get("spark.databricks.clusterUsageTags.clusterId") @@ -111,8 +136,6 @@ def __init__( elif self.cloud_provider == "azure": self._configure_automatic_pricing( pricing_workload_type=pricing_workload_type, - dbu_rate=dbu_rate, - infrastructure_cost_per_hour=infrastructure_cost_per_hour, ) else: self.extended_engine_metadata["pricing_source"] = "not_configured" @@ -120,8 +143,6 @@ def __init__( def _configure_automatic_pricing( self, pricing_workload_type: str, - dbu_rate: Optional[float], - infrastructure_cost_per_hour: Optional[float], ) -> None: if ( self.spark.conf.get( @@ -166,8 +187,6 @@ def _configure_automatic_pricing( worker_instance_type=worker_sku, worker_count=int(worker_count), workload_type=pricing_workload_type, - dbu_rate_override=dbu_rate, - infrastructure_cost_per_hour_override=infrastructure_cost_per_hour, ) except DatabricksPricingError as exc: logger.warning("Estimated job cost will not be reported: %s", exc) diff --git a/src/lakebench/engines/databricks_sql_warehouse.py b/src/lakebench/engines/databricks_sql_warehouse.py index 60f3833..08b10db 100644 --- a/src/lakebench/engines/databricks_sql_warehouse.py +++ b/src/lakebench/engines/databricks_sql_warehouse.py @@ -8,7 +8,7 @@ from typing import Mapping, Optional, Sequence from .base import BaseEngine -from .databricks_pricing import ( +from .resources.databricks_pricing import ( DatabricksPricingError, estimate_sql_warehouse_cost, normalize_cloud_provider, @@ -37,7 +37,39 @@ class DatabricksSQLWarehouse(BaseEngine): - """Databricks SQL Warehouse engine.""" + """Execute LakeBench SQL workloads on a Databricks SQL warehouse. + + Attributes + ---------- + server_hostname : str + Hostname of the Databricks workspace. + warehouse_name : str + Name of the SQL warehouse used for benchmark execution. + catalog_name : str + Databricks catalog containing the benchmark schema. + schema_name : str + Name of the Databricks schema used for benchmark tables. + access_token : str + Databricks access token used for API and SQL authentication. + schema_uri : str, optional + Storage location assigned to the schema. + enable_result_caching : bool, optional + Whether Databricks SQL result caching remains enabled. + cost_per_hour : float, optional + Total hourly warehouse cost used for estimated job cost. + dbu_rate : float, optional + Explicit serverless price per DBU. + compute_stats_all_cols : bool, optional + Deprecated compatibility option for post-load column statistics. + + Notes + ----- + Cloud provider and region are detected automatically from + ``current_metastore()``. Serverless warehouse pricing is resolved from + Databricks billing tables when available, with supported public-catalog + fallbacks. Classic and pro warehouses require an explicit total hourly + cost. + """ SQLGLOT_DIALECT = "spark" SUPPORTS_MOUNT_PATH = False diff --git a/src/lakebench/engines/delta_rs.py b/src/lakebench/engines/delta_rs.py index 59ad0f6..21c913d 100644 --- a/src/lakebench/engines/delta_rs.py +++ b/src/lakebench/engines/delta_rs.py @@ -2,8 +2,14 @@ class DeltaRs(BaseEngine): - """ - Delta-Rs Engine + """Provide shared Delta Lake writes for non-Spark engines. + + Notes + ----- + This is an internal engine helper rather than a standalone benchmark + execution engine. Non-Spark engines delegate Delta table persistence to + this class. It exposes the delta-rs ``write_deltalake`` function and + ``DeltaTable`` class after construction. """ def __init__(self): diff --git a/src/lakebench/engines/duckdb.py b/src/lakebench/engines/duckdb.py index 33dd84c..1ea0bff 100644 --- a/src/lakebench/engines/duckdb.py +++ b/src/lakebench/engines/duckdb.py @@ -10,8 +10,25 @@ class DuckDB(BaseEngine): - """ - DuckDB Engine + """Execute LakeBench workloads with an embedded DuckDB connection. + + Attributes + ---------- + schema_or_working_directory_uri : str + Base URI used to store Delta tables. + cost_per_vcore_hour : float, optional + Retail cost per vCore hour used for estimated job cost. + cost_per_hour : float, optional + Total hourly compute cost, mutually exclusive with + ``cost_per_vcore_hour``. + storage_options : dict, optional + Filesystem credentials and options passed to Delta Lake clients. + + Notes + ----- + Cloud storage credentials are passed through ``storage_options`` or + discovered from the runtime environment. Delta tables are written through + delta-rs and queried through DuckDB. """ SQLGLOT_DIALECT = "duckdb" diff --git a/src/lakebench/engines/fabric_data_warehouse.py b/src/lakebench/engines/fabric_data_warehouse.py index 2a50d41..7bfb5db 100644 --- a/src/lakebench/engines/fabric_data_warehouse.py +++ b/src/lakebench/engines/fabric_data_warehouse.py @@ -21,16 +21,12 @@ class FabricDataWarehouse(BaseEngine): Attributes ---------- - SQLGLOT_DIALECT : str - Specifies the SQL dialect to be used by the engine when SQL transpiling - is required. - SUPPORTS_ONELAKE : bool - Indicates if the engine supports OneLake URIs - (e.g., abfss://workspace@onelake.dfs.fabric.microsoft.com/...) - SUPPORTS_SCHEMA_PREP : bool - Indicates if the engine supports schema preparation (creation of empty table with defined schema) - SUPPORTS_MOUNT_PATH : bool - Indicates if the engine supports mount URIs (e.g., /mnt/...) + warehouse_name : str + Name of the Fabric Data Warehouse. + warehouse_server : str + SQL endpoint hostname for the Fabric Data Warehouse. + schema_name : str + Name of the warehouse schema used for benchmark tables. Notes ----- diff --git a/src/lakebench/engines/fabric_spark.py b/src/lakebench/engines/fabric_spark.py index 70c837e..9642fa4 100644 --- a/src/lakebench/engines/fabric_spark.py +++ b/src/lakebench/engines/fabric_spark.py @@ -8,8 +8,33 @@ class FabricSpark(Spark): - """ - Fabric Spark Engine + """Execute LakeBench workloads in a Microsoft Fabric Spark runtime. + + Attributes + ---------- + lakehouse_name : str + Name of the Fabric Lakehouse used as the Spark catalog. + lakehouse_schema_name : str + Name of the Lakehouse schema used for benchmark tables. + spark_measure_telemetry : bool, optional + Whether to collect sparkMeasure execution telemetry. + cost_per_vcore_hour : float, optional + Retail cost per vCore hour used for estimated job cost. + cost_per_hour : float, optional + Total hourly compute cost, mutually exclusive with + ``cost_per_vcore_hour``. + collect_stats_on_write : bool, optional + Whether Fabric extended Delta statistics are collected during writes. + compute_stats_all_cols : bool, optional + Deprecated alias for ``collect_stats_on_write``. + tblproperties : dict, optional + Delta table properties added to table creation statements. + + Notes + ----- + The engine uses the Fabric-attached Lakehouse as its catalog. Fabric write + statistics are configured at the Spark session level and replace a + separate analyze-after-load step. """ _FAST_OPTIMIZE_CONFIG = "spark.microsoft.delta.optimize.fast.enabled" diff --git a/src/lakebench/engines/hdi_spark.py b/src/lakebench/engines/hdi_spark.py index 416af26..1caf5dc 100644 --- a/src/lakebench/engines/hdi_spark.py +++ b/src/lakebench/engines/hdi_spark.py @@ -4,8 +4,27 @@ class HDISpark(Spark): - """ - HDInsight Spark Engine + """Execute LakeBench workloads on an Azure HDInsight Spark cluster. + + Attributes + ---------- + schema_name : str + Name of the Spark database used for benchmark tables. + spark_measure_telemetry : bool, optional + Whether sparkMeasure telemetry is collected for timed operations. + cost_per_vcore_hour : float, optional + Retail cost per vCore hour used for estimated job cost. + cost_per_hour : float, optional + Total hourly compute cost, mutually exclusive with + ``cost_per_vcore_hour``. + tblproperties : dict, optional + Delta table properties added to table creation statements. + + Notes + ----- + HDInsight uses the generic Spark loading and query implementation without + a separate catalog. Cost reporting can be configured as either a total + hourly rate or a per-vCore hourly rate. """ def __init__( diff --git a/src/lakebench/engines/polars.py b/src/lakebench/engines/polars.py index a096f91..d8cd9f9 100644 --- a/src/lakebench/engines/polars.py +++ b/src/lakebench/engines/polars.py @@ -9,8 +9,25 @@ class Polars(BaseEngine): - """ - Polars Engine + """Execute LakeBench workloads with the Polars SQL engine. + + Attributes + ---------- + schema_or_working_directory_uri : str + Base URI used to store Delta tables. + cost_per_vcore_hour : float, optional + Retail cost per vCore hour used for estimated job cost. + cost_per_hour : float, optional + Total hourly compute cost, mutually exclusive with + ``cost_per_vcore_hour``. + storage_options : dict, optional + Filesystem credentials and options passed to Delta Lake clients. + + Notes + ----- + Polars reads benchmark data into DataFrames and delegates Delta table + persistence to delta-rs. SQL is rendered with the DuckDB SQLGlot dialect + for compatibility with the Polars SQL surface. """ SQLGLOT_DIALECT = "duckdb" diff --git a/src/lakebench/engines/resources/__init__.py b/src/lakebench/engines/resources/__init__.py new file mode 100644 index 0000000..9d4d158 --- /dev/null +++ b/src/lakebench/engines/resources/__init__.py @@ -0,0 +1 @@ +"""Internal resources used by LakeBench engines.""" diff --git a/src/lakebench/engines/databricks_azure_dbu_map.py b/src/lakebench/engines/resources/databricks_azure_dbu_map.py similarity index 99% rename from src/lakebench/engines/databricks_azure_dbu_map.py rename to src/lakebench/engines/resources/databricks_azure_dbu_map.py index 251134c..3f24ea8 100644 --- a/src/lakebench/engines/databricks_azure_dbu_map.py +++ b/src/lakebench/engines/resources/databricks_azure_dbu_map.py @@ -1,4 +1,4 @@ -"""Generated Azure Databricks DBU consumption mappings. +"""Generated Azure Databricks DBU consumption mapping resource. Source: https://azure.microsoft.com/en-us/pricing/details/databricks/ Refresh with: uv run python scripts/refresh_databricks_azure_dbu_map.py diff --git a/src/lakebench/engines/databricks_pricing.py b/src/lakebench/engines/resources/databricks_pricing.py similarity index 91% rename from src/lakebench/engines/databricks_pricing.py rename to src/lakebench/engines/resources/databricks_pricing.py index 2b80040..c16785c 100644 --- a/src/lakebench/engines/databricks_pricing.py +++ b/src/lakebench/engines/resources/databricks_pricing.py @@ -204,8 +204,6 @@ def estimate_azure_spark_cost( worker_instance_type: str, worker_count: int, workload_type: str, - dbu_rate_override: Optional[float] = None, - infrastructure_cost_per_hour_override: Optional[float] = None, ) -> DatabricksPriceEstimate: driver_sku = ( driver_instance_type[len("Standard_") :] @@ -229,21 +227,10 @@ def estimate_azure_spark_cost( ) from exc dbus_per_hour = driver_dbus + (worker_dbus * worker_count) - if dbu_rate_override is not None: - dbu_rate = Decimal(str(dbu_rate_override)) - dbu_source = "explicit_dbu_rate" - else: - dbu_rate = get_azure_dbu_hourly_rate(region, workload_type) - dbu_source = "azure_retail_catalog_dbu" - - if infrastructure_cost_per_hour_override is not None: - infrastructure_cost = Decimal(str(infrastructure_cost_per_hour_override)) - infrastructure_source = "explicit_infrastructure_rate" - else: - driver_rate = get_azure_vm_hourly_rate(region, driver_instance_type) - worker_rate = get_azure_vm_hourly_rate(region, worker_instance_type) - infrastructure_cost = driver_rate + (worker_rate * worker_count) - infrastructure_source = "azure_retail_catalog_vm" + dbu_rate = get_azure_dbu_hourly_rate(region, workload_type) + driver_rate = get_azure_vm_hourly_rate(region, driver_instance_type) + worker_rate = get_azure_vm_hourly_rate(region, worker_instance_type) + infrastructure_cost = driver_rate + (worker_rate * worker_count) dbu_cost = dbus_per_hour * dbu_rate return DatabricksPriceEstimate( @@ -255,7 +242,7 @@ def estimate_azure_spark_cost( dbu_rate=dbu_rate, dbu_cost_per_hour=dbu_cost, infrastructure_cost_per_hour=infrastructure_cost, - pricing_source=f"{dbu_source}+{infrastructure_source}", + pricing_source="azure_retail_catalog_dbu+azure_retail_catalog_vm", ) diff --git a/src/lakebench/engines/sail.py b/src/lakebench/engines/sail.py index 589c879..4603acd 100644 --- a/src/lakebench/engines/sail.py +++ b/src/lakebench/engines/sail.py @@ -10,10 +10,25 @@ class Sail(BaseEngine): - """ - Sail Engine - - File system support: https://docs.lakesail.com/sail/main/guide/storage/ + """Execute LakeBench workloads through the Sail Spark Connect server. + + Attributes + ---------- + schema_or_working_directory_uri : str + Base URI used to store Delta tables. + cost_per_vcore_hour : float, optional + Retail cost per vCore hour used for estimated job cost. + cost_per_hour : float, optional + Total hourly compute cost, mutually exclusive with + ``cost_per_vcore_hour``. + storage_options : dict, optional + Filesystem credentials and options passed to Delta Lake clients. + + Notes + ----- + Sail starts a shared Spark Connect server and session on first use. Delta + writes use delta-rs because Sail subclasses ``BaseEngine`` rather than the + generic ``Spark`` engine. """ _SAIL_SERVER = None diff --git a/src/lakebench/engines/spark.py b/src/lakebench/engines/spark.py index 3ee9d0b..9c2e407 100644 --- a/src/lakebench/engines/spark.py +++ b/src/lakebench/engines/spark.py @@ -13,24 +13,23 @@ class Spark(BaseEngine): Attributes ---------- - SQLGLOT_DIALECT : str, optional - Specifies the SQL dialect to be used by the engine when SQL transpiling - is required. Default is None. - SUPPORTS_ONELAKE : bool - Indicates if the engine supports OneLake URIs (e.g., abfss://workspace@onelake.dfs.fabric.microsoft.com/...) - SUPPORTS_SCHEMA_PREP : bool - Indicates if the engine supports schema preparation (creation of empty table with defined schema) - SUPPORTS_MOUNT_PATH : bool - Indicates if the engine supports mount URIs (e.g., /mnt/...) - - Methods - ------- - get_total_cores() - Returns the total number of CPU cores available on the system. - get_compute_size() - Returns a formatted string with the compute size. - append_array_to_delta(abfss_path: str, array: list) - Appends a list of data to a Delta table at the specified path. + schema_name : str + Name of the Spark database used for benchmark tables. + catalog_name : str, optional + Catalog containing the benchmark schema. + schema_uri : str, optional + Storage location assigned to the Spark schema. + spark_measure_telemetry : bool, optional + Whether to collect sparkMeasure execution telemetry. + cost_per_vcore_hour : float, optional + Retail cost per vCore hour used for estimated job cost. + cost_per_hour : float, optional + Total hourly compute cost, mutually exclusive with + ``cost_per_vcore_hour``. + compute_stats_all_cols : bool, optional + Deprecated compatibility option for post-load column statistics. + tblproperties : dict, optional + Delta table properties added to table creation statements. """ SQLGLOT_DIALECT = "spark" diff --git a/src/lakebench/engines/synapse_spark.py b/src/lakebench/engines/synapse_spark.py index ec52ec8..588d622 100644 --- a/src/lakebench/engines/synapse_spark.py +++ b/src/lakebench/engines/synapse_spark.py @@ -5,8 +5,29 @@ class SynapseSpark(Spark): - """ - Synapse Spark Engine + """Execute LakeBench workloads in an Azure Synapse Spark pool. + + Attributes + ---------- + schema_name : str + Name of the Spark database used for benchmark tables. + schema_uri : str, optional + Storage location assigned to the Spark schema. + spark_measure_telemetry : bool, optional + Whether to collect sparkMeasure execution telemetry. + cost_per_vcore_hour : float, optional + Retail cost per vCore hour used for estimated job cost. + cost_per_hour : float, optional + Total hourly compute cost, mutually exclusive with + ``cost_per_vcore_hour``. + tblproperties : dict, optional + Delta table properties added to table creation statements. + + Notes + ----- + Construction is restricted to detected Synapse runtimes. When no manual + cost is supplied, the engine queries Azure retail pricing for the regional + Synapse vCore rate and records the resulting hourly estimate. """ def __init__( diff --git a/tests/test_databricks_engines.py b/tests/test_databricks_engines.py index 15b16c4..5abb1ba 100644 --- a/tests/test_databricks_engines.py +++ b/tests/test_databricks_engines.py @@ -7,8 +7,10 @@ import pytest from sqlglot import exp -from lakebench.engines.databricks_azure_dbu_map import AZURE_SPARK_DBU_MAP -from lakebench.engines.databricks_pricing import ( +from lakebench.engines.databricks_spark import DatabricksSpark +from lakebench.engines.databricks_sql_warehouse import DatabricksSQLWarehouse +from lakebench.engines.resources.databricks_azure_dbu_map import AZURE_SPARK_DBU_MAP +from lakebench.engines.resources.databricks_pricing import ( DatabricksPricingError, estimate_azure_spark_cost, estimate_sql_warehouse_cost, @@ -18,8 +20,6 @@ log_if_manual_compute_cost_missing, sql_warehouse_dbus_per_hour, ) -from lakebench.engines.databricks_spark import DatabricksSpark -from lakebench.engines.databricks_sql_warehouse import DatabricksSQLWarehouse @pytest.mark.parametrize( @@ -81,7 +81,7 @@ def test_serverless_sql_warehouse_estimate_uses_account_list_price(cloud_provide assert estimate.pricing_source == "databricks_system_billing_list_prices" -@patch("lakebench.engines.databricks_pricing.get_azure_serverless_sql_dbu_hourly_rate") +@patch("lakebench.engines.resources.databricks_pricing.get_azure_serverless_sql_dbu_hourly_rate") def test_azure_serverless_sql_warehouse_falls_back_to_retail_api(mock_retail_rate): mock_retail_rate.return_value = Decimal("0.70") @@ -99,7 +99,7 @@ def test_azure_serverless_sql_warehouse_falls_back_to_retail_api(mock_retail_rat mock_retail_rate.assert_called_once_with("eastus") -@patch("lakebench.engines.databricks_pricing._get_azure_retail_price") +@patch("lakebench.engines.resources.databricks_pricing._get_azure_retail_price") def test_azure_serverless_sql_rate_uses_exact_regional_meter( mock_retail_price, ): @@ -117,8 +117,8 @@ def test_azure_serverless_sql_rate_uses_exact_regional_meter( assert "type eq 'Consumption'" in query -@patch("lakebench.engines.databricks_pricing.get_azure_dbu_hourly_rate") -@patch("lakebench.engines.databricks_pricing.get_azure_vm_hourly_rate") +@patch("lakebench.engines.resources.databricks_pricing.get_azure_dbu_hourly_rate") +@patch("lakebench.engines.resources.databricks_pricing.get_azure_vm_hourly_rate") def test_azure_cluster_estimate_combines_mapped_dbus_and_infrastructure_cost( mock_vm_rate, mock_dbu_rate, @@ -156,7 +156,7 @@ def test_azure_dbu_mapping_distinguishes_workload_and_photon(): ("all-purpose-compute-with-photon", "Premium All-purpose Photon DBU"), ], ) -@patch("lakebench.engines.databricks_pricing._get_azure_retail_price") +@patch("lakebench.engines.resources.databricks_pricing._get_azure_retail_price") def test_azure_dbu_rate_uses_exact_workload_meter( mock_retail_price, workload_type, @@ -170,7 +170,10 @@ def test_azure_dbu_rate_uses_exact_workload_meter( @pytest.mark.parametrize("cloud_provider", ["aws", "gcp"]) def test_aws_and_gcp_log_when_manual_compute_cost_is_missing(cloud_provider, caplog): - with caplog.at_level(logging.WARNING, logger="lakebench.engines.databricks_pricing"): + with caplog.at_level( + logging.WARNING, + logger="lakebench.engines.resources.databricks_pricing", + ): log_if_manual_compute_cost_missing(cloud_provider, None, None) assert "Estimated job cost will not be reported" in caplog.text @@ -204,6 +207,8 @@ def test_databricks_spark_exposes_both_manual_cost_inputs(): parameters = inspect.signature(DatabricksSpark.__init__).parameters assert "cost_per_vcore_hour" in parameters assert "cost_per_hour" in parameters + assert "dbu_rate" not in parameters + assert "infrastructure_cost_per_hour" not in parameters def test_azure_cluster_estimate_explains_how_to_refresh_missing_sku(): @@ -294,7 +299,7 @@ def test_sql_warehouse_pricing_permission_failure_logs_without_crashing(caplog): ) stack.enter_context( patch( - "lakebench.engines.databricks_pricing.get_azure_serverless_sql_dbu_hourly_rate", + "lakebench.engines.resources.databricks_pricing.get_azure_serverless_sql_dbu_hourly_rate", return_value=Decimal("0.70"), ) ) diff --git a/tests/test_engine.py b/tests/test_engine.py index b036a1a..4f06f14 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -6,6 +6,7 @@ import pytest +import lakebench.engines as engines from lakebench.engines import ( Daft, DuckDB, @@ -20,6 +21,51 @@ from lakebench.engines.base import BaseEngine, MissingDependenciesError +def _documented_attributes(engine_class): + docstring = inspect.cleandoc(engine_class.__doc__ or "") + lines = docstring.splitlines() + if "Attributes" not in lines: + return {} + + start = lines.index("Attributes") + 2 + attributes = {} + for index in range(start, len(lines)): + line = lines[index] + if index + 1 < len(lines) and line and set(lines[index + 1]) == {"-"}: + break + if line and not line.startswith(" ") and " : " in line: + name, type_description = line.split(" : ", 1) + attributes[name] = type_description + return attributes + + +def test_public_engine_docstrings_only_document_constructor_inputs(): + engine_classes = { + name: value for name, value in vars(engines).items() if inspect.isclass(value) and issubclass(value, BaseEngine) + } + errors = [] + for name, engine_class in engine_classes.items(): + parameters = { + parameter_name: parameter + for parameter_name, parameter in inspect.signature(engine_class.__init__).parameters.items() + if parameter_name != "self" + and parameter.kind not in {inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD} + } + attributes = _documented_attributes(engine_class) + if set(attributes) != set(parameters): + errors.append( + f"{name}: documented attributes {sorted(attributes)} do not match " + f"constructor inputs {sorted(parameters)}" + ) + continue + + for parameter_name, parameter in parameters.items(): + if parameter.default is not inspect.Parameter.empty and "optional" not in attributes[parameter_name]: + errors.append(f"{name}.{parameter_name}: defaulted input must be marked optional") + + assert errors == [] + + class _MinimalEngine(BaseEngine): """Minimal concrete subclass for testing BaseEngine without optional deps.""" diff --git a/uv.lock b/uv.lock index 5a18e8c..b93292d 100644 --- a/uv.lock +++ b/uv.lock @@ -1409,11 +1409,6 @@ daft = [ { name = "deltalake", version = "1.5.1", source = { registry = "https://packagefeedproxy.microsoft.io/pypi/simple" }, marker = "python_full_version >= '3.10'" }, { name = "pyarrow", version = "21.0.0", source = { registry = "https://packagefeedproxy.microsoft.io/pypi/simple" }, marker = "python_full_version >= '3.10'" }, ] -databricks-spark = [ - { name = "pyspark", version = "3.5.8", source = { registry = "https://packagefeedproxy.microsoft.io/pypi/simple" }, marker = "python_full_version >= '3.9' or (extra == 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-duckdb') or (extra == 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-polars') or (extra == 'extra-9-lakebench-duckdb' and extra == 'extra-9-lakebench-fabric-data-warehouse') or (extra == 'extra-9-lakebench-duckdb' and extra == 'extra-9-lakebench-fabric-warehouse') or (extra == 'extra-9-lakebench-duckdb' and extra == 'extra-9-lakebench-fabricwarehouse') or (extra == 'extra-9-lakebench-fabric-data-warehouse' and extra == 'extra-9-lakebench-polars') or (extra == 'extra-9-lakebench-fabric-warehouse' and extra == 'extra-9-lakebench-polars') or (extra == 'extra-9-lakebench-fabricwarehouse' and extra == 'extra-9-lakebench-polars')" }, - { name = "requests", version = "2.32.5", source = { registry = "https://packagefeedproxy.microsoft.io/pypi/simple" }, marker = "(python_full_version == '3.9.*' and extra == 'extra-9-lakebench-databricks-spark') or (python_full_version != '3.9.*' and extra == 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-duckdb') or (python_full_version != '3.9.*' and extra == 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-polars') or (python_full_version != '3.9.*' and extra == 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-sail') or (extra != 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-sail') or (extra != 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-duckdb' and extra == 'extra-9-lakebench-fabric-data-warehouse') or (extra != 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-duckdb' and extra == 'extra-9-lakebench-fabric-warehouse') or (extra != 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-duckdb' and extra == 'extra-9-lakebench-fabricwarehouse') or (extra != 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-fabric-data-warehouse' and extra == 'extra-9-lakebench-polars') or (extra != 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-fabric-data-warehouse' and extra == 'extra-9-lakebench-sail') or (extra != 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-fabric-warehouse' and extra == 'extra-9-lakebench-polars') or (extra != 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-fabric-warehouse' and extra == 'extra-9-lakebench-sail') or (extra != 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-fabricwarehouse' and extra == 'extra-9-lakebench-polars') or (extra != 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-fabricwarehouse' and extra == 'extra-9-lakebench-sail') or (extra != 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-sail' and extra == 'extra-9-lakebench-spark') or (extra == 'extra-9-lakebench-daft' and extra != 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-duckdb') or (extra == 'extra-9-lakebench-daft' and extra != 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-polars') or (extra == 'extra-9-lakebench-daft' and extra != 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-sail') or (extra != 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-duckdb' and extra == 'extra-9-lakebench-fabric-data-warehouse') or (extra != 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-duckdb' and extra == 'extra-9-lakebench-fabric-warehouse') or (extra != 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-duckdb' and extra == 'extra-9-lakebench-fabricwarehouse') or (extra != 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-fabric-data-warehouse' and extra == 'extra-9-lakebench-polars') or (extra != 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-fabric-data-warehouse' and extra == 'extra-9-lakebench-sail') or (extra != 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-fabric-warehouse' and extra == 'extra-9-lakebench-polars') or (extra != 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-fabric-warehouse' and extra == 'extra-9-lakebench-sail') or (extra != 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-fabricwarehouse' and extra == 'extra-9-lakebench-polars') or (extra != 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-fabricwarehouse' and extra == 'extra-9-lakebench-sail') or (extra != 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-sail' and extra == 'extra-9-lakebench-spark')" }, - { name = "requests", version = "2.34.2", source = { registry = "https://packagefeedproxy.microsoft.io/pypi/simple" }, marker = "(python_full_version >= '3.10' and extra == 'extra-9-lakebench-databricks-spark') or (python_full_version < '3.10' and extra == 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-duckdb') or (python_full_version < '3.10' and extra == 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-polars') or (python_full_version < '3.10' and extra == 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-sail') or (extra != 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-sail') or (extra != 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-duckdb' and extra == 'extra-9-lakebench-fabric-data-warehouse') or (extra != 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-duckdb' and extra == 'extra-9-lakebench-fabric-warehouse') or (extra != 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-duckdb' and extra == 'extra-9-lakebench-fabricwarehouse') or (extra != 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-fabric-data-warehouse' and extra == 'extra-9-lakebench-polars') or (extra != 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-fabric-data-warehouse' and extra == 'extra-9-lakebench-sail') or (extra != 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-fabric-warehouse' and extra == 'extra-9-lakebench-polars') or (extra != 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-fabric-warehouse' and extra == 'extra-9-lakebench-sail') or (extra != 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-fabricwarehouse' and extra == 'extra-9-lakebench-polars') or (extra != 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-fabricwarehouse' and extra == 'extra-9-lakebench-sail') or (extra != 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-sail' and extra == 'extra-9-lakebench-spark') or (extra == 'extra-9-lakebench-daft' and extra != 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-duckdb') or (extra == 'extra-9-lakebench-daft' and extra != 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-polars') or (extra == 'extra-9-lakebench-daft' and extra != 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-sail') or (extra != 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-duckdb' and extra == 'extra-9-lakebench-fabric-data-warehouse') or (extra != 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-duckdb' and extra == 'extra-9-lakebench-fabric-warehouse') or (extra != 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-duckdb' and extra == 'extra-9-lakebench-fabricwarehouse') or (extra != 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-fabric-data-warehouse' and extra == 'extra-9-lakebench-polars') or (extra != 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-fabric-data-warehouse' and extra == 'extra-9-lakebench-sail') or (extra != 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-fabric-warehouse' and extra == 'extra-9-lakebench-polars') or (extra != 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-fabric-warehouse' and extra == 'extra-9-lakebench-sail') or (extra != 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-fabricwarehouse' and extra == 'extra-9-lakebench-polars') or (extra != 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-fabricwarehouse' and extra == 'extra-9-lakebench-sail') or (extra != 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-sail' and extra == 'extra-9-lakebench-spark')" }, -] databricks-sql-warehouse = [ { name = "databricks-sql-connector", version = "4.3.0", source = { registry = "https://packagefeedproxy.microsoft.io/pypi/simple" }, marker = "python_full_version == '3.9.*' or (extra == 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-duckdb') or (extra == 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-polars') or (extra == 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-sail') or (extra == 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-sail') or (extra == 'extra-9-lakebench-duckdb' and extra == 'extra-9-lakebench-fabric-data-warehouse') or (extra == 'extra-9-lakebench-duckdb' and extra == 'extra-9-lakebench-fabric-warehouse') or (extra == 'extra-9-lakebench-duckdb' and extra == 'extra-9-lakebench-fabricwarehouse') or (extra == 'extra-9-lakebench-fabric-data-warehouse' and extra == 'extra-9-lakebench-polars') or (extra == 'extra-9-lakebench-fabric-data-warehouse' and extra == 'extra-9-lakebench-sail') or (extra == 'extra-9-lakebench-fabric-warehouse' and extra == 'extra-9-lakebench-polars') or (extra == 'extra-9-lakebench-fabric-warehouse' and extra == 'extra-9-lakebench-sail') or (extra == 'extra-9-lakebench-fabricwarehouse' and extra == 'extra-9-lakebench-polars') or (extra == 'extra-9-lakebench-fabricwarehouse' and extra == 'extra-9-lakebench-sail') or (extra == 'extra-9-lakebench-sail' and extra == 'extra-9-lakebench-spark')" }, { name = "databricks-sql-connector", version = "4.6.0", source = { registry = "https://packagefeedproxy.microsoft.io/pypi/simple" }, marker = "python_full_version >= '3.10' or (extra == 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-duckdb') or (extra == 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-polars') or (extra == 'extra-9-lakebench-daft' and extra == 'extra-9-lakebench-sail') or (extra == 'extra-9-lakebench-databricks-spark' and extra == 'extra-9-lakebench-sail') or (extra == 'extra-9-lakebench-duckdb' and extra == 'extra-9-lakebench-fabric-data-warehouse') or (extra == 'extra-9-lakebench-duckdb' and extra == 'extra-9-lakebench-fabric-warehouse') or (extra == 'extra-9-lakebench-duckdb' and extra == 'extra-9-lakebench-fabricwarehouse') or (extra == 'extra-9-lakebench-fabric-data-warehouse' and extra == 'extra-9-lakebench-polars') or (extra == 'extra-9-lakebench-fabric-data-warehouse' and extra == 'extra-9-lakebench-sail') or (extra == 'extra-9-lakebench-fabric-warehouse' and extra == 'extra-9-lakebench-polars') or (extra == 'extra-9-lakebench-fabric-warehouse' and extra == 'extra-9-lakebench-sail') or (extra == 'extra-9-lakebench-fabricwarehouse' and extra == 'extra-9-lakebench-polars') or (extra == 'extra-9-lakebench-fabricwarehouse' and extra == 'extra-9-lakebench-sail') or (extra == 'extra-9-lakebench-sail' and extra == 'extra-9-lakebench-spark')" }, @@ -1518,10 +1513,8 @@ requires-dist = [ { name = "pyarrow", marker = "python_full_version >= '3.10' and extra == 'sail'", specifier = ">=16.0.0" }, { name = "pyodbc", marker = "python_full_version >= '3.9' and extra == 'fabric-data-warehouse'", specifier = ">=5.0.0" }, { name = "pysail", marker = "python_full_version >= '3.10' and extra == 'sail'", specifier = ">=0.7.1" }, - { name = "pyspark", marker = "python_full_version >= '3.9' and extra == 'databricks-spark'", specifier = ">=3.5.0,<4.0.0" }, { name = "pyspark", marker = "python_full_version >= '3.9' and extra == 'spark'", specifier = ">=3.5.0,<4.0.0" }, { name = "pyspark", extras = ["connect"], marker = "python_full_version >= '3.10' and extra == 'sail'", specifier = ">=4.0.0" }, - { name = "requests", marker = "python_full_version >= '3.9' and extra == 'databricks-spark'", specifier = ">=2.28.0" }, { name = "requests", marker = "python_full_version >= '3.9' and extra == 'databricks-sql-warehouse'", specifier = ">=2.28.0" }, { name = "requests", marker = "extra == 'fabric-data-warehouse'", specifier = ">=2.28.0" }, { name = "sparkmeasure", marker = "python_full_version >= '3.9' and extra == 'sparkmeasure'", specifier = "==0.28.0" }, @@ -1531,7 +1524,7 @@ requires-dist = [ { name = "tenacity", marker = "python_full_version < '3.9'", specifier = ">=8.2.3,<9" }, { name = "tenacity", marker = "python_full_version >= '3.9'", specifier = "==9.1.2" }, ] -provides-extras = ["duckdb", "polars", "daft", "tpcds-datagen", "tpch-datagen", "tpcds-duckdb-datagen", "sparkmeasure", "spark", "databricks-spark", "databricks-sql-warehouse", "sail", "fabric-data-warehouse", "fabric-warehouse", "fabricwarehouse"] +provides-extras = ["duckdb", "polars", "daft", "tpcds-datagen", "tpch-datagen", "tpcds-duckdb-datagen", "sparkmeasure", "spark", "databricks-sql-warehouse", "sail", "fabric-data-warehouse", "fabric-warehouse", "fabricwarehouse"] [package.metadata.requires-dev] dev = [ From 4bb7b06bf2d49a610e11fabd2a2d858c8be80f1d Mon Sep 17 00:00:00 2001 From: Miles Cole Date: Wed, 7 Oct 2026 13:56:21 -0600 Subject: [PATCH 2/2] chore: bump version --- pyproject.toml | 2 +- uv.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 44389a3..3dba671 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "lakebench" -version = "2.1.1" +version = "2.2.0" authors = [ { name="Miles Cole" }, ] diff --git a/uv.lock b/uv.lock index b93292d..ac0f882 100644 --- a/uv.lock +++ b/uv.lock @@ -864,8 +864,8 @@ wheels = [ { url = "https://ms-feed-25.pkgs.visualstudio.com/6f084628-a36d-42cb-934d-057357e379dc/_packaging/49d7402f-07bb-4b18-a9ce-086e6e98a554/pypi/download/deltalake/1.6.2/deltalake-1.6.2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:6dd5efcf165c5418a02afe6de4b9d88c268c6c4eae2a4c38d40eab72bc69c8ee", size = 44734772, upload-time = "2026-08-05T09:42:12Z" }, { url = "https://ms-feed-25.pkgs.visualstudio.com/6f084628-a36d-42cb-934d-057357e379dc/_packaging/49d7402f-07bb-4b18-a9ce-086e6e98a554/pypi/download/deltalake/1.6.2/deltalake-1.6.2-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00ca11b3081345a922173efc5e6582edf2019554f184d9a1e961e4f888974dae", size = 49591746, upload-time = "2026-08-04T13:33:51Z" }, { url = "https://ms-feed-25.pkgs.visualstudio.com/6f084628-a36d-42cb-934d-057357e379dc/_packaging/49d7402f-07bb-4b18-a9ce-086e6e98a554/pypi/download/deltalake/1.6.2/deltalake-1.6.2-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:46caeee8ea14b89975305d6d36abef99246f7ff7b1fc1194b9395d92f6db3533", size = 48810511, upload-time = "2026-08-12T14:19:06Z" }, - { url = "https://ms-feed-25.pkgs.visualstudio.com/6f084628-a36d-42cb-934d-057357e379dc/_packaging/49d7402f-07bb-4b18-a9ce-086e6e98a554/pypi/download/deltalake/1.6.2/deltalake-1.6.2-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:bec019bbe82b06d0947758342849978dc3167d374edf99c8ae8af0ae4bf3ac6b", size = 48779586, upload-time = "2026-07-08T13:21:27Z" }, - { url = "https://ms-feed-25.pkgs.visualstudio.com/6f084628-a36d-42cb-934d-057357e379dc/_packaging/49d7402f-07bb-4b18-a9ce-086e6e98a554/pypi/download/deltalake/1.6.2/deltalake-1.6.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e5bf6d4ba40023e45e7b3eb2f948bcd648af7cc38463f8b60af0bf5f6278231c", size = 49550955, upload-time = "2026-07-08T13:35:38Z" }, + { url = "https://ms-feed-25.pkgs.visualstudio.com/6f084628-a36d-42cb-934d-057357e379dc/_packaging/49d7402f-07bb-4b18-a9ce-086e6e98a554/pypi/download/deltalake/1.6.2/deltalake-1.6.2-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:bec019bbe82b06d0947758342849978dc3167d374edf99c8ae8af0ae4bf3ac6b", size = 48779586, upload-time = "2026-10-07T17:59:04Z" }, + { url = "https://ms-feed-25.pkgs.visualstudio.com/6f084628-a36d-42cb-934d-057357e379dc/_packaging/49d7402f-07bb-4b18-a9ce-086e6e98a554/pypi/download/deltalake/1.6.2/deltalake-1.6.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e5bf6d4ba40023e45e7b3eb2f948bcd648af7cc38463f8b60af0bf5f6278231c", size = 49550955, upload-time = "2026-10-07T18:00:09Z" }, { url = "https://ms-feed-25.pkgs.visualstudio.com/6f084628-a36d-42cb-934d-057357e379dc/_packaging/49d7402f-07bb-4b18-a9ce-086e6e98a554/pypi/download/deltalake/1.6.2/deltalake-1.6.2-cp310-abi3-win_amd64.whl", hash = "sha256:235c06c1a0041afb1a52d98672d67a2fe6a38813bbea889335ff1eea6dbea5b5", size = 52640891, upload-time = "2026-07-29T21:56:55Z" }, ] @@ -1387,7 +1387,7 @@ wheels = [ [[package]] name = "lakebench" -version = "2.1.1" +version = "2.2.0" source = { editable = "." } dependencies = [ { name = "fsspec" },