Merge/v1.5 variegata into main

.github/workflows/packaging_wheels.yml

@@ -32,7 +32,7 @@ jobs:
       matrix:
         python: [ cp314 ]
         platform:
-          - { os: windows-2025,     arch: amd64,      cibw_system: win }
+          - { os: windows-2022,     arch: amd64,      cibw_system: win }
           - { os: windows-11-arm,   arch: ARM64,      cibw_system: win }
           - { os: ubuntu-24.04,     arch: x86_64,     cibw_system: manylinux }
           - { os: ubuntu-24.04-arm, arch: aarch64,    cibw_system: manylinux }

.github/workflows/targeted_test.yml

@@ -8,7 +8,7 @@ on:
         required: true
         type: choice
         options:
-          - 'windows-2025'
+          - 'windows-2022'
           - 'windows-11-arm'
           - 'ubuntu-24.04'
           - 'ubuntu-24.04-arm'

src/duckdb_py/arrow/arrow_array_stream.cpp

@@ -14,11 +14,11 @@
 
 namespace duckdb {
 
-void TransformDuckToArrowChunk(ArrowSchema &arrow_schema, ArrowArray &data, py::list &batches) {
+void TransformDuckToArrowChunk(py::object pyarrow_schema, ArrowArray &data, py::list &batches) {
 	py::gil_assert();
 	auto pyarrow_lib_module = py::module::import("pyarrow").attr("lib");
 	auto batch_import_func = pyarrow_lib_module.attr("RecordBatch").attr("_import_from_c");
-	batches.append(batch_import_func(reinterpret_cast<uint64_t>(&data), reinterpret_cast<uint64_t>(&arrow_schema)));
+	batches.append(batch_import_func(reinterpret_cast<uint64_t>(&data), pyarrow_schema));
 }
 
 void VerifyArrowDatasetLoaded() {

src/duckdb_py/arrow/arrow_export_utils.cpp

@@ -17,18 +17,28 @@ namespace duckdb {
 
 namespace pyarrow {
 
-py::object ToArrowTable(const vector<LogicalType> &types, const vector<string> &names, const py::list &batches,
-                        ClientProperties &options) {
+py::object ToPyArrowSchema(const ArrowSchema &schema) {
 	py::gil_scoped_acquire acquire;
 
 	auto pyarrow_lib_module = py::module::import("pyarrow").attr("lib");
-	auto from_batches_func = pyarrow_lib_module.attr("Table").attr("from_batches");
 	auto schema_import_func = pyarrow_lib_module.attr("Schema").attr("_import_from_c");
+	return schema_import_func(reinterpret_cast<uint64_t>(&schema));
+}
+
+py::object ToArrowTable(const py::list &batches, py::object pyarrow_schema) {
+	py::gil_scoped_acquire acquire;
+
+	auto pyarrow_lib_module = py::module::import("pyarrow").attr("lib");
+	auto from_batches_func = pyarrow_lib_module.attr("Table").attr("from_batches");
+
+	return py::cast<duckdb::pyarrow::Table>(from_batches_func(batches, pyarrow_schema));
+}
+
+py::object ToArrowTable(const vector<LogicalType> &types, const vector<string> &names, const py::list &batches,
+                        ClientProperties &options) {
 	ArrowSchema schema;
 	ArrowConverter::ToArrowSchema(&schema, types, names, options);
-	auto schema_obj = schema_import_func(reinterpret_cast<uint64_t>(&schema));
-
-	return py::cast<duckdb::pyarrow::Table>(from_batches_func(batches, schema_obj));
+	return ToArrowTable(batches, ToPyArrowSchema(schema));
 }
 
 } // namespace pyarrow

src/duckdb_py/include/duckdb_python/arrow/arrow_array_stream.hpp

@@ -62,7 +62,7 @@ enum class PyArrowObjectType {
 	PolarsLazyFrame
 };
 
-void TransformDuckToArrowChunk(ArrowSchema &arrow_schema, ArrowArray &data, py::list &batches);
+void TransformDuckToArrowChunk(py::object pyarrow_schema, ArrowArray &data, py::list &batches);
 
 PyArrowObjectType GetArrowType(const py::handle &obj);
 

src/duckdb_py/include/duckdb_python/arrow/arrow_export_utils.hpp

@@ -6,9 +6,13 @@ namespace duckdb {
 
 namespace pyarrow {
 
+py::object ToPyArrowSchema(const ArrowSchema &schema);
+
 py::object ToArrowTable(const vector<LogicalType> &types, const vector<string> &names, const py::list &batches,
                         ClientProperties &options);
 
+py::object ToArrowTable(const py::list &batches, py::object pyarrow_schema);
+
 } // namespace pyarrow
 
 } // namespace duckdb

src/duckdb_py/include/duckdb_python/pyresult.hpp

@@ -36,15 +36,13 @@ struct DuckDBPyResult {
 
 	PandasDataFrame FetchDF(bool date_as_object);
 
-	duckdb::pyarrow::Table FetchArrowTable(idx_t rows_per_batch, bool to_polars);
-
 	PandasDataFrame FetchDFChunk(const idx_t vectors_per_chunk = 1, bool date_as_object = false);
 
 	py::dict FetchPyTorch();
 
 	py::dict FetchTF();
 
-	ArrowArrayStream FetchArrowArrayStream(idx_t rows_per_batch = 1000000);
+	duckdb::pyarrow::Table FetchArrowTable(idx_t rows_per_batch, bool to_polars);
 	duckdb::pyarrow::RecordBatchReader FetchRecordBatchReader(idx_t rows_per_batch = 1000000);
 	py::object FetchArrowCapsule(idx_t rows_per_batch = 1000000);
 
@@ -71,6 +69,19 @@ struct DuckDBPyResult {
 	unique_ptr<DataChunk> FetchNextRaw(QueryResult &result);
 	unique_ptr<NumpyResultConversion> InitializeNumpyConversion(bool pandas = false);
 
+	//! Re-feed an already-MATERIALIZED result (a ColumnDataCollection, e.g. from
+	//! rel.execute()) back through the engine on the user's own context. The eager
+	//! variant installs a PhysicalArrowCollector to produce an ArrowQueryResult
+	//! (parallel); the stream variant produces a lazy StreamQueryResult that co-owns
+	//! the context (so it survives `del conn`). Never call these on a StreamQueryResult:
+	//! a lazy result already has a live context and is converted/wrapped directly.
+	void PromoteMaterializedToArrow(idx_t batch_size);
+
+	template <typename T>
+	T RunWithArrowSchema(const std::function<T(const ArrowSchema &)> &fun, bool dedup_col_names);
+	duckdb::pyarrow::Table MaterializedResultToArrowTable(const ArrowSchema &arrow_schema, idx_t rows_per_batch);
+	ArrowArrayStream FetchArrowArrayStream(idx_t rows_per_batch);
+
 private:
 	idx_t chunk_offset = 0;
 

src/duckdb_py/pyconnection.cpp

@@ -413,7 +413,7 @@ shared_ptr<DuckDBPyConnection> DuckDBPyConnection::UnregisterUDF(const string &n
 		auto &catalog = Catalog::GetCatalog(context, SYSTEM_CATALOG);
 		DropInfo info;
 		info.type = CatalogType::SCALAR_FUNCTION_ENTRY;
-		info.name = Identifier(name);
+		info.NameMutable() = Identifier(name);
 		info.allow_drop_internal = true;
 		info.cascade = false;
 		info.if_not_found = OnEntryNotFound::THROW_EXCEPTION;
@@ -1651,11 +1651,12 @@ unique_ptr<DuckDBPyRelation> DuckDBPyConnection::RunQuery(const py::object &quer
 unique_ptr<DuckDBPyRelation> DuckDBPyConnection::Table(const string &tname) {
 	auto &connection = con.GetConnection();
 	auto qualified_name = QualifiedName::Parse(tname);
-	if (qualified_name.schema.empty()) {
-		qualified_name.schema = DEFAULT_SCHEMA;
+	if (qualified_name.Schema().empty()) {
+		qualified_name.SchemaMutable() = DEFAULT_SCHEMA;
 	}
 	try {
-		return CreateRelation(connection.Table(qualified_name.catalog, qualified_name.schema, qualified_name.name));
+		return CreateRelation(
+		    connection.Table(qualified_name.Catalog(), qualified_name.Schema(), qualified_name.Name()));
 	} catch (const CatalogException &) {
 		// CatalogException will be of the type '... is not a table'
 		// Not a table in the database, make a query relation that can perform replacement scans

src/duckdb_py/pyexpression.cpp

@@ -325,13 +325,13 @@ shared_ptr<DuckDBPyExpression> DuckDBPyExpression::ColumnExpression(const py::ar
 		}
 
 		auto qualified_name = QualifiedName::Parse(column_name);
-		if (!qualified_name.catalog.empty()) {
-			column_names.push_back(qualified_name.catalog);
+		if (!qualified_name.Catalog().empty()) {
+			column_names.push_back(qualified_name.Catalog());
 		}
-		if (!qualified_name.schema.empty()) {
-			column_names.push_back(qualified_name.schema);
+		if (!qualified_name.Schema().empty()) {
+			column_names.push_back(qualified_name.Schema());
 		}
-		column_names.push_back(qualified_name.name);
+		column_names.push_back(qualified_name.Name());
 	} else {
 		for (auto &part : names) {
 			column_names.push_back(Identifier(py::str(part)));

src/duckdb_py/pyrelation.cpp

@@ -960,11 +960,11 @@ PandasDataFrame DuckDBPyRelation::FetchDFChunk(idx_t vectors_per_chunk, bool dat
 	return result->FetchDFChunk(vectors_per_chunk, date_as_object);
 }
 
-duckdb::pyarrow::Table DuckDBPyRelation::ToArrowTableInternal(idx_t batch_size, bool to_polars) {
+pyarrow::Table DuckDBPyRelation::ToArrowTableInternal(idx_t batch_size, bool to_polars) {
+	if (!result && !rel) {
+		return py::none();
+	}
 	if (!result) {
-		if (!rel) {
-			return py::none();
-		}
 		auto &config = ClientConfig::GetConfig(*rel->context->GetContext());
 		ScopedConfigSetting scoped_setting(
 		    config,
@@ -991,19 +991,9 @@ py::object DuckDBPyRelation::ToArrowCapsule(const py::object &requested_schema)
 		if (!rel) {
 			return py::none();
 		}
-		// The PyCapsule protocol doesn't allow custom parameters, so we use the same
-		// default batch size as fetch_arrow_table / fetch_record_batch.
-		idx_t batch_size = 1000000;
-		auto &config = ClientConfig::GetConfig(*rel->context->GetContext());
-		ScopedConfigSetting scoped_setting(
-		    config,
-		    [&batch_size](ClientConfig &config) {
-			    config.get_result_collector = [&batch_size](ClientContext &context, PreparedStatementData &data) {
-				    return PhysicalArrowCollector::Create(context, data, batch_size);
-			    };
-		    },
-		    [](ClientConfig &config) { config.get_result_collector = nullptr; });
-		ExecuteOrThrow();
+		// Fresh relation: stream lazily on the user's context (capsule survives `del conn`,
+		// but shares the single active-stream slot - consume before reusing the connection).
+		ExecuteOrThrow(true);
 	}
 	AssertResultOpen();
 	auto res = result->FetchArrowCapsule();
@@ -1049,6 +1039,7 @@ duckdb::pyarrow::RecordBatchReader DuckDBPyRelation::ToRecordBatch(idx_t batch_s
 		if (!rel) {
 			return py::none();
 		}
+		// Fresh relation: stream lazily on the user's own context (survives `del conn`).
 		ExecuteOrThrow(true);
 	}
 	AssertResultOpen();
@@ -1590,7 +1581,7 @@ DuckDBPyRelation &DuckDBPyRelation::Execute() {
 void DuckDBPyRelation::InsertInto(const string &table) {
 	AssertRelation();
 	auto parsed_info = QualifiedName::Parse(table);
-	auto insert = rel->InsertRel(parsed_info.catalog, parsed_info.schema, parsed_info.name);
+	auto insert = rel->InsertRel(parsed_info.Catalog(), parsed_info.Schema(), parsed_info.Name());
 	PyExecuteRelation(insert);
 }
 
@@ -1654,7 +1645,7 @@ void DuckDBPyRelation::Insert(const py::object &params) const {
 void DuckDBPyRelation::Create(const string &table) {
 	AssertRelation();
 	auto parsed_info = QualifiedName::Parse(table);
-	auto create = rel->CreateRel(parsed_info.schema, parsed_info.name, false);
+	auto create = rel->CreateRel(parsed_info.Schema(), parsed_info.Name(), false);
 	PyExecuteRelation(create);
 }
 
@@ -1756,7 +1747,7 @@ string DuckDBPyRelation::Explain(ExplainType type, const string &format) {
 
 	// An empty format means "auto": the default format, or HTML when running under Jupyter.
 	const bool auto_format = format.empty();
-	auto explain_format = auto_format ? GetExplainFormat(type) : ProfilerPrintFormat::FromString(format);
+	auto explain_format = auto_format ? GetExplainFormat(type) : ProfilerPrintFormat(format);
 	auto res = rel->Explain(type, explain_format);
 	D_ASSERT(res->type == duckdb::QueryResultType::MATERIALIZED_RESULT);
 	auto &materialized = res->Cast<MaterializedQueryResult>();