diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4f006d4a..581c5ca4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -93,9 +93,10 @@ when the database runs in a container. For local Oracle runs, add `-- --test-threads=2` to avoid overwhelming the listener. Check affected matrix results for SQL execution changes; SQLite alone cannot establish portability. -ODBC databases can also run on sqlx 0.9, through the [sqlx-odbc](https://github.com/lovasoa/sqlx-odbc) driver. This is the unstable migration path for [#1056](https://github.com/sqlpage/SQLPage/issues/1056), and releases do not include it. To use it, build with the `unstable-sqlx-next` feature and set `SQLPAGE_UNSTABLE_SQLX_NEXT=true`. With this driver, migrations, `sqlpage_files` and CSV imports stop with an error that names the missing feature. +SQL Server and ODBC databases can also run on sqlx 0.9, through the [sqlx-sqlserver](https://github.com/lovasoa/sqlx-sqlserver) and [sqlx-odbc](https://github.com/lovasoa/sqlx-odbc) drivers. This is the unstable migration path for [#1056](https://github.com/sqlpage/SQLPage/issues/1056), and releases do not include it. To use it, build with the `unstable-sqlx-next` feature and set `SQLPAGE_UNSTABLE_SQLX_NEXT=true`. The feature needs OpenSSL, which the Nix shell provides. With these drivers, migrations, `sqlpage_files` and CSV imports stop with an error that names the missing feature. ```bash +SQLPAGE_UNSTABLE_SQLX_NEXT=true DATABASE_URL='mssql://root:Password123!@localhost/sqlpage' cargo test --features unstable-sqlx-next SQLPAGE_UNSTABLE_SQLX_NEXT=true DATABASE_URL='Driver=DuckDB;Database=/tmp/sqlpage-test.duckdb' cargo test --features unstable-sqlx-next ``` diff --git a/Cargo.lock b/Cargo.lock index d9924b16..a2c36b04 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1057,7 +1057,7 @@ dependencies = [ "bitflags 1.3.2", "core-foundation 0.9.4", "core-graphics-types", - "foreign-types", + "foreign-types 0.5.0", "libc", ] @@ -1639,6 +1639,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "etcetera" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de48cc4d1c1d97a20fd819def54b890cadde72ed3ad0c614822a0a433361be96" +dependencies = [ + "cfg-if", + "windows-sys 0.61.2", +] + [[package]] name = "event-listener" version = "5.4.2" @@ -1721,6 +1731,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared 0.1.1", +] + [[package]] name = "foreign-types" version = "0.5.0" @@ -1728,7 +1747,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" dependencies = [ "foreign-types-macros", - "foreign-types-shared", + "foreign-types-shared 0.3.1", ] [[package]] @@ -1742,6 +1761,12 @@ dependencies = [ "syn 3.0.10", ] +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "foreign-types-shared" version = "0.3.1" @@ -2815,6 +2840,23 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e94e1e6445d314f972ff7395df2de295fe51b71821694f0b0e1e79c4f12c8577" +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "ndk" version = "0.9.0" @@ -3296,12 +3338,49 @@ dependencies = [ "url", ] +[[package]] +name = "openssl" +version = "0.10.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4de8111e94a56adca7ebfe6c365b6d42f99deb69b74b6becacd4204e2f193aa2" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "foreign-types 0.3.2", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "openssl-probe" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e036b411754132fd6b74e94b74e9a13c5cf606fa93d1885c35860d3060820b1" +[[package]] +name = "openssl-sys" +version = "0.9.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19c413d6ec9ebaf5e1c829d355cb3e9f783778d700f135be0921d52fce3c60ae" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "opentelemetry" version = "0.33.1" @@ -4648,6 +4727,7 @@ dependencies = [ "sqlx", "sqlx-core-oldapi", "sqlx-odbc", + "sqlx-sqlserver", "tempfile", "tokio", "tokio-stream", @@ -4688,6 +4768,9 @@ checksum = "378620ccc25c62c89d8be1c819e76a88d59bdcc3304733330788948e619bfd71" dependencies = [ "sqlx-core", "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", ] [[package]] @@ -4700,6 +4783,7 @@ dependencies = [ "bigdecimal", "bytes", "cfg-if", + "chrono", "crc", "crossbeam-queue", "either", @@ -4715,6 +4799,7 @@ dependencies = [ "memchr", "percent-encoding", "serde", + "serde_json", "sha2 0.10.9", "smallvec", "thiserror 2.0.21", @@ -4722,6 +4807,7 @@ dependencies = [ "tokio-stream", "tracing", "url", + "uuid", ] [[package]] @@ -4817,11 +4903,41 @@ dependencies = [ "serde_json", "sha2 0.10.9", "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", "syn 2.0.119", "tokio", "url", ] +[[package]] +name = "sqlx-mysql" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90b8020fe17c5f2c245bfa2505d7ef59c5604839527c740266ad2214acebea27" +dependencies = [ + "bitflags 2.13.2", + "byteorder", + "bytes", + "crc", + "digest 0.11.3", + "dotenvy", + "either", + "futures-core", + "futures-util", + "generic-array", + "log", + "percent-encoding", + "serde", + "sha1 0.11.0", + "sha2 0.11.1", + "sqlx-core", + "thiserror 2.0.21", + "tracing", + "uuid", +] + [[package]] name = "sqlx-odbc" version = "0.0.3" @@ -4839,6 +4955,42 @@ dependencies = [ "url", ] +[[package]] +name = "sqlx-postgres" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87a2bdd6e83f6b3ea525ca9fee568030508b58355a43d0b2c1674d5f79dcd65e" +dependencies = [ + "atoi 2.0.0", + "base64 0.22.1", + "bitflags 2.13.2", + "byteorder", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf 0.13.0", + "hmac 0.13.0", + "itoa", + "log", + "md-5", + "memchr", + "rand 0.10.3", + "serde", + "serde_json", + "sha2 0.11.1", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.21", + "tracing", + "uuid", + "whoami", +] + [[package]] name = "sqlx-rt-oldapi" version = "0.6.59" @@ -4850,6 +5002,53 @@ dependencies = [ "tokio-rustls", ] +[[package]] +name = "sqlx-sqlite" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488e99c397a62007e4229aec669a179816339afc6d2620ca6fa420dbee2e982c" +dependencies = [ + "atoi 2.0.0", + "flume", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "sqlx-core", + "thiserror 2.0.21", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "sqlx-sqlserver" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d07a49eceb2cfa3ab6ae42765cd54063201b4120667d6da9ff58ec5e2cbf3301" +dependencies = [ + "bigdecimal", + "chrono", + "futures-core", + "futures-util", + "log", + "native-tls", + "num-bigint", + "percent-encoding", + "sqlx-core", + "thiserror 2.0.21", + "tokio", + "tokio-native-tls", + "url", + "uuid", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -5078,6 +5277,16 @@ dependencies = [ "syn 3.0.10", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.6" diff --git a/Cargo.toml b/Cargo.toml index 7e18d6ce..ae3c6f9a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -112,8 +112,9 @@ tokio-util = "0.7.12" openidconnect = { version = "4.0.0", default-features = false, features = ["accept-rfc3339-timestamps"] } encoding_rs = "0.8.35" odbc-sys = { version = "0", optional = true } -sqlx-next = { package = "sqlx", version = "0.9", default-features = false, features = ["runtime-tokio"], optional = true } +sqlx-next = { package = "sqlx", version = "0.9", default-features = false, features = ["runtime-tokio", "uuid"], optional = true } sqlx-odbc = { version = "0.0.3", default-features = false, features = ["bigdecimal", "chrono"], optional = true } +sqlx-sqlserver = { version = "0.0.3", default-features = false, features = ["bigdecimal", "chrono", "migrate", "uuid"], optional = true } regex = "1" lettre = { version = "0.11", default-features = false, features = [ "aws-lc-rs", @@ -139,7 +140,7 @@ opentelemetry-semantic-conventions = { version = "0.33", features = ["semconv_ex [features] default = [] -unstable-sqlx-next = ["dep:sqlx-next", "dep:sqlx-odbc"] +unstable-sqlx-next = ["dep:sqlx-next", "dep:sqlx-odbc", "dep:sqlx-sqlserver"] odbc-static = ["odbc-sys", "odbc-sys/vendored-unix-odbc"] lambda-web = [ "dep:actix-http", diff --git a/flake.nix b/flake.nix index 62d37a25..94420327 100644 --- a/flake.nix +++ b/flake.nix @@ -95,9 +95,10 @@ rustToolchainFor = pkgs: pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; # odbc-sys links against a driver manager, and the C sources vendored by - # aws-lc-sys, libsqlite3-sys and zstd-sys need a compiler. Nothing here - # reaches for OpenSSL: SQLPage speaks TLS through rustls. build.rs embeds - # the browser assets, so every cargo gate builds them with npm first. + # aws-lc-sys, libsqlite3-sys and zstd-sys need a compiler. SQLPage speaks + # TLS through rustls; only the unstable-sqlx-next feature links OpenSSL, + # because sqlx-sqlserver requires native-tls. build.rs embeds the browser + # assets, so every cargo gate builds them with npm first. cargoRuntime = pkgs: [ (rustToolchainFor pkgs) pkgs.stdenv.cc @@ -128,6 +129,8 @@ cargoEnv = pkgs: { LIBRARY_PATH = "${pkgs.unixodbc}/lib"; + OPENSSL_LIB_DIR = "${lib.getLib pkgs.openssl}/lib"; + OPENSSL_INCLUDE_DIR = "${lib.getDev pkgs.openssl}/include"; }; # Playwright is installed via npm, we we need to set the browser paths. @@ -270,6 +273,7 @@ ++ lib.attrValues (gatesFor pkgs); buildInputs = with pkgs; [ + openssl unixodbc zstd ]; diff --git a/src/webserver/database/sqlx_next/mod.rs b/src/webserver/database/sqlx_next/mod.rs index 7fdffd6f..3b7fc245 100644 --- a/src/webserver/database/sqlx_next/mod.rs +++ b/src/webserver/database/sqlx_next/mod.rs @@ -1,4 +1,5 @@ mod odbc; +mod sql_server; use std::time::Duration; @@ -15,6 +16,7 @@ use sqlx_next::{ Arguments, AssertSqlSafe, ColumnIndex, Either, Execute, Executor, Row, SqlSafeStr, SqlStr, Type, }; use sqlx_odbc::Odbc; +use sqlx_sqlserver::Mssql; use super::connect::{ConnectionHooks, max_connections}; use super::driver::FetchedItem; @@ -25,22 +27,26 @@ pub(super) use sqlx_next::Error; #[derive(Clone, Debug)] pub enum Pool { Odbc(sqlx_odbc::OdbcPool), + SqlServer(sqlx_sqlserver::MssqlPool), } pub enum PooledConnection { Odbc(PoolConnection), + SqlServer(PoolConnection), } #[derive(Clone, Copy)] pub(super) enum Driver { Odbc, + SqlServer, } impl Driver { pub(super) fn serving(kind: AnyKind) -> Option { match kind { AnyKind::Odbc => Some(Self::Odbc), - AnyKind::Postgres | AnyKind::MySql | AnyKind::Sqlite | AnyKind::Mssql => None, + AnyKind::Mssql => Some(Self::SqlServer), + AnyKind::Postgres | AnyKind::MySql | AnyKind::Sqlite => None, } } @@ -48,6 +54,7 @@ impl Driver { pub(super) async fn connect(self, config: &AppConfig) -> anyhow::Result<(Pool, String)> { match self { Self::Odbc => odbc::connect(config).await, + Self::SqlServer => sql_server::connect(config).await, } } } @@ -56,24 +63,28 @@ impl Pool { pub(super) async fn acquire(&self) -> Result { Ok(match self { Self::Odbc(pool) => PooledConnection::Odbc(pool.acquire().await?), + Self::SqlServer(pool) => PooledConnection::SqlServer(pool.acquire().await?), }) } pub(super) fn size(&self) -> u32 { match self { Self::Odbc(pool) => pool.size(), + Self::SqlServer(pool) => pool.size(), } } pub(super) fn num_idle(&self) -> usize { match self { Self::Odbc(pool) => pool.num_idle(), + Self::SqlServer(pool) => pool.num_idle(), } } pub(super) async fn close(&self) { match self { Self::Odbc(pool) => pool.close().await, + Self::SqlServer(pool) => pool.close().await, } } } @@ -88,6 +99,9 @@ impl PooledConnection { Self::Odbc(connection) => { fetch_decoded::(&mut **connection, sql, params, odbc::decode_row) } + Self::SqlServer(connection) => { + fetch_decoded::(&mut **connection, sql, params, sql_server::decode_row) + } } } @@ -95,6 +109,7 @@ impl PooledConnection { let sql = AssertSqlSafe(sql.to_owned()); match self { Self::Odbc(connection) => connection.execute(sql).await.map(drop), + Self::SqlServer(connection) => connection.execute(sql).await.map(drop), } } } diff --git a/src/webserver/database/sqlx_next/sql_server.rs b/src/webserver/database/sqlx_next/sql_server.rs new file mode 100644 index 00000000..f9f70fa2 --- /dev/null +++ b/src/webserver/database/sqlx_next/sql_server.rs @@ -0,0 +1,82 @@ +use anyhow::Context; +use bigdecimal::BigDecimal; +use chrono::{DateTime, FixedOffset, NaiveDate, NaiveDateTime, NaiveTime}; +use serde_json::Value; +use sqlx::any::AnyKind; +use sqlx_next::types::Uuid; +use sqlx_next::{Column, ConnectOptions, Row, ValueRef}; +use sqlx_sqlserver::{Mssql, MssqlColumn, MssqlConnectOptions, MssqlRow, MssqlType}; + +use super::{Pool, decode_or_default, pool_options}; +use crate::app_config::AppConfig; +use crate::webserver::database::blob_to_data_url::vec_to_data_uri_value; +use crate::webserver::database::connect::retry_connecting; +use crate::webserver::database::sql_to_json::decimal_to_json; + +const DBMS_NAME: &str = "Microsoft SQL Server"; + +pub(super) async fn connect(config: &AppConfig) -> anyhow::Result<(Pool, String)> { + anyhow::ensure!( + config.database_password.is_none(), + "The unstable sqlx 0.9 SQL Server driver does not support database_password yet. \ + Put the password in database_url, or set unstable_sqlx_next to false." + ); + let options: MssqlConnectOptions = config.database_url.parse()?; + drop(retry_connecting(config, || options.connect()).await?); + let pool = pool_options::(config, AnyKind::Mssql) + .connect_with(options) + .await + .with_context(|| format!("Unable to open connection pool to {}", config.database_url))?; + Ok((Pool::SqlServer(pool), DBMS_NAME.to_owned())) +} + +pub(super) fn decode_row(row: &MssqlRow) -> Vec<(String, Value)> { + row.columns() + .iter() + .map(|column| (column.name().to_owned(), column_to_json(row, column))) + .collect() +} + +fn column_to_json(row: &MssqlRow, column: &MssqlColumn) -> Value { + let value = match row.try_get_raw(column.ordinal()) { + Ok(value) if value.is_null() => return Value::Null, + Ok(value) => value, + Err(error) => { + log::warn!("Unable to extract value from row: {error:?}"); + return Value::Null; + } + }; + let kind = value.type_info().kind().clone(); + match kind { + MssqlType::Null => Value::Null, + MssqlType::Real | MssqlType::Float | MssqlType::Money => { + decode_or_default::(value).into() + } + MssqlType::Decimal => decimal_to_json(&decode_or_default::(value)), + MssqlType::BigInt => decode_or_default::(value).into(), + MssqlType::Int => decode_or_default::(value).into(), + MssqlType::SmallInt => decode_or_default::(value).into(), + MssqlType::TinyInt => decode_or_default::(value).into(), + MssqlType::Bit => decode_or_default::(value).into(), + MssqlType::Date => decode_or_default::(value) + .to_string() + .into(), + MssqlType::Time => decode_or_default::(value) + .to_string() + .into(), + MssqlType::DateTimeOffset => decode_or_default::>(value) + .to_rfc3339() + .into(), + MssqlType::DateTime | MssqlType::DateTime2 => { + decode_or_default::(value) + .format("%FT%T%.f") + .to_string() + .into() + } + MssqlType::UniqueIdentifier => decode_or_default::(value).to_string().into(), + MssqlType::VarBinary => vec_to_data_uri_value(&decode_or_default::>(value)), + MssqlType::NVarChar | MssqlType::VarChar | MssqlType::Other(_) => { + decode_or_default::(value).into() + } + } +}