Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1,154 changes: 325 additions & 829 deletions Cargo.lock

Large diffs are not rendered by default.

5 changes: 4 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ edition = "2024"
pgdog-plugin = { path = "./pgdog-plugin", version = "0.5.0", default-features = false }
pgdog-config = { path = "./pgdog-config", version = "0.1.0" }
pgdog-postgres-types = { path = "./pgdog-postgres-types"}
pg_raw_parse = { git = "https://github.com/pgdogdev/pg_raw_parse.git", rev = "89937b1" }
pg_raw_parse = { git = "https://github.com/pgdogdev/pg_raw_parse.git", rev = "a4784fc603563d3db8e9f381347f039f39b1dd85" }
pgdog-util = { path = "./pgdog-util", version = "0.1.0" }
bon = "3.9"
derive_more = { version = "2", features = ["display", "from"] }
Expand All @@ -34,6 +34,9 @@ reqwest = { version = "0.13", features = ["json"] }
bytes = "1"
uuid = { version = "1", features = ["v4", "v7", "serde"] }
parking_lot = "0.12"
rand = "0.10"
serde = { version = "1", features = ["derive"] }
toml = "0.8"
futures-util = "0.3.32"
tokio = { version = "1.54", features = ["full"]}

Expand Down
6 changes: 3 additions & 3 deletions integration/rust/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ test = true
[dependencies]
tokio-postgres = {version = "0.7.13", features = ["with-uuid-1"]}
native-tls = "0.2"
sqlx = { version = "0.8.6", features = ["postgres", "runtime-tokio", "tls-native-tls", "bigdecimal", "chrono", "json", "rust_decimal", "uuid"]}
sqlx = { version = "0.9.1", default-features = false, features = ["derive", "postgres", "runtime-tokio", "tls-native-tls", "bigdecimal", "chrono", "json", "rust_decimal", "uuid"]}
tokio.workspace = true
futures-util.workspace = true
uuid.workspace = true
Expand All @@ -22,9 +22,9 @@ rust_decimal = { version = "1.42.0", features = ["macros"] }

[dev-dependencies]
postgres-native-tls = "0.5"
serial_test = "3"
serial_test = "4"
reqwest.workspace = true
rand = "0.9"
rand.workspace = true
chrono-tz = "0.10.4"
pgdog-stats = { path = "../../pgdog-stats" }
pg_stream = "0.2.1"
Expand Down
7 changes: 7 additions & 0 deletions integration/rust/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,2 +1,9 @@
#[macro_export]
macro_rules! unsafe_sql {
($($arg:tt)*) => {
::sqlx::SqlSafeStr::into_sql_str(::sqlx::AssertSqlSafe(::std::format!($($arg)*)))
};
}

pub mod setup;
pub mod utils;
11 changes: 4 additions & 7 deletions integration/rust/src/setup.rs
Original file line number Diff line number Diff line change
Expand Up @@ -73,17 +73,14 @@ pub struct Backend {
}

pub async fn backends(application_name: &str, pool: &Pool<Postgres>) -> Vec<Backend> {
pool.fetch_all(
format!(
"SELECT pid::INTEGER,
pool.fetch_all(unsafe_sql!(
"SELECT pid::INTEGER,
backend_start::TEXT
FROM pg_stat_activity
WHERE application_name = '{}'
ORDER BY backend_start",
application_name
)
.as_str(),
)
application_name
))
.await
.unwrap()
.into_iter()
Expand Down
5 changes: 3 additions & 2 deletions integration/rust/tests/bench/prepared_stress.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,9 @@ async fn slam_with_prepared() -> Result<(), Box<dyn std::error::Error>> {
loop {
let r = rand::random_range(0..statements);

let query = format!("SELECT 1, 2, 3, 4, $1, 'apples and oranges', 'blah', {}", r);
let query = sqlx::query(query.as_str()).bind(r).execute(&mut conn);
let query =
unsafe_sql!("SELECT 1, 2, 3, 4, $1, 'apples and oranges', 'blah', {}", r);
let query = sqlx::query(query).bind(r).execute(&mut conn);

select! {
res = query => {
Expand Down
79 changes: 41 additions & 38 deletions integration/rust/tests/integration/admin/resharding/copy_data.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,22 +18,21 @@ fn numbered_table(i: usize) -> String {
}

async fn create_table(pool: &Pool<Postgres>, table: &str) {
pool.execute(format!("CREATE SCHEMA IF NOT EXISTS {TEST_SCHEMA}").as_str())
pool.execute(unsafe_sql!("CREATE SCHEMA IF NOT EXISTS {TEST_SCHEMA}"))
.await
.expect("test schema creation must succeed");
pool.execute(
format!("CREATE TABLE {TEST_SCHEMA}.{table} (id BIGSERIAL PRIMARY KEY, val TEXT)").as_str(),
)
pool.execute(unsafe_sql!(
"CREATE TABLE {TEST_SCHEMA}.{table} (id BIGSERIAL PRIMARY KEY, val TEXT)"
))
.await
.unwrap();
}

async fn seed_table(direct: &Pool<Postgres>, table: &str, n: i64) {
direct
.execute(
format!("INSERT INTO {TEST_SCHEMA}.{table} (val) SELECT 'v' || g FROM generate_series(1, {n}) g")
.as_str(),
)
.execute(unsafe_sql!(
"INSERT INTO {TEST_SCHEMA}.{table} (val) SELECT 'v' || g FROM generate_series(1, {n}) g"
))
.await
.unwrap();
}
Expand All @@ -44,7 +43,9 @@ fn table_progress(row: &PgRow) -> TaskProgress {

async fn set_parallel_within_table(admin: &Pool<Postgres>, readers: usize) {
admin
.execute(format!("SET resharding_parallel_within_table_copies TO {readers}").as_str())
.execute(unsafe_sql!(
"SET resharding_parallel_within_table_copies TO {readers}"
))
.await
.unwrap();
}
Expand All @@ -67,20 +68,19 @@ async fn copy_data(parallel_within_table: usize) {
// errors silently leaves the table with its default identity.
direct
.execute(
format!(
unsafe_sql!(
"ALTER TABLE {TEST_SCHEMA}.{TEST_TABLE} ALTER COLUMN val SET NOT NULL;
ALTER TABLE {TEST_SCHEMA}.{TEST_TABLE} ADD CONSTRAINT {secondary_index} UNIQUE (val);
ALTER TABLE {TEST_SCHEMA}.{TEST_TABLE} REPLICA IDENTITY USING INDEX {secondary_index}"
)
.as_str(),
),
)
.await
.unwrap();
seed_rows(&direct, ROW_COUNT).await;
create_publication(&direct).await;

let row = admin
.fetch_one(format!("COPY_DATA pgdog pgdog_sharded {TEST_PUB}").as_str())
.fetch_one(unsafe_sql!("COPY_DATA pgdog pgdog_sharded {TEST_PUB}"))
.await
.unwrap();
let task_id: i64 = row.get::<String, _>("task_id").parse().unwrap();
Expand Down Expand Up @@ -109,8 +109,7 @@ async fn copy_data(parallel_within_table: usize) {
assert_eq!(identity, "i");
let error = shard
.execute(
format!("INSERT INTO {TEST_SCHEMA}.{TEST_TABLE} (id, val) VALUES (21, 'v1')")
.as_str(),
unsafe_sql!("INSERT INTO {TEST_SCHEMA}.{TEST_TABLE} (id, val) VALUES (21, 'v1')"),
)
.await
.unwrap_err();
Expand All @@ -136,7 +135,7 @@ async fn test_copy_data_parallel() {
}

/// Runs `seed` to create the rows within the test table that will be copied to the dest.
async fn copy_seeded(seed: &str, expected: i64) {
async fn copy_seeded(seed: impl sqlx::SqlSafeStr + Send, expected: i64) {
let direct = connection_sqlx_direct().await;
let admin = admin_sqlx().await;
cleanup(&admin, &direct).await;
Expand All @@ -148,8 +147,11 @@ async fn copy_seeded(seed: &str, expected: i64) {
direct.execute(seed).await.unwrap();
create_publication(&direct).await;

let task_id =
run_task_command(&admin, &format!("COPY_DATA pgdog pgdog_sharded {TEST_PUB}")).await;
let task_id = run_task_command(
&admin,
unsafe_sql!("COPY_DATA pgdog pgdog_sharded {TEST_PUB}"),
)
.await;

wait_for_relation_on_shards(&admin, task_id, TEST_TABLE).await;
wait_for_rows_each_shard(&admin, task_id, TEST_TABLE, expected).await;
Expand All @@ -170,15 +172,15 @@ async fn test_copy_data_empty_table() {

#[tokio::test]
async fn test_copy_data_leading_dead_blocks() {
let seed = format!(
let seed = unsafe_sql!(
"INSERT INTO {TEST_SCHEMA}.{TEST_TABLE} (val) SELECT 'v' || g FROM generate_series(1, 10000) g;
DELETE FROM {TEST_SCHEMA}.{TEST_TABLE} WHERE id <= 5000"
);
copy_seeded(&seed, 5000).await;
copy_seeded(seed, 5000).await;
}

async fn table_checksum(pool: &Pool<Postgres>) -> (i64, String) {
sqlx::query_as(&format!(
sqlx::query_as(unsafe_sql!(
"SELECT count(*), coalesce(sum(hashtextextended(id::text || ':' || val, 0)), 0)::text \
FROM {TEST_SCHEMA}.{TEST_TABLE}"
))
Expand Down Expand Up @@ -208,23 +210,20 @@ async fn copy_data_with_concurrent_writes(parallel_within_table: usize) {
let writes = async {
for i in 0..WRITES {
direct
.execute(
format!(
"INSERT INTO {TEST_SCHEMA}.{TEST_TABLE} (val) VALUES ('new' || {i});
.execute(unsafe_sql!(
"INSERT INTO {TEST_SCHEMA}.{TEST_TABLE} (val) VALUES ('new' || {i});
UPDATE {TEST_SCHEMA}.{TEST_TABLE} SET val = val || 'x' WHERE id = {};
DELETE FROM {TEST_SCHEMA}.{TEST_TABLE} WHERE id = {}",
i * 37 % ROW_COUNT + 1,
i * 101 % ROW_COUNT + 1,
)
.as_str(),
)
i * 37 % ROW_COUNT + 1,
i * 101 % ROW_COUNT + 1,
))
.await
.unwrap();
}
};

let command = format!("COPY_DATA pgdog pgdog_sharded {TEST_PUB}");
let copy = run_task_command(&admin, &command);
let command = unsafe_sql!("COPY_DATA pgdog pgdog_sharded {TEST_PUB}");
let copy = run_task_command(&admin, command);
let (task_id, ()) = tokio::join!(copy, writes);

let expected = table_checksum(&direct).await;
Expand Down Expand Up @@ -274,7 +273,9 @@ async fn test_failed_copy_cancels_siblings_and_rolls_back() {
.collect::<Vec<_>>()
.join(", ");
direct
.execute(format!("CREATE PUBLICATION {TEST_PUB} FOR TABLE {tables}").as_str())
.execute(unsafe_sql!(
"CREATE PUBLICATION {TEST_PUB} FOR TABLE {tables}"
))
.await
.unwrap();

Expand All @@ -283,16 +284,18 @@ async fn test_failed_copy_cancels_siblings_and_rolls_back() {
let shard = connection_sqlx_direct_db(db).await;
create_table(&shard, &poisoned).await;
shard
.execute(
format!("INSERT INTO {TEST_SCHEMA}.{poisoned} (id, val) VALUES (1, 'poison')")
.as_str(),
)
.execute(unsafe_sql!(
"INSERT INTO {TEST_SCHEMA}.{poisoned} (id, val) VALUES (1, 'poison')"
))
.await
.unwrap();
}

let task_id =
run_task_command(&admin, &format!("COPY_DATA pgdog pgdog_sharded {TEST_PUB}")).await;
let task_id = run_task_command(
&admin,
unsafe_sql!("COPY_DATA pgdog pgdog_sharded {TEST_PUB}"),
)
.await;

wait_for_task(&admin, "the copy-data run to fail", |task| {
task.id == Some(task_id) && matches!(task.status, TaskProgress::Error { .. })
Expand Down
Loading
Loading