From 1307a7075239c892fc4329523b46c617b530462a Mon Sep 17 00:00:00 2001 From: Abit Gurung Date: Sun, 4 Oct 2026 12:06:29 -0500 Subject: [PATCH 1/3] fix: send LUW LOB parameters beyond 32767 bytes as EXTDTA Parameters bound to CLOB/BLOB/DBCLOB targets declared larger than 32767 bytes were described in SQLDTA as VARCHAR/VARBINARY/VARGRAPHIC with lengths such as 0x8000 or 0xFFFF. An FD:OCA varying-length descriptor only has 15 bits of length, so LUW 11.5 and 12.1 answered with DSCINVRM (0x220A). The driver ignored that reply, so the statement reported rowCount=0 without an error and nothing was written (#31). LUW input descriptors for such LOBs now use DRDA LOB types with the same 9-byte reference LUW returns for LOB results: status 0x02 and an eight-byte big-endian length (UTF-16 code units for DBCLOB). Values follow SQLDTA as same-correlation EXTDTA objects in parameter order, each with the nullable indicator. Smaller LOB targets keep the existing encoding, and z/OS descriptors are unchanged. DSCINVRM is now classified as a protocol reply error, so rejected descriptors can no longer report success. --- crates/db2-client/src/connection.rs | 384 +++++++++++++++++++++- crates/db2-client/src/statement.rs | 8 +- crates/db2-proto/src/codepoints.rs | 1 + tests/integration/lob_regressions_test.rs | 76 ++++- tests/node/lob-regressions.test.ts | 59 ++++ 5 files changed, 507 insertions(+), 21 deletions(-) diff --git a/crates/db2-client/src/connection.rs b/crates/db2-client/src/connection.rs index 4a06711..df74d95 100644 --- a/crates/db2-client/src/connection.rs +++ b/crates/db2-client/src/connection.rs @@ -1261,7 +1261,7 @@ impl ClientInner { } input_descriptors = self.describe_input(&pkgnamcsn).await?; - let sqldta_data = build_sqldta(params, &input_descriptors)?; + let sqldta = build_sqldta(params, &input_descriptors)?; let corr_id = self.next_correlation_id(); let opnqry_data = { let mut ddm = db2_proto::ddm::DdmBuilder::new(codepoints::OPNQRY); @@ -1275,7 +1275,7 @@ impl ClientInner { let mut writer = DssWriter::new(corr_id); writer.write_request_next_same_corr(&opnqry_data, true); - writer.write_object(&sqldta_data, false); + sqldta.write(&mut writer, false); let send_buf = writer.finish(); self.send_bytes(&send_buf).await?; @@ -2263,7 +2263,7 @@ impl ClientInner { writer.write_request(&command, false); } else { writer.write_request_next_same_corr(&command, true); - writer.write_object(&build_sqldta(params, descriptors)?, false); + build_sqldta(params, descriptors)?.write(&mut writer, false); } self.send_bytes(&writer.finish()).await?; let frames = self.read_reply_frames().await?; @@ -2382,12 +2382,12 @@ impl ClientInner { } else { db2_proto::commands::excsqlstt::build_excsqlstt_default(pkgnamcsn) }; - let sqldta_data = build_sqldta(params, descriptors)?; + let sqldta = build_sqldta(params, descriptors)?; let rdbcmm_data = db2_proto::commands::rdbcmm::build_rdbcmm(); let mut writer = DssWriter::new(corr_id); writer.write_request_next_same_corr(&excsqlstt_data, true); - writer.write_object(&sqldta_data, self.auto_commit); + sqldta.write(&mut writer, self.auto_commit); if self.auto_commit { writer.write_request(&rdbcmm_data, false); } @@ -2441,11 +2441,11 @@ impl ClientInner { let excsqlstt_data = db2_proto::commands::excsqlstt::build_excsqlstt_default(pkgnamcsn); - let sqldta_data = build_sqldta(row, descriptors)?; + let sqldta = build_sqldta(row, descriptors)?; let mut writer = DssWriter::new(corr_id); writer.write_request_next_same_corr(&excsqlstt_data, true); - writer.write_object(&sqldta_data, !is_last); + sqldta.write(&mut writer, !is_last); send_buf.extend_from_slice(&writer.finish()); } @@ -3367,7 +3367,12 @@ impl ClientInner { format_hex_preview(&obj.data, 160) ); } - descriptors.extend(parse_input_sqldard_descriptors(&obj)); + let mut parsed = parse_input_sqldard_descriptors(&obj); + externalize_luw_lob_input_descriptors( + &mut parsed, + self.server_info.as_ref(), + ); + descriptors.extend(parsed); } codepoints::SQLCARD => { let card = db2_proto::replies::sqlcard::parse_sqlcard(&obj) @@ -6245,10 +6250,42 @@ fn quote_sql_identifier(name: &str) -> String { format!("\"{}\"", name.replace('"', "\"\"")) } +/// FD:OCA varying-length descriptors carry a 15-bit maximum; a set high bit +/// declares a LOB placeholder instead, which LUW rejects with DSCINVRM when +/// sent as VARCHAR/VARBINARY/VARGRAPHIC. +const MAX_VARYING_INPUT_LEN: u16 = 0x7FFF; +/// LUW LOB placeholder: a status byte and an eight-byte big-endian length, +/// the same reference shape LUW returns for LOB result columns. +const LUW_LOB_INPUT_PLACEHOLDER_LEN: u16 = 9; +/// Placeholder status for a value that follows in an EXTDTA object. +const LUW_LOB_INPUT_EXTDTA_STATUS: u8 = 0x02; + +/// Encoded SQLDTA plus the EXTDTA objects for its externalized LOB values. +#[derive(Debug)] +pub(crate) struct SqlDta { + pub(crate) sqldta: Vec, + pub(crate) extdta: Vec>, +} + +impl SqlDta { + /// Write SQLDTA and its EXTDTA objects as one same-correlation chain. + pub(crate) fn write(&self, writer: &mut DssWriter, chained: bool) { + let Some((last, leading)) = self.extdta.split_last() else { + writer.write_object(&self.sqldta, chained); + return; + }; + writer.write_object_same_corr(&self.sqldta, true); + for extdta in leading { + writer.write_object_same_corr(extdta, true); + } + writer.write_object(last, chained); + } +} + pub(crate) fn build_sqldta( params: &[&dyn ToSql], descriptors: &[db2_proto::fdoca::ColumnDescriptor], -) -> Result, Error> { +) -> Result { let descriptors = if descriptors.is_empty() { infer_parameter_descriptors(params)? } else { @@ -6267,6 +6304,9 @@ pub(crate) fn build_sqldta( .iter() .zip(descriptors) .map(|(param, mut descriptor)| { + if descriptor.extdta_reference_length.is_some() { + return descriptor; + } let value = param.to_db2_value(); if matches!( value, @@ -6298,20 +6338,52 @@ pub(crate) fn build_sqldta( let mut builder = db2_proto::ddm::DdmBuilder::new(codepoints::SQLDTA); builder.add_raw(&build_sqldta_fdoca_prefix(&descriptors)?); - let data = build_sqldta_row_data(params, &descriptors)?; + let (data, extdta) = build_sqldta_row_data(params, &descriptors)?; let mut inner = db2_proto::ddm::DdmBuilder::new(codepoints::FDODTA); inner.add_raw(&data); builder.add_raw(&inner.build()); let sqldta = builder.build(); if debug_hex_enabled() { eprintln!( - "[db2-wire] built SQLDTA with {} descriptor(s), total={} bytes, preview={}", + "[db2-wire] built SQLDTA with {} descriptor(s), total={} bytes, extdta={:?}, preview={}", descriptors.len(), sqldta.len(), + extdta.iter().map(Vec::len).collect::>(), format_hex_preview(&sqldta, 128) ); } - Ok(sqldta) + Ok(SqlDta { sqldta, extdta }) +} + +/// Describe LUW LOB inputs that exceed a varying-length descriptor as DRDA +/// LOB placeholders, so their values are sent in EXTDTA after SQLDTA. +/// Smaller LOB targets keep the VARCHAR/VARBINARY/VARGRAPHIC encoding. +fn externalize_luw_lob_input_descriptors( + descriptors: &mut [db2_proto::fdoca::ColumnDescriptor], + server: Option<&ServerInfo>, +) { + use db2_proto::types::Db2Type; + + if server.is_some_and(is_db2_zos_server) { + return; + } + for descriptor in descriptors { + if descriptor.length <= MAX_VARYING_INPUT_LEN { + continue; + } + let base = match descriptor.db2_type { + Db2Type::Blob => db2_proto::types::DRDA_TYPE_BLOB, + Db2Type::Clob => DRDA_TYPE_CLOB_MIXED, + Db2Type::DbClob => db2_proto::types::DRDA_TYPE_DBCLOB, + _ => continue, + }; + descriptor.drda_type = if descriptor.nullable { + base | 0x01 + } else { + base + }; + descriptor.extdta_reference_length = Some(LUW_LOB_INPUT_PLACEHOLDER_LEN); + } } fn build_sqldta_fdoca_prefix( @@ -6336,8 +6408,11 @@ fn build_sqldta_fdoca_prefix( gda.push(TRIPLET_TYPE_GDA); gda.push(GDA_PREFIX); for descriptor in descriptors { + let length = descriptor + .extdta_reference_length + .map_or(descriptor.length, |width| 0x8000 | width); gda.push(descriptor.drda_type); - gda.extend_from_slice(&descriptor.length.to_be_bytes()); + gda.extend_from_slice(&length.to_be_bytes()); } let rlo = [ @@ -6630,6 +6705,8 @@ fn looks_like_zos_input_descriptor_start(data: &[u8]) -> bool { && matches!(ccsid, 0 | 37 | 500 | 819 | 1200 | 1208) } +/// DRDA LOBCMIXED: CLOB data in the negotiated mixed-byte CCSID. +const DRDA_TYPE_CLOB_MIXED: u8 = 0xCE; const NULL_LID: u8 = 0x00; const NULL_DATA: u8 = 0xFF; const INDICATOR_NOT_NULL: u8 = 0x00; @@ -6846,8 +6923,9 @@ fn infer_parameter_descriptors( fn build_sqldta_row_data( params: &[&dyn ToSql], descriptors: &[db2_proto::fdoca::ColumnDescriptor], -) -> Result, Error> { +) -> Result<(Vec, Vec>), Error> { let mut data = Vec::with_capacity(1 + params.len() * 8); + let mut extdta = Vec::new(); data.push(NULL_LID); for (index, (param, descriptor)) in params.iter().zip(descriptors.iter()).enumerate() { @@ -6865,10 +6943,58 @@ fn build_sqldta_row_data( ))); } + if descriptor.extdta_reference_length.is_some() { + let (bytes, length) = encode_lob_parameter_value(&value, descriptor)?; + data.push(LUW_LOB_INPUT_EXTDTA_STATUS); + data.extend_from_slice(&length.to_be_bytes()); + // Like LOB result values, EXTDTA repeats the nullable indicator. + let mut payload = Vec::with_capacity(1 + bytes.len()); + if descriptor.nullable { + payload.push(INDICATOR_NOT_NULL); + } + payload.extend_from_slice(&bytes); + extdta.push( + db2_proto::ddm::DdmBuilder::new(codepoints::EXTDTA) + .add_raw(&payload) + .build(), + ); + continue; + } + data.extend_from_slice(&encode_parameter_value(&value, descriptor)?); } - Ok(data) + Ok((data, extdta)) +} + +/// Encode an externalized LOB value and the length its placeholder declares. +fn encode_lob_parameter_value( + value: &db2_proto::types::Db2Value, + descriptor: &db2_proto::fdoca::ColumnDescriptor, +) -> Result<(Vec, u64), Error> { + use db2_proto::types::Db2Type; + + let bytes = match descriptor.db2_type { + Db2Type::Blob => extract_binary(value)?, + // LOBCMIXED data uses the UTF-8 mixed-byte CCSID requested at ACCRDB. + Db2Type::Clob => extract_text(value)?.as_bytes().to_vec(), + Db2Type::DbClob => encode_graphic_text_bytes(value)?, + _ => { + return Err(Error::ParameterType(format!( + "unsupported LOB parameter type: {:?}", + descriptor.db2_type + ))) + } + }; + // DBCLOB lengths count UTF-16 code units, as for LUW VARGRAPHIC input. + let length = if matches!(descriptor.db2_type, Db2Type::DbClob) + && graphic_length_is_character_count(descriptor) + { + bytes.len() / 2 + } else { + bytes.len() + }; + Ok((bytes, length as u64)) } fn encode_parameter_value( @@ -7691,6 +7817,7 @@ fn frames_have_data_or_terminal_reply(frames: &[DssFrame]) -> bool { | codepoints::PRMNSPRM | codepoints::VALNSPRM | codepoints::DTAMCHRM + | codepoints::DSCINVRM | codepoints::QRYNOPRM ) }) @@ -7715,6 +7842,7 @@ fn frames_have_query_data_or_query_end_reply(frames: &[DssFrame]) -> bool { | codepoints::PRMNSPRM | codepoints::VALNSPRM | codepoints::DTAMCHRM + | codepoints::DSCINVRM | codepoints::QRYNOPRM ) }) @@ -7975,6 +8103,7 @@ fn ddm_codepoint_name(code_point: u16) -> String { codepoints::ENDQRYRM => "ENDQRYRM", codepoints::QRYNOPRM => "QRYNOPRM", codepoints::DTAMCHRM => "DTAMCHRM", + codepoints::DSCINVRM => "DSCINVRM", codepoints::RDBUPDRM => "RDBUPDRM", codepoints::SQLERRRM => "SQLERRRM", codepoints::SYNTAXRM => "SYNTAXRM", @@ -8017,6 +8146,7 @@ fn reply_codepoint_name(code_point: u16) -> Option<&'static str> { codepoints::SQLERRRM => Some("SQLERRRM"), codepoints::CMDCHKRM => Some("CMDCHKRM"), codepoints::DTAMCHRM => Some("DTAMCHRM"), + codepoints::DSCINVRM => Some("DSCINVRM"), codepoints::QRYNOPRM => Some("QRYNOPRM"), codepoints::OBJNSPRM => Some("OBJNSPRM"), codepoints::RDBNACRM => Some("RDBNACRM"), @@ -8540,6 +8670,228 @@ mod tests { assert_eq!(descriptors[3].db2_type, db2_proto::types::Db2Type::BigInt); } + fn split_sqldta(sqldta: &[u8]) -> (Vec<(u8, u16)>, Vec) { + let obj = ClientInner::parse_ddm(sqldta).unwrap(); + assert_eq!(obj.code_point, codepoints::SQLDTA); + let prefix_len = usize::from(u16::from_be_bytes([obj.data[0], obj.data[1]])); + let gda_len = usize::from(obj.data[4]); + let triplets = obj.data[7..4 + gda_len] + .chunks(3) + .map(|item| (item[0], u16::from_be_bytes([item[1], item[2]]))) + .collect(); + let fdodta = ClientInner::parse_ddm(&obj.data[prefix_len..]).unwrap(); + assert_eq!(fdodta.code_point, codepoints::FDODTA); + (triplets, fdodta.data) + } + + fn lob_placeholder(length: u64) -> Vec { + let mut placeholder = vec![0x00, LUW_LOB_INPUT_EXTDTA_STATUS]; + placeholder.extend_from_slice(&length.to_be_bytes()); + placeholder + } + + #[tokio::test] + async fn luw_lob_inputs_beyond_varying_limit_use_placeholders_and_extdta() { + // Issue #31: LUW rejects VARCHAR/VARBINARY/VARGRAPHIC descriptors + // above 0x7FFF with DSCINVRM, so these inputs must be externalized. + let frames = vec![reply_frame( + input_sqldard(&[ + (409, 32767, 1208), + (409, 32768, 1208), + (405, 1 << 20, 0), + (413, 32768, 1200), + (497, 4, 0), + (409, 1 << 20, 1208), + (405, 32768, 0), + ]), + 1, + )]; + let client = Client::new(Config::default()); + let inner = client.inner.lock().await; + let descriptors = inner.parse_input_descriptors(&frames).unwrap(); + assert_eq!( + descriptors + .iter() + .map(|descriptor| descriptor.extdta_reference_length) + .collect::>(), + [None, Some(9), Some(9), Some(9), None, Some(9), Some(9)] + ); + + let small = "small"; + let clob = "é"; + let blob = vec![0x00, 0xFF, 0x07]; + let dbclob = "d\u{1D11E}"; + let id = 1i32; + let null_clob: Option<&str> = None; + let empty_blob: Vec = Vec::new(); + let params: [&dyn ToSql; 7] = [&small, &clob, &blob, &dbclob, &id, &null_clob, &empty_blob]; + let encoded = build_sqldta(¶ms, &descriptors).unwrap(); + let (triplets, fdodta) = split_sqldta(&encoded.sqldta); + assert_eq!( + triplets, + [ + (0x33, 0x7FFF), + (0xCF, 0x8009), + (0xC9, 0x8009), + (0xCD, 0x8009), + (0x03, 0x0004), + (0xCF, 0x8009), + (0xC9, 0x8009), + ] + ); + + let mut expected = vec![NULL_LID, 0x00, 0x00, 0x05]; + expected.extend_from_slice(b"small"); + expected.extend(lob_placeholder(2)); + expected.extend(lob_placeholder(3)); + // DBCLOB lengths count UTF-16 code units: d plus a surrogate pair. + expected.extend(lob_placeholder(3)); + expected.extend_from_slice(&[0x00, 0x01, 0x00, 0x00, 0x00]); + expected.push(NULL_DATA); + expected.extend(lob_placeholder(0)); + assert_eq!(fdodta, expected); + + let extdta = encoded + .extdta + .iter() + .map(|object| { + let object = ClientInner::parse_ddm(object).unwrap(); + assert_eq!(object.code_point, codepoints::EXTDTA); + object.data + }) + .collect::>(); + assert_eq!( + extdta, + [ + vec![0x00, 0xC3, 0xA9], + vec![0x00, 0x00, 0xFF, 0x07], + vec![0x00, 0x00, 0x64, 0xD8, 0x34, 0xDD, 0x1E], + vec![0x00], + ] + ); + } + + #[test] + fn zos_lob_input_descriptors_are_not_externalized() { + let descriptor = db2_proto::fdoca::ColumnDescriptor { + column_index: 0, + drda_type: 0x33, + length: u16::MAX, + precision: 0, + scale: 0, + nullable: true, + ccsid: 37, + db2_type: db2_proto::types::Db2Type::Clob, + extdta_reference_length: None, + byte_order: db2_proto::fdoca::ByteOrder::LittleEndian, + }; + let zos = ServerInfo { + product_name: "DB2".into(), + server_release: "DSN13015".into(), + server_class: "QDB2".into(), + manager_levels: Vec::new(), + }; + let mut descriptors = [descriptor.clone()]; + externalize_luw_lob_input_descriptors(&mut descriptors, Some(&zos)); + assert_eq!(descriptors[0].drda_type, 0x33); + assert_eq!(descriptors[0].extdta_reference_length, None); + + externalize_luw_lob_input_descriptors(&mut descriptors, None); + assert_eq!(descriptors[0].drda_type, 0xCF); + assert_eq!(descriptors[0].extdta_reference_length, Some(9)); + } + + #[test] + fn sqldta_chains_large_extdta_objects_with_its_correlation() { + let lob = |db2_type, drda_type| db2_proto::fdoca::ColumnDescriptor { + column_index: 0, + drda_type, + length: u16::MAX, + precision: 0, + scale: 0, + nullable: true, + ccsid: 1208, + db2_type, + extdta_reference_length: Some(LUW_LOB_INPUT_PLACEHOLDER_LEN), + byte_order: db2_proto::fdoca::ByteOrder::LittleEndian, + }; + let descriptors = [ + lob(db2_proto::types::Db2Type::Blob, 0xC9), + lob(db2_proto::types::Db2Type::Clob, 0xCF), + ]; + let blob = (0..100_000).map(|index| index as u8).collect::>(); + let clob = "x".repeat(40_000); + let params: [&dyn ToSql; 2] = [&blob, &clob]; + let encoded = build_sqldta(¶ms, &descriptors).unwrap(); + + let mut writer = DssWriter::new(7); + writer.write_request_next_same_corr(&[0, 4, 0x20, 0x0B], true); + encoded.write(&mut writer, true); + writer.write_request(&db2_proto::commands::rdbcmm::build_rdbcmm(), false); + let mut reader = DssReader::new(writer.finish()); + let mut frames = Vec::new(); + while let Some(frame) = reader.next_frame().unwrap() { + frames.push(frame); + } + + let shape = frames + .iter() + .map(|frame| { + ( + frame.header.dss_type, + frame.header.flags.chained, + frame.header.flags.same_correlation, + frame.header.correlation_id, + ) + }) + .collect::>(); + use db2_proto::dss::DssType::{Object, Request}; + assert_eq!( + shape, + [ + (Request, true, true, 7), + (Object, true, true, 7), + (Object, true, true, 7), + (Object, true, false, 7), + (Request, false, false, 7), + ] + ); + assert_eq!(frames[1].payload, encoded.sqldta); + let blob_extdta = ClientInner::parse_ddm(&frames[2].payload).unwrap(); + let clob_extdta = ClientInner::parse_ddm(&frames[3].payload).unwrap(); + assert_eq!(blob_extdta.code_point, codepoints::EXTDTA); + assert_eq!(blob_extdta.data[0], INDICATOR_NOT_NULL); + assert_eq!(&blob_extdta.data[1..], blob.as_slice()); + assert_eq!(clob_extdta.data[0], INDICATOR_NOT_NULL); + assert_eq!(&clob_extdta.data[1..], clob.as_bytes()); + } + + #[tokio::test] + async fn invalid_description_reply_fails_execute() { + // DSCINVRM was previously ignored, so a rejected SQLDTA reported + // rowCount=0 without an error. + let mut payload = db2_proto::ddm::DdmBuilder::new(codepoints::DSCINVRM) + .add_u16(codepoints::SVRCOD, 8) + .build(); + payload.extend( + db2_proto::ddm::DdmBuilder::new(codepoints::SQLCARD) + .add_raw(&[0xFF]) + .build(), + ); + let client = Client::new(Config::default()); + let mut inner = client.inner.lock().await; + + let err = inner + .process_execute_reply(&[reply_frame(payload, 1)]) + .await + .unwrap_err(); + + assert!( + matches!(&err, Error::Protocol(message) if message.contains("execute failed with DSCINVRM")), + "{err:?}" + ); + } + #[tokio::test] async fn parse_input_descriptors_reads_zos_descriptor_table() { let frames = vec![reply_frame( @@ -8566,7 +8918,7 @@ mod tests { let state = "43"; let reason = "2 REQ SEPARATED"; let params: [&dyn ToSql; 4] = [&partition, &processed_date, &state, &reason]; - let sqldta = build_sqldta(¶ms, &descriptors).unwrap(); + let sqldta = build_sqldta(¶ms, &descriptors).unwrap().sqldta; assert_eq!(sqldta[11], db2_proto::types::DRDA_TYPE_NVARCHAR); assert_eq!(sqldta[14], db2_proto::types::DRDA_TYPE_NVARCHAR); diff --git a/crates/db2-client/src/statement.rs b/crates/db2-client/src/statement.rs index 66df134..b94f3b1 100644 --- a/crates/db2-client/src/statement.rs +++ b/crates/db2-client/src/statement.rs @@ -118,11 +118,11 @@ impl PreparedStatement { ddm.add_code_point(0x215D, &[0x01]); // QRYCLSIMP = 1 ddm.build() }; - let sqldta_data = build_sqldta(params, &self.param_descriptors)?; + let sqldta = build_sqldta(params, &self.param_descriptors)?; let mut writer = DssWriter::new(corr_id); writer.write_request_next_same_corr(&opnqry_data, true); - writer.write_object(&sqldta_data, false); + sqldta.write(&mut writer, false); let send_buf = writer.finish(); if env::var_os("DB2_WIRE_DEBUG_HEX").is_some() { eprintln!( @@ -171,12 +171,12 @@ impl PreparedStatement { } else { db2_proto::commands::excsqlstt::build_excsqlstt_default(&pkgnamcsn) }; - let sqldta_data = build_sqldta(params, &self.param_descriptors)?; + let sqldta = build_sqldta(params, &self.param_descriptors)?; let rdbcmm_data = db2_proto::commands::rdbcmm::build_rdbcmm(); let mut writer = DssWriter::new(corr_id); writer.write_request_next_same_corr(&excsqlstt_data, true); - writer.write_object(&sqldta_data, guard.auto_commit); + sqldta.write(&mut writer, guard.auto_commit); if guard.auto_commit { writer.write_request(&rdbcmm_data, false); } diff --git a/crates/db2-proto/src/codepoints.rs b/crates/db2-proto/src/codepoints.rs index 4456205..9463e52 100644 --- a/crates/db2-proto/src/codepoints.rs +++ b/crates/db2-proto/src/codepoints.rs @@ -73,6 +73,7 @@ pub const ENDQRYRM: u16 = 0x220B; // End of Query Reply Message pub const QRYNOPRM: u16 = 0x2202; // Query Not Open Reply Message pub const OPNQRYRM: u16 = 0x2205; // Open Query Complete Reply Message pub const DTAMCHRM: u16 = 0x220E; // Data Descriptor Mismatch Reply Message +pub const DSCINVRM: u16 = 0x220A; // Invalid Description Reply Message pub const RDBUPDRM: u16 = 0x2218; // RDB Update Reply Message pub const SYNTAXRM: u16 = 0x124C; // Data Stream Syntax Error Reply Message pub const PRCCNVRM: u16 = 0x1245; // Conversational Protocol Error Reply Message diff --git a/tests/integration/lob_regressions_test.rs b/tests/integration/lob_regressions_test.rs index 6c7081a..35ba529 100644 --- a/tests/integration/lob_regressions_test.rs +++ b/tests/integration/lob_regressions_test.rs @@ -1,4 +1,4 @@ -//! Live regressions for GitHub issues #19 and #20 (Db2 LUW 11.5 and 12.1). +//! Live regressions for GitHub issues #19, #20 and #31 (Db2 LUW 11.5 and 12.1). #[path = "../common/mod.rs"] mod common; use common::*; @@ -196,3 +196,77 @@ async fn unicode_lobs_and_graphic_values_survive_single_row_direct_and_prepared_ assert_eq!(followup.rows[0].get_by_index::(0), Some(42)); client.close().await.unwrap(); } + +#[tokio::test] +async fn lob_parameters_beyond_varying_descriptor_limit_are_written() { + use db2_proto::types::Db2Value::*; + let mut client = db2_client::Client::new(test_config()); + client.connect().await.unwrap(); + let table = unique_table(31); + client + .query( + &format!( + "CREATE TABLE {table} (ID INTEGER NOT NULL PRIMARY KEY, SMALL CLOB(32767), \ + BIG CLOB(32768), BB BLOB(1M), DB DBCLOB(32768))" + ), + &[], + ) + .await + .unwrap(); + let check = async { + client + .query( + &format!("INSERT INTO {table} (ID, SMALL, BIG) VALUES (1, 'old', 'old')"), + &[], + ) + .await?; + // Exactly the declared 32768 bytes. + let big = "β".repeat(16_384); + let blob = (0..70_000).map(|index| index as u8).collect::>(); + let graphic = "漢\u{1D11E}".repeat(5_000); + let id = 1i32; + let update = format!("UPDATE {table} SET SMALL = ?, BIG = ?, BB = ?, DB = ? WHERE ID = ?"); + let direct = client + .query(&update, &[&"small", &big, &blob, &graphic, &id]) + .await?; + assert_eq!(direct.row_count, 1); + let select = format!("SELECT SMALL, BIG, BB, DB FROM {table} WHERE ID = 1"); + let row = client.query(&select, &[]).await?.rows[0].values().to_vec(); + assert_eq!( + row, + [Clob("small".into()), Clob(big), Blob(blob), Clob(graphic)] + ); + + let overflow = "β".repeat(16_385); + let err = client + .query( + &format!("UPDATE {table} SET BIG = ? WHERE ID = ?"), + &[&overflow, &id], + ) + .await + .unwrap_err(); + assert!( + matches!(&err, db2_client::Error::Sql { sqlstate, .. } if sqlstate == "22001"), + "{err:?}" + ); + + let statement = client.prepare(&update).await?; + let empty: Vec = Vec::new(); + let null_text: Option<&str> = None; + let prepared = statement + .execute(&[&"", &"", &empty, &null_text, &id]) + .await?; + statement.close().await?; + assert_eq!(prepared.row_count, 1); + let row = client.query(&select, &[]).await?.rows[0].values().to_vec(); + assert_eq!(row, [Clob("".into()), Clob("".into()), Blob(vec![]), Null]); + Ok::<_, db2_client::Error>(()) + } + .await; + client + .query(&format!("DROP TABLE {table}"), &[]) + .await + .unwrap(); + check.unwrap(); + client.close().await.unwrap(); +} diff --git a/tests/node/lob-regressions.test.ts b/tests/node/lob-regressions.test.ts index dc666a6..658d74c 100644 --- a/tests/node/lob-regressions.test.ts +++ b/tests/node/lob-regressions.test.ts @@ -82,3 +82,62 @@ test('issue #20: 25-column rows retain DOUBLE, GRAPHIC and mixed large/NULL LOBs assert.deepEqual((await client.query('VALUES 1')).rows, [{ '1': 1 }]); } finally { await client.close(); } }); + +test('issue #31: parameters bound to LOBs declared beyond 32767 bytes are written', { timeout: 120_000 }, async () => { + const client = new Client(getConfig()); + await client.connect(); + const table = uniqueTable(31); + const select = `SELECT ID, SMALL, BIG, BB, DB FROM ${table} ORDER BY ID`; + try { + await client.query(`CREATE TABLE ${table} (ID INTEGER NOT NULL PRIMARY KEY, SMALL CLOB(32767), BIG CLOB(32768), BB BLOB(1M), DB DBCLOB(32768))`); + try { + await client.query(`INSERT INTO ${table} (ID, SMALL, BIG) VALUES (1, 'old', 'old')`); + for (const [sql, params] of [ + [`UPDATE ${table} SET SMALL = ? WHERE ID = 1`, ['new']], + [`UPDATE ${table} SET BIG = ? WHERE ID = 1`, ['new']], + [`UPDATE ${table} SET BB = ? WHERE ID = 1`, [Buffer.from([1, 2, 3])]], + [`UPDATE ${table} SET DB = ? WHERE ID = 1`, ['漢é\u{1D11E}']], + ] as const) { + assert.equal((await client.query(sql, [...params])).rowCount, 1, sql); + } + assert.deepEqual((await client.query(select)).rows, [ + { ID: 1, SMALL: 'new', BIG: 'new', BB: Buffer.from([1, 2, 3]), DB: '漢é\u{1D11E}' }, + ]); + + assert.equal((await client.query(`UPDATE ${table} SET BB = CAST(? AS BLOB(1M)) WHERE ID = ?`, [Buffer.from([4, 5]), 1])).rowCount, 1); + assert.equal((await client.query(`UPDATE ${table} SET BIG = CAST(? AS VARCHAR(100)) WHERE ID = 1`, ['cast'])).rowCount, 1); + assert.deepEqual((await client.query(`SELECT BIG, BB FROM ${table} WHERE ID = 1`)).rows, [{ BIG: 'cast', BB: Buffer.from([4, 5]) }]); + + // Boundary-sized and genuinely large values, several LOB parameters in + // one statement, followed by a non-LOB parameter. + const big = 'é'.repeat(16_384); + const blob = Buffer.from(Array.from({ length: 600_000 }, (_, index) => (index * 7) % 256)); + const graphic = '漢\u{1D11E}'.repeat(10_000); + const update = `UPDATE ${table} SET SMALL = ?, BIG = ?, BB = ?, DB = ? WHERE ID = ?`; + assert.equal((await client.query(update, ['small', big, blob, graphic, 1])).rowCount, 1); + assert.deepEqual((await client.query(select)).rows, [{ ID: 1, SMALL: 'small', BIG: big, BB: blob, DB: graphic }]); + await assert.rejects(client.query(`UPDATE ${table} SET BIG = ? WHERE ID = 1`, [`${big}x`]), { sqlstate: '22001' }); + assert.equal((await client.query(`SELECT LENGTH(BIG) AS L FROM ${table}`)).rows[0].L, 32_768); + + const statement = await client.prepare(update); + try { + assert.equal((await statement.execute(['', '', Buffer.alloc(0), null, 1])).rowCount, 1); + assert.deepEqual((await client.query(select)).rows, [{ ID: 1, SMALL: '', BIG: '', BB: Buffer.alloc(0), DB: null }]); + } finally { await statement.close(); } + + const insert = await client.prepare(`INSERT INTO ${table} (ID, BIG, BB, DB) VALUES (?, ?, ?, ?)`); + try { + assert.equal((await insert.executeBatch([[2, 'two', Buffer.from([2]), 'b'], [3, null, null, '']])).rowCount, 2); + } finally { await insert.close(); } + assert.deepEqual((await client.query(`SELECT ID, BIG, BB, DB FROM ${table} WHERE ID > 1 ORDER BY ID`)).rows, [ + { ID: 2, BIG: 'two', BB: Buffer.from([2]), DB: 'b' }, + { ID: 3, BIG: null, BB: null, DB: '' }, + ]); + + assert.deepEqual((await client.query(`SELECT ID FROM ${table} WHERE LENGTH(CAST(? AS CLOB(1M))) = ? ORDER BY ID`, [big, 32_768])).rows, [ + { ID: 1 }, { ID: 2 }, { ID: 3 }, + ]); + } finally { await client.query(`DROP TABLE ${table}`); } + assert.deepEqual((await client.query('VALUES 1')).rows, [{ '1': 1 }]); + } finally { await client.close(); } +}); From 334f59ba0a879718f4da7a7378663ed816a42068 Mon Sep 17 00:00:00 2001 From: Abit Gurung Date: Sun, 4 Oct 2026 19:58:49 -0500 Subject: [PATCH 2/3] fix: keep EXTDTA aligned after empty LOBs and externalize XML inputs LUW pairs EXTDTA objects only with nonempty LOB placeholders. Sending an indicator-only EXTDTA for an empty value shifted every later LOB value and LUW rejected the request ("lob lengths don't match"). A zero-length placeholder is now complete on its own. XML inputs declare no length, so values beyond 32767 bytes could not fit the VARCHAR descriptor and failed client-side. On LUW they are now sent as LOBCMIXED placeholders with EXTDTA, which Db2 parses into the XML target exactly like the VARCHAR form. --- crates/db2-client/src/connection.rs | 65 ++++++++++++++-- tests/integration/lob_regressions_test.rs | 25 +++++- tests/node/lob-regressions.test.ts | 93 +++++++++++++++++++++++ 3 files changed, 174 insertions(+), 9 deletions(-) diff --git a/crates/db2-client/src/connection.rs b/crates/db2-client/src/connection.rs index df74d95..7fb46b1 100644 --- a/crates/db2-client/src/connection.rs +++ b/crates/db2-client/src/connection.rs @@ -6358,6 +6358,8 @@ pub(crate) fn build_sqldta( /// Describe LUW LOB inputs that exceed a varying-length descriptor as DRDA /// LOB placeholders, so their values are sent in EXTDTA after SQLDTA. /// Smaller LOB targets keep the VARCHAR/VARBINARY/VARGRAPHIC encoding. +/// XML inputs declare no length, so they are always sent as LOBCMIXED +/// text, which LUW parses into the XML target like a VARCHAR value. fn externalize_luw_lob_input_descriptors( descriptors: &mut [db2_proto::fdoca::ColumnDescriptor], server: Option<&ServerInfo>, @@ -6368,12 +6370,14 @@ fn externalize_luw_lob_input_descriptors( return; } for descriptor in descriptors { - if descriptor.length <= MAX_VARYING_INPUT_LEN { + if descriptor.length <= MAX_VARYING_INPUT_LEN + && !matches!(descriptor.db2_type, Db2Type::Xml) + { continue; } let base = match descriptor.db2_type { Db2Type::Blob => db2_proto::types::DRDA_TYPE_BLOB, - Db2Type::Clob => DRDA_TYPE_CLOB_MIXED, + Db2Type::Clob | Db2Type::Xml => DRDA_TYPE_CLOB_MIXED, Db2Type::DbClob => db2_proto::types::DRDA_TYPE_DBCLOB, _ => continue, }; @@ -6947,6 +6951,12 @@ fn build_sqldta_row_data( let (bytes, length) = encode_lob_parameter_value(&value, descriptor)?; data.push(LUW_LOB_INPUT_EXTDTA_STATUS); data.extend_from_slice(&length.to_be_bytes()); + // A zero-length value is complete in its placeholder. LUW pairs + // EXTDTA objects only with nonempty LOBs and rejects an empty + // object before a later LOB ("lob lengths don't match"). + if bytes.is_empty() { + continue; + } // Like LOB result values, EXTDTA repeats the nullable indicator. let mut payload = Vec::with_capacity(1 + bytes.len()); if descriptor.nullable { @@ -6977,7 +6987,7 @@ fn encode_lob_parameter_value( let bytes = match descriptor.db2_type { Db2Type::Blob => extract_binary(value)?, // LOBCMIXED data uses the UTF-8 mixed-byte CCSID requested at ACCRDB. - Db2Type::Clob => extract_text(value)?.as_bytes().to_vec(), + Db2Type::Clob | Db2Type::Xml => extract_text(value)?.as_bytes().to_vec(), Db2Type::DbClob => encode_graphic_text_bytes(value)?, _ => { return Err(Error::ParameterType(format!( @@ -8724,7 +8734,8 @@ mod tests { let id = 1i32; let null_clob: Option<&str> = None; let empty_blob: Vec = Vec::new(); - let params: [&dyn ToSql; 7] = [&small, &clob, &blob, &dbclob, &id, &null_clob, &empty_blob]; + // The empty BLOB precedes nonempty LOBs, which must not shift EXTDTA. + let params: [&dyn ToSql; 7] = [&small, &clob, &empty_blob, &dbclob, &id, &null_clob, &blob]; let encoded = build_sqldta(¶ms, &descriptors).unwrap(); let (triplets, fdodta) = split_sqldta(&encoded.sqldta); assert_eq!( @@ -8743,12 +8754,12 @@ mod tests { let mut expected = vec![NULL_LID, 0x00, 0x00, 0x05]; expected.extend_from_slice(b"small"); expected.extend(lob_placeholder(2)); - expected.extend(lob_placeholder(3)); + expected.extend(lob_placeholder(0)); // DBCLOB lengths count UTF-16 code units: d plus a surrogate pair. expected.extend(lob_placeholder(3)); expected.extend_from_slice(&[0x00, 0x01, 0x00, 0x00, 0x00]); expected.push(NULL_DATA); - expected.extend(lob_placeholder(0)); + expected.extend(lob_placeholder(3)); assert_eq!(fdodta, expected); let extdta = encoded @@ -8764,13 +8775,51 @@ mod tests { extdta, [ vec![0x00, 0xC3, 0xA9], - vec![0x00, 0x00, 0xFF, 0x07], vec![0x00, 0x00, 0x64, 0xD8, 0x34, 0xDD, 0x1E], - vec![0x00], + vec![0x00, 0x00, 0xFF, 0x07], ] ); } + #[tokio::test] + async fn luw_xml_inputs_always_use_placeholders_and_extdta() { + // XML has no declared input length, so values beyond 32767 bytes + // cannot fit a VARCHAR descriptor; LUW parses LOBCMIXED text instead. + let frames = vec![reply_frame(input_sqldard(&[(989, 0, 0), (497, 4, 0)]), 1)]; + let client = Client::new(Config::default()); + let inner = client.inner.lock().await; + let descriptors = inner.parse_input_descriptors(&frames).unwrap(); + assert_eq!(descriptors[0].extdta_reference_length, Some(9)); + + let large = format!("{}", "é".repeat(20_000)); + for (xml, id) in [(Some(""), 1i32), (Some(large.as_str()), 2), (None, 3)] { + let params: [&dyn ToSql; 2] = [&xml, &id]; + let encoded = build_sqldta(¶ms, &descriptors).unwrap(); + let (triplets, fdodta) = split_sqldta(&encoded.sqldta); + assert_eq!(triplets, [(0xCF, 0x8009), (0x03, 0x0004)]); + + let mut expected = match xml { + Some(xml) => lob_placeholder(xml.len() as u64), + None => vec![NULL_DATA], + }; + expected.insert(0, NULL_LID); + expected.push(INDICATOR_NOT_NULL); + expected.extend_from_slice(&id.to_le_bytes()); + assert_eq!(fdodta, expected); + + let extdta = encoded + .extdta + .iter() + .map(|object| ClientInner::parse_ddm(object).unwrap().data) + .collect::>(); + let expected = xml + .map(|xml| [&[0x00][..], xml.as_bytes()].concat()) + .into_iter() + .collect::>(); + assert_eq!(extdta, expected); + } + } + #[test] fn zos_lob_input_descriptors_are_not_externalized() { let descriptor = db2_proto::fdoca::ColumnDescriptor { diff --git a/tests/integration/lob_regressions_test.rs b/tests/integration/lob_regressions_test.rs index 35ba529..f62167c 100644 --- a/tests/integration/lob_regressions_test.rs +++ b/tests/integration/lob_regressions_test.rs @@ -234,7 +234,12 @@ async fn lob_parameters_beyond_varying_descriptor_limit_are_written() { let row = client.query(&select, &[]).await?.rows[0].values().to_vec(); assert_eq!( row, - [Clob("small".into()), Clob(big), Blob(blob), Clob(graphic)] + [ + Clob("small".into()), + Clob(big), + Blob(blob), + Clob(graphic.clone()) + ] ); let overflow = "β".repeat(16_385); @@ -260,6 +265,24 @@ async fn lob_parameters_beyond_varying_descriptor_limit_are_written() { assert_eq!(prepared.row_count, 1); let row = client.query(&select, &[]).await?.rows[0].values().to_vec(); assert_eq!(row, [Clob("".into()), Clob("".into()), Blob(vec![]), Null]); + + // Empty LOBs before a nonempty LOB send no EXTDTA of their own. + let statement = client.prepare(&update).await?; + let prepared = statement + .execute(&[&"s", &"", &empty, &graphic, &id]) + .await?; + statement.close().await?; + assert_eq!(prepared.row_count, 1); + let row = client.query(&select, &[]).await?.rows[0].values().to_vec(); + assert_eq!( + row, + [ + Clob("s".into()), + Clob("".into()), + Blob(vec![]), + Clob(graphic) + ] + ); Ok::<_, db2_client::Error>(()) } .await; diff --git a/tests/node/lob-regressions.test.ts b/tests/node/lob-regressions.test.ts index 658d74c..7e50053 100644 --- a/tests/node/lob-regressions.test.ts +++ b/tests/node/lob-regressions.test.ts @@ -141,3 +141,96 @@ test('issue #31: parameters bound to LOBs declared beyond 32767 bytes are writte assert.deepEqual((await client.query('VALUES 1')).rows, [{ '1': 1 }]); } finally { await client.close(); } }); + +test('issue #31: empty LOB parameters do not shift later EXTDTA values', { timeout: 180_000 }, async () => { + const client = new Client(getConfig()); + await client.connect(); + const table = uniqueTable(31); + const routine = `${table}_P`; + const select = `SELECT ID, C, X, B, G FROM ${table} ORDER BY ID`; + let routineCreated = false; + try { + await client.query(`CREATE TABLE ${table} (ID INTEGER NOT NULL PRIMARY KEY, C CLOB(32768), X CLOB(2M), B BLOB(2M), G DBCLOB(1M))`); + try { + // Every empty/NULL/small/large order of the externalized columns; + // empty and NULL values must stay distinct. + const kinds = { + empty: ['', '', Buffer.alloc(0), ''], + null: [null, null, null, null], + small: ['é', 'x€\u{1D11E}', Buffer.from([0, 255, 1]), '漢\u{1D11E}'], + large: ['c'.repeat(32_768), 'x€'.repeat(20_000), Buffer.alloc(40_000, 3), '漢'.repeat(20_000)], + } as const; + const names = Object.keys(kinds) as (keyof typeof kinds)[]; + const combinations = names.flatMap(c => names.flatMap(x => names.flatMap(b => names.map(g => + [kinds[c][0], kinds[x][1], kinds[b][2], kinds[g][3]])))); + let rows = combinations.map((values, index) => [index + 1, ...values]); + const shift = (by: number) => rows.map(([id], index) => [id, ...combinations[(index + by) % combinations.length]]); + + const insert = await client.prepare(`INSERT INTO ${table} VALUES (?, ?, ?, ?, ?)`); + try { + assert.equal((await insert.executeBatch(rows)).rowCount, rows.length); + // Reviewer shape: 10 rows of 33-60KB values with NULL and empty mixed in. + const mixed = Array.from({ length: 10 }, (_, i) => [ + i + 1_000, i % 2 ? 'b' : null, i % 3 ? 'b€'.repeat(12_000) : null, + i % 4 ? Buffer.alloc(33_000 + i, i) : Buffer.alloc(0), i % 5 ? '漢'.repeat(20_000) : null, + ]); + assert.equal((await insert.executeBatch(mixed)).rowCount, mixed.length); + assert.deepEqual((await client.query(select, [], { rowMode: 'array' })).rows, [...rows, ...mixed]); + await client.query(`DELETE FROM ${table} WHERE ID >= 1000`); + } finally { await insert.close(); } + + // Direct and prepared single executions alternate per row. + rows = shift(1); + const updateSql = `UPDATE ${table} SET C = ?, X = ?, B = ?, G = ? WHERE ID = ?`; + const update = await client.prepare(updateSql); + try { + for (const [index, [id, ...values]] of rows.entries()) { + const result = index % 2 ? await update.execute([...values, id]) : await client.query(updateSql, [...values, id]); + assert.equal(result.rowCount, 1); + } + } finally { await update.close(); } + assert.deepEqual((await client.query(select, [], { rowMode: 'array' })).rows, rows); + + await client.query(`CREATE PROCEDURE ${routine} (IN P_ID INTEGER, IN P_C CLOB(32768), IN P_X CLOB(2M), IN P_B BLOB(2M), IN P_G DBCLOB(1M)) LANGUAGE SQL BEGIN UPDATE ${table} SET C = P_C, X = P_X, B = P_B, G = P_G WHERE ID = P_ID; END`); + routineCreated = true; + rows = shift(2); + for (const [id, ...values] of rows) { + await client.query(`CALL ${routine}(?, ?, ?, ?, ?)`, [id, ...values]); + } + assert.deepEqual((await client.query(select, [], { rowMode: 'array' })).rows, rows); + } finally { + if (routineCreated) await client.query(`DROP PROCEDURE ${routine}`); + await client.query(`DROP TABLE ${table}`); + } + assert.deepEqual((await client.query('VALUES 1')).rows, [{ '1': 1 }]); + } finally { await client.close(); } +}); + +test('issue #31: XML parameters beyond 32767 bytes are written', { timeout: 120_000 }, async () => { + const client = new Client(getConfig()); + await client.connect(); + const table = uniqueTable(31); + try { + await client.query(`CREATE TABLE ${table} (ID INTEGER NOT NULL PRIMARY KEY, X XML, C CLOB(1M))`); + try { + const small = 'é'; + const large = `${'x€'.repeat(30_000)}`; + const select = `SELECT ID, XMLSERIALIZE(X AS CLOB(1M)) AS X FROM ${table} ORDER BY ID`; + assert.equal((await client.query(`INSERT INTO ${table} (ID, X, C) VALUES (?, ?, ?)`, [1, large, 'c'.repeat(40_000)])).rowCount, 1); + assert.equal((await client.query(`UPDATE ${table} SET X = ? WHERE ID = ?`, [large.replace('b>', 'c>').replace('${'x€'.repeat(30_000)}` }, + { ID: 2, X: small }, + { ID: 3, X: null }, + { ID: 4, X: large }, + ]); + assert.deepEqual((await client.query(`SELECT ID FROM ${table} WHERE XMLEXISTS('$d/b' PASSING CAST(? AS XML) AS "d") AND ID = ?`, [large, 4])).rows, [{ ID: 4 }]); + await assert.rejects(client.query(`UPDATE ${table} SET X = ? WHERE ID = 1`, ['']), { sqlstate: '2200M' }); + } finally { await client.query(`DROP TABLE ${table}`); } + assert.deepEqual((await client.query('VALUES 1')).rows, [{ '1': 1 }]); + } finally { await client.close(); } +}); From 37bae9a1eb78543a2e2f869a6195e6d1903681b6 Mon Sep 17 00:00:00 2001 From: Abit Gurung Date: Sun, 4 Oct 2026 20:17:07 -0500 Subject: [PATCH 3/3] perf: pass binary parameters to the addon without per-byte arrays normalizeParam turned every Buffer, typed array and ArrayBuffer into a JS array, which the addon then read one element at a time and held as one JSON value per byte. With LOB parameters beyond 32 KB now written, a 50 MB BLOB took ~3 s and blocked the event loop for most of it. Byte views now reach the addon as Uint8Array, which copies them once into a Vec, and parameters move into Db2Value instead of being cloned. A 50 MB BLOB update drops from ~3.0 s to ~0.5 s, on par with CLOB. --- crates/db2-napi/index.cjs | 11 +++--- crates/db2-napi/src/js_connection.rs | 5 +-- crates/db2-napi/src/js_pool.rs | 5 +-- crates/db2-napi/src/js_statement.rs | 7 +--- crates/db2-napi/src/js_transaction.rs | 5 +-- crates/db2-napi/src/js_types.rs | 56 ++++++++++++--------------- tests/node/lob-regressions.test.ts | 31 +++++++++++++++ 7 files changed, 66 insertions(+), 54 deletions(-) diff --git a/crates/db2-napi/index.cjs b/crates/db2-napi/index.cjs index 95bbda6..abdf70f 100644 --- a/crates/db2-napi/index.cjs +++ b/crates/db2-napi/index.cjs @@ -276,14 +276,15 @@ function normalizeParam(value) { } return value.toISOString().replace('T', ' ').replace(/Z$/, '') } - if (typeof Buffer !== 'undefined' && Buffer.isBuffer(value)) { - return Array.from(value) - } + // Byte views reach the addon as-is; copying them into a JS array would cost + // one property access per byte for large BLOB parameters. if (typeof ArrayBuffer !== 'undefined' && ArrayBuffer.isView(value)) { - return Array.from(new Uint8Array(value.buffer, value.byteOffset, value.byteLength)) + return value instanceof Uint8Array + ? value + : new Uint8Array(value.buffer, value.byteOffset, value.byteLength) } if (typeof ArrayBuffer !== 'undefined' && value instanceof ArrayBuffer) { - return Array.from(new Uint8Array(value)) + return new Uint8Array(value) } return value } diff --git a/crates/db2-napi/src/js_connection.rs b/crates/db2-napi/src/js_connection.rs index 03f7f7e..9d06cf3 100644 --- a/crates/db2-napi/src/js_connection.rs +++ b/crates/db2-napi/src/js_connection.rs @@ -150,10 +150,7 @@ impl JsClient { .as_mut() .ok_or_else(|| napi::Error::from_reason("Client is not connected"))?; - let db2_params = match ¶ms { - Some(p) => js_params_to_db2(p), - None => Vec::new(), - }; + let db2_params = params.map(js_params_to_db2).unwrap_or_default(); let param_refs: Vec<&dyn db2_client::ToSql> = db2_params .iter() diff --git a/crates/db2-napi/src/js_pool.rs b/crates/db2-napi/src/js_pool.rs index c76ed71..9845d21 100644 --- a/crates/db2-napi/src/js_pool.rs +++ b/crates/db2-napi/src/js_pool.rs @@ -156,10 +156,7 @@ impl JsPool { let mut napi_diagnostics = Vec::new(); let params_started = collect_diagnostics.then(Instant::now); - let db2_params = match ¶ms { - Some(p) => js_params_to_db2(p), - None => Vec::new(), - }; + let db2_params = params.map(js_params_to_db2).unwrap_or_default(); push_elapsed_diagnostic(&mut napi_diagnostics, "napi_pool_params_ms", params_started); let refs_started = collect_diagnostics.then(Instant::now); diff --git a/crates/db2-napi/src/js_statement.rs b/crates/db2-napi/src/js_statement.rs index 8963502..f9948f7 100644 --- a/crates/db2-napi/src/js_statement.rs +++ b/crates/db2-napi/src/js_statement.rs @@ -43,10 +43,7 @@ impl JsPreparedStatement { .as_mut() .ok_or_else(|| napi::Error::from_reason("PreparedStatement is closed"))?; - let db2_params = match ¶ms { - Some(p) => js_params_to_db2(p), - None => Vec::new(), - }; + let db2_params = params.map(js_params_to_db2).unwrap_or_default(); let param_refs: Vec<&dyn db2_client::ToSql> = db2_params .iter() @@ -86,7 +83,7 @@ impl JsPreparedStatement { // Convert all rows from JSON to Db2Value let db2_rows: Vec> = - param_rows.iter().map(|row| js_params_to_db2(row)).collect(); + param_rows.into_iter().map(js_params_to_db2).collect(); // Build references for each row let param_ref_rows: Vec> = db2_rows diff --git a/crates/db2-napi/src/js_transaction.rs b/crates/db2-napi/src/js_transaction.rs index 5a97a16..400b755 100644 --- a/crates/db2-napi/src/js_transaction.rs +++ b/crates/db2-napi/src/js_transaction.rs @@ -44,10 +44,7 @@ impl JsTransaction { napi::Error::from_reason("Transaction is already committed or rolled back") })?; - let db2_params = match ¶ms { - Some(p) => js_params_to_db2(p), - None => Vec::new(), - }; + let db2_params = params.map(js_params_to_db2).unwrap_or_default(); let param_refs: Vec<&dyn db2_client::ToSql> = db2_params .iter() diff --git a/crates/db2-napi/src/js_types.rs b/crates/db2-napi/src/js_types.rs index 864a982..646b8d4 100644 --- a/crates/db2-napi/src/js_types.rs +++ b/crates/db2-napi/src/js_types.rs @@ -623,7 +623,11 @@ fn env_truthy(name: &str) -> bool { } /// Convert parameters before napi's JSON decoder can panic on JavaScript BigInt. -pub struct JsParameter(serde_json::Value); +/// Binary values bypass JSON so a large Buffer is copied once, not per byte. +pub enum JsParameter { + Json(serde_json::Value), + Bytes(Vec), +} impl TypeName for JsParameter { fn type_name() -> &'static str { @@ -661,21 +665,11 @@ impl FromNapiValue for JsParameter { } ValueType::Object if value.is_buffer()? => { let buffer = unsafe { napi::bindgen_prelude::Buffer::from_napi_value(env, raw)? }; - serde_json::Value::Array( - buffer - .iter() - .map(|byte| serde_json::Value::from(*byte)) - .collect(), - ) + return Ok(Self::Bytes(buffer.to_vec())); } ValueType::Object if value.is_typedarray()? => { let bytes = napi::bindgen_prelude::Uint8Array::from_napi_value(env, raw)?; - serde_json::Value::Array( - bytes - .iter() - .map(|byte| serde_json::Value::from(*byte)) - .collect(), - ) + return Ok(Self::Bytes(bytes.to_vec())); } ValueType::Object if { @@ -697,12 +691,7 @@ impl FromNapiValue for JsParameter { } else { std::slice::from_raw_parts(data.cast::(), length) }; - serde_json::Value::Array( - bytes - .iter() - .map(|byte| serde_json::Value::from(*byte)) - .collect(), - ) + return Ok(Self::Bytes(bytes.to_vec())); } ValueType::Object if value.is_array()? => { let object = value.coerce_to_object()?; @@ -722,9 +711,9 @@ impl FromNapiValue for JsParameter { "Array parameters must contain only integer bytes (0..255)", )); } - bytes.push(serde_json::Value::from(byte as u8)); + bytes.push(byte as u8); } - serde_json::Value::Array(bytes) + return Ok(Self::Bytes(bytes)); } ValueType::Object => { return Err(parameter_error( @@ -743,23 +732,26 @@ impl FromNapiValue for JsParameter { ValueType::String | ValueType::Boolean => serde_json::Value::from_napi_value(env, raw)?, _ => return Err(parameter_error(env, "Unsupported parameter type")), }; - Ok(Self(json)) + Ok(Self::Json(json)) } } -pub fn js_params_to_db2(params: &[JsParameter]) -> Vec { +pub fn js_params_to_db2(params: Vec) -> Vec { params - .iter() - .map(|param| json_to_db2_value(¶m.0)) + .into_iter() + .map(|param| match param { + JsParameter::Json(value) => json_to_db2_value(value), + JsParameter::Bytes(bytes) => db2_proto::types::Db2Value::Binary(bytes), + }) .collect() } /// Convert a single JSON value to a Db2Value. -fn json_to_db2_value(val: &serde_json::Value) -> db2_proto::types::Db2Value { +fn json_to_db2_value(val: serde_json::Value) -> db2_proto::types::Db2Value { use db2_proto::types::Db2Value; match val { serde_json::Value::Null => Db2Value::Null, - serde_json::Value::Bool(b) => Db2Value::Boolean(*b), + serde_json::Value::Bool(b) => Db2Value::Boolean(b), serde_json::Value::Number(n) => { if let Some(i) = n.as_i64() { if i >= i32::MIN as i64 && i <= i32::MAX as i64 { @@ -773,7 +765,7 @@ fn json_to_db2_value(val: &serde_json::Value) -> db2_proto::types::Db2Value { Db2Value::Null } } - serde_json::Value::String(s) => Db2Value::VarChar(s.clone()), + serde_json::Value::String(s) => Db2Value::VarChar(s), serde_json::Value::Array(arr) => { // Treat arrays of numbers as binary data let bytes: Vec = arr @@ -782,9 +774,9 @@ fn json_to_db2_value(val: &serde_json::Value) -> db2_proto::types::Db2Value { .collect(); Db2Value::Binary(bytes) } - serde_json::Value::Object(obj) => buffer_like_object_to_bytes(obj) + serde_json::Value::Object(obj) => buffer_like_object_to_bytes(&obj) .map(Db2Value::Binary) - .unwrap_or_else(|| Db2Value::VarChar(val.to_string())), + .unwrap_or_else(|| Db2Value::VarChar(serde_json::Value::Object(obj).to_string())), } } @@ -1052,7 +1044,7 @@ mod tests { "data": [0, 1, 127, 255] }); assert_eq!( - json_to_db2_value(&node_buffer_json), + json_to_db2_value(node_buffer_json), db2_proto::types::Db2Value::Binary(vec![0, 1, 127, 255]) ); @@ -1063,7 +1055,7 @@ mod tests { "3": 239 }); assert_eq!( - json_to_db2_value(&indexed_object), + json_to_db2_value(indexed_object), db2_proto::types::Db2Value::Binary(vec![222, 173, 190, 239]) ); } diff --git a/tests/node/lob-regressions.test.ts b/tests/node/lob-regressions.test.ts index 7e50053..2b87b97 100644 --- a/tests/node/lob-regressions.test.ts +++ b/tests/node/lob-regressions.test.ts @@ -234,3 +234,34 @@ test('issue #31: XML parameters beyond 32767 bytes are written', { timeout: 120_ assert.deepEqual((await client.query('VALUES 1')).rows, [{ '1': 1 }]); } finally { await client.close(); } }); + +test('binary parameters pass byte views without per-byte conversion', { timeout: 120_000 }, async () => { + const client = new Client(getConfig()); + await client.connect(); + const table = uniqueTable(31); + try { + await client.query(`CREATE TABLE ${table} (ID INTEGER NOT NULL PRIMARY KEY, B BLOB(100M))`); + try { + const backing = Buffer.from([9, 1, 2, 3, 4, 9]); + const words = new Int16Array([0x0201, 0x0403]); + const large = Buffer.alloc(8 << 20); + for (let index = 0; index < large.length; index += 4096) large[index] = index >> 12; + const values: [unknown, Buffer][] = [ + [backing.subarray(1, 5), Buffer.from([1, 2, 3, 4])], + [new Uint8Array(backing.buffer, backing.byteOffset + 2, 3), Buffer.from([2, 3, 4])], + [words, Buffer.from(words.buffer)], + [new DataView(backing.buffer, backing.byteOffset + 1, 2), Buffer.from([1, 2])], + [new Uint8Array([7, 8]).buffer, Buffer.from([7, 8])], + [[5, 6], Buffer.from([5, 6])], + [large, large], + ]; + for (const [index, [value]] of values.entries()) { + assert.equal((await client.query(`INSERT INTO ${table} VALUES (?, ?)`, [index, value as Buffer])).rowCount, 1); + } + const rows = (await client.query(`SELECT B FROM ${table} ORDER BY ID`, [], { rowMode: 'array' })).rows; + assert.equal(rows.length, values.length); + for (const [index, [, expected]] of values.entries()) assert.ok(expected.equals(rows[index][0] as Buffer), `value ${index}`); + await assert.rejects(async () => client.query(`INSERT INTO ${table} VALUES (?, ?)`, [99, [1, 256] as unknown as Buffer]), /integer bytes/); + } finally { await client.query(`DROP TABLE ${table}`); } + } finally { await client.close(); } +});