From 1fbc67192a7c15b1618041cb907eeda181a40684 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Mon, 26 Oct 2026 00:59:18 +0000 Subject: [PATCH 01/11] fix(update): refuse an update whose result nests beyond 32 levels UpdateItem and the TransactWriteItems Update checked the nesting depth of ExpressionAttributeValues but not of the item the update produced, so a SET of a 2-level map at a 30-level path stored an attribute nested past the limit, on every backend. Amazon DynamoDB refuses an update whose result nests an attribute more than 32 levels (31 M or L wrappers around a scalar) and writes nothing: UpdateItem answers ValidationException `Nesting Levels have exceeded supported limits`, and a TransactWriteItems Update is cancelled with a ValidationError reason carrying that text (measured 2026-10-25 at 19:53 UTC and 2026-10-26 at 00:55 UTC, each case twice). A failing condition is reported first. apply_update_validated, which every backend's update and transactional update path calls after its condition check, now checks the top-level attributes the SET actions wrote. REMOVE, ADD and DELETE cannot deepen a value, and other attributes are not checked, so an item stored deeper than the limit before this change stays updatable through them. --- crates/core/src/expression/mod.rs | 1 + .../core/src/expression/update_evaluator.rs | 1 + crates/core/src/expression/update_nesting.rs | 128 ++++++++++++++++++ 3 files changed, 130 insertions(+) create mode 100644 crates/core/src/expression/update_nesting.rs diff --git a/crates/core/src/expression/mod.rs b/crates/core/src/expression/mod.rs index db2613a7e..e6d61d4cd 100755 --- a/crates/core/src/expression/mod.rs +++ b/crates/core/src/expression/mod.rs @@ -20,6 +20,7 @@ mod resolver; mod search_condition; mod tokenizer; mod update_evaluator; +mod update_nesting; mod update_parser; pub use ast::{ArithOp, CompareOp, Expr, PathElement, UpdateAction}; diff --git a/crates/core/src/expression/update_evaluator.rs b/crates/core/src/expression/update_evaluator.rs index 52f2a149b..ec278d8c4 100755 --- a/crates/core/src/expression/update_evaluator.rs +++ b/crates/core/src/expression/update_evaluator.rs @@ -113,6 +113,7 @@ pub fn apply_update_validated( } } apply_update(actions, item, maps)?; + super::update_nesting::validate_written_nesting_depth(actions, item, maps)?; crate::validation::validate_vector_write_changed( item, &before, diff --git a/crates/core/src/expression/update_nesting.rs b/crates/core/src/expression/update_nesting.rs new file mode 100644 index 000000000..8ec7bc417 --- /dev/null +++ b/crates/core/src/expression/update_nesting.rs @@ -0,0 +1,128 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Nesting depth of the attributes an update expression writes. + +use std::collections::BTreeMap; + +use crate::error::DynamoDbError; +use crate::types::AttributeValue; + +use super::ast::UpdateAction; +use super::resolver::{ExpressionMaps, resolve_element_name}; + +/// Most M or L wrappers an attribute value may have: 31 wrappers around a +/// scalar is 32 levels, the service's limit. +const MAX_WRAPPERS: usize = 31; + +/// Refuse an updated item whose SET targets now nest beyond the limit. +/// +/// Only the top-level attributes a SET writes are checked. REMOVE, ADD and +/// DELETE cannot deepen a value, and an attribute the update does not touch +/// is left alone, so an item stored deeper than the limit before this check +/// existed stays updatable through its other attributes. +/// +/// # Errors +/// +/// `ValidationException` `Nesting Levels have exceeded supported limits`, the +/// service's text for UpdateItem and for a TransactWriteItems Update +/// cancellation reason (measured 2026-10-25). +pub(super) fn validate_written_nesting_depth( + actions: &[UpdateAction], + item: &BTreeMap, + maps: &ExpressionMaps, +) -> Result<(), DynamoDbError> { + for action in actions { + let UpdateAction::Set { path, .. } = action else { + continue; + }; + let Some(first) = path.first() else { + continue; + }; + let name = resolve_element_name(first, maps)?; + if item + .get(name.as_ref()) + .is_some_and(|v| wrappers_exceed(v, 0)) + { + return Err(DynamoDbError::ValidationException( + "Nesting Levels have exceeded supported limits".to_owned(), + )); + } + } + Ok(()) +} + +/// Whether `value`, sitting under `above` wrappers, has more than +/// [`MAX_WRAPPERS`] wrappers on any path down to a scalar. +fn wrappers_exceed(value: &AttributeValue, above: usize) -> bool { + let children: Box> = match value { + AttributeValue::M(map) => Box::new(map.values()), + AttributeValue::L(list) => Box::new(list.iter()), + _ => return false, + }; + let here = above + 1; + here > MAX_WRAPPERS || children.into_iter().any(|v| wrappers_exceed(v, here)) +} + +#[cfg(test)] +mod tests { + use std::collections::{BTreeMap, HashMap}; + + use super::*; + use crate::expression::{parse_update, tokenize}; + + fn nmap(wrappers: usize) -> AttributeValue { + let mut v = AttributeValue::S("leaf".to_owned()); + for _ in 0..wrappers { + v = AttributeValue::M(BTreeMap::from([("k".to_owned(), v)])); + } + v + } + + fn check(expr: &str, item: &BTreeMap) -> Result<(), DynamoDbError> { + let actions = parse_update(&tokenize(expr).unwrap()).unwrap(); + // The names map is keyed without the leading `#`. + let maps = ExpressionMaps::new( + HashMap::from([("m".to_owned(), "m".to_owned())]), + HashMap::new(), + ); + validate_written_nesting_depth(&actions, item, &maps) + } + + fn is_nesting_error(result: Result<(), DynamoDbError>) -> bool { + matches!( + result, + Err(DynamoDbError::ValidationException(m)) + if m == "Nesting Levels have exceeded supported limits" + ) + } + + #[test] + fn thirty_one_wrappers_pass_and_thirty_two_are_refused() { + let at = BTreeMap::from([("m".to_owned(), nmap(31))]); + assert!(check("SET #m = :v", &at).is_ok()); + let over = BTreeMap::from([("m".to_owned(), nmap(32))]); + assert!(is_nesting_error(check("SET #m.k = :v", &over))); + } + + #[test] + fn a_list_counts_as_a_wrapper_and_every_branch_is_checked() { + let mut deep = AttributeValue::S("leaf".to_owned()); + for _ in 0..32 { + deep = AttributeValue::L(vec![AttributeValue::N("1".to_owned()), deep]); + } + let item = BTreeMap::from([("m".to_owned(), deep)]); + assert!(is_nesting_error(check("SET #m[0] = :v", &item))); + } + + #[test] + fn attributes_the_update_does_not_set_are_not_checked() { + let item = BTreeMap::from([ + ("m".to_owned(), nmap(40)), + ("x".to_owned(), AttributeValue::S("y".to_owned())), + ]); + assert!(check("SET x = :v", &item).is_ok()); + assert!(check("REMOVE #m.k", &item).is_ok()); + assert!(is_nesting_error(check("SET #m.k = :v", &item))); + } +} From 8fd65a044d0a3e7ea271cd43f23834f8f5abb7a2 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Mon, 26 Oct 2026 00:59:18 +0000 Subject: [PATCH 02/11] test(update): nesting depth of the item an update produces Dual-target wire tests from the 2026-10-25 and 2026-10-26 measurements: SET into a deep map or list, at the top level, through list_append and through if_not_exists, up to and past the limit; a failing condition next to a result that is too deep; an update of another attribute on an item at the limit; and the TransactWriteItems Update cancellation reason. --- tests/test_update_result_nesting_depth.py | 185 ++++++++++++++++++++++ 1 file changed, 185 insertions(+) create mode 100644 tests/test_update_result_nesting_depth.py diff --git a/tests/test_update_result_nesting_depth.py b/tests/test_update_result_nesting_depth.py new file mode 100644 index 000000000..095d9381d --- /dev/null +++ b/tests/test_update_result_nesting_depth.py @@ -0,0 +1,185 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Nesting depth of the item an UpdateItem or a TransactWriteItems Update produces. + +Amazon DynamoDB refuses an update whose result nests an attribute value more +than 32 levels (31 M or L wrappers around a scalar) and writes nothing: +UpdateItem answers ValidationException ``Nesting Levels have exceeded +supported limits``, and a TransactWriteItems Update is cancelled with a +``ValidationError`` reason carrying the same text. A failing condition is +reported first. Measured on 2026-10-25 at 19:53 UTC and 2026-10-26 at 00:55 +UTC (us-east-1), each case twice. + +Depth below is counted in wrappers: a value with w wrappers has w + 1 levels. + +Dual-target against Amazon DynamoDB and extenddb. +""" + +from __future__ import annotations + +import pytest +from botocore.exceptions import ClientError + +from conftest import scoped_table + +MESSAGE = "Nesting Levels have exceeded supported limits" +NAMES = {"#m": "m", "#k": "k"} + + +def nmap(wrappers: int, leaf: dict | None = None) -> dict: + value = leaf or {"S": "leaf"} + for _ in range(wrappers): + value = {"M": {"k": value}} + return value + + +def nlist(wrappers: int) -> dict: + value = {"S": "leaf"} + for _ in range(wrappers): + value = {"L": [value]} + return value + + +def wrappers(value: dict) -> int: + depth = 0 + while True: + if value.get("M"): + value = next(iter(value["M"].values())) + elif value.get("L"): + value = value["L"][0] + else: + return depth + depth += 1 + + +def path(j: int) -> str: + return "#m" + ".#k" * j + + +@pytest.fixture(scope="module") +def table(dynamodb_client): + with scoped_table( + dynamodb_client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + key_schema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + ) as name: + yield name + + +def _put(client, table, sk, m): + key = {"pk": {"S": "nest"}, "sk": {"S": sk}} + item = {**key, "x": {"S": "x"}} + if m is not None: + item["m"] = m + client.put_item(TableName=table, Item=item) + return key + + +def _stored_wrappers(client, table, key): + item = client.get_item(TableName=table, Key=key, ConsistentRead=True)["Item"] + return wrappers(item["m"]) if "m" in item else None + + +# (case, base value of m, update expression, names, values, wrappers stored after) +ACCEPTED = [ + ("j29_w2", nmap(30), f"SET {path(29)} = :v", NAMES, {":v": nmap(2)}, 31), + ("j30_w1", nmap(30), f"SET {path(30)} = :v", NAMES, {":v": nmap(1)}, 31), + ("j1_w30", nmap(30), f"SET {path(1)} = :v", NAMES, {":v": nmap(30)}, 31), + ("j15_w16", nmap(30), f"SET {path(15)} = :v", NAMES, {":v": nmap(16)}, 31), + ("top_w31", None, "SET #m = :v", {"#m": "m"}, {":v": nmap(31)}, 31), + ("new_j1_w30", {"M": {}}, f"SET {path(1)} = :v", NAMES, {":v": nmap(30)}, 31), + ("list_j30_w1", nlist(30), "SET #m" + "[0]" * 30 + " = :v", {"#m": "m"}, {":v": nlist(1)}, 31), + ("if_not_exists_kept", nmap(30), f"SET {path(30)} = if_not_exists({path(30)}, :v)", NAMES, + {":v": nmap(2)}, 30), + ("at_limit_scalar", nmap(31), f"SET {path(31)} = :v", NAMES, {":v": {"S": "y"}}, 31), +] + +REFUSED = [ + ("j29_w3", nmap(30), f"SET {path(29)} = :v", NAMES, {":v": nmap(3)}, 30), + ("j30_w2", nmap(30), f"SET {path(30)} = :v", NAMES, {":v": nmap(2)}, 30), + ("j1_w31", nmap(30), f"SET {path(1)} = :v", NAMES, {":v": nmap(31)}, 30), + ("j15_w17", nmap(30), f"SET {path(15)} = :v", NAMES, {":v": nmap(17)}, 30), + ("new_j1_w31", {"M": {}}, f"SET {path(1)} = :v", NAMES, {":v": nmap(31)}, 0), + ("list_j30_w2", nlist(30), "SET #m" + "[0]" * 30 + " = :v", {"#m": "m"}, {":v": nlist(2)}, 30), + ("list_append", nmap(29, {"L": [{"S": "a"}]}), f"SET {path(29)} = list_append({path(29)}, :v)", NAMES, + {":v": {"L": [nmap(2)]}}, 30), + ("if_not_exists_default", nmap(29, {"M": {}}), f"SET {path(30)} = if_not_exists(#m.#z, :v)", + {**NAMES, "#z": "z"}, {":v": nmap(2)}, 29), +] + + +@pytest.mark.parametrize(("case", "base", "expr", "names", "values", "after"), ACCEPTED, + ids=[c[0] for c in ACCEPTED]) +def test_update_up_to_the_limit_is_applied(dynamodb_client, table, case, base, expr, names, values, after): + key = _put(dynamodb_client, table, f"ok-{case}", base) + dynamodb_client.update_item( + TableName=table, Key=key, UpdateExpression=expr, + ExpressionAttributeNames=names, ExpressionAttributeValues=values, + ) + assert _stored_wrappers(dynamodb_client, table, key) == after + + +@pytest.mark.parametrize(("case", "base", "expr", "names", "values", "after"), REFUSED, + ids=[c[0] for c in REFUSED]) +def test_update_past_the_limit_is_refused(dynamodb_client, table, case, base, expr, names, values, after): + key = _put(dynamodb_client, table, f"no-{case}", base) + with pytest.raises(ClientError) as exc_info: + dynamodb_client.update_item( + TableName=table, Key=key, UpdateExpression=expr, + ExpressionAttributeNames=names, ExpressionAttributeValues=values, + ) + err = exc_info.value.response["Error"] + assert (err["Code"], err["Message"]) == ("ValidationException", MESSAGE) + assert _stored_wrappers(dynamodb_client, table, key) == after + + +def test_a_failing_condition_is_reported_before_the_depth(dynamodb_client, table): + key = _put(dynamodb_client, table, "condfail", nmap(30)) + with pytest.raises(ClientError) as exc_info: + dynamodb_client.update_item( + TableName=table, Key=key, UpdateExpression=f"SET {path(30)} = :v", + ConditionExpression="#x = :no", + ExpressionAttributeNames={**NAMES, "#x": "x"}, + ExpressionAttributeValues={":v": nmap(2), ":no": {"S": "no"}}, + ) + assert exc_info.value.response["Error"]["Code"] == "ConditionalCheckFailedException" + assert _stored_wrappers(dynamodb_client, table, key) == 30 + + +def test_an_item_at_the_limit_takes_an_update_of_another_attribute(dynamodb_client, table): + key = _put(dynamodb_client, table, "other-attr", nmap(31)) + dynamodb_client.update_item( + TableName=table, Key=key, UpdateExpression="SET #x = :v", + ExpressionAttributeNames={"#x": "x"}, ExpressionAttributeValues={":v": {"S": "y"}}, + ) + assert _stored_wrappers(dynamodb_client, table, key) == 31 + + +def test_transact_update_past_the_limit_is_cancelled(dynamodb_client, table): + key = _put(dynamodb_client, table, "tx-no", nmap(30)) + with pytest.raises(ClientError) as exc_info: + dynamodb_client.transact_write_items(TransactItems=[{"Update": { + "TableName": table, "Key": key, "UpdateExpression": f"SET {path(30)} = :v", + "ExpressionAttributeNames": NAMES, "ExpressionAttributeValues": {":v": nmap(2)}, + }}]) + resp = exc_info.value.response + assert resp["Error"]["Code"] == "TransactionCanceledException" + reasons = [(r.get("Code"), r.get("Message")) for r in resp["CancellationReasons"]] + assert reasons == [("ValidationError", MESSAGE)] + assert _stored_wrappers(dynamodb_client, table, key) == 30 + + +def test_transact_update_up_to_the_limit_is_applied(dynamodb_client, table): + key = _put(dynamodb_client, table, "tx-ok", nmap(30)) + dynamodb_client.transact_write_items(TransactItems=[{"Update": { + "TableName": table, "Key": key, "UpdateExpression": f"SET {path(29)} = :v", + "ExpressionAttributeNames": NAMES, "ExpressionAttributeValues": {":v": nmap(2)}, + }}]) + assert _stored_wrappers(dynamodb_client, table, key) == 31 From 564e3149f769fd1305081519aa8a180ac521ceb9 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Mon, 26 Oct 2026 02:03:38 +0000 Subject: [PATCH 03/11] fix(update): check ADD targets for depth, walk each attribute once ExtendDB's ADD creates the missing maps on its path, so an ADD can nest an attribute past the limit (`ADD #m.#k...#k.#a.#b.#c :one` on a value 30 wrappers deep stored 32). The check now covers the top-level attribute of each ADD target as well as each SET target. Each top-level attribute is walked once however many actions write into it, and the walk matches maps and lists directly instead of boxing an iterator per node. With 200 SET actions into one list of 4,000 maps, UpdateItem took 78 ms against 36 ms on main before this; the walk now runs once. --- crates/core/src/expression/update_nesting.rs | 52 +++++++++++++++----- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/crates/core/src/expression/update_nesting.rs b/crates/core/src/expression/update_nesting.rs index 8ec7bc417..48e848379 100644 --- a/crates/core/src/expression/update_nesting.rs +++ b/crates/core/src/expression/update_nesting.rs @@ -15,12 +15,14 @@ use super::resolver::{ExpressionMaps, resolve_element_name}; /// scalar is 32 levels, the service's limit. const MAX_WRAPPERS: usize = 31; -/// Refuse an updated item whose SET targets now nest beyond the limit. +/// Refuse an updated item whose SET or ADD targets now nest beyond the limit. /// -/// Only the top-level attributes a SET writes are checked. REMOVE, ADD and -/// DELETE cannot deepen a value, and an attribute the update does not touch -/// is left alone, so an item stored deeper than the limit before this check -/// existed stays updatable through its other attributes. +/// Only the top-level attributes a SET or an ADD writes are checked, each +/// once however many actions write into it. ADD is checked because it +/// creates the missing maps on its path. REMOVE and DELETE cannot deepen a +/// value, and an attribute the update does not touch is left alone, so an +/// item stored deeper than the limit before this check existed stays +/// updatable through its other attributes. /// /// # Errors /// @@ -32,14 +34,20 @@ pub(super) fn validate_written_nesting_depth( item: &BTreeMap, maps: &ExpressionMaps, ) -> Result<(), DynamoDbError> { + let mut written = Vec::new(); for action in actions { - let UpdateAction::Set { path, .. } = action else { + let (UpdateAction::Set { path, .. } | UpdateAction::Add { path, .. }) = action else { continue; }; let Some(first) = path.first() else { continue; }; let name = resolve_element_name(first, maps)?; + if !written.contains(&name) { + written.push(name); + } + } + for name in &written { if item .get(name.as_ref()) .is_some_and(|v| wrappers_exceed(v, 0)) @@ -55,13 +63,16 @@ pub(super) fn validate_written_nesting_depth( /// Whether `value`, sitting under `above` wrappers, has more than /// [`MAX_WRAPPERS`] wrappers on any path down to a scalar. fn wrappers_exceed(value: &AttributeValue, above: usize) -> bool { - let children: Box> = match value { - AttributeValue::M(map) => Box::new(map.values()), - AttributeValue::L(list) => Box::new(list.iter()), - _ => return false, - }; let here = above + 1; - here > MAX_WRAPPERS || children.into_iter().any(|v| wrappers_exceed(v, here)) + match value { + AttributeValue::M(map) => { + here > MAX_WRAPPERS || map.values().any(|v| wrappers_exceed(v, here)) + } + AttributeValue::L(list) => { + here > MAX_WRAPPERS || list.iter().any(|v| wrappers_exceed(v, here)) + } + _ => false, + } } #[cfg(test)] @@ -124,5 +135,22 @@ mod tests { assert!(check("SET x = :v", &item).is_ok()); assert!(check("REMOVE #m.k", &item).is_ok()); assert!(is_nesting_error(check("SET #m.k = :v", &item))); + assert!(check("DELETE #m.k :v", &item).is_ok()); + } + + #[test] + fn an_add_target_is_checked() { + let over = BTreeMap::from([("m".to_owned(), nmap(32))]); + assert!(is_nesting_error(check("ADD #m.k :v", &over))); + let at = BTreeMap::from([("m".to_owned(), nmap(31))]); + assert!(check("ADD #m.k :v", &at).is_ok()); + } + + #[test] + fn two_actions_into_one_attribute_are_checked_as_one() { + let over = BTreeMap::from([("m".to_owned(), nmap(32))]); + assert!(is_nesting_error(check("SET #m.k = :v, #m.j = :v", &over))); + let at = BTreeMap::from([("m".to_owned(), nmap(31))]); + assert!(check("SET #m.k = :v, #m.j = :v ADD #m.i :v", &at).is_ok()); } } From c05cbc66567523b84a59962b7ecbc703704227a3 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Mon, 26 Oct 2026 02:03:38 +0000 Subject: [PATCH 04/11] test(update): an ADD that would create maps past the nesting limit The service refuses `ADD path(29).#a.#b.#c :one` on a value 30 wrappers deep for its 33-element path (measured 2026-10-26 at 01:26 UTC, twice); ExtendDB refuses it for the depth of the maps the ADD would create. The test compares the error code and the stored item only. The stored-depth helper now takes the deepest branch, so the list_append case sees an appended element. --- tests/test_update_result_nesting_depth.py | 27 +++++++++++++++-------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/tests/test_update_result_nesting_depth.py b/tests/test_update_result_nesting_depth.py index 095d9381d..b2cce5978 100644 --- a/tests/test_update_result_nesting_depth.py +++ b/tests/test_update_result_nesting_depth.py @@ -42,15 +42,9 @@ def nlist(wrappers: int) -> dict: def wrappers(value: dict) -> int: - depth = 0 - while True: - if value.get("M"): - value = next(iter(value["M"].values())) - elif value.get("L"): - value = value["L"][0] - else: - return depth - depth += 1 + """Most wrappers on any branch; an empty map or list adds none.""" + children = list(value.get("M", {}).values()) or value.get("L", []) + return 1 + max(wrappers(c) for c in children) if children else 0 def path(j: int) -> str: @@ -140,6 +134,21 @@ def test_update_past_the_limit_is_refused(dynamodb_client, table, case, base, ex assert _stored_wrappers(dynamodb_client, table, key) == after +def test_an_add_that_would_create_maps_past_the_limit_is_refused(dynamodb_client, table): + # The service refuses this path for its 33 elements (measured 2026-10-26 + # at 01:26 UTC) and ExtendDB for the depth of the maps ADD would create, + # so only the code and the stored item are compared. + key = _put(dynamodb_client, table, "add-deep", nmap(30)) + with pytest.raises(ClientError) as exc_info: + dynamodb_client.update_item( + TableName=table, Key=key, UpdateExpression=f"ADD {path(29)}.#a.#b.#c :one", + ExpressionAttributeNames={**NAMES, "#a": "a", "#b": "b", "#c": "c"}, + ExpressionAttributeValues={":one": {"N": "1"}}, + ) + assert exc_info.value.response["Error"]["Code"] == "ValidationException" + assert _stored_wrappers(dynamodb_client, table, key) == 30 + + def test_a_failing_condition_is_reported_before_the_depth(dynamodb_client, table): key = _put(dynamodb_client, table, "condfail", nmap(30)) with pytest.raises(ClientError) as exc_info: From 38ef819a299d163da5fb025bca929b3fab64931e Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:07:46 +0000 Subject: [PATCH 05/11] refactor(update): check written attributes with the item depth walker crates/core/src/expression/update_nesting.rs had its own copy of the walker and the limit in crates/core/src/validation/mod.rs. It counted 31 wrappers where validation/mod.rs counts MAX_ITEM_NESTING_DEPTH levels. The walker in validation/mod.rs is now a predicate, nests_beyond_item_limit, and the update check calls it. The limit now has one definition. validate_item_nesting_depth and validate_attribute_values_nesting_depth give the same error as before. The update check keeps its own text, "Nesting Levels have exceeded supported limits". Both walkers refused a value at its 32nd M or L wrapper on any path, so no answer changes. The unit tests of both files pass unchanged. --- crates/core/src/expression/update_nesting.rs | 28 +++---------- crates/core/src/validation/mod.rs | 42 +++++++++----------- 2 files changed, 23 insertions(+), 47 deletions(-) diff --git a/crates/core/src/expression/update_nesting.rs b/crates/core/src/expression/update_nesting.rs index 48e848379..639be5abb 100644 --- a/crates/core/src/expression/update_nesting.rs +++ b/crates/core/src/expression/update_nesting.rs @@ -7,16 +7,16 @@ use std::collections::BTreeMap; use crate::error::DynamoDbError; use crate::types::AttributeValue; +use crate::validation::nests_beyond_item_limit; use super::ast::UpdateAction; use super::resolver::{ExpressionMaps, resolve_element_name}; -/// Most M or L wrappers an attribute value may have: 31 wrappers around a -/// scalar is 32 levels, the service's limit. -const MAX_WRAPPERS: usize = 31; - /// Refuse an updated item whose SET or ADD targets now nest beyond the limit. /// +/// The limit is the item limit of `crate::validation`: 31 M or L wrappers +/// around a scalar is 32 levels, the service's limit. +/// /// Only the top-level attributes a SET or an ADD writes are checked, each /// once however many actions write into it. ADD is checked because it /// creates the missing maps on its path. REMOVE and DELETE cannot deepen a @@ -48,10 +48,7 @@ pub(super) fn validate_written_nesting_depth( } } for name in &written { - if item - .get(name.as_ref()) - .is_some_and(|v| wrappers_exceed(v, 0)) - { + if item.get(name.as_ref()).is_some_and(nests_beyond_item_limit) { return Err(DynamoDbError::ValidationException( "Nesting Levels have exceeded supported limits".to_owned(), )); @@ -60,21 +57,6 @@ pub(super) fn validate_written_nesting_depth( Ok(()) } -/// Whether `value`, sitting under `above` wrappers, has more than -/// [`MAX_WRAPPERS`] wrappers on any path down to a scalar. -fn wrappers_exceed(value: &AttributeValue, above: usize) -> bool { - let here = above + 1; - match value { - AttributeValue::M(map) => { - here > MAX_WRAPPERS || map.values().any(|v| wrappers_exceed(v, here)) - } - AttributeValue::L(list) => { - here > MAX_WRAPPERS || list.iter().any(|v| wrappers_exceed(v, here)) - } - _ => false, - } -} - #[cfg(test)] mod tests { use std::collections::{BTreeMap, HashMap}; diff --git a/crates/core/src/validation/mod.rs b/crates/core/src/validation/mod.rs index 50fb2a0f3..ede1982f2 100755 --- a/crates/core/src/validation/mod.rs +++ b/crates/core/src/validation/mod.rs @@ -2540,8 +2540,8 @@ pub(crate) const MAX_ITEM_NESTING_DEPTH: usize = 32; /// Validate that no attribute value in `item` nests beyond `MAX_ITEM_NESTING_DEPTH`. pub fn validate_item_nesting_depth(item: &Item) -> Result<(), DynamoDbError> { - for value in item.values() { - check_attribute_value_depth(value, 0)?; + if item.values().any(nests_beyond_item_limit) { + return Err(nesting_depth_error()); } Ok(()) } @@ -2552,38 +2552,32 @@ pub fn validate_attribute_values_nesting_depth<'a, I>(values: I) -> Result<(), D where I: IntoIterator, { - for v in values { - check_attribute_value_depth(v, 0)?; + if values.into_iter().any(nests_beyond_item_limit) { + return Err(nesting_depth_error()); } Ok(()) } -fn check_attribute_value_depth( - value: &AttributeValue, - current_depth: usize, -) -> Result<(), DynamoDbError> { +/// Whether `value` nests beyond `MAX_ITEM_NESTING_DEPTH` levels. A check that +/// refuses such a value with its own text calls this, so the limit has one +/// definition. +pub(crate) fn nests_beyond_item_limit(value: &AttributeValue) -> bool { + wrappers_reach_limit(value, 0) +} + +/// Whether `value`, under `current_depth` M or L wrappers, brings the number +/// of wrappers on some path to `MAX_ITEM_NESTING_DEPTH`. +fn wrappers_reach_limit(value: &AttributeValue, current_depth: usize) -> bool { + let next = current_depth + 1; match value { AttributeValue::M(map) => { - let next = current_depth + 1; - if next >= MAX_ITEM_NESTING_DEPTH { - return Err(nesting_depth_error()); - } - for v in map.values() { - check_attribute_value_depth(v, next)?; - } + next >= MAX_ITEM_NESTING_DEPTH || map.values().any(|v| wrappers_reach_limit(v, next)) } AttributeValue::L(list) => { - let next = current_depth + 1; - if next >= MAX_ITEM_NESTING_DEPTH { - return Err(nesting_depth_error()); - } - for v in list { - check_attribute_value_depth(v, next)?; - } + next >= MAX_ITEM_NESTING_DEPTH || list.iter().any(|v| wrappers_reach_limit(v, next)) } - _ => {} + _ => false, } - Ok(()) } fn nesting_depth_error() -> DynamoDbError { From 81768cc52fa18f739d6e2f67c962a2157c45dfda Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:49:25 +0000 Subject: [PATCH 06/11] test(update): the size checks on either side of the depth check The service sizes an UpdateItem statement before it checks the nesting depth of the updated item. It sizes the updated item after that check (measured on 2026-11-10 from 19:28 to 19:37 UTC in us-east-1, each case twice). - A SET of a 409,500-byte string next to a SET that takes m to 32 wrappers puts the statement over the 409,600-byte limit. UpdateItem answers "Item size to update has exceeded the maximum allowed size", with the SET actions in either order. - An item holds a 405,000-byte string. A SET of 8,000 bytes next to the same SET of m keeps the statement under the limit and puts the updated item over it. UpdateItem answers the nesting message in either order. A SET of 120,000 bytes on an item of 300,000 bytes gets the same answer. A TransactWriteItems Update is cancelled with one ValidationError reason that carries the nesting message, in either order. The same UpdateItem through a shallow path answers the size message. - A PartiQL UPDATE that sets a value of 2 maps at a path 30 maps deep answers the nesting message and changes nothing. A value of 1 map is applied. The tests that expect the size message skip MongoDB and Cassandra, as tests/test_item_size_at_limit.py does. Against a server built from the parent commit, the two statement tests fail, because ExtendDB answers the nesting message there. The other new tests pass. --- tests/test_partiql_item_depth.py | 48 +++++++-- tests/test_update_result_nesting_depth.py | 123 ++++++++++++++++++++++ 2 files changed, 164 insertions(+), 7 deletions(-) diff --git a/tests/test_partiql_item_depth.py b/tests/test_partiql_item_depth.py index 4ae3f3501..8542f23ab 100644 --- a/tests/test_partiql_item_depth.py +++ b/tests/test_partiql_item_depth.py @@ -9,17 +9,19 @@ UpdateItem give the same value. An UPDATE meets the item check only once its condition holds and its SET applies: on a missing item it fails its condition, and a SET through a path the item does not have answers the update's own -message. A SET of a key attribute is refused ahead of all of this, and a -32-level value the statement does not store (an if_not_exists value the item -does not take, a value only in the WHERE clause) is accepted. In -ExecuteTransaction two members on one item are refused first; otherwise the -item check cancels the transaction with a ValidationError reason at the -member's position, beside each other member's own reason. A 33-level +message. A SET of a shallow parameter through a path that takes a stored value +past the limit meets the same item check. A SET of a key attribute is refused +ahead of all of this, and a 32-level value the statement does not store (an +if_not_exists value the item does not take, a value only in the WHERE clause) +is accepted. In ExecuteTransaction two members on one item are refused first; +otherwise the item check cancels the transaction with a ValidationError reason +at the member's position, beside each other member's own reason. A 33-level parameter is refused by the parameter check in every form. Runs against real DynamoDB (EXTENDDB_TEST_ENDPOINT unset, AWS credentials) and against an extenddb server (EXTENDDB_TEST_ENDPOINT set) with identical -assertions. Every expected value was measured on the service (2026-11-02). +assertions. Every expected value was measured on the service (2026-11-02; the +SET through a path 30 maps deep on 2026-11-10). Tables use the ``extenddb-measure-pqd-`` prefix and are deleted at the end. """ @@ -259,6 +261,38 @@ def test_a_deep_value_only_in_the_where_clause_is_accepted( assert stored_item(dynamodb_client, table, stored)["y"] == {"N": "1"} +class TestPartiqlUpdateThroughADeepPath: + """A SET of a shallow parameter at a path 30 maps deep (measured 2026-11-10).""" + + def set_leaf(self, client, table: str, key: str, levels: int): + return client.execute_statement( + Statement=f"UPDATE \"{table}\" SET m" + ".n" * 30 + " = ? WHERE pk = ?", + Parameters=[deep(levels), {"S": key}], + ) + + @pytest.fixture + def deep_item(self, dynamodb_client, table) -> str: + """The key of a fresh item whose ``m`` holds 30 maps.""" + key = f"deep-path-{uuid.uuid4().hex[:8]}" + dynamodb_client.put_item(TableName=table, Item={"pk": {"S": key}, "m": deep(30)}) + return key + + def test_a_value_that_brings_the_item_to_the_limit_is_applied( + self, dynamodb_client, table, deep_item + ): + assert self.set_leaf(dynamodb_client, table, deep_item, 1)["Items"] == [] + assert stored_item(dynamodb_client, table, deep_item)["m"] == deep(31) + + def test_a_value_that_takes_the_item_past_the_limit_meets_the_item_check( + self, dynamodb_client, table, deep_item + ): + assert ( + validation_message(lambda: self.set_leaf(dynamodb_client, table, deep_item, 2)) + == ITEM_CHECK + ) + assert stored_item(dynamodb_client, table, deep_item)["m"] == deep(30) + + class TestPartiqlDeleteAndSelect: """A 32-level value in the WHERE clause of a DELETE or a SELECT.""" diff --git a/tests/test_update_result_nesting_depth.py b/tests/test_update_result_nesting_depth.py index b2cce5978..8200dc43c 100644 --- a/tests/test_update_result_nesting_depth.py +++ b/tests/test_update_result_nesting_depth.py @@ -11,6 +11,15 @@ reported first. Measured on 2026-10-25 at 19:53 UTC and 2026-10-26 at 00:55 UTC (us-east-1), each case twice. +The item size checks come on either side of the depth check (measured on +2026-11-10 from 19:28 to 19:37 UTC, us-east-1, each case twice). UpdateItem +sizes its statement before the depth check: a statement over the 409,600-byte +limit is refused with ``Item size to update has exceeded the maximum allowed +size``, with the SET actions in either order. The updated item is sized after +the depth check: when only the updated item is over the limit, the depth check +answers, for UpdateItem in either SET order and for a TransactWriteItems +Update. + Depth below is counted in wrappers: a value with w wrappers has w + 1 levels. Dual-target against Amazon DynamoDB and extenddb. @@ -18,6 +27,8 @@ from __future__ import annotations +import os + import pytest from botocore.exceptions import ClientError @@ -192,3 +203,115 @@ def test_transact_update_up_to_the_limit_is_applied(dynamodb_client, table): "ExpressionAttributeNames": NAMES, "ExpressionAttributeValues": {":v": nmap(2)}, }}]) assert _stored_wrappers(dynamodb_client, table, key) == 31 + + +# The tests below that expect the size answer run on SQLite and PostgreSQL. The +# MongoDB layer applies no item size limit on updates, and the Cassandra layer +# sizes every update by the updated item with other wording, both documented in +# docs/differences-from-dynamodb.md. With the backend unset (a run against the +# service), they run. +SIZED_UPDATES = pytest.mark.skipif( + os.environ.get("EXTENDDB_TEST_BACKEND", "") in {"mongodb", "cassandra"}, + reason="UpdateItem is sized on the finished item (Cassandra) or unbounded (MongoDB); " + "see docs/differences-from-dynamodb.md", +) +SIZE_MESSAGE = "Item size to update has exceeded the maximum allowed size" +GROW_NAMES = {**NAMES, "#g": "grow"} + + +def _sets(first: str, other: str, depth_path: str = path(30)) -> list[str]: + sets = [other, f"{depth_path} = :v"] + return sets[::-1] if first == "depth" else sets + + +def _put_large(client, table, sk, stored_bytes): + """An item under the size limit: m with 30 wrappers and a string of stored_bytes.""" + key = {"pk": {"S": "nest"}, "sk": {"S": sk}} + client.put_item(TableName=table, Item={ + **key, "x": {"S": "x"}, "m": nmap(30), "big": {"S": "a" * stored_bytes}, + }) + return key + + +def _assert_unchanged(client, table, key, added): + item = client.get_item(TableName=table, Key=key, ConsistentRead=True)["Item"] + assert added not in item + assert wrappers(item["m"]) == 30 + + +@SIZED_UPDATES +@pytest.mark.parametrize("first", ["size", "depth"]) +def test_a_statement_over_the_size_limit_is_refused_ahead_of_the_depth( + dynamodb_client, table, first, +): + key = _put(dynamodb_client, table, f"statement-{first}-first", nmap(30)) + with pytest.raises(ClientError) as exc_info: + dynamodb_client.update_item( + TableName=table, Key=key, UpdateExpression="SET " + ", ".join(_sets(first, "#big = :big")), + ExpressionAttributeNames={**NAMES, "#big": "big"}, + ExpressionAttributeValues={":v": nmap(2), ":big": {"S": "a" * 409_500}}, + ) + err = exc_info.value.response["Error"] + assert (err["Code"], err["Message"]) == ("ValidationException", SIZE_MESSAGE) + _assert_unchanged(dynamodb_client, table, key, "big") + + +# (case, bytes stored, bytes the update adds, the SET action written first) +UPDATED_ITEM_OVER = [ + ("405000_8000_grow_first", 405_000, 8_000, "grow"), + ("405000_8000_depth_first", 405_000, 8_000, "depth"), + ("300000_120000_grow_first", 300_000, 120_000, "grow"), +] + + +@pytest.mark.parametrize(("case", "stored", "grow", "first"), UPDATED_ITEM_OVER, + ids=[c[0] for c in UPDATED_ITEM_OVER]) +def test_an_updated_item_over_the_size_limit_meets_the_depth_check_first( + dynamodb_client, table, case, stored, grow, first, +): + key = _put_large(dynamodb_client, table, f"updated-{case}", stored) + with pytest.raises(ClientError) as exc_info: + dynamodb_client.update_item( + TableName=table, Key=key, UpdateExpression="SET " + ", ".join(_sets(first, "#g = :g")), + ExpressionAttributeNames=GROW_NAMES, + ExpressionAttributeValues={":v": nmap(2), ":g": {"S": "b" * grow}}, + ) + err = exc_info.value.response["Error"] + assert (err["Code"], err["Message"]) == ("ValidationException", MESSAGE) + _assert_unchanged(dynamodb_client, table, key, "grow") + + +@SIZED_UPDATES +def test_the_same_update_through_a_shallow_path_is_refused_for_its_size(dynamodb_client, table): + # The control for the test above: a SET at 28 keys keeps m at 30 wrappers, + # so only the size of the updated item is past a limit. + key = _put_large(dynamodb_client, table, "updated-shallow", 405_000) + with pytest.raises(ClientError) as exc_info: + dynamodb_client.update_item( + TableName=table, Key=key, + UpdateExpression="SET " + ", ".join(_sets("grow", "#g = :g", path(28))), + ExpressionAttributeNames=GROW_NAMES, + ExpressionAttributeValues={":v": nmap(2), ":g": {"S": "b" * 8_000}}, + ) + err = exc_info.value.response["Error"] + assert (err["Code"], err["Message"]) == ("ValidationException", SIZE_MESSAGE) + _assert_unchanged(dynamodb_client, table, key, "grow") + + +@pytest.mark.parametrize("first", ["grow", "depth"]) +def test_transact_update_of_an_item_over_the_size_limit_meets_the_depth_check_first( + dynamodb_client, table, first, +): + key = _put_large(dynamodb_client, table, f"tx-updated-{first}-first", 405_000) + with pytest.raises(ClientError) as exc_info: + dynamodb_client.transact_write_items(TransactItems=[{"Update": { + "TableName": table, "Key": key, + "UpdateExpression": "SET " + ", ".join(_sets(first, "#g = :g")), + "ExpressionAttributeNames": GROW_NAMES, + "ExpressionAttributeValues": {":v": nmap(2), ":g": {"S": "b" * 8_000}}, + }}]) + resp = exc_info.value.response + assert resp["Error"]["Code"] == "TransactionCanceledException" + reasons = [(r.get("Code"), r.get("Message")) for r in resp["CancellationReasons"]] + assert reasons == [("ValidationError", MESSAGE)] + _assert_unchanged(dynamodb_client, table, key, "grow") From f29f8e1b4c8b4d4d69a100f93c8307f604f7c3a7 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:58:31 +0000 Subject: [PATCH 07/11] fix(update): size the UpdateItem statement ahead of the depth check The service sizes an UpdateItem statement before it checks the nesting depth of the updated item. It sizes the updated item after that check (measured on 2026-11-10 from 19:28 to 19:37 UTC in us-east-1, each case twice). Before this change, apply_update_validated refused a deep result first. An update whose statement was over the item size limit got the nesting message. The service answers "Item size to update has exceeded the maximum allowed size". The new apply_update_item_validated in crates/core/src/expression/update_evaluator.rs applies a standalone UpdateItem and checks the result as apply_update_validated does. Then it sizes the update with validate_update_item_size. When the depth check refuses, it first checks the statement figure with the new validate_update_statement_size in crates/core/src/validation/mod.rs, the statement half of validate_update_item_size. The SQLite and PostgreSQL UpdateItem paths call it in place of their two calls. The measurements give two orders: the statement size ahead of the depth, and the depth ahead of the size of the updated item. The answer changes only for an update whose statement is over the limit and whose result nests too deep. The vector check keeps its place after the depth check and ahead of the size check. A PartiQL UPDATE and the legacy AttributeUpdates form are not sized by their statement, so their checks keep their order. TransactWriteItems sizes the updated item after the depth check already. The MongoDB and Cassandra UpdateItem paths do not size the statement, and do not change. --- crates/core/src/expression/mod.rs | 2 +- .../core/src/expression/update_evaluator.rs | 67 ++++++++- .../src/expression/update_evaluator_tests.rs | 136 ++++++++++++++++++ crates/core/src/validation/mod.rs | 26 ++++ .../storage-postgres/src/data/update_item.rs | 14 +- crates/storage-sqlite/src/data/update_item.rs | 11 +- 6 files changed, 242 insertions(+), 14 deletions(-) diff --git a/crates/core/src/expression/mod.rs b/crates/core/src/expression/mod.rs index 07272c0aa..c3b345d8a 100755 --- a/crates/core/src/expression/mod.rs +++ b/crates/core/src/expression/mod.rs @@ -44,5 +44,5 @@ pub use search_condition::{ validate_search_condition_expression, }; pub use tokenizer::{Token, tokenize, tokenize_for, tokenize_with_limit}; -pub use update_evaluator::apply_update_validated; +pub use update_evaluator::{apply_update_item_validated, apply_update_validated}; pub use update_parser::{parse_update, parse_update_from}; diff --git a/crates/core/src/expression/update_evaluator.rs b/crates/core/src/expression/update_evaluator.rs index ec278d8c4..121078f38 100755 --- a/crates/core/src/expression/update_evaluator.rs +++ b/crates/core/src/expression/update_evaluator.rs @@ -93,6 +93,66 @@ pub fn apply_update_validated( maps: &ExpressionMaps, vector_indexes: &[VectorIndexKeyInfo], attribute_definitions: &[AttributeDefinition], +) -> Result<(), DynamoDbError> { + apply_update_checked( + actions, + item, + maps, + vector_indexes, + attribute_definitions, + None, + ) +} + +/// Apply the actions of a standalone `UpdateItem`, validate the result as +/// [`apply_update_validated`] does, and size the update as +/// [`crate::validation::validate_update_item_size`] does. +/// +/// The service sizes the statement before it checks the nesting depth of the +/// updated item, and sizes the updated item after that check (measured +/// us-east-1, 2026-11-10). An update whose statement is over the limit is +/// refused for its size, whatever the depth. An update whose statement is +/// within the limit and whose result nests too deep gets the depth refusal, +/// even when the updated item is over the limit. Only these orders are +/// measured, so the statement is sized ahead of the depth check only when that +/// check refuses. The vector check keeps its place after the depth check and +/// ahead of the size check. An update from a PartiQL `UPDATE` and the legacy +/// `AttributeUpdates` form are not sized by their statement, and their checks +/// run in the order of [`apply_update_validated`]. +/// +/// # Errors +/// +/// The errors of [`apply_update_validated`], and `ValidationException` `Item +/// size to update has exceeded the maximum allowed size` when the update is +/// over `max_item_size_bytes`. +pub fn apply_update_item_validated( + actions: &[UpdateAction], + item: &mut BTreeMap, + maps: &ExpressionMaps, + vector_indexes: &[VectorIndexKeyInfo], + attribute_definitions: &[AttributeDefinition], + max_item_size_bytes: usize, +) -> Result<(), DynamoDbError> { + apply_update_checked( + actions, + item, + maps, + vector_indexes, + attribute_definitions, + Some(max_item_size_bytes), + )?; + crate::validation::validate_update_item_size(item, actions, maps, max_item_size_bytes) +} + +/// [`apply_update_validated`]. With `statement_limit`, a depth refusal gives +/// way to the size refusal of a statement over that limit. +fn apply_update_checked( + actions: &[UpdateAction], + item: &mut BTreeMap, + maps: &ExpressionMaps, + vector_indexes: &[VectorIndexKeyInfo], + attribute_definitions: &[AttributeDefinition], + statement_limit: Option, ) -> Result<(), DynamoDbError> { // Snapshot only the vector-relevant attributes (the vector attribute and // the search-schema attributes of each index) rather than cloning the @@ -113,7 +173,12 @@ pub fn apply_update_validated( } } apply_update(actions, item, maps)?; - super::update_nesting::validate_written_nesting_depth(actions, item, maps)?; + if let Err(depth) = super::update_nesting::validate_written_nesting_depth(actions, item, maps) { + if let Some(max_bytes) = statement_limit { + crate::validation::validate_update_statement_size(actions, maps, max_bytes)?; + } + return Err(depth); + } crate::validation::validate_vector_write_changed( item, &before, diff --git a/crates/core/src/expression/update_evaluator_tests.rs b/crates/core/src/expression/update_evaluator_tests.rs index 968e6d356..e2f0f13a1 100755 --- a/crates/core/src/expression/update_evaluator_tests.rs +++ b/crates/core/src/expression/update_evaluator_tests.rs @@ -660,3 +660,139 @@ fn set_parenthesised_arithmetic() { apply("SET c = (c - :v)", &mut item, HashMap::new(), values).unwrap(); assert_eq!(item.get("c"), Some(&AttributeValue::N("7".into()))); } + +/// The size checks of a standalone `UpdateItem` on either side of the depth +/// check (measured us-east-1, 2026-11-10). +mod update_item_sizing { + use super::*; + use crate::validation::ITEM_SIZE_TO_UPDATE_EXCEEDED; + + const LIMIT: usize = 409_600; + const DEPTH: &str = "Nesting Levels have exceeded supported limits"; + + fn nmap(wrappers: usize) -> AttributeValue { + let mut value = AttributeValue::S("leaf".to_owned()); + for _ in 0..wrappers { + value = AttributeValue::M(BTreeMap::from([("k".to_owned(), value)])); + } + value + } + + fn text(bytes: usize) -> AttributeValue { + AttributeValue::S("a".repeat(bytes)) + } + + /// An item whose `m` holds 30 wrappers, beside a string of `stored` bytes. + fn item(stored: usize) -> BTreeMap { + BTreeMap::from([ + ("pk".to_owned(), AttributeValue::S("k1".to_owned())), + ("m".to_owned(), nmap(30)), + ("stored".to_owned(), text(stored)), + ]) + } + + /// `SET #added = :added` of `added` bytes and a SET of 2 wrappers at `depth` + /// keys below `m`, in the order `depth_first` gives. + fn update( + added: usize, + depth: usize, + depth_first: bool, + ) -> (Vec, ExpressionMaps) { + let deep = format!("#m{} = :v", ".k".repeat(depth)); + let sets = if depth_first { + format!("SET {deep}, #added = :added") + } else { + format!("SET #added = :added, {deep}") + }; + let maps = ExpressionMaps::new( + HashMap::from([ + ("m".to_owned(), "m".to_owned()), + ("added".to_owned(), "added".to_owned()), + ]), + HashMap::from([("v".to_owned(), nmap(2)), ("added".to_owned(), text(added))]), + ); + (parse_update(&tokenize(&sets).unwrap()).unwrap(), maps) + } + + fn message(result: Result<(), DynamoDbError>) -> String { + match result { + Err(DynamoDbError::ValidationException(message)) => message, + other => panic!("expected a ValidationException, got {other:?}"), + } + } + + /// The answer to the update on `item` as a standalone `UpdateItem`. + fn update_item( + mut item: BTreeMap, + added: usize, + depth: usize, + depth_first: bool, + ) -> String { + let (actions, maps) = update(added, depth, depth_first); + message(apply_update_item_validated( + &actions, + &mut item, + &maps, + &[], + &[], + LIMIT, + )) + } + + #[test] + fn a_statement_over_the_limit_is_refused_for_its_size_ahead_of_the_depth() { + for depth_first in [false, true] { + assert_eq!( + update_item(item(0), 409_500, 30, depth_first), + ITEM_SIZE_TO_UPDATE_EXCEEDED + ); + } + } + + #[test] + fn an_updated_item_over_the_limit_meets_the_depth_check_first() { + for depth_first in [false, true] { + assert_eq!(update_item(item(405_000), 8_000, 30, depth_first), DEPTH); + } + assert_eq!(update_item(item(300_000), 120_000, 30, false), DEPTH); + } + + #[test] + fn the_same_update_through_a_shallow_path_is_refused_for_its_size() { + assert_eq!( + update_item(item(405_000), 8_000, 28, false), + ITEM_SIZE_TO_UPDATE_EXCEEDED + ); + } + + #[test] + fn a_form_not_sized_by_its_statement_keeps_the_depth_refusal() { + // A PartiQL UPDATE and the legacy AttributeUpdates form: the statement + // is over the limit, and the answer is the one apply_update_validated + // gives. + for partiql in [true, false] { + let (actions, mut maps) = update(409_500, 30, false); + maps.finished_item_sizing = partiql; + maps.legacy_attribute_updates = !partiql; + let plain = message(apply_update_validated( + &actions, + &mut item(0), + &maps, + &[], + &[], + )); + assert_eq!(plain, DEPTH); + assert_eq!( + message(apply_update_item_validated( + &actions, + &mut item(0), + &maps, + &[], + &[], + LIMIT + )), + plain + ); + } + } +} diff --git a/crates/core/src/validation/mod.rs b/crates/core/src/validation/mod.rs index ede1982f2..c5489819a 100755 --- a/crates/core/src/validation/mod.rs +++ b/crates/core/src/validation/mod.rs @@ -2317,6 +2317,32 @@ pub fn validate_update_item_size( Ok(()) } +/// The statement half of [`validate_update_item_size`], for a standalone +/// `UpdateItem` written as an expression. An update from a PartiQL `UPDATE` +/// (`maps.finished_item_sizing`) and the legacy `AttributeUpdates` form +/// (`maps.legacy_attribute_updates`) are not sized by their statement, and +/// pass here. +/// +/// # Errors +/// +/// Returns `DynamoDbError::ValidationException` with the update wording when +/// the statement figure exceeds `max_bytes`. +pub(crate) fn validate_update_statement_size( + actions: &[crate::expression::UpdateAction], + maps: &crate::expression::ExpressionMaps, + max_bytes: usize, +) -> Result<(), DynamoDbError> { + if maps.finished_item_sizing || maps.legacy_attribute_updates { + return Ok(()); + } + if update_statement_size(actions, maps)? > max_bytes { + return Err(DynamoDbError::ValidationException( + ITEM_SIZE_TO_UPDATE_EXCEEDED.to_owned(), + )); + } + Ok(()) +} + /// The bytes the service charges the operand expression of a `SET` clause: /// value placeholders at their size, function calls, arithmetic operators and /// document paths at their fixed costs. diff --git a/crates/storage-postgres/src/data/update_item.rs b/crates/storage-postgres/src/data/update_item.rs index a92ae9660..8f933a31d 100755 --- a/crates/storage-postgres/src/data/update_item.rs +++ b/crates/storage-postgres/src/data/update_item.rs @@ -192,21 +192,21 @@ impl PostgresEngine { // Apply update actions and validate the resulting image against the // table's vector indexes: vector validity is a property of the stored // value, and expression-form checks cannot cover if_not_exists / - // list_append / copy-from-attribute forms. - expression::apply_update_validated( + // list_append / copy-from-attribute forms. Size the update the way + // the service does: what the statement writes plus a fixed cost per + // clause, against the item limit. A statement over the limit is + // refused ahead of the depth check, and an updated item over the + // limit after it. + expression::apply_update_item_validated( actions, &mut item, maps, &key_info.vector_indexes, &key_info.attribute_definitions, + self.max_item_size_bytes, ) .map_err(|e| StorageError::Validation(e.to_string()))?; - // Size the update the way the service does: what the statement - // writes plus a fixed cost per clause, against the item limit. - validation::validate_update_item_size(&item, actions, maps, self.max_item_size_bytes) - .map_err(|e| StorageError::Validation(e.to_string()))?; - // Secondary-index keys on the written item, as the transacted Update // path and the SQLite backend already do. Literal operands were // checked ahead of the condition, above; a value computed from the diff --git a/crates/storage-sqlite/src/data/update_item.rs b/crates/storage-sqlite/src/data/update_item.rs index 0eabdde21..2ac84bbc5 100644 --- a/crates/storage-sqlite/src/data/update_item.rs +++ b/crates/storage-sqlite/src/data/update_item.rs @@ -80,19 +80,20 @@ impl SqliteEngine { } // Start from the existing image, or from the key for a fresh upsert. + // Size the update the way the service does: what the statement writes + // plus a fixed cost per clause, against the item limit. A statement + // over the limit is refused ahead of the depth check, and an updated + // item over the limit after it. let mut item = old.clone().unwrap_or_else(|| key.clone()); - expression::apply_update_validated( + expression::apply_update_item_validated( actions, &mut item, maps, &key_info.vector_indexes, &key_info.attribute_definitions, + self.max_item_size_bytes, ) .map_err(|e| StorageError::Validation(e.to_string()))?; - // Size the update the way the service does: what the statement writes - // plus a fixed cost per clause, against the item limit. - validation::validate_update_item_size(&item, actions, maps, self.max_item_size_bytes) - .map_err(|e| StorageError::Validation(e.to_string()))?; // Secondary-index key validation on the post-update item, matching the // TransactWriteItems update path: a wrong-typed index key attribute or // an index key set to an empty value is a ValidationException up front, From f7e3da13f3ce4634ddd1c573739fc900799eef82 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:18:52 +0000 Subject: [PATCH 08/11] docs(partiql): say which check refuses a deep UPDATE value first The doc comment of unwritable_update_error said that the item check on the updated item answers once the condition holds and the update applies. With this branch, apply_update_validated refuses first when an update stores a value past the nesting limit. It checks each attribute that a SET or an ADD writes, and its text is the text of ITEM_NESTING_MESSAGE. The comment now says that. It also says that the item check, which runs next, refuses only an attribute stored past the limit that the update does not write. The code does not change. --- crates/engine/src/partiql/item_depth.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/crates/engine/src/partiql/item_depth.rs b/crates/engine/src/partiql/item_depth.rs index 1b37fc461..ee5acc74f 100644 --- a/crates/engine/src/partiql/item_depth.rs +++ b/crates/engine/src/partiql/item_depth.rs @@ -106,13 +106,17 @@ fn holds_a_value_past_the_item_limit(operand: &Operand, params: &[AttributeValue /// key attribute is refused first. The update is then evaluated as the /// storage layer evaluates any update, against the stored item, and nothing /// is written: a condition that fails or an update that does not apply is the -/// answer, then the item check on the updated item. +/// answer. An update that stores the deep value is refused by +/// `apply_update_validated` itself, which checks each attribute a `SET` or an +/// `ADD` writes and answers the text of `ITEM_NESTING_MESSAGE`. The item check +/// on the updated item runs last. It refuses only an item that already held +/// an attribute past the limit, one the update does not write. /// -/// `None` when the updated item passes the item check, which happens only -/// when the deep value is not stored, such as `if_not_exists` on an attribute -/// the item has. The caller then sends the update to `UpdateItem`, whose -/// nesting check of every `ExpressionAttributeValue` refuses it, where the -/// service applies the update (measured 2026-11-02). +/// `None` when neither check refuses, which happens only when the deep value +/// is not stored, such as `if_not_exists` on an attribute the item has. The +/// caller then sends the update to `UpdateItem`, whose nesting check of every +/// `ExpressionAttributeValue` refuses it, where the service applies the +/// update (measured 2026-11-02). pub(super) async fn unwritable_update_error( ctx: &OperationContext, parts: &UpdateParts, From e47a2ef76e6919e231842ea00b590c43a4dc8708 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:51:27 +0000 Subject: [PATCH 09/11] test(partiql): a deep SET in a batch, a transaction and beside 400 KB The service was measured on 2026-11-10 at 20:39 UTC in us-east-1, two sends each. Each item held x and 30 maps in m. - BatchExecuteStatement: the first member sets x on one item and answers no error. The second member sets a value of 2 maps at 30 keys below m on another item. Its entry is the ValidationError "Nesting Levels have exceeded supported limits". The first item takes the update, and the second item does not change. - ExecuteTransaction with the same two members is cancelled with the reasons None and ValidationError, the second with the nesting message. Neither item changes. - ExecuteStatement with SET big = ? (409,500 bytes) and the same deep SET answers ValidationException with the nesting message, with the SET clauses in either order. The item does not change. - With the deep SET at 28 keys, the same statement answers "Item size to update has exceeded the maximum allowed size". Five dual-target tests in tests/test_partiql_item_depth.py encode these answers. The size test skips MongoDB and Cassandra, as tests/test_partiql_item_size.py does. The items of the two earlier tests in the class now hold x, as the measured items did. Against a SQLite server built from the parent commit, the five tests pass, so the code does not change. --- tests/test_partiql_item_depth.py | 119 ++++++++++++++++++++++++++++--- 1 file changed, 111 insertions(+), 8 deletions(-) diff --git a/tests/test_partiql_item_depth.py b/tests/test_partiql_item_depth.py index 8542f23ab..b8bc03678 100644 --- a/tests/test_partiql_item_depth.py +++ b/tests/test_partiql_item_depth.py @@ -10,7 +10,10 @@ condition holds and its SET applies: on a missing item it fails its condition, and a SET through a path the item does not have answers the update's own message. A SET of a shallow parameter through a path that takes a stored value -past the limit meets the same item check. A SET of a key attribute is refused +past the limit meets the same item check: in ExecuteStatement, as the +statement's ValidationError entry in BatchExecuteStatement, as a ValidationError +reason in ExecuteTransaction, and ahead of the size check when the statement +also sets a 409,500-byte parameter. A SET of a key attribute is refused ahead of all of this, and a 32-level value the statement does not store (an if_not_exists value the item does not take, a value only in the WHERE clause) is accepted. In ExecuteTransaction two members on one item are refused first; @@ -38,6 +41,7 @@ ITEM_CHECK = "Nesting Levels have exceeded supported limits" NESTED = f"{ITEM_CHECK}: Attributes in the item have nested levels beyond supported limit" BAD_PATH = "The document path provided in the update expression is invalid for update" +SIZE_TO_UPDATE = "Item size to update has exceeded the maximum allowed size" CONDITION_FAILED = "The conditional request failed" CANCELLED = "Transaction cancelled, please refer cancellation reasons for specific reasons" ITEM_REASON = {"Code": "ValidationError", "Message": ITEM_CHECK} @@ -58,6 +62,19 @@ strict=True, ) +# The PostgreSQL and SQLite storage layers size updates the way the service +# does. The MongoDB layer applies no limit to updates, and the Cassandra layer +# sizes every update against a fixed 409,600 bytes (see +# docs/differences-from-dynamodb.md) and words the refusal "Item size has +# exceeded the maximum allowed size" (validate_item_size in +# crates/core/src/validation/mod.rs). The case that needs the size refusal is +# skipped on those two backends, as in tests/test_partiql_item_size.py. +refused_for_size_on_default_backends_only = pytest.mark.skipif( + os.environ.get("EXTENDDB_TEST_BACKEND", "") in {"mongodb", "cassandra"}, + reason="updates are sized against a fixed limit (Cassandra) or not at all (MongoDB); " + "see docs/differences-from-dynamodb.md", +) + def deep(levels: int) -> dict: """``levels`` maps wrapped around a string.""" @@ -262,20 +279,57 @@ def test_a_deep_value_only_in_the_where_clause_is_accepted( class TestPartiqlUpdateThroughADeepPath: - """A SET of a shallow parameter at a path 30 maps deep (measured 2026-11-10).""" + """A SET of a shallow parameter at a path 30 maps deep, on an item that holds + ``x`` and 30 maps in ``m``: in ExecuteStatement, as a BatchExecuteStatement + member and as an ExecuteTransaction member, and beside a SET of a + 409,500-byte parameter (measured 2026-11-10).""" + + @staticmethod + def deep_set(table: str, key: str, levels: int) -> dict: + """The UPDATE that sets ``levels`` maps at 30 keys below ``m``.""" + return { + "Statement": f"UPDATE \"{table}\" SET m" + ".n" * 30 + " = ? WHERE pk = ?", + "Parameters": [deep(levels), {"S": key}], + } + + @staticmethod + def x_set(table: str, key: str) -> dict: + return { + "Statement": f"UPDATE \"{table}\" SET x = ? WHERE pk = ?", + "Parameters": [{"S": "y"}, {"S": key}], + } + + @staticmethod + def fresh(client, table: str) -> str: + """The key of a fresh item that holds ``x`` and 30 maps in ``m``.""" + key = f"deep-path-{uuid.uuid4().hex[:8]}" + client.put_item( + TableName=table, Item={"pk": {"S": key}, "x": {"S": "x"}, "m": deep(30)} + ) + return key + + @staticmethod + def unchanged(key: str) -> dict: + return {"pk": {"S": key}, "x": {"S": "x"}, "m": deep(30)} def set_leaf(self, client, table: str, key: str, levels: int): + return client.execute_statement(**self.deep_set(table, key, levels)) + + def set_big_beside(self, client, table: str, key: str, depth: int, big_first: bool): + """SET a 409,500-byte ``big`` and SET 2 maps at ``depth`` keys below ``m``, + as two SET clauses in the order ``big_first`` gives.""" + clauses = ["big = ?", "m" + ".n" * depth + " = ?"] + params = [{"S": "a" * 409_500}, deep(2)] + if not big_first: + clauses, params = clauses[::-1], params[::-1] return client.execute_statement( - Statement=f"UPDATE \"{table}\" SET m" + ".n" * 30 + " = ? WHERE pk = ?", - Parameters=[deep(levels), {"S": key}], + Statement=f"UPDATE \"{table}\" SET {clauses[0]} SET {clauses[1]} WHERE pk = ?", + Parameters=[*params, {"S": key}], ) @pytest.fixture def deep_item(self, dynamodb_client, table) -> str: - """The key of a fresh item whose ``m`` holds 30 maps.""" - key = f"deep-path-{uuid.uuid4().hex[:8]}" - dynamodb_client.put_item(TableName=table, Item={"pk": {"S": key}, "m": deep(30)}) - return key + return self.fresh(dynamodb_client, table) def test_a_value_that_brings_the_item_to_the_limit_is_applied( self, dynamodb_client, table, deep_item @@ -292,6 +346,55 @@ def test_a_value_that_takes_the_item_past_the_limit_meets_the_item_check( ) assert stored_item(dynamodb_client, table, deep_item)["m"] == deep(30) + def test_a_batch_member_past_the_limit_fails_only_that_statement( + self, dynamodb_client, table, deep_item + ): + other = self.fresh(dynamodb_client, table) + resp = dynamodb_client.batch_execute_statement( + Statements=[self.x_set(table, other), self.deep_set(table, deep_item, 2)] + ) + assert [entry.get("Error") for entry in resp["Responses"]] == [ + None, + {"Code": "ValidationError", "Message": ITEM_CHECK}, + ] + assert stored_item(dynamodb_client, table, other)["x"] == {"S": "y"} + assert stored_item(dynamodb_client, table, deep_item) == self.unchanged(deep_item) + + def test_a_transaction_member_past_the_limit_cancels_the_transaction( + self, dynamodb_client, table, deep_item + ): + other = self.fresh(dynamodb_client, table) + statements = [self.x_set(table, other), self.deep_set(table, deep_item, 2)] + assert cancellation(dynamodb_client, statements) == [{"Code": "None"}, ITEM_REASON] + assert stored_item(dynamodb_client, table, other) == self.unchanged(other) + assert stored_item(dynamodb_client, table, deep_item) == self.unchanged(deep_item) + + @pytest.mark.parametrize("big_first", [True, False], ids=["big_first", "deep_first"]) + def test_a_large_parameter_beside_the_deep_set_meets_the_item_check( + self, dynamodb_client, table, deep_item, big_first + ): + assert ( + validation_message( + lambda: self.set_big_beside(dynamodb_client, table, deep_item, 30, big_first) + ) + == ITEM_CHECK + ) + assert stored_item(dynamodb_client, table, deep_item) == self.unchanged(deep_item) + + @refused_for_size_on_default_backends_only + def test_the_large_parameter_beside_a_set_at_28_keys_is_refused_for_its_size( + self, dynamodb_client, table, deep_item + ): + # The control for the test above: m keeps 30 maps, so only the size of + # the updated item is past a limit. + assert ( + validation_message( + lambda: self.set_big_beside(dynamodb_client, table, deep_item, 28, True) + ) + == SIZE_TO_UPDATE + ) + assert stored_item(dynamodb_client, table, deep_item) == self.unchanged(deep_item) + class TestPartiqlDeleteAndSelect: """A 32-level value in the WHERE clause of a DELETE or a SELECT.""" From b14e5c80e92691cc5de17a876b4e04e9dceeeb5f Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:58:31 +0000 Subject: [PATCH 10/11] docs: record the transacted Update the service refuses before the run docs/differences-from-dynamodb.md said that a transacted Update is sized by the finished item alone. The service refused one transacted Update for its size before the transaction ran (measured on 2026-11-10 at 19:28 UTC in us-east-1, two sends). The Update set a 409,500-byte value, and a second SET action took a map past the nesting limit. The answer was ValidationException "Item size to update has exceeded the maximum allowed size", and the item did not change. ExtendDB runs that transaction and cancels it with one ValidationError reason that carries "Nesting Levels have exceeded supported limits". Before this branch, PostgreSQL and SQLite cancelled it with a reason that carried the size message. The rule behind the check of the service is not measured. #937 tracks the measurement and the check. The paragraph now gives this case, and a new row of the table records the request, both answers and the issue. The row above it no longer says that every transacted Update is sized on the finished item. --- docs/differences-from-dynamodb.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/differences-from-dynamodb.md b/docs/differences-from-dynamodb.md index b768e0a76..c3e0492dd 100755 --- a/docs/differences-from-dynamodb.md +++ b/docs/differences-from-dynamodb.md @@ -178,14 +178,19 @@ remains. The key and untouched attributes are left out. Both the statement figur and the finished item are checked against the limit, and a breach refuses with `Item size to update has exceeded the maximum allowed size`. -A transacted `Update` and a PartiQL `UPDATE` are sized by the finished item -alone, with the same wording. For PartiQL this holds in `ExecuteStatement`, +A transacted `Update` and a PartiQL `UPDATE` are sized by the finished item, +with the same wording. For PartiQL this holds in `ExecuteStatement`, `BatchExecuteStatement` and `ExecuteTransaction` (measured in us-east-1 on -2026-11-04). +2026-11-04). The service also refused one transacted `Update` for its size +before the transaction ran (measured in us-east-1 on 2026-11-10). That +`Update` set a 409,500-byte value beside a `SET` that took a map past the +nesting limit. The rule behind that check is not measured, and ExtendDB does +not have it (the second row below). | Area | DynamoDB | ExtendDB | |------|----------|------| -| `UpdateItem` item-size accounting | In 31 of 33 commercial regions a standalone `UpdateItem` is sized by the statement (the rule above). At a one-byte key the largest finished item it can produce is 409,581 bytes. Two regions (`eu-south-2`, `me-central-1`) size an `UpdateItem` on the finished item, like a `PutItem`. | PostgreSQL and SQLite follow the 31-region statement rule. A transacted `Update` and a PartiQL `UPDATE` are sized on the finished item on both, as on the service. Cassandra sizes every update on the finished item against a fixed 409,600 bytes; MongoDB applies no item-size limit on updates. | +| `UpdateItem` item-size accounting | In 31 of 33 commercial regions a standalone `UpdateItem` is sized by the statement (the rule above). At a one-byte key the largest finished item it can produce is 409,581 bytes. Two regions (`eu-south-2`, `me-central-1`) size an `UpdateItem` on the finished item, like a `PutItem`. | PostgreSQL and SQLite follow the 31-region statement rule. On both, a transacted `Update` and a PartiQL `UPDATE` are sized on the finished item, as on the service. The next row gives a transacted `Update` that the service refuses before the transaction runs. Cassandra sizes every update on the finished item against a fixed 409,600 bytes. MongoDB applies no item-size limit on updates. | +| Transacted `Update` refused before the transaction | A TransactWriteItems `Update` sets a 409,500-byte value, and a second `SET` action takes a map past the nesting limit. The service refuses it with `ValidationException` `Item size to update has exceeded the maximum allowed size` before the transaction runs. The item is unchanged (measured in us-east-1 on 2026-11-10). The rule behind this check is not measured. | The transaction runs and is cancelled with one `ValidationError` reason that carries `Nesting Levels have exceeded supported limits`. In builds before #590, PostgreSQL and SQLite cancelled it with a reason that carried the size message. Tracked in #937. | ## Capacity and Throttling From e91da2e03dfcfd5567627b6e275734b1a4d99170 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:58:34 +0000 Subject: [PATCH 11/11] docs(update): name the nesting refusal and where the update check runs - The "# Errors" section of apply_update_validated now gives the nesting refusal. Its text is "Nesting Levels have exceeded supported limits", for an attribute that a SET or an ADD writes past the limit. - Three comments said that the legacy AttributeUpdates form is not sized by its statement. They are the doc comments of apply_update_item_validated and validate_update_statement_size, and the comment of a unit test. validate_update_item_size sizes that form with a figure that it computes from the finished values of the attributes the form names. The comments now say that, and that the statement check does not apply to that form or to a PartiQL UPDATE. The unit test is renamed to a_partiql_or_legacy_update_keeps_the_depth_refusal. The comment of apply_update_item_validated also says that the order of the depth check against the vector check is not measured. - docs/dynamodb-limits.md named validate_item_nesting_depth as the nesting check of UpdateItem. The row now names validate_attribute_values_nesting_depth for the values of UpdateItem. It names validate_written_nesting_depth for the attributes that UpdateItem, a TransactWriteItems Update or a PartiQL UPDATE writes. - A comment in tests/test_update_result_nesting_depth.py cited docs/differences-from-dynamodb.md for the wording of the Cassandra size refusal, which that file does not give. The comment now gives the text, "Item size has exceeded the maximum allowed size", and names validate_item_size as its source. The code does not change. --- crates/core/src/expression/update_evaluator.rs | 16 ++++++++++++---- .../src/expression/update_evaluator_tests.rs | 10 ++++++---- crates/core/src/validation/mod.rs | 9 +++++---- docs/dynamodb-limits.md | 2 +- tests/test_update_result_nesting_depth.py | 6 ++++-- 5 files changed, 28 insertions(+), 15 deletions(-) diff --git a/crates/core/src/expression/update_evaluator.rs b/crates/core/src/expression/update_evaluator.rs index 121078f38..fb8a2193b 100755 --- a/crates/core/src/expression/update_evaluator.rs +++ b/crates/core/src/expression/update_evaluator.rs @@ -87,6 +87,10 @@ pub(crate) fn apply_update( /// not reject an unrelated update to an item carrying a pre-existing invalid /// value (measured 2026-08-14), and whole-image validation would make items /// the backfill deliberately skipped permanently un-updatable. +/// +/// Returns `ValidationException` `Nesting Levels have exceeded supported +/// limits` when an attribute that a `SET` or an `ADD` writes nests past the +/// limit, before the vector check. pub fn apply_update_validated( actions: &[UpdateAction], item: &mut BTreeMap, @@ -115,10 +119,14 @@ pub fn apply_update_validated( /// within the limit and whose result nests too deep gets the depth refusal, /// even when the updated item is over the limit. Only these orders are /// measured, so the statement is sized ahead of the depth check only when that -/// check refuses. The vector check keeps its place after the depth check and -/// ahead of the size check. An update from a PartiQL `UPDATE` and the legacy -/// `AttributeUpdates` form are not sized by their statement, and their checks -/// run in the order of [`apply_update_validated`]. +/// check refuses. The order of the depth check against the vector check is not +/// measured, and the vector check keeps its place after the depth check and +/// ahead of the size check. A PartiQL `UPDATE` (`maps.finished_item_sizing`) +/// is sized by its updated item alone. The figure of the legacy +/// `AttributeUpdates` form (`maps.legacy_attribute_updates`) is computed from +/// the finished values of the attributes it names. The statement check does +/// not apply to either form, so their checks run in the order of +/// [`apply_update_validated`]. /// /// # Errors /// diff --git a/crates/core/src/expression/update_evaluator_tests.rs b/crates/core/src/expression/update_evaluator_tests.rs index e2f0f13a1..bb2a3955b 100755 --- a/crates/core/src/expression/update_evaluator_tests.rs +++ b/crates/core/src/expression/update_evaluator_tests.rs @@ -766,10 +766,12 @@ mod update_item_sizing { } #[test] - fn a_form_not_sized_by_its_statement_keeps_the_depth_refusal() { - // A PartiQL UPDATE and the legacy AttributeUpdates form: the statement - // is over the limit, and the answer is the one apply_update_validated - // gives. + fn a_partiql_or_legacy_update_keeps_the_depth_refusal() { + // A PartiQL UPDATE is sized by its updated item alone, and the figure + // of the legacy AttributeUpdates form is computed from the finished + // values of the attributes it names, so the statement check does not + // apply to either. The expression's statement figure is over the + // limit, and the answer is the one apply_update_validated gives. for partiql in [true, false] { let (actions, mut maps) = update(409_500, 30, false); maps.finished_item_sizing = partiql; diff --git a/crates/core/src/validation/mod.rs b/crates/core/src/validation/mod.rs index c5489819a..5c15c0d32 100755 --- a/crates/core/src/validation/mod.rs +++ b/crates/core/src/validation/mod.rs @@ -2318,10 +2318,11 @@ pub fn validate_update_item_size( } /// The statement half of [`validate_update_item_size`], for a standalone -/// `UpdateItem` written as an expression. An update from a PartiQL `UPDATE` -/// (`maps.finished_item_sizing`) and the legacy `AttributeUpdates` form -/// (`maps.legacy_attribute_updates`) are not sized by their statement, and -/// pass here. +/// `UpdateItem` written as an expression. It passes an update from a PartiQL +/// `UPDATE` (`maps.finished_item_sizing`), which is sized by its updated item +/// alone. It also passes the legacy `AttributeUpdates` form +/// (`maps.legacy_attribute_updates`), whose figure is computed from the +/// finished values of the attributes it names. /// /// # Errors /// diff --git a/docs/dynamodb-limits.md b/docs/dynamodb-limits.md index 39167fb48..37214e2b5 100755 --- a/docs/dynamodb-limits.md +++ b/docs/dynamodb-limits.md @@ -42,7 +42,7 @@ Source: [AWS DynamoDB Service Quotas](https://docs.aws.amazon.com/amazondynamodb | Partition key size | 1–2,048 bytes | Enforced | `validate_key_sizes`, `LimitsConfig::max_partition_key_size_bytes` | | Sort key size | 1–1,024 bytes | Enforced | `validate_key_sizes`, `LimitsConfig::max_sort_key_size_bytes` | | Attribute name size | 1 to 65,535 bytes (64 KB) | Enforced | `validate_attribute_name_sizes`, `validate_expression_attribute_name_sizes`, `validate_attribute_updates_name_sizes`, `LimitsConfig::max_attribute_name_bytes`; UTF-8 bytes, top-level names and map keys at every depth. PutItem, BatchWriteItem.PutRequest, TransactWriteItems.Put and ImportTable check the item. PutItem, UpdateItem, DeleteItem and every TransactWriteItems member (Put, Update, Delete, ConditionCheck) check every `ExpressionAttributeNames` value and every map key inside every `ExpressionAttributeValues` value; UpdateItem also checks the names and values of `AttributeUpdates`. PutItem and UpdateItem report the name ahead of the table lookup; UpdateItem also reports it ahead of the condition and the index-key checks. TransactWriteItems checks every member, and BatchWriteItem every PutRequest, before any table lookup, so neither a missing table (in any member) nor a PutRequest item missing its key is reported first; a transaction fails as a whole with a top-level ValidationException. With an over-limit name in both expression maps, UpdateItem reports the `ExpressionAttributeNames` placeholder and a transacted Update or Put reports the `ExpressionAttributeValues` placeholder; a transacted Put with a long item name and a long `ExpressionAttributeNames` value reports the placeholder, a standalone PutItem the item. The message depends on the request form (`AttributeNameRequestForm`): PutItem, UpdateItem and DeleteItem answer `1 validation error detected: Attribute name is too large, must be less than 65536 bytes`, followed by ` for key #n` when the name came through `ExpressionAttributeNames`; a TransactWriteItems member answers `ExpressionAttributeNames contains invalid value: Attribute name is too large, must be less than 65536 bytes for key #n`, or `ExpressionAttributeValues contains invalid value: ... for key :v` for a map key inside a value; TransactWriteItems.Put and BatchWriteItem.PutRequest answer `Attribute name is too large, must be less than 65536 bytes` for a name in the item. | -| Attribute nesting depth | 32 levels | Enforced | `validate_item_nesting_depth`, applied on PutItem, UpdateItem, BatchWriteItem.PutRequest, TransactWriteItems.Put, ImportTable | +| Attribute nesting depth | 32 levels | Enforced | `validate_item_nesting_depth`, applied on PutItem, BatchWriteItem.PutRequest, TransactWriteItems.Put, ImportTable. UpdateItem checks its `ExpressionAttributeValues` and `AttributeUpdates` values with `validate_attribute_values_nesting_depth`. After an update is applied, `validate_written_nesting_depth` checks each attribute that a `SET` or an `ADD` writes with `nests_beyond_item_limit`, on UpdateItem, the TransactWriteItems `Update` and the PartiQL `UPDATE`, and refuses with `Nesting Levels have exceeded supported limits` | | Number of attributes per item | No practical limit | Enforced | ExtendDB has no per-item attribute count limit | ## Secondary Indexes diff --git a/tests/test_update_result_nesting_depth.py b/tests/test_update_result_nesting_depth.py index 8200dc43c..26ee57c3a 100644 --- a/tests/test_update_result_nesting_depth.py +++ b/tests/test_update_result_nesting_depth.py @@ -207,8 +207,10 @@ def test_transact_update_up_to_the_limit_is_applied(dynamodb_client, table): # The tests below that expect the size answer run on SQLite and PostgreSQL. The # MongoDB layer applies no item size limit on updates, and the Cassandra layer -# sizes every update by the updated item with other wording, both documented in -# docs/differences-from-dynamodb.md. With the backend unset (a run against the +# sizes every update by the updated item, both documented in +# docs/differences-from-dynamodb.md. The Cassandra refusal is worded "Item size +# has exceeded the maximum allowed size" (validate_item_size in +# crates/core/src/validation/mod.rs). With the backend unset (a run against the # service), they run. SIZED_UPDATES = pytest.mark.skipif( os.environ.get("EXTENDDB_TEST_BACKEND", "") in {"mongodb", "cassandra"},