[v2] Enhancements for CRT S3 integration

.changes/next-release/bugfix-crt-24627.json

@@ -0,0 +1,5 @@
+{
+  "type": "bugfix",
+  "category": "crt",
+  "description": "Return error when final rename task fails on downloads"
+}

.changes/next-release/enhancement-crt-31704.json

@@ -0,0 +1,5 @@
+{
+  "type": "enhancement",
+  "category": "crt",
+  "description": "Support ``multipart_threshold`` (upload only) and ``max_concurrent_requests`` config options for CRT client."
+}

.changes/next-release/enhancement-crt-44299.json

@@ -0,0 +1,5 @@
+{
+  "type": "enhancement",
+  "category": "crt",
+  "description": "Enforce minimum 10gbps target throughput for explicitly configured crt environments"
+}

.changes/next-release/enhancement-crt-53269.json

@@ -0,0 +1,5 @@
+{
+  "type": "enhancement",
+  "category": "crt",
+  "description": "Set lower 4gbps target throughput default for non-EC2 hosts."
+}

.changes/next-release/enhancement-crt-69912.json

@@ -0,0 +1,5 @@
+{
+  "type": "enhancement",
+  "category": "crt",
+  "description": "Configure CRT client to download with single GET when object size is below ``multipart_threshold``"
+}

.changes/next-release/enhancement-crt-76250.json

@@ -0,0 +1,5 @@
+{
+  "type": "enhancement",
+  "category": "crt",
+  "description": "Pass resolved ``max_attempts`` value to the CRT client."
+}

.changes/next-release/enhancement-s3-42614.json

@@ -0,0 +1,5 @@
+{
+  "type": "enhancement",
+  "category": "``s3``",
+  "description": "Follow bucket region redirects for the ``s3`` commands when CRT is enabled."
+}

awscli/botocore/configprovider.py

@@ -21,6 +21,7 @@
 from botocore import utils
 from botocore.exceptions import InvalidConfigError
 
+
 def _resolve_new_retries():
     _env_new_retries = os.environ.get('AWS_NEW_RETRIES_2026')
     if _env_new_retries is not None:
@@ -451,6 +452,24 @@ def get_config_variable(self, logical_name):
         provider = self._mapping[logical_name]
         return provider.provide()
 
+    def is_explicitly_set(self, logical_name):
+        """
+        Determine whether a value was configured for the specified
+        logical_name, as opposed to resolving to a built-in default.
+
+        :type logical_name: str
+        :param logical_name: The logical name of the session variable
+            you want to check.
+
+        :returns: True if a value was configured, False otherwise.
+        """
+        if logical_name in self._overrides:
+            return True
+        if logical_name not in self._mapping:
+            return False
+        resolved = self._mapping[logical_name].resolve()
+        return resolved is not None and resolved.is_configured
+
     def get_config_provider(self, logical_name):
         """
         Retrieve the provider associated with the specified logical_name.
@@ -525,6 +544,14 @@ def set_config_provider(self, logical_name, provider):
         self._mapping[logical_name] = provider
 
 
+class ConfigValue:
+    """A resolved config value and whether it came from a configured source."""
+
+    def __init__(self, value, is_configured=True):
+        self.value = value
+        self.is_configured = is_configured
+
+
 class BaseProvider:
     """Base class for configuration value providers.
 
@@ -536,6 +563,17 @@ def provide(self):
         """Provide a config value."""
         raise NotImplementedError('provide')
 
+    def resolve(self):
+        """Provide a config value along with where it came from.
+
+        :rtype: Optional[ConfigValue]
+        :returns: The resolved value, or None if this provider has none.
+        """
+        value = self.provide()
+        if value is None:
+            return None
+        return ConfigValue(value)
+
 
 class ChainProvider(BaseProvider):
     """This provider wraps one or more other providers.
@@ -568,10 +606,16 @@ def provide(self):
         one in the chain to return a non-None value is the returned from the
         ChainProvider. When no non-None value is found, None is returned.
         """
+        resolved = self.resolve()
+        return resolved.value if resolved is not None else None
+
+    def resolve(self):
         for provider in self._providers:
-            value = provider.provide()
-            if value is not None:
-                return self._convert_type(value)
+            resolved = provider.resolve()
+            if resolved is not None:
+                return ConfigValue(
+                    self._convert_type(resolved.value), resolved.is_configured
+                )
         return None
 
     def _convert_type(self, value):
@@ -717,6 +761,9 @@ def provide(self):
         """Provide the constant value given during initialization."""
         return self._value
 
+    def resolve(self):
+        return ConfigValue(self._value, is_configured=False)
+
     def __repr__(self):
         return f'ConstantProvider(value={self._value})'
 

awscli/botocore/utils.py

@@ -1576,7 +1576,7 @@ class S3RegionRedirectorv2:
     """
 
     def __init__(self, endpoint_bridge, client, cache=None):
-        self._cache = cache or {}
+        self._cache = {} if cache is None else cache
         self._client = weakref.proxy(client)
 
     def register(self, event_emitter=None):
@@ -1616,48 +1616,13 @@ def redirect_from_error(self, request_dict, response, operation, **kwargs):
             )
             return
 
-        error = response[1].get('Error', {})
-        error_code = error.get('Code')
-        response_metadata = response[1].get('ResponseMetadata', {})
-
-        # We have to account for 400 responses because
-        # if we sign a Head* request with the wrong region,
-        # we'll get a 400 Bad Request but we won't get a
-        # body saying it's an "AuthorizationHeaderMalformed".
-        is_special_head_object = (
-            error_code in ('301', '400') and operation.name == 'HeadObject'
-        )
-        is_special_head_bucket = (
-            error_code in ('301', '400')
-            and operation.name == 'HeadBucket'
-            and 'x-amz-bucket-region'
-            in response_metadata.get('HTTPHeaders', {})
-        )
-        is_wrong_signing_region = (
-            error_code == 'AuthorizationHeaderMalformed' and 'Region' in error
-        )
-        is_redirect_status = response[0] is not None and response[
-            0
-        ].status_code in (301, 302, 307)
-        is_permanent_redirect = error_code == 'PermanentRedirect'
-        is_opt_in_region_redirect = (
-            error_code == 'IllegalLocationConstraintException'
-            and operation.name != 'CreateBucket'
-        )
-        if not any(
-            [
-                is_special_head_object,
-                is_wrong_signing_region,
-                is_permanent_redirect,
-                is_special_head_bucket,
-                is_redirect_status,
-                is_opt_in_region_redirect,
-            ]
-        ):
+        bucket = redirect_ctx.get('bucket')
+        if bucket is None:
+            return
+        if not self.is_redirect_response(response, operation):
             return
 
-        bucket = request_dict['context']['s3_redirect']['bucket']
-        client_region = request_dict['context'].get('client_region')
+        client_region = request_dict.get('context', {}).get('client_region')
         new_region = self.get_bucket_region(bucket, response)
 
         if new_region is None:
@@ -1702,6 +1667,55 @@ def redirect_from_error(self, request_dict, response, operation, **kwargs):
         # Return 0 so it doesn't wait to retry
         return 0
 
+    def get_redirect_region(self, bucket, response, operation):
+        """Return the region a response redirects a bucket to, if any."""
+        if bucket is None or ArnParser.is_arn(bucket):
+            return None
+        if not self.is_redirect_response(response, operation):
+            return None
+        return self.get_bucket_region(bucket, response)
+
+    def is_redirect_response(self, response, operation):
+        """Return whether a response says the bucket is in another region."""
+        error = response[1].get('Error', {})
+        error_code = error.get('Code')
+        response_metadata = response[1].get('ResponseMetadata', {})
+
+        # We have to account for 400 responses because
+        # if we sign a Head* request with the wrong region,
+        # we'll get a 400 Bad Request but we won't get a
+        # body saying it's an "AuthorizationHeaderMalformed".
+        is_special_head_object = (
+            error_code in ('301', '400') and operation.name == 'HeadObject'
+        )
+        is_special_head_bucket = (
+            error_code in ('301', '400')
+            and operation.name == 'HeadBucket'
+            and 'x-amz-bucket-region'
+            in response_metadata.get('HTTPHeaders', {})
+        )
+        is_wrong_signing_region = (
+            error_code == 'AuthorizationHeaderMalformed' and 'Region' in error
+        )
+        is_redirect_status = response[0] is not None and response[
+            0
+        ].status_code in (301, 302, 307)
+        is_permanent_redirect = error_code == 'PermanentRedirect'
+        is_opt_in_region_redirect = (
+            error_code == 'IllegalLocationConstraintException'
+            and operation.name != 'CreateBucket'
+        )
+        return any(
+            [
+                is_special_head_object,
+                is_wrong_signing_region,
+                is_permanent_redirect,
+                is_special_head_bucket,
+                is_redirect_status,
+                is_opt_in_region_redirect,
+            ]
+        )
+
     def get_bucket_region(self, bucket, response):
         """
         There are multiple potential sources for the new region to redirect to,