-
-
Notifications
You must be signed in to change notification settings - Fork 505
security: rewind file bodies before 401 retry to prevent empty uploads #409
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -18,6 +18,7 @@ | |
| import re | ||
| import threading | ||
| import time | ||
| from collections.abc import Iterator, Mapping | ||
| from pathlib import Path | ||
| from typing import Any, cast | ||
|
|
||
|
|
@@ -185,6 +186,61 @@ def _build_basic_auth(client_id: str) -> str: | |
| return "Basic " + base64.b64encode(f"{client_id}:".encode()).decode() | ||
|
|
||
|
|
||
| def _iter_file_objects(kwargs: dict[str, Any]) -> Iterator[Any]: | ||
| """Yield file-like objects referenced by request kwargs (files/data).""" | ||
| files = kwargs.get("files") | ||
| if isinstance(files, Mapping): | ||
| values = list(files.values()) | ||
| elif isinstance(files, (list, tuple)): | ||
| values = [v for _, v in files] | ||
| else: | ||
| values = [] | ||
| for value in values: | ||
| # requests accepts fileobj or (name, fileobj[, content_type[, headers]]) | ||
| fileobj = ( | ||
| value[1] if isinstance(value, (tuple, list)) and len(value) >= 2 else value | ||
| ) | ||
| if hasattr(fileobj, "read"): | ||
| yield fileobj | ||
| data = kwargs.get("data") | ||
| if hasattr(data, "read"): | ||
| yield data | ||
|
|
||
|
|
||
| def _capture_file_positions( | ||
| kwargs: dict[str, Any], | ||
| ) -> list[tuple[Any, int]] | None: | ||
| """Record the stream position of each file-like body before a request. | ||
|
|
||
| Returns None when any body cannot be repositioned, meaning a retry | ||
| would re-read from EOF and silently send empty/truncated content. | ||
| """ | ||
| positions: list[tuple[Any, int]] = [] | ||
| for fileobj in _iter_file_objects(kwargs): | ||
| try: | ||
| if not fileobj.seekable(): | ||
| return None | ||
| positions.append((fileobj, fileobj.tell())) | ||
| except (OSError, ValueError, AttributeError): | ||
| return None | ||
| data = kwargs.get("data") | ||
| if data is not None and not hasattr(data, "read") and isinstance(data, Iterator): | ||
| # Streamed iterable body (e.g. a generator): attempt #1 consumes it | ||
| # and it cannot be rewound, so a retry must not be attempted. | ||
| return None | ||
|
Comment on lines
+226
to
+230
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🗄️ Data Integrity & Integration | 🟠 Major | ⚡ Quick win 🧩 Analysis chain🏁 Script executed: #!/bin/bash
set -euo pipefail
python3 - <<'PY'
from collections.abc import Iterable, Iterator
class OneShotBody:
def __init__(self):
self._body = iter((b"chunk1", b"chunk2"))
def __iter__(self):
return self._body
body = OneShotBody()
assert isinstance(body, Iterable)
assert not isinstance(body, Iterator)
assert list(body) == [b"chunk1", b"chunk2"]
assert list(body) == []
print("One-shot Iterable is missed by an Iterator-only check")
PY
python3 - <<'PY'
import inspect
import requests
from requests.models import PreparedRequest
print("requests:", requests.__version__)
print(inspect.getsource(PreparedRequest.prepare_body))
PYRepository: cyberjunky/python-garminconnect Length of output: 344 🏁 Script executed: #!/bin/bash
set -euo pipefail
echo "== repo files =="
git ls-files | sed -n '1,120p'
echo "== locate client.py and test files =="
fd -a 'client.py|test_garmin_unit.py' . | sed -n '1,80p'
echo "== inspect client.py relevant section =="
client=$(fd 'client.py' . | head -1)
if [ -n "$client" ]; then
wc -l "$client"
sed -n '180,260p' "$client" | nl -ba -v180
fi
echo "== search retry/capture helpers and tests =="
rg -n "_capture_file_positions|_restore_file_positions|_run_request|streamed|generator_data_skips|requests" -S .gitignore README.md tests garminconnect 2>/dev/null | sed -n '1,220p'Repository: cyberjunky/python-garminconnect Length of output: 1614 🌐 Web query:
💡 Result: In the Python requests library, the PreparedRequest.prepare_body method handles the preparation of request bodies, including detecting whether the provided data is a streamed body [1][2]. When data is passed to prepare_body, the library identifies it as a stream if it is iterable (typically checked via collections.abc.Iterable or the presence of a iter attribute) and is not a standard type like a string, bytes, list, tuple, or Mapping [1][3][2]. Key mechanisms for streamed bodies include: 1. Stream Detection: Modern versions of requests use a check similar to isinstance(data, Iterable) or hasattr(data, "iter") to identify streams [3][2]. Using both ensures compatibility with objects that rely on getattr delegation for their interface, such as file-like wrappers [4][3]. 2. Rewind Capability: If the streamed object has a tell method, requests records the current position in the _body_position attribute [2][5]. This allows the library to attempt to rewind the stream if the request needs to be resent, such as following a 307 or 308 redirect [4][2]. 3. Content Length: If a length can be determined for the stream (using super_len), it is set in the Content-Length header; otherwise, the library uses Transfer-Encoding: chunked [1][2]. A recent regression in version 2.34.0 (addressed in PR Citations:
🏁 Script executed: #!/bin/bash
set -euo pipefail
client="$(fd 'client.py' . | head -1)"
test_file="$(fd 'test_garmin_unit.py' . | head -1)"
echo "== client imports and retry helpers =="
sed -n '1,40p' "$client"
sed -n '190,245p' "$client"
echo "== test generator/stream body context =="
sed -n '1,80p' "$test_file"
rg -n "def test_generator_data_skips_retry_and_raises|_capture_file_positions|_run_request|Iterab|Iterator|OneShot|Generator|stream" "$test_file" "$client"
echo "== Python standard library probe for current Iterator check =="
python3 - <<'PY'
from collections.abc import Iterable, Iterator, Mapping
class OneShotBody:
def __init__(self):
self._body = iter((b"chunk1", b"chunk2"))
def __iter__(self):
return self._body
# Mirrors the current request body classification before retry decisions
class Current:
pass
Current.__bases__ = ()
def current_is_iterable_file(data):
return data is not None and hasattr(data, "read") and Current.__bases__ == ()
def current_is_stream(data):
return data is not None and not hasattr(data, "read") and isinstance(data, Iterator)
body = OneShotBody()
print("object", type(body).__name__)
print("is_iterable_file()", current_is_iterable_file(body))
print("is_stream()", current_is_stream(body))
print("first list()", list(body))
print("second list()", list(body))
# Modern equivalent with materialized container exclusion
def fixed_is_stream(data):
return (data is not None
and not hasattr(data, "read")
and (isinstance(data, Iterable) or hasattr(data, "iter"))
and not isinstance(data, (str, bytes, list, tuple, Mapping)))
print("fixed_is_stream()", fixed_is_stream(body))
print("fixed first list()", list(body))
print("fixed second list()", list(body))
PYRepository: cyberjunky/python-garminconnect Length of output: 9409 🌐 Web query:
💡 Result: In the requests library, the PreparedRequest.prepare_body method determines if the provided body data should be treated as a stream (iterable) by checking if the data is an instance of collections.abc.Iterable or has the iter attribute [1][2]. For many years, the library exclusively used hasattr(data, "iter") to detect streams [3][2]. However, a change was introduced to use isinstance(data, Iterable) for better adherence to modern Python practices [3][2]. This change caused a regression for objects that use getattr-based delegation (such as file-like wrappers like tqdm.wrapattr), because isinstance only checks the class MRO and ignores attributes resolved via getattr [3][2]. To address this, as of May 2026, the implementation in the main branch of the requests library was updated to use both checks in conjunction to ensure compatibility: if (isinstance(data, Iterable) or hasattr(data, "iter")) and not isinstance(data, (str, bytes, list, tuple, Mapping)): # logic for handling streamed bodies [1][2] This ensures that the library correctly identifies both standard iterables and objects that proxy their iterator interface through dynamic attribute delegation [2][4]. Citations:
🏁 Script executed: #!/bin/bash
set -euo pipefail
client="$(fd 'client.py' . | head -1)"
test_file="$(fd 'test_garmin_unit.py' . | head -1)"
echo "== full _capture/_restore helper =="
sed -n '189,229p' "$client"
echo "== _run_request relevant section =="
sed -n '1560,1660p' "$client"
echo "== generator test context =="
sed -n '1728,1770p' "$test_file"
echo "== deterministic probe mirroring current helper predicate =="
python3 - <<'PY'
from collections.abc import Iterator, Mapping, Iterable
class OneShotData:
def __iter__(self):
return iter([b"x", b"y"])
def current_capture_returns_none(data):
return data is not None and not hasattr(data, "read") and isinstance(data, Iterator)
print("current_capture_returns_none(generator_function_called)", current_capture_returns_none((lambda: (x for x in []))()))
print("current_capture_returns_none(one_shot_iterable)", current_capture_returns_none(OneShotData()))
print("current_capture_returns_none(generator_function_called)", current_capture_returns_none((lambda: (x for x in []))()))
print("current_capture_returns_none(one_shot_iterable)", current_capture_returns_none(OneShotData()))
class ModernPredicate:
pass
ModernPredicate.__bases__ = ()
def current_like(data):
return data is not None and not hasattr(data, "read") and isinstance(data, Iterator)
def fixed_like(data):
return (
data is not None
and not hasattr(data, "read")
and (isinstance(data, Iterable) or hasattr(data, "iter"))
and not isinstance(data, (str, bytes, list, tuple, Mapping))
)
for body in [(lambda: (x for x in []))(), OneShotData(), iter((b"one", b"two"))]:
print(type(body).__name__, "current=", current_like(body), "fixed=", fixed_like(body))
print(_iter_file_objects.__self__)
PYRepository: cyberjunky/python-garminconnect Length of output: 7514 Reject one-shot
🤖 Prompt for AI Agents |
||
| return positions | ||
|
|
||
|
|
||
| def _restore_file_positions(positions: list[tuple[Any, int]]) -> bool: | ||
| """Rewind file-like bodies to their pre-request positions. False on failure.""" | ||
| try: | ||
| for fileobj, pos in positions: | ||
| fileobj.seek(pos) | ||
| except (OSError, ValueError, AttributeError): | ||
| return False | ||
| return True | ||
|
coderabbitai[bot] marked this conversation as resolved.
|
||
|
|
||
|
|
||
| def _decode_jwt_payload(token: str) -> dict[str, Any] | None: | ||
| """Decode a JWT payload without verifying the signature. | ||
|
|
||
|
|
@@ -1532,12 +1588,23 @@ def _run_request(self, method: str, path: str, **kwargs: Any) -> Any: | |
| headers.update(custom_headers) | ||
|
|
||
| sess = self._api_session | ||
| # Snapshot stream positions of any file bodies so a 401 retry can | ||
| # rewind them; attempt #1 reads file handles to EOF, and re-sending | ||
| # the same kwargs would otherwise upload an empty/truncated body. | ||
| file_positions = _capture_file_positions(kwargs) | ||
| resp = sess.request(method, url, headers=headers, **kwargs) | ||
|
|
||
| if resp.status_code == 401: | ||
| with self._token_lock: | ||
| self._refresh_session() | ||
| resp = sess.request(method, url, headers=self.get_api_headers(), **kwargs) | ||
| if file_positions is not None and _restore_file_positions(file_positions): | ||
| headers = self.get_api_headers() | ||
| headers.update(custom_headers) | ||
| resp = sess.request(method, url, headers=headers, **kwargs) | ||
| else: | ||
| # Unseekable/unrewindable file body: retrying would silently | ||
| # send an empty part. Fall through so the 401 raises below. | ||
| _LOGGER.debug("Skipping 401 retry: request body is not rewindable") | ||
|
|
||
| if resp.status_code == 204: | ||
|
|
||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.