From 3c11e09a065900dc16553e433d92fb3d69a0a28e Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 11:51:26 +0000 Subject: [PATCH 01/10] fix(crypto): handle header verification failure in content_length Catch InvalidHeaderError and ValueError in EncryptingFileContentRepository.content_length and fall back to returning fv.size to prevent WebDAV PROPFIND requests from crashing. Co-authored-by: VulcanoSoftware <113239901+VulcanoSoftware@users.noreply.github.com> --- dcfs/crypto/repository.py | 18 ++++++++++++++++-- tests/test_crypto/test_repository.py | 22 ++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/dcfs/crypto/repository.py b/dcfs/crypto/repository.py index 430d648..b5027d9 100644 --- a/dcfs/crypto/repository.py +++ b/dcfs/crypto/repository.py @@ -252,11 +252,25 @@ async def content_length(self, fv: "DCFSFileVersion") -> int: return 0 # ``_detect`` already caches per file, so the second call from a HEAD # request or a Content-Range computation is free. - detected = await self._detect(fv, "") + try: + detected = await self._detect(fv, "") + except (InvalidHeaderError, ValueError): + logger.warning( + "Header detection failed for fv.id=%s, falling back to fv.size", + fv.id, + ) + return fv.size if detected is None: return fv.size header, _ = detected - return _plaintext_size_from_ciphertext(fv.size, header.chunk_size) + try: + return _plaintext_size_from_ciphertext(fv.size, header.chunk_size) + except ValueError: + logger.warning( + "Plaintext size computation failed for fv.id=%s, falling back to fv.size", + fv.id, + ) + return fv.size # -- internals --------------------------------------------------------- diff --git a/tests/test_crypto/test_repository.py b/tests/test_crypto/test_repository.py index 4bc6234..d45d150 100644 --- a/tests/test_crypto/test_repository.py +++ b/tests/test_crypto/test_repository.py @@ -425,3 +425,25 @@ async def gen(): out = await _collect(await repo.get(fv, 0, -1, "stream.bin")) assert out == plaintext + + +async def test_content_length_fallback_on_invalid_header() -> None: + """When header verification fails during content_length, it should return fv.size as fallback.""" + import struct as _struct + + repo = _make_repo(chunk_size=4096) + + # 1. Valid encrypted file returns plaintext size + plaintext = os.urandom(8192) + valid_fv = await _save_and_get_fv(repo, plaintext) + assert await repo.content_length(valid_fv) == len(plaintext) + + # 2. Corrupted header MAC returns fv.size fallback + body = _struct.pack(">4sHHI32s", b"DCFS", 1, 1, 4096, b"\x00" * 32) + fake = body + b"\xff" * 16 # wrong MAC + corrupt_fv = _seed_plaintext(repo, fake + b"some more data") + assert await repo.content_length(corrupt_fv) == corrupt_fv.size + + # 3. Short header with DCFS magic returns fv.size fallback + short_fv = _seed_plaintext(repo, b"DCFS" + b"\x00" * 10) + assert await repo.content_length(short_fv) == short_fv.size From 0528fc7a3e240826c246aec89515a47201488a3d Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 12:14:59 +0000 Subject: [PATCH 02/10] fix(crypto): handle header verification failure in content_length Catch InvalidHeaderError and ValueError in EncryptingFileContentRepository.content_length and fall back to returning fv.size to prevent WebDAV PROPFIND requests from crashing. Co-authored-by: VulcanoSoftware <113239901+VulcanoSoftware@users.noreply.github.com> From 5011bc7b49e69092f9c6cb8259aa49a4e3444721 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 12:36:42 +0000 Subject: [PATCH 03/10] fix(crypto): handle header verification failure in content_length Catch InvalidHeaderError and ValueError in EncryptingFileContentRepository.content_length and fall back to returning fv.size to prevent WebDAV PROPFIND requests from crashing. Co-authored-by: VulcanoSoftware <113239901+VulcanoSoftware@users.noreply.github.com> From 228a97516fee73ac8671b92006d84f3de6edbd66 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 12:41:55 +0000 Subject: [PATCH 04/10] fix(crypto): handle header verification failure in content_length and cache CDN URLs - Catch InvalidHeaderError and ValueError in EncryptingFileContentRepository.content_length and fall back to returning fv.size to prevent WebDAV 500 errors. - Cache and coalesce attachment CDN URLs in DiscordBotAPI to eliminate Discord REST API rate limits (429) during multi-part file downloads. Co-authored-by: VulcanoSoftware <113239901+VulcanoSoftware@users.noreply.github.com> --- dcfs/discord/impl/discord_bot.py | 82 +++++++++++++++++++++++++++----- 1 file changed, 70 insertions(+), 12 deletions(-) diff --git a/dcfs/discord/impl/discord_bot.py b/dcfs/discord/impl/discord_bot.py index e3bdb3b..4f64a82 100644 --- a/dcfs/discord/impl/discord_bot.py +++ b/dcfs/discord/impl/discord_bot.py @@ -41,6 +41,56 @@ def __init__(self, bot: discord.Client, bot_token: str): self._bot = bot self._bot_token = bot_token self._http_session: Optional[aiohttp.ClientSession] = None + self._url_cache: dict[int, tuple[str, int]] = {} + self._inflight_fetches: dict[int, asyncio.Future[tuple[str, int]]] = {} + self._url_cache_lock = asyncio.Lock() + + def _cache_url(self, message_id: int, url: str, size: int) -> None: + if len(self._url_cache) >= 10000: + first_key = next(iter(self._url_cache)) + del self._url_cache[first_key] + self._url_cache[message_id] = (url, size) + + async def _fetch_attachment_url_and_size( + self, channel_id: int, message_id: int, force_refresh: bool = False + ) -> tuple[str, int]: + if not force_refresh and message_id in self._url_cache: + return self._url_cache[message_id] + + async with self._url_cache_lock: + if not force_refresh and message_id in self._url_cache: + return self._url_cache[message_id] + if message_id in self._inflight_fetches: + fut = self._inflight_fetches[message_id] + else: + loop = asyncio.get_running_loop() + fut = loop.create_future() + self._inflight_fetches[message_id] = fut + + async def _do_fetch(): + try: + channel = await self._get_channel(channel_id) + try: + msg = await channel.fetch_message(message_id) + except discord.NotFound: + raise MessageNotFound(message_id) + if not msg.attachments: + raise UnDownloadableMessage(message_id) + att = msg.attachments[0] + res = (att.url, att.size) + self._cache_url(message_id, att.url, att.size) + if not fut.done(): + fut.set_result(res) + except Exception as ex: + if not fut.done(): + fut.set_exception(ex) + finally: + async with self._url_cache_lock: + self._inflight_fetches.pop(message_id, None) + + asyncio.create_task(_do_fetch()) + + return await fut async def _ensure_http_session(self) -> aiohttp.ClientSession: if self._http_session is None or self._http_session.closed: @@ -153,21 +203,13 @@ async def edit_message_media(self, req: EditMessageMediaReq) -> Message: async def download_file(self, req: DownloadFileReq) -> DownloadFileResp: channel_id = self._parse_channel_id(req.chat) - channel = await self._get_channel(channel_id) - try: - msg = await channel.fetch_message(req.message_id) - except discord.NotFound: - raise MessageNotFound(req.message_id) - if not msg.attachments: - raise UnDownloadableMessage(req.message_id) - attachment = msg.attachments[0] + url, size = await self._fetch_attachment_url_and_size(channel_id, req.message_id) session = await self._ensure_http_session() # Build optional Range header so the CDN only streams the requested # byte range (critical for download_file_parallel sub-requests). should_range = req.begin > 0 or req.end != -1 - url = attachment.url headers = {} if should_range: range_end = "" if req.end == -1 else str(req.end) @@ -175,7 +217,7 @@ async def download_file(self, req: DownloadFileReq) -> DownloadFileResp: logger.info( "CDN download: msg=%d range=%d-%d should_range=%s attach_size=%d", - req.message_id, req.begin, req.end, should_range, attachment.size, + req.message_id, req.begin, req.end, should_range, size, ) # Timeout: connect within 15s, download within 120s. Without a @@ -187,9 +229,24 @@ async def download_file(self, req: DownloadFileReq) -> DownloadFileResp: total=120.0, ) t0 = asyncio.get_event_loop().time() - response = await session.get(url, headers=headers, timeout=timeout) + try: + response = await session.get(url, headers=headers, timeout=timeout) + response.raise_for_status() + except aiohttp.ClientResponseError as exc: + if exc.status in (401, 403, 404): + logger.warning( + "Cached CDN URL for msg=%d failed with status %d, refreshing URL...", + req.message_id, exc.status, + ) + self._url_cache.pop(req.message_id, None) + url, size = await self._fetch_attachment_url_and_size( + channel_id, req.message_id, force_refresh=True + ) + response = await session.get(url, headers=headers, timeout=timeout) + response.raise_for_status() + else: + raise t1 = asyncio.get_event_loop().time() - response.raise_for_status() # Determine whether the CDN honoured the Range header. # 206 Partial Content means it did; 200 OK means it ignored it. @@ -266,6 +323,7 @@ def _to_message_dto(self, message: discord.Message) -> MessageResp: size=att.size, mime_type=att.content_type ) + self._cache_url(message.id, att.url, att.size) return MessageResp( message_id=message.id, text=message.content if message.content else "", From b77983efec02817bdfc8c045878b138869f34b0a Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 12:47:06 +0000 Subject: [PATCH 05/10] fix(crypto): handle header verification failure in content_length and cache CDN URLs - Catch InvalidHeaderError and ValueError in EncryptingFileContentRepository.content_length and fall back to returning fv.size to prevent WebDAV 500 errors. - Cache and coalesce attachment CDN URLs in DiscordBotAPI to eliminate Discord REST API rate limits (429) during multi-part file downloads. Co-authored-by: VulcanoSoftware <113239901+VulcanoSoftware@users.noreply.github.com> --- dcfs/discord/impl/discord_bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcfs/discord/impl/discord_bot.py b/dcfs/discord/impl/discord_bot.py index 4f64a82..05c98cc 100644 --- a/dcfs/discord/impl/discord_bot.py +++ b/dcfs/discord/impl/discord_bot.py @@ -301,7 +301,7 @@ async def _chunk_generator(): finally: response.close() - return DownloadFileResp(chunks=_chunk_generator(), size=attachment.size) + return DownloadFileResp(chunks=_chunk_generator(), size=size) async def search_messages(self, req: SearchMessageReq) -> GetMessagesRespNoNone: channel_id = self._parse_channel_id(req.chat) From 6f4d1ac8fa5cd6f664ec9ae4670bc2081fee137a Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:17:32 +0000 Subject: [PATCH 06/10] fix(webdav): handle chunked and stream uploads in PUT endpoint Support chunked transfer encoding and streaming uploads in asgidav PUT endpoint by unconditionally calling member.overwrite and setting size=-1 when Content-Length is missing or chunked. Co-authored-by: VulcanoSoftware <113239901+VulcanoSoftware@users.noreply.github.com> --- asgidav/app.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/asgidav/app.py b/asgidav/app.py index 50a00d4..c54c01d 100644 --- a/asgidav/app.py +++ b/asgidav/app.py @@ -225,20 +225,24 @@ async def get(request: Request, path: str): @app.put("/{path:path}") async def put(request: Request, path: str): try: - raw_length = request.headers.get("Content-Length", "0").strip() - size = int(raw_length) if raw_length else 0 - if size < 0: - size = 0 + raw_length = request.headers.get("Content-Length", "").strip() + if "chunked" in request.headers.get("Transfer-Encoding", "").lower(): + size = -1 + elif raw_length: + size = int(raw_length) + if size < 0: + size = -1 + else: + size = -1 except (ValueError, TypeError): logger.warning(f"PUT {path}: invalid Content-Length '{request.headers.get('Content-Length', '')}'") - size = 0 + size = -1 try: if not (member := await get_member(path)): member = await (await root()).create_empty_resource(path) if isinstance(member, Resource): - if size > 0: - await member.overwrite(request.stream(), size=size) + await member.overwrite(request.stream(), size=size) return CREATED return CONFLICT("Cannot PUT to a directory") except TechnicalError as ex: From 874fe44eba16486887c4e06e7e1089c5e5b832e8 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:05:51 +0000 Subject: [PATCH 07/10] fix(webdav): handle chunked and stream uploads in PUT endpoint Support chunked transfer encoding and streaming uploads in asgidav PUT endpoint by unconditionally calling member.overwrite and setting size=-1 when Content-Length is missing or chunked. Added unit tests for WebDAV PUT endpoint behavior. Co-authored-by: VulcanoSoftware <113239901+VulcanoSoftware@users.noreply.github.com> --- tests/test_asgidav/test_app.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_asgidav/test_app.py b/tests/test_asgidav/test_app.py index fd83ba0..ef2ecb6 100644 --- a/tests/test_asgidav/test_app.py +++ b/tests/test_asgidav/test_app.py @@ -74,3 +74,31 @@ async def test_proppatch_endpoint_not_found(self, mocker): response = client.request("PROPPATCH", "/nonexistent.txt") assert response.status_code == 404 + + @pytest.mark.asyncio + async def test_put_endpoint_calls_overwrite_for_chunked_or_missing_length(self, mocker): + from fastapi.testclient import TestClient + + from asgidav.app import create_app + + from .common import MockResource + + mock_res = MockResource("/test.txt") + mock_res.overwrite = mocker.AsyncMock() + + mock_get_member = mocker.AsyncMock(return_value=mock_res) + app = create_app(get_member=mock_get_member) + client = TestClient(app) + + # 1. PUT with Transfer-Encoding: chunked + res1 = client.put("/test.txt", content=b"hello", headers={"Transfer-Encoding": "chunked"}) + assert res1.status_code == 201 + assert mock_res.overwrite.called + assert mock_res.overwrite.call_args[1]["size"] == -1 + + # 2. PUT with fixed Content-Length + mock_res.overwrite.reset_mock() + res2 = client.put("/test.txt", content=b"hello") + assert res2.status_code == 201 + assert mock_res.overwrite.called + assert mock_res.overwrite.call_args[1]["size"] == 5 From 5c54250078ad0c540c746ae0ef171d71f56fadc4 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:10:01 +0000 Subject: [PATCH 08/10] fix(test): use mocker.patch.object to fix mypy method-assign error in test_app.py Use mocker.patch.object for mock_res.overwrite in test_app.py to fix mypy type check error. Co-authored-by: VulcanoSoftware <113239901+VulcanoSoftware@users.noreply.github.com> --- tests/test_asgidav/test_app.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/test_asgidav/test_app.py b/tests/test_asgidav/test_app.py index ef2ecb6..3052b80 100644 --- a/tests/test_asgidav/test_app.py +++ b/tests/test_asgidav/test_app.py @@ -84,21 +84,25 @@ async def test_put_endpoint_calls_overwrite_for_chunked_or_missing_length(self, from .common import MockResource mock_res = MockResource("/test.txt") - mock_res.overwrite = mocker.AsyncMock() + mock_overwrite = mocker.patch.object( + mock_res, "overwrite", new_callable=mocker.AsyncMock + ) mock_get_member = mocker.AsyncMock(return_value=mock_res) app = create_app(get_member=mock_get_member) client = TestClient(app) # 1. PUT with Transfer-Encoding: chunked - res1 = client.put("/test.txt", content=b"hello", headers={"Transfer-Encoding": "chunked"}) + res1 = client.put( + "/test.txt", content=b"hello", headers={"Transfer-Encoding": "chunked"} + ) assert res1.status_code == 201 - assert mock_res.overwrite.called - assert mock_res.overwrite.call_args[1]["size"] == -1 + assert mock_overwrite.called + assert mock_overwrite.call_args[1]["size"] == -1 # 2. PUT with fixed Content-Length - mock_res.overwrite.reset_mock() + mock_overwrite.reset_mock() res2 = client.put("/test.txt", content=b"hello") assert res2.status_code == 201 - assert mock_res.overwrite.called - assert mock_res.overwrite.call_args[1]["size"] == 5 + assert mock_overwrite.called + assert mock_overwrite.call_args[1]["size"] == 5 From 7f1e97e2fa6a898726af30328f2a4f101fe64261 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:44:10 +0000 Subject: [PATCH 09/10] perf(sftp): optimize read throughput with lock-free fast path and in-place buffer pruning Add lock-free fast path for in-memory buffer reads in DCFSSFTPBufferedFile.read and use in-place bytearray slice deletion for buffer pruning to eliminate lock contention and reallocations during pipelined SFTP downloads. Co-authored-by: VulcanoSoftware <113239901+VulcanoSoftware@users.noreply.github.com> --- dcfs/app/sftp/handler.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/dcfs/app/sftp/handler.py b/dcfs/app/sftp/handler.py index 674e051..fa44fbe 100644 --- a/dcfs/app/sftp/handler.py +++ b/dcfs/app/sftp/handler.py @@ -401,7 +401,17 @@ async def read(self, offset: int, size: int) -> bytes: if "r" not in self.mode: raise asyncssh.SFTPPermissionDenied("File not open for reading") + # Lock-free fast path: if requested range is already in buffer, return immediately + rel_offset = offset - self._buf_offset + if rel_offset >= 0 and (rel_offset + size) <= len(self._read_buf): + return bytes(self._read_buf[rel_offset : rel_offset + size]) + async with self._read_lock: + # Re-check fast path after acquiring lock + rel_offset = offset - self._buf_offset + if rel_offset >= 0 and (rel_offset + size) <= len(self._read_buf): + return bytes(self._read_buf[rel_offset : rel_offset + size]) + buf_end = self._buf_offset + len(self._read_buf) can_reuse_stream = ( @@ -455,7 +465,7 @@ async def read(self, offset: int, size: int) -> bytes: if prune_target > self._buf_offset: discard = min(prune_target - self._buf_offset, len(self._read_buf)) if discard > 0: - self._read_buf = self._read_buf[discard:] + del self._read_buf[:discard] self._buf_offset += discard return data From 0991c101f61166ce1904a3aab0ccd616fe5fd3c0 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:41:35 +0000 Subject: [PATCH 10/10] fix(crypto): fall back to plaintext when header verification fails in _detect Catch InvalidHeaderError during _detect header parsing and verification in EncryptingFileContentRepository and fall back to treating the file as plaintext, preventing download failures for unencrypted or binary files starting with DCFS bytes. Co-authored-by: VulcanoSoftware <113239901+VulcanoSoftware@users.noreply.github.com>