From 682d9d8bec3296d927ab7e047f063410eb1ab0ee Mon Sep 17 00:00:00 2001 From: mateofuentespombo Date: Tue, 14 Apr 2026 11:32:59 +0200 Subject: [PATCH 01/33] fix: fix folder creation logic to ensure CurrentFolder is valid before proceeding --- .../MobileFileManager.razor.cs | 34 +++++++++---------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/TelegramDownloader/Shared/MobileFileManager/MobileFileManager.razor.cs b/TelegramDownloader/Shared/MobileFileManager/MobileFileManager.razor.cs index cf0347a..d6ad4f1 100644 --- a/TelegramDownloader/Shared/MobileFileManager/MobileFileManager.razor.cs +++ b/TelegramDownloader/Shared/MobileFileManager/MobileFileManager.razor.cs @@ -892,29 +892,27 @@ private async Task ConfirmNewFolder() { if (string.IsNullOrWhiteSpace(NewFolderName)) return; - // Use CurrentFolder which has the correct Id from LoadFiles response - FileManagerDirectoryContent parentFolder; - if (CurrentFolder != null) + // If CurrentFolder is missing or has no Id, refresh from the server first + if (CurrentFolder == null || string.IsNullOrEmpty(CurrentFolder.Id)) { - parentFolder = new FileManagerDirectoryContent - { - Id = CurrentFolder.Id, - Name = CurrentFolder.Name, - FilterPath = NormalizePath(CurrentFolder.FilterPath ?? ""), - FilterId = CurrentFolder.FilterId, - IsFile = false - }; + await LoadFiles(); } - else + + // After refresh, if we still don't have a valid CurrentFolder with Id, abort + if (CurrentFolder == null || string.IsNullOrEmpty(CurrentFolder.Id)) { - // Fallback for root - this shouldn't normally happen as LoadFiles sets CurrentFolder - parentFolder = new FileManagerDirectoryContent - { - FilterPath = NormalizePath(CurrentPath), - IsFile = false - }; + return; } + var parentFolder = new FileManagerDirectoryContent + { + Id = CurrentFolder.Id, + Name = CurrentFolder.Name, + FilterPath = string.IsNullOrEmpty(CurrentFolder.FilterPath) ? "" : NormalizePath(CurrentFolder.FilterPath), + FilterId = CurrentFolder.FilterId ?? "", + IsFile = false + }; + var args = new MfmFolderCreateEventArgs { FolderName = NewFolderName, From 8baf50c2de5ba5915c0c786ec15a18403f24c1f1 Mon Sep 17 00:00:00 2001 From: mateofuentespombo Date: Tue, 14 Apr 2026 12:09:14 +0200 Subject: [PATCH 02/33] fix: solve upload to empty folder and upload to root folder --- TelegramDownloader/Data/FileService.cs | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/TelegramDownloader/Data/FileService.cs b/TelegramDownloader/Data/FileService.cs index f33befd..c1cbecf 100644 --- a/TelegramDownloader/Data/FileService.cs +++ b/TelegramDownloader/Data/FileService.cs @@ -1073,12 +1073,18 @@ public async Task importSharedData(ShareFilesModel sfm, GenericNotificationProgr public async Task> createFolder(string dbName, FolderCreateEventArgs args) { - return (await _db.createEntry(dbName, await _db.toBasonFile(args.Path, args.FolderName, args.ParentFolder))).Select(x => x.toFileManagerContent()).ToList(); + var result = (await _db.createEntry(dbName, await _db.toBasonFile(args.Path, args.FolderName, args.ParentFolder))).Select(x => x.toFileManagerContent()).ToList(); + if (!string.IsNullOrEmpty(args.ParentFolder?.Id)) + await _db.setDirectoryHasChild(dbName, args.ParentFolder.Id); + return result; } public async Task> createFolder(string dbName, string path, string folderName, Syncfusion.Blazor.FileManager.FileManagerDirectoryContent? parentFolder) { - return (await _db.createEntry(dbName, await _db.toBasonFile(path, folderName, parentFolder))).Select(x => x.toFileManagerContent()).ToList(); + var result = (await _db.createEntry(dbName, await _db.toBasonFile(path, folderName, parentFolder))).Select(x => x.toFileManagerContent()).ToList(); + if (!string.IsNullOrEmpty(parentFolder?.Id)) + await _db.setDirectoryHasChild(dbName, parentFolder.Id); + return result; } public async Task CreateDatabase(string id) @@ -1608,12 +1614,18 @@ public async Task UploadFileFromServer(string dbName, string currentPath, List Date: Tue, 14 Apr 2026 13:06:55 +0200 Subject: [PATCH 03/33] fix: solve folder problem --- TelegramDownloader/Data/FileService.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TelegramDownloader/Data/FileService.cs b/TelegramDownloader/Data/FileService.cs index c1cbecf..e469e6d 100644 --- a/TelegramDownloader/Data/FileService.cs +++ b/TelegramDownloader/Data/FileService.cs @@ -1623,7 +1623,7 @@ public async Task UploadFileFromServer(string dbName, string currentPath, List Date: Tue, 14 Apr 2026 16:21:58 +0200 Subject: [PATCH 04/33] Sync develop with main after v3.6.3 (#93) * Develop to main (#90) * fix: fix folder creation logic to ensure CurrentFolder is valid before proceeding * fix: solve upload to empty folder and upload to root folder * fix: solve folder problem * Bump version to 3.6.3 --------- Co-authored-by: Mateo Co-authored-by: github-actions[bot] --- TFMAudioApp/TFMAudioApp.csproj | 2 +- TelegramDownloader/TelegramDownloader.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/TFMAudioApp/TFMAudioApp.csproj b/TFMAudioApp/TFMAudioApp.csproj index 35883e0..a93a84e 100644 --- a/TFMAudioApp/TFMAudioApp.csproj +++ b/TFMAudioApp/TFMAudioApp.csproj @@ -34,7 +34,7 @@ com.tfm.audioapp - 3.6.2 + 3.6.3 1 diff --git a/TelegramDownloader/TelegramDownloader.csproj b/TelegramDownloader/TelegramDownloader.csproj index 4bbcc70..b74540a 100644 --- a/TelegramDownloader/TelegramDownloader.csproj +++ b/TelegramDownloader/TelegramDownloader.csproj @@ -1,7 +1,7 @@ - 3.6.2.0 + 3.6.3.0 Mateo TelegramFileManager net10.0 From d1c60e10accc2baf6c07814c65cdcb11970596e7 Mon Sep 17 00:00:00 2001 From: mateofuentespombo Date: Sat, 23 May 2026 13:31:05 +0200 Subject: [PATCH 05/33] fix: update MongoDB.Driver and System.IO.Hashing package versions --- TelegramDownloader/TelegramDownloader.csproj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/TelegramDownloader/TelegramDownloader.csproj b/TelegramDownloader/TelegramDownloader.csproj index b74540a..972b7fc 100644 --- a/TelegramDownloader/TelegramDownloader.csproj +++ b/TelegramDownloader/TelegramDownloader.csproj @@ -23,7 +23,7 @@ - + @@ -33,8 +33,8 @@ - - + + From f37761cc3e982a5c3ff4fed60a8304b944556eee Mon Sep 17 00:00:00 2001 From: mateofuentespombo Date: Mon, 13 Jul 2026 00:39:44 +0200 Subject: [PATCH 06/33] feat: wire progressive download cache into tfm streaming endpoint - StreamAudioByTfmId now starts the background ProgressiveDownloadService download (previously injected but never invoked) so every streamed track is fetched from Telegram once and persisted to the disk cache - Serve ranges from the growing cache file, waiting briefly when the background download is close instead of opening duplicate fetches - Direct Telegram fetches (far seeks) now stream 512KB chunks to the response via new DownloadFileStreamChunks and are limited by a semaphore - Robust Range parsing: TryParse, suffix ranges (bytes=-N), 416 for unsatisfiable ranges, removed ambiguous to==0 sentinel - ProgressiveDownloadService: fix IsRangeAvailable null check, drop stale entries when cache files are evicted, cap retries with backoff, and trim the cache directory to 10GB (oldest first, throttled) Co-Authored-By: Claude Fable 5 --- .../Mobile/MobileStreamController.cs | 166 +++++++++++++----- TelegramDownloader/Data/ITelegramService.cs | 1 + TelegramDownloader/Data/TelegramService.cs | 57 ++++++ .../Services/ProgressiveDownloadService.cs | 107 +++++++++-- 4 files changed, 275 insertions(+), 56 deletions(-) diff --git a/TelegramDownloader/Controllers/Mobile/MobileStreamController.cs b/TelegramDownloader/Controllers/Mobile/MobileStreamController.cs index b0b841a..7ee6000 100644 --- a/TelegramDownloader/Controllers/Mobile/MobileStreamController.cs +++ b/TelegramDownloader/Controllers/Mobile/MobileStreamController.cs @@ -26,6 +26,13 @@ public class MobileStreamController : ControllerBase private readonly ILogger _logger; private static readonly SemaphoreSlim _downloadSemaphore = new(5); // Allow 5 concurrent downloads for preload + // Limit concurrent direct range fetches against Telegram (seeks ahead of the cache) + private static readonly SemaphoreSlim _telegramRangeSemaphore = new(4); + + // If the requested range starts within this distance of the background download + // position, wait for the download instead of opening a duplicate Telegram fetch + private const long WAIT_PROXIMITY_BYTES = 4 * 1024 * 1024; + public MobileStreamController( ITelegramService ts, IDbService db, @@ -238,28 +245,43 @@ public async Task StreamAudioByTfmId(string channelId, string tfm } } - // Parse range header + // Parse range header (RFC 7233: bytes=X-, bytes=X-Y, bytes=-N) var rangeHeader = Request.Headers["Range"].ToString(); + long totalLength = dbFile.Size; long from = 0; - long to = 0; + long? to = null; bool hasRange = false; - if (!string.IsNullOrEmpty(rangeHeader) && rangeHeader.StartsWith("bytes=")) + if (!string.IsNullOrEmpty(rangeHeader) && rangeHeader.StartsWith("bytes=") && !rangeHeader.Contains(',')) { - hasRange = true; - var range = rangeHeader.Replace("bytes=", "").Split('-'); - from = long.Parse(range[0]); - if (range.Length > 1 && !string.IsNullOrEmpty(range[1])) - to = long.Parse(range[1]); + var parts = rangeHeader.Substring("bytes=".Length).Split('-'); + if (parts.Length == 2) + { + if (string.IsNullOrEmpty(parts[0])) + { + // Suffix range: last N bytes + if (long.TryParse(parts[1], out var suffixLength) && suffixLength > 0) + { + from = Math.Max(0, totalLength - suffixLength); + to = totalLength - 1; + hasRange = true; + } + } + else if (long.TryParse(parts[0], out var parsedFrom) && parsedFrom >= 0) + { + from = parsedFrom; + hasRange = true; + if (!string.IsNullOrEmpty(parts[1]) && long.TryParse(parts[1], out var parsedTo)) + to = parsedTo; + } + } + // Malformed ranges fall through as "no range" (initial chunk) } - long totalLength = dbFile.Size; - - // Check how much is already cached - long cachedBytes = 0; - if (System.IO.File.Exists(filePath)) + if (hasRange && (from >= totalLength || (to.HasValue && to.Value < from))) { - cachedBytes = new FileInfo(filePath).Length; + Response.Headers["Content-Range"] = $"bytes */{totalLength}"; + return StatusCode(StatusCodes.Status416RangeNotSatisfiable); } // For initial request without Range, return first chunk as 206 with full size info @@ -270,23 +292,59 @@ public async Task StreamAudioByTfmId(string channelId, string tfm to = Math.Min(2 * 1024 * 1024, totalLength - 1); // First 2MB } - // Handle Range request - if (to == 0 || to >= totalLength) + // Open-ended or oversized ranges: cap the response to ~2.5MB + long rangeEnd = (!to.HasValue || to.Value >= totalLength) + ? Math.Min(from + (5 * 524288), totalLength - 1) + : to.Value; + + // Kick off (or attach to) the background download that fills the disk cache, + // so every streamed track ends up cached and is downloaded from Telegram once + ProgressiveDownloadInfo downloadInfo = null; + if (dbFile.MessageId.HasValue) { - // Open-ended range - to = Math.Min(from + (5 * 524288), totalLength - 1); // ~2.5MB chunk + try + { + downloadInfo = await _progressiveDownload.StartOrGetDownloadAsync(cacheFileName, channelId, dbFile, filePath); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Could not start background cache download for {FileName}", name); + } } - // Check if range is available locally + long CachedBytes() + { + long onDisk = System.IO.File.Exists(filePath) ? new FileInfo(filePath).Length : 0; + return Math.Max(onDisk, downloadInfo?.DownloadedBytes ?? 0); + } + + // If the background download is nearby, wait briefly for it to cover the + // range start instead of opening a duplicate Telegram download. This is the + // normal sequential-playback path: same latency as a direct fetch (both pull + // sequential 512KB chunks), but the bytes get persisted. + if (downloadInfo != null && downloadInfo.IsDownloading && + from - CachedBytes() <= WAIT_PROXIMITY_BYTES) + { + var waitTarget = Math.Min(rangeEnd, from + 524288); // at least 512KB past the start + var deadline = DateTime.UtcNow.AddSeconds(12); + while (downloadInfo.IsDownloading && + downloadInfo.DownloadedBytes <= waitTarget && + DateTime.UtcNow < deadline) + { + await Task.Delay(150, HttpContext.RequestAborted); + } + } + + var cachedBytes = CachedBytes(); + + // Serve from the (possibly still growing) cache file if (from < cachedBytes) { - // Part or all of range is in cache - var availableEnd = Math.Min(to, cachedBytes - 1); + var availableEnd = Math.Min(rangeEnd, cachedBytes - 1); var length = availableEnd - from + 1; _logger.LogDebug("Serving from cache: bytes {From}-{To} of {Total}", from, availableEnd, totalLength); - // Read from cache file using var cacheStream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); cacheStream.Seek(from, SeekOrigin.Begin); var buffer = new byte[length]; @@ -303,10 +361,10 @@ public async Task StreamAudioByTfmId(string channelId, string tfm return new EmptyResult(); } - // Range not in cache - stream from Telegram - _logger.LogDebug("Streaming from Telegram: bytes {From}-{To} of {Total}", from, to, totalLength); + // Range far ahead of the background download (seek): fetch it directly from + // Telegram, streaming each 512KB chunk to the response as it arrives + _logger.LogDebug("Streaming from Telegram: bytes {From}-{To} of {Total}", from, rangeEnd, totalLength); - // Get message from Telegram if (!dbFile.MessageId.HasValue) { return NotFound(ApiResponse.Fail("File has no MessageId")); @@ -318,34 +376,52 @@ public async Task StreamAudioByTfmId(string channelId, string tfm return NotFound(ApiResponse.Fail("Message not found in Telegram")); } - // Align to 512KB boundaries for Telegram + // Align down to 512KB for Telegram; skip the prefix when writing the response var alignedFrom = (from / 524288) * 524288; - var alignedTo = ((to + 524288) / 524288) * 524288; - if (alignedTo > totalLength) alignedTo = totalLength; + var skipBytes = from - alignedFrom; + var responseLength = rangeEnd - from + 1; + + await _telegramRangeSemaphore.WaitAsync(HttpContext.RequestAborted); + try + { + Response.StatusCode = StatusCodes.Status206PartialContent; + Response.ContentType = mimeType; + Response.ContentLength = responseLength; + Response.Headers["Content-Range"] = $"bytes {from}-{rangeEnd}/{totalLength}"; + Response.Headers["Accept-Ranges"] = "bytes"; + Response.Headers["Content-Disposition"] = $"inline; filename=\"{HttpUtility.UrlEncode(name)}\""; - var downloadLength = alignedTo - alignedFrom; + long remainingSkip = skipBytes; + long remainingWrite = responseLength; - // Download the range from Telegram - byte[] data = await _ts.DownloadFileStream(message, alignedFrom, (int)downloadLength); + await foreach (var chunk in _ts.DownloadFileStreamChunks( + message, alignedFrom, skipBytes + responseLength, HttpContext.RequestAborted)) + { + int start = 0; + int length = chunk.Length; - // Calculate skip bytes to get to the exact requested position - var skipBytes = from - alignedFrom; - var responseLength = Math.Min(to - from + 1, data.Length - skipBytes); + if (remainingSkip > 0) + { + var toSkip = (int)Math.Min(remainingSkip, length); + start += toSkip; + length -= toSkip; + remainingSkip -= toSkip; + } + + if (length <= 0) continue; - if (skipBytes < 0 || skipBytes >= data.Length) + var toWrite = (int)Math.Min(length, remainingWrite); + await Response.Body.WriteAsync(chunk, start, toWrite, HttpContext.RequestAborted); + remainingWrite -= toWrite; + + if (remainingWrite <= 0) break; + } + } + finally { - _logger.LogError("Invalid skip calculation: skipBytes={Skip}, dataLength={DataLen}", skipBytes, data.Length); - return StatusCode(500, "Error calculating response range"); + _telegramRangeSemaphore.Release(); } - Response.StatusCode = StatusCodes.Status206PartialContent; - Response.ContentType = mimeType; - Response.ContentLength = responseLength; - Response.Headers["Content-Range"] = $"bytes {from}-{from + responseLength - 1}/{totalLength}"; - Response.Headers["Accept-Ranges"] = "bytes"; - Response.Headers["Content-Disposition"] = $"inline; filename=\"{HttpUtility.UrlEncode(name)}\""; - - await Response.Body.WriteAsync(data, (int)skipBytes, (int)responseLength); return new EmptyResult(); } catch (OperationCanceledException) diff --git a/TelegramDownloader/Data/ITelegramService.cs b/TelegramDownloader/Data/ITelegramService.cs index 48a9d6a..5963b83 100644 --- a/TelegramDownloader/Data/ITelegramService.cs +++ b/TelegramDownloader/Data/ITelegramService.cs @@ -21,6 +21,7 @@ public interface ITelegramService Task CallQrGenerator(Action func, CancellationToken ct, bool logoutFirst = false); Task DownloadFile(ChatMessages message, string fileName = null, string folder = null, DownloadModel model = null, bool shouldAddToList = false); Task DownloadFileStream(Message message, long offset, int limit); + IAsyncEnumerable DownloadFileStreamChunks(Message message, long offset, long limit, CancellationToken ct = default); Task DownloadFileAndReturn(ChatMessages message, Stream ms = null, string fileName = null, string folder = null, DownloadModel model = null); Task DownloadFileAndReturnWithOffset(ChatMessages message, Stream ms = null, string fileName = null, string folder = null, DownloadModel model = null, long offset = 0); Task> GetFouriteChannels(bool mustRefresh = true); diff --git a/TelegramDownloader/Data/TelegramService.cs b/TelegramDownloader/Data/TelegramService.cs index 186bc62..3f20d01 100644 --- a/TelegramDownloader/Data/TelegramService.cs +++ b/TelegramDownloader/Data/TelegramService.cs @@ -8,6 +8,7 @@ using Syncfusion.Blazor.Sparkline.Internal; using System.Collections.Generic; using System.Reflection.Metadata; +using System.Runtime.CompilerServices; using System.Threading.Channels; using System.Threading.Tasks; using TelegramDownloader.Data.db; @@ -1126,6 +1127,62 @@ public async Task DownloadFileStream(Message message, long offset, int l throw new ArgumentException("Invalid message or media type."); } + /// + /// Streams a byte range from Telegram in 512KB chunks, yielding each chunk as it + /// arrives instead of buffering the whole range in memory. Offset must be 4KB-aligned + /// (callers align to 512KB). Stops early if Telegram signals end of file. + /// + public async IAsyncEnumerable DownloadFileStreamChunks(Message message, long offset, long limit, [EnumeratorCancellation] CancellationToken ct = default) + { + _logger.LogDebug("DownloadFileStreamChunks - Offset: {Offset}, Limit: {Limit}", offset, limit); + + if (message is not Message msg || msg.media is not MessageMediaDocument doc) + throw new ArgumentException("Invalid message or media type."); + + InputDocument inputFile = doc.document; + long currentOffset = offset; + long remaining = limit; + + while (remaining > 0) + { + ct.ThrowIfCancellationRequested(); + + var location = new InputDocumentFileLocation + { + id = inputFile.id, + access_hash = inputFile.access_hash, + file_reference = inputFile.file_reference, + thumb_size = "" + }; + + Upload_FileBase file; + try + { + file = await client.Upload_GetFile(location, currentOffset, limit: FILESPLITSIZE); + } + catch (RpcException ex) when (ex.Code == 303 && ex.Message == "FILE_MIGRATE_X") + { + var dcClient = await client.GetClientForDC(-ex.X, true); + file = await dcClient.Upload_GetFile(location, currentOffset, limit: FILESPLITSIZE); + } + + if (file is not Upload_File uploadFile) + throw new InvalidOperationException("Unexpected file type returned."); + + if (uploadFile.bytes.Length == 0) + yield break; + + yield return uploadFile.bytes; + + currentOffset += uploadFile.bytes.Length; + remaining -= uploadFile.bytes.Length; + + // Telegram returns fewer bytes than requested only at end of file + if (uploadFile.bytes.Length < FILESPLITSIZE) + yield break; + } + } + public async Task DownloadFileAndReturn(ChatMessages message, Stream ms = null, string fileName = null, string folder = null, DownloadModel model = null) { if (model == null) diff --git a/TelegramDownloader/Services/ProgressiveDownloadService.cs b/TelegramDownloader/Services/ProgressiveDownloadService.cs index 0cb8974..cc532dd 100644 --- a/TelegramDownloader/Services/ProgressiveDownloadService.cs +++ b/TelegramDownloader/Services/ProgressiveDownloadService.cs @@ -60,6 +60,11 @@ public class ProgressiveDownloadService : IProgressiveDownloadService // Lock for starting downloads private readonly ConcurrentDictionary _downloadLocks = new(); + // Cache directory size cap: oldest files are evicted first (by write time) + private const long MAX_CACHE_BYTES = 10L * 1024 * 1024 * 1024; // 10 GB + private static DateTime _lastCleanup = DateTime.MinValue; + private static readonly object _cleanupLock = new(); + public ProgressiveDownloadService( ITelegramService ts, TransactionInfoService tis, @@ -89,12 +94,6 @@ public bool IsRangeAvailable(string cacheKey, long start, long end) { if (!_activeDownloads.TryGetValue(cacheKey, out var info)) { - // Check if file exists and is complete - if (File.Exists(info?.FilePath ?? "")) - { - var fileInfo = new FileInfo(info!.FilePath); - return end <= fileInfo.Length; - } return false; } @@ -111,7 +110,12 @@ public async Task StartOrGetDownloadAsync( // Quick check if already complete if (_activeDownloads.TryGetValue(cacheKey, out var existingInfo) && existingInfo.IsComplete) { - return existingInfo; + if (File.Exists(existingInfo.FilePath)) + { + return existingInfo; + } + // Cache file was evicted: forget the stale entry and re-download + _activeDownloads.TryRemove(cacheKey, out _); } // Check if file already exists and is complete @@ -143,7 +147,8 @@ public async Task StartOrGetDownloadAsync( // Double-check after acquiring lock if (_activeDownloads.TryGetValue(cacheKey, out existingInfo)) { - if (existingInfo.IsComplete || existingInfo.IsDownloading) + if (existingInfo.IsDownloading || + (existingInfo.IsComplete && File.Exists(existingInfo.FilePath))) { return existingInfo; } @@ -234,7 +239,9 @@ private async Task DownloadInBackgroundAsync( // Download in chunks const int chunkSize = 512 * 1024; // 512KB chunks + const int maxConsecutiveErrors = 5; var chatMessage = new ChatMessages { message = message }; + var consecutiveErrors = 0; while (info.DownloadedBytes < info.TotalSize && !info.CancellationTokenSource!.Token.IsCancellationRequested) { @@ -255,6 +262,7 @@ private async Task DownloadInBackgroundAsync( info.DownloadedBytes += chunk.Length; dm._transmitted = info.DownloadedBytes; + consecutiveErrors = 0; // Log progress every 10% var progress = (double)info.DownloadedBytes / info.TotalSize * 100; @@ -266,9 +274,19 @@ private async Task DownloadInBackgroundAsync( } catch (Exception ex) when (ex is not OperationCanceledException) { - _logger.LogError(ex, "Error downloading chunk at offset {Offset}", info.DownloadedBytes); - // Wait a bit and retry - await Task.Delay(1000); + consecutiveErrors++; + _logger.LogError(ex, "Error downloading chunk at offset {Offset} (attempt {Attempt}/{Max})", + info.DownloadedBytes, consecutiveErrors, maxConsecutiveErrors); + + if (consecutiveErrors >= maxConsecutiveErrors) + { + _logger.LogError("Aborting background download after {Max} consecutive failures: {CacheKey}", + maxConsecutiveErrors, info.CacheKey); + break; + } + + // Back off progressively before retrying + await Task.Delay(1000 * consecutiveErrors); } } @@ -280,6 +298,11 @@ private async Task DownloadInBackgroundAsync( Path.GetFileName(info.FilePath), info.DownloadedBytes, info.TotalSize); + + if (info.IsComplete && !string.IsNullOrEmpty(directory)) + { + _ = Task.Run(() => TrimCacheDirectory(directory)); + } } catch (OperationCanceledException) { @@ -292,5 +315,67 @@ private async Task DownloadInBackgroundAsync( info.IsDownloading = false; } } + + /// + /// Keeps the streaming cache directory under MAX_CACHE_BYTES, deleting the + /// oldest files first (by write time). Files being downloaded are skipped. + /// Throttled to run at most every 30 minutes. + /// + private void TrimCacheDirectory(string directory) + { + lock (_cleanupLock) + { + if (DateTime.UtcNow - _lastCleanup < TimeSpan.FromMinutes(30)) return; + _lastCleanup = DateTime.UtcNow; + } + + try + { + var files = new DirectoryInfo(directory).GetFiles(); + var totalSize = files.Sum(f => f.Length); + if (totalSize <= MAX_CACHE_BYTES) return; + + var activePaths = _activeDownloads.Values + .Where(i => i.IsDownloading) + .Select(i => i.FilePath) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + var removed = 0; + foreach (var file in files.OrderBy(f => f.LastWriteTimeUtc)) + { + if (totalSize <= MAX_CACHE_BYTES) break; + if (activePaths.Contains(file.FullName)) continue; + + try + { + var size = file.Length; + file.Delete(); + totalSize -= size; + removed++; + + // Drop any stale tracking entry pointing at the deleted file + var stale = _activeDownloads.FirstOrDefault(kv => + string.Equals(kv.Value.FilePath, file.FullName, StringComparison.OrdinalIgnoreCase)); + if (stale.Key != null) + { + _activeDownloads.TryRemove(stale.Key, out _); + } + } + catch (IOException) + { + // File in use (e.g. being served): skip it + } + } + + if (removed > 0) + { + _logger.LogInformation("Cache trim: removed {Count} files from {Directory}", removed, directory); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Cache trim failed for {Directory}", directory); + } + } } } From 128776ac9b4d90fb3dd51b5cb21677b9349b248c Mon Sep 17 00:00:00 2001 From: mateofuentespombo Date: Mon, 13 Jul 2026 09:00:04 +0200 Subject: [PATCH 07/33] feat: configurable STRM streaming mode with progressive disk cache Add a 'Streaming Mode' setting (Direct streaming / Progressive cache / Full preload) selectable in the web Config page, used when generating STRM files for Emby/Kodi. - New GetFileStreamCached endpoint: streams from Telegram with Range support while a background download fills the local cache; later ranges and replays are served from disk. - ProgressiveDownloadService now supports split (multi-message) files, caching parts sequentially into a single file with resume support. - Direct seeks ahead of the cache locate the Telegram part containing the requested range and cap the response at the part boundary. - STRM generation picks the endpoint from the configured mode; files smaller than MaxPreloadFileSizeInMb are still fully preloaded. - Backwards compatible with the legacy PreloadFilesOnStream flag (kept in sync on save; used as fallback when the new setting is unset). --- .../Controllers/FileController.cs | 268 +++++++++++++++++- TelegramDownloader/Data/FileService.cs | 27 +- TelegramDownloader/Models/GeneralConfig.cs | 38 +++ TelegramDownloader/Pages/Config.razor | 33 ++- .../Services/ProgressiveDownloadService.cs | 140 +++++---- 5 files changed, 443 insertions(+), 63 deletions(-) diff --git a/TelegramDownloader/Controllers/FileController.cs b/TelegramDownloader/Controllers/FileController.cs index e507a2f..c24c799 100644 --- a/TelegramDownloader/Controllers/FileController.cs +++ b/TelegramDownloader/Controllers/FileController.cs @@ -31,13 +31,22 @@ public class FileController : ControllerBase string root = FileService.RELATIVELOCALDIR; private static SemaphoreSlim semaphoreSlim = new SemaphoreSlim(1); + + // Limit concurrent direct range fetches against Telegram (seeks ahead of the cache) + private static readonly SemaphoreSlim telegramRangeSemaphore = new SemaphoreSlim(4); + + // If the requested range starts within this distance of the background download + // position, wait for the download instead of opening a duplicate Telegram fetch + private const long WAIT_PROXIMITY_BYTES = 8 * 1024 * 1024; + IDbService _db { get; set; } ITelegramService _ts { get; set; } IFileService _fs { get; set; } TransactionInfoService _tis { get; set; } + private readonly IProgressiveDownloadService _progressiveDownload; private ILogger _logger { get; set; } - public FileController(IDbService db, ITelegramService ts, IFileService fs, TransactionInfoService tis, ILogger logger) + public FileController(IDbService db, ITelegramService ts, IFileService fs, TransactionInfoService tis, IProgressiveDownloadService progressiveDownload, ILogger logger) { this.basePath = Environment.CurrentDirectory; if (!System.IO.Directory.Exists(Path.Combine(basePath, root))) @@ -48,6 +57,7 @@ public FileController(IDbService db, ITelegramService ts, IFileService fs, Trans _ts = ts; _db = db; _tis = tis; + _progressiveDownload = progressiveDownload; _logger = logger; this.operation = new PhysicalFileProvider(); @@ -662,6 +672,262 @@ public async Task GetFileStream(string idChannel, string idFile, } + /// + /// Stream file from Telegram while caching it to disk in the background (progressive streaming). + /// Playback starts immediately; once fully cached, ranges are served from disk. + /// Supports split (multi-message) files: parts are cached sequentially into a single file + /// and seeks ahead of the cache are fetched from the part that contains the requested range. + /// + /// Telegram channel ID + /// TFM database file ID + /// File name (used for mime type / Content-Disposition) + [HttpGet] + [Route("GetFileStreamCached/{idChannel}/{idFile}/{name}")] + [ProducesResponseType(typeof(FileStreamResult), 200)] + [ProducesResponseType(206)] + [ProducesResponseType(404)] + [ProducesResponseType(416)] + public async Task GetFileStreamCached(string idChannel, string idFile, string name) + { + var mimeType = FileService.getMimeType(name.Split(".").Last()); + + var dbFile = await _fs.getItemById(idChannel, idFile); + if (dbFile == null) + { + return NotFound(); + } + + // Ordered Telegram messages that make up the file (several for split files) + var messageIds = ProgressiveDownloadService.GetMessageIds(dbFile); + if (messageIds == null || messageIds.Count == 0) + { + return NotFound(); + } + + var fileName = dbFile.Name; + var cacheFileName = $"{idChannel}-{(dbFile.MessageId != null ? dbFile.MessageId.ToString() : dbFile.Id)}-{fileName}"; + var tempPath = Path.Combine(FileService.TEMPDIR, "_temp"); + var filePath = Path.Combine(tempPath, cacheFileName); + Directory.CreateDirectory(tempPath); + + long totalLength = dbFile.Size; + + // Fully cached: serve straight from disk with full range support + if (System.IO.File.Exists(filePath) && new FileInfo(filePath).Length >= totalLength) + { + _logger.LogDebug("GetFileStreamCached - serving fully cached file {FileName}", fileName); + Response.Headers["Content-Disposition"] = $"inline; filename=\"{HttpUtility.UrlEncode(fileName)}\""; + return PhysicalFile(filePath, mimeType, enableRangeProcessing: true); + } + + // Parse range header (RFC 7233: bytes=X-, bytes=X-Y, bytes=-N) + var rangeHeader = Request.Headers["Range"].ToString(); + long from = 0; + long? to = null; + bool hasRange = false; + + if (!string.IsNullOrEmpty(rangeHeader) && rangeHeader.StartsWith("bytes=") && !rangeHeader.Contains(',')) + { + var parts = rangeHeader.Substring("bytes=".Length).Split('-'); + if (parts.Length == 2) + { + if (string.IsNullOrEmpty(parts[0])) + { + // Suffix range: last N bytes + if (long.TryParse(parts[1], out var suffixLength) && suffixLength > 0) + { + from = Math.Max(0, totalLength - suffixLength); + to = totalLength - 1; + hasRange = true; + } + } + else if (long.TryParse(parts[0], out var parsedFrom) && parsedFrom >= 0) + { + from = parsedFrom; + hasRange = true; + if (!string.IsNullOrEmpty(parts[1]) && long.TryParse(parts[1], out var parsedTo)) + to = parsedTo; + } + } + // Malformed ranges fall through as "no range" (initial chunk) + } + + if (hasRange && (from >= totalLength || (to.HasValue && to.Value < from))) + { + Response.Headers["Content-Range"] = $"bytes */{totalLength}"; + return StatusCode(StatusCodes.Status416RangeNotSatisfiable); + } + + // Initial request without Range: return the first chunk as 206 with total size info + if (!hasRange) + { + from = 0; + to = Math.Min(6 * 1024 * 1024, totalLength - 1); + } + + // Open-ended or oversized ranges: cap the response so the player keeps asking + long rangeEnd = (!to.HasValue || to.Value >= totalLength) + ? Math.Min(from + (4 * 1024 * 1024), totalLength - 1) + : to.Value; + + // Kick off (or attach to) the background download that fills the disk cache, + // so the file is downloaded from Telegram only once + ProgressiveDownloadInfo downloadInfo = null; + try + { + downloadInfo = await _progressiveDownload.StartOrGetDownloadAsync(cacheFileName, idChannel, dbFile, filePath); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Could not start background cache download for {FileName}", fileName); + } + + long CachedBytes() + { + long onDisk = System.IO.File.Exists(filePath) ? new FileInfo(filePath).Length : 0; + return Math.Max(onDisk, downloadInfo?.DownloadedBytes ?? 0); + } + + try + { + // If the background download is nearby, wait briefly for it to cover the range + // start instead of opening a duplicate Telegram fetch (normal sequential playback) + if (downloadInfo != null && downloadInfo.IsDownloading && + from - CachedBytes() <= WAIT_PROXIMITY_BYTES) + { + var waitTarget = Math.Min(rangeEnd, from + 524288); // at least 512KB past the start + var deadline = DateTime.UtcNow.AddSeconds(15); + while (downloadInfo.IsDownloading && + downloadInfo.DownloadedBytes <= waitTarget && + DateTime.UtcNow < deadline) + { + await Task.Delay(150, HttpContext.RequestAborted); + } + } + + var cachedBytes = CachedBytes(); + + // Serve from the (possibly still growing) cache file + if (from < cachedBytes) + { + var availableEnd = Math.Min(rangeEnd, cachedBytes - 1); + var length = availableEnd - from + 1; + + _logger.LogDebug("GetFileStreamCached - serving from cache: bytes {From}-{To} of {Total}", from, availableEnd, totalLength); + + using var cacheStream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + cacheStream.Seek(from, SeekOrigin.Begin); + var buffer = new byte[length]; + var bytesRead = await cacheStream.ReadAsync(buffer, 0, (int)length, HttpContext.RequestAborted); + + Response.StatusCode = StatusCodes.Status206PartialContent; + Response.ContentType = mimeType; + Response.ContentLength = bytesRead; + Response.Headers["Content-Range"] = $"bytes {from}-{from + bytesRead - 1}/{totalLength}"; + Response.Headers["Accept-Ranges"] = "bytes"; + Response.Headers["Content-Disposition"] = $"inline; filename=\"{HttpUtility.UrlEncode(fileName)}\""; + + await Response.Body.WriteAsync(buffer, 0, bytesRead, HttpContext.RequestAborted); + return new EmptyResult(); + } + + // Range far ahead of the background download (seek): fetch it directly from + // Telegram, streaming each 512KB chunk to the response as it arrives + _logger.LogDebug("GetFileStreamCached - streaming from Telegram: bytes {From}-{To} of {Total}", from, rangeEnd, totalLength); + + // Locate the part (Telegram message) containing `from`. Like GetFileStream, + // assume uniform part sizes (the splitter uses a fixed split size) + TL.Message message; + long partStart = 0; + long partSize; + if (messageIds.Count == 1) + { + message = await _ts.getMessageFile(idChannel, messageIds[0]); + partSize = ProgressiveDownloadService.GetDocumentSize(message); + } + else + { + var firstMessage = await _ts.getMessageFile(idChannel, messageIds[0]); + long firstPartSize = ProgressiveDownloadService.GetDocumentSize(firstMessage); + if (firstPartSize <= 0) + { + return NotFound(); + } + int partIndex = (int)Math.Min(from / firstPartSize, messageIds.Count - 1); + partStart = (long)partIndex * firstPartSize; + message = partIndex == 0 ? firstMessage : await _ts.getMessageFile(idChannel, messageIds[partIndex]); + partSize = ProgressiveDownloadService.GetDocumentSize(message); + } + + if (message == null || partSize <= 0) + { + return NotFound(); + } + + // Serve only up to the end of this part; the player asks for the rest itself + rangeEnd = Math.Min(rangeEnd, partStart + partSize - 1); + if (rangeEnd < from) + { + Response.Headers["Content-Range"] = $"bytes */{totalLength}"; + return StatusCode(StatusCodes.Status416RangeNotSatisfiable); + } + + // Align down to 512KB for Telegram; skip the prefix when writing the response + var localFrom = from - partStart; + var alignedFrom = (localFrom / 524288) * 524288; + var skipBytes = localFrom - alignedFrom; + var responseLength = rangeEnd - from + 1; + + await telegramRangeSemaphore.WaitAsync(HttpContext.RequestAborted); + try + { + Response.StatusCode = StatusCodes.Status206PartialContent; + Response.ContentType = mimeType; + Response.ContentLength = responseLength; + Response.Headers["Content-Range"] = $"bytes {from}-{rangeEnd}/{totalLength}"; + Response.Headers["Accept-Ranges"] = "bytes"; + Response.Headers["Content-Disposition"] = $"inline; filename=\"{HttpUtility.UrlEncode(fileName)}\""; + + long remainingSkip = skipBytes; + long remainingWrite = responseLength; + + await foreach (var chunk in _ts.DownloadFileStreamChunks( + message, alignedFrom, skipBytes + responseLength, HttpContext.RequestAborted)) + { + int start = 0; + int length = chunk.Length; + + if (remainingSkip > 0) + { + var toSkip = (int)Math.Min(remainingSkip, length); + start += toSkip; + length -= toSkip; + remainingSkip -= toSkip; + } + + if (length <= 0) continue; + + var toWrite = (int)Math.Min(length, remainingWrite); + await Response.Body.WriteAsync(chunk, start, toWrite, HttpContext.RequestAborted); + remainingWrite -= toWrite; + + if (remainingWrite <= 0) break; + } + } + finally + { + telegramRangeSemaphore.Release(); + } + + return new EmptyResult(); + } + catch (OperationCanceledException) + { + _logger.LogDebug("Client closed connection during cached stream - File: {FileName}", fileName); + return new EmptyResult(); + } + } + /// /// Export channel database to JSON file /// diff --git a/TelegramDownloader/Data/FileService.cs b/TelegramDownloader/Data/FileService.cs index e469e6d..28695d7 100644 --- a/TelegramDownloader/Data/FileService.cs +++ b/TelegramDownloader/Data/FileService.cs @@ -1252,6 +1252,21 @@ public async Task AddUploadFileFromServer(string dbName, string currentPath, Lis _tis.CheckPendingUploadInfoTasks(); } + /// + /// Builds the URL written inside a .strm file according to the configured streaming mode. + /// Files smaller than MaxPreloadFileSizeInMb are always fully preloaded. + /// + private static string BuildStrmUrl(string host, string dbName, BsonFileManagerModel file) + { + StreamingMode mode = GeneralConfigStatic.config.GetEffectiveStreamingMode(); + if (mode == StreamingMode.Preload || HelperService.bytesToMegaBytes(file.Size) < GeneralConfigStatic.config.MaxPreloadFileSizeInMb) + { + return Path.Combine(host, "api/file/GetFileByTfmId", Uri.EscapeDataString(file.Name)).Replace("\\", "/") + $"?idChannel={dbName}&idFile={file.Id}"; + } + string endpoint = mode == StreamingMode.ProgressiveCache ? "GetFileStreamCached" : "GetFileStream"; + return Path.Combine(host, "api/file", endpoint, dbName, file.Id, "file" + file.Type).Replace("\\", "/"); + } + public async Task CreateStrmFiles(string path, string dbName, string host) { String folderPathName = Path.GetFileName(path.TrimEnd('/')); @@ -1300,11 +1315,7 @@ public async Task CreateStrmFiles(string path, string dbName, string hos { if (FileExtensionTypeTest.isVideoExtension(file.Type) || FileExtensionTypeTest.isAudioExtension(file.Type)) { - string contenido = Path.Combine(host, "api/file/GetFileStream", dbName, file.Id, "file" + file.Type).Replace("\\", "/"); - if (GeneralConfigStatic.config.PreloadFilesOnStream || HelperService.bytesToMegaBytes(file.Size) < GeneralConfigStatic.config.MaxPreloadFileSizeInMb) - { - contenido = Path.Combine(host, "api/file/GetFileByTfmId", Uri.EscapeDataString(file.Name)).Replace("\\", "/") + $"?idChannel={dbName}&idFile={file.Id}"; - } + string contenido = BuildStrmUrl(host, dbName, file); string pattern = $@"\.({file.Type.Replace(".", "")})$"; File.WriteAllText(Regex.Replace(filePath, pattern, ".strm"), contenido); } @@ -1338,11 +1349,7 @@ public async Task CreateStrmFilesToLocal(string path, string dbName, string host { if (FileExtensionTypeTest.isVideoExtension(file.Type) || FileExtensionTypeTest.isAudioExtension(file.Type)) { - string contenido = Path.Combine(host, "api/file/GetFileStream", dbName, file.Id, "file" + file.Type).Replace("\\", "/"); - if (GeneralConfigStatic.config.PreloadFilesOnStream || HelperService.bytesToMegaBytes(file.Size) < GeneralConfigStatic.config.MaxPreloadFileSizeInMb) - { - contenido = Path.Combine(host, "api/file/GetFileByTfmId", Uri.EscapeDataString(file.Name)).Replace("\\", "/") + $"?idChannel={dbName}&idFile={file.Id}"; - } + string contenido = BuildStrmUrl(host, dbName, file); string pattern = $@"\.({file.Type.Replace(".", "")})$"; // Ensure parent directory exists var parentDir = Path.GetDirectoryName(Regex.Replace(filePath, pattern, ".strm")); diff --git a/TelegramDownloader/Models/GeneralConfig.cs b/TelegramDownloader/Models/GeneralConfig.cs index 4d1a715..d51d90d 100644 --- a/TelegramDownloader/Models/GeneralConfig.cs +++ b/TelegramDownloader/Models/GeneralConfig.cs @@ -12,6 +12,27 @@ namespace TelegramDownloader.Models { + /// + /// How exported STRM files (Emby/Kodi/etc.) play media from Telegram. + /// + public enum StreamingMode + { + /// + /// Stream chunks directly from Telegram on demand. Playback starts immediately, + /// nothing is stored on disk, but every play re-downloads from Telegram. + /// + DirectStream = 0, + /// + /// Stream immediately while a background download fills the local cache. + /// Playback starts from second one and subsequent plays are served from disk. + /// + ProgressiveCache = 1, + /// + /// Download the whole file to the local cache before playback starts (legacy behavior). + /// + Preload = 2 + } + public class GeneralConfigStatic { public static GeneralConfig config { get; set; } = new GeneralConfig(); @@ -31,6 +52,10 @@ public static async Task SaveChanges(IDbService db, GeneralConfig gc) if (gc.MemorySplitSizeGB < 1) gc.MemorySplitSizeGB = 1; if (gc.MemorySplitSizeGB > maxAllowedSize) gc.MemorySplitSizeGB = maxAllowedSize; + // Keep the legacy flag in sync so older builds reading this config behave the same + if (gc.StrmStreamingMode.HasValue) + gc.PreloadFilesOnStream = gc.StrmStreamingMode.Value == StreamingMode.Preload; + await db.SaveConfig(gc); config = gc; if (gc.SplitSize > 0) @@ -97,6 +122,19 @@ public class GeneralConfig public bool ShouldShowCaptionPath { get; set; } = false; public bool ShouldShowLogInTerminal { get; set; } = false; public bool PreloadFilesOnStream { get; set; } = false; + /// + /// Streaming mode used by exported STRM files. Null means "not set yet": + /// the effective mode is then derived from the legacy PreloadFilesOnStream flag. + /// + [BsonRepresentation(BsonType.String)] + public StreamingMode? StrmStreamingMode { get; set; } = null; + + public StreamingMode GetEffectiveStreamingMode() + { + if (StrmStreamingMode.HasValue) + return StrmStreamingMode.Value; + return PreloadFilesOnStream ? StreamingMode.Preload : StreamingMode.DirectStream; + } public bool ShouldShowPaginatedFileChannel { get; set; } = false; public bool hasFileManagerVirtualScroll { get; set; } = false; public bool UseMobileFileManagerAlways { get; set; } = false; diff --git a/TelegramDownloader/Pages/Config.razor b/TelegramDownloader/Pages/Config.razor index 2d0fcad..c7f88f0 100644 --- a/TelegramDownloader/Pages/Config.razor +++ b/TelegramDownloader/Pages/Config.razor @@ -252,14 +252,24 @@
- Preload Files on Stream + Streaming Mode (STRM / Emby / Kodi)
- Automatically preload files to cache when streaming + How exported .strm files play media from Telegram: +
    +
  • Direct streaming: plays instantly, chunks are fetched from Telegram on demand, nothing is stored on disk. Each play re-downloads from Telegram.
    • +
  • Progressive cache: plays instantly while the file is downloaded to the local cache in the background. Next plays are served from disk. Recommended
    • +
  • Full preload: downloads the whole file before playback starts (legacy behavior).
    • +
+ Applies to newly exported STRM files and to playback through the streaming endpoints. Files smaller than "Max. Preload File Size" are always fully preloaded.
- + + + + +
@@ -913,6 +923,23 @@ @code { public GeneralConfig Model { get; set; } + + /// + /// Proxy for the STRM streaming mode: reads the effective mode (falling back to the + /// legacy PreloadFilesOnStream flag) and keeps that flag in sync when changed. + /// + private StreamingMode StreamingModeSetting + { + get => Model?.GetEffectiveStreamingMode() ?? StreamingMode.DirectStream; + set + { + if (Model != null) + { + Model.StrmStreamingMode = value; + Model.PreloadFilesOnStream = value == StreamingMode.Preload; + } + } + } private bool _disposed = false; private bool isRestarting = false; diff --git a/TelegramDownloader/Services/ProgressiveDownloadService.cs b/TelegramDownloader/Services/ProgressiveDownloadService.cs index cc532dd..6126f3e 100644 --- a/TelegramDownloader/Services/ProgressiveDownloadService.cs +++ b/TelegramDownloader/Services/ProgressiveDownloadService.cs @@ -2,6 +2,7 @@ using TelegramDownloader.Data; using TelegramDownloader.Data.db; using TelegramDownloader.Models; +using TL; namespace TelegramDownloader.Services { @@ -197,22 +198,16 @@ private async Task DownloadInBackgroundAsync( Directory.CreateDirectory(directory); } - // Get message from Telegram - if (!dbFile.MessageId.HasValue) + // Ordered list of Telegram messages that make up this file + // (split files span several messages, one document per part) + var messageIds = GetMessageIds(dbFile); + if (messageIds == null || messageIds.Count == 0) { _logger.LogError("File has no MessageId: {CacheKey}", info.CacheKey); info.IsDownloading = false; return; } - var message = await _ts.getMessageFile(channelId, dbFile.MessageId.Value); - if (message == null) - { - _logger.LogError("Message not found in Telegram: {CacheKey}", info.CacheKey); - info.IsDownloading = false; - return; - } - // Create or open file for writing using var fileStream = new FileStream( info.FilePath, @@ -220,9 +215,10 @@ private async Task DownloadInBackgroundAsync( FileAccess.Write, FileShare.Read); - // Resume from where we left off if file partially exists - var startOffset = fileStream.Length; - fileStream.Seek(0, SeekOrigin.End); + // Resume from where we left off if file partially exists. + // Align down to 512KB so per-part offsets stay Telegram-aligned. + var startOffset = (fileStream.Length / 524288) * 524288; + fileStream.Seek(startOffset, SeekOrigin.Begin); info.DownloadedBytes = startOffset; // Create download model for progress tracking @@ -237,57 +233,87 @@ private async Task DownloadInBackgroundAsync( }; _tis.addToDownloadList(dm); - // Download in chunks + // Download in chunks, part by part (a single-message file is just one part) const int chunkSize = 512 * 1024; // 512KB chunks const int maxConsecutiveErrors = 5; - var chatMessage = new ChatMessages { message = message }; var consecutiveErrors = 0; + var aborted = false; + long partStartGlobal = 0; + var ct = info.CancellationTokenSource!.Token; - while (info.DownloadedBytes < info.TotalSize && !info.CancellationTokenSource!.Token.IsCancellationRequested) + foreach (var msgId in messageIds) { - var remaining = info.TotalSize - info.DownloadedBytes; - var toDownload = (int)Math.Min(chunkSize, remaining); + if (aborted || ct.IsCancellationRequested) + break; - try + var message = await _ts.getMessageFile(channelId, msgId); + var partSize = GetDocumentSize(message); + if (message == null || partSize <= 0) { - var chunk = await _ts.DownloadFileStream(message, info.DownloadedBytes, toDownload); - if (chunk.Length == 0) - { - _logger.LogWarning("Received empty chunk at offset {Offset}", info.DownloadedBytes); - break; - } + _logger.LogError("Message {MessageId} not found in Telegram or has no document: {CacheKey}", msgId, info.CacheKey); + aborted = true; + break; + } - await fileStream.WriteAsync(chunk, 0, chunk.Length, info.CancellationTokenSource.Token); - await fileStream.FlushAsync(info.CancellationTokenSource.Token); + // Part already fully cached (resume): skip it + if (info.DownloadedBytes >= partStartGlobal + partSize) + { + partStartGlobal += partSize; + continue; + } - info.DownloadedBytes += chunk.Length; - dm._transmitted = info.DownloadedBytes; - consecutiveErrors = 0; + var localOffset = info.DownloadedBytes - partStartGlobal; - // Log progress every 10% - var progress = (double)info.DownloadedBytes / info.TotalSize * 100; - if ((int)progress % 10 == 0) - { - _logger.LogDebug("Download progress: {FileName} - {Progress:F1}%", - Path.GetFileName(info.FilePath), progress); - } - } - catch (Exception ex) when (ex is not OperationCanceledException) + while (localOffset < partSize && !ct.IsCancellationRequested) { - consecutiveErrors++; - _logger.LogError(ex, "Error downloading chunk at offset {Offset} (attempt {Attempt}/{Max})", - info.DownloadedBytes, consecutiveErrors, maxConsecutiveErrors); + var toDownload = (int)Math.Min(chunkSize, partSize - localOffset); - if (consecutiveErrors >= maxConsecutiveErrors) + try { - _logger.LogError("Aborting background download after {Max} consecutive failures: {CacheKey}", - maxConsecutiveErrors, info.CacheKey); - break; + var chunk = await _ts.DownloadFileStream(message, localOffset, toDownload); + if (chunk.Length == 0) + { + _logger.LogWarning("Received empty chunk at offset {Offset} of message {MessageId}", localOffset, msgId); + aborted = true; + break; + } + + await fileStream.WriteAsync(chunk, 0, chunk.Length, ct); + await fileStream.FlushAsync(ct); + + localOffset += chunk.Length; + info.DownloadedBytes = partStartGlobal + localOffset; + dm._transmitted = info.DownloadedBytes; + consecutiveErrors = 0; + + // Log progress every 10% + var progress = (double)info.DownloadedBytes / info.TotalSize * 100; + if ((int)progress % 10 == 0) + { + _logger.LogDebug("Download progress: {FileName} - {Progress:F1}%", + Path.GetFileName(info.FilePath), progress); + } + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + consecutiveErrors++; + _logger.LogError(ex, "Error downloading chunk at offset {Offset} of message {MessageId} (attempt {Attempt}/{Max})", + localOffset, msgId, consecutiveErrors, maxConsecutiveErrors); + + if (consecutiveErrors >= maxConsecutiveErrors) + { + _logger.LogError("Aborting background download after {Max} consecutive failures: {CacheKey}", + maxConsecutiveErrors, info.CacheKey); + aborted = true; + break; + } + + // Back off progressively before retrying + await Task.Delay(1000 * consecutiveErrors); } - - // Back off progressively before retrying - await Task.Delay(1000 * consecutiveErrors); } + + partStartGlobal += partSize; } info.IsComplete = info.DownloadedBytes >= info.TotalSize; @@ -316,6 +342,22 @@ private async Task DownloadInBackgroundAsync( } } + /// + /// Ordered list of Telegram message IDs that make up a file: a single message for + /// regular files, several (one per part) for split files. + /// + internal static List? GetMessageIds(BsonFileManagerModel dbFile) + { + if (dbFile.MessageId.HasValue && (dbFile.ListMessageId == null || dbFile.ListMessageId.Count <= 1)) + return new List { dbFile.MessageId.Value }; + return dbFile.ListMessageId; + } + + internal static long GetDocumentSize(Message? message) + { + return message?.media is MessageMediaDocument { document: Document doc } ? doc.size : 0; + } + /// /// Keeps the streaming cache directory under MAX_CACHE_BYTES, deleting the /// oldest files first (by write time). Files being downloaded are skipped. From 13bf92fefa101b774826ae3486a0256ba95ac0bf Mon Sep 17 00:00:00 2001 From: mateofuentespombo Date: Mon, 13 Jul 2026 12:46:56 +0200 Subject: [PATCH 08/33] fix: show progress and honor cancel for progressive cache downloads The background cache download appeared in the downloads list but never updated its progress and could not be stopped: - Report progress through DownloadModel.ProgressCallback (updates percentage, transmitted/size strings, global speed stats and fires the UI events) instead of writing _transmitted directly. The task is now also marked Completed when the download finishes. - Check the task state on every chunk so pressing Cancel (or Pause) in the downloads list actually stops the background download. - A user cancel is remembered for 1 hour per file so ongoing playback range requests don't immediately restart the cache download; the stream keeps working through direct Telegram fetches. A later playback caches again. - Start the progress counter at the resume offset so resumed downloads don't inflate the global speed stats. - Incomplete downloads no longer linger as 'Working' in the list. --- .../Services/ProgressiveDownloadService.cs | 72 ++++++++++++++++--- 1 file changed, 62 insertions(+), 10 deletions(-) diff --git a/TelegramDownloader/Services/ProgressiveDownloadService.cs b/TelegramDownloader/Services/ProgressiveDownloadService.cs index 6126f3e..1f5e904 100644 --- a/TelegramDownloader/Services/ProgressiveDownloadService.cs +++ b/TelegramDownloader/Services/ProgressiveDownloadService.cs @@ -47,6 +47,13 @@ public class ProgressiveDownloadInfo public bool IsDownloading { get; set; } public DateTime StartTime { get; set; } public CancellationTokenSource? CancellationTokenSource { get; set; } + /// + /// Set when the user cancelled the background download from the downloads list. + /// While set (and not expired), new range requests won't restart the cache download; + /// playback keeps working through direct Telegram fetches. + /// + public bool CancelledByUser { get; set; } + public DateTime CancelledAtUtc { get; set; } } public class ProgressiveDownloadService : IProgressiveDownloadService @@ -109,14 +116,23 @@ public async Task StartOrGetDownloadAsync( string filePath) { // Quick check if already complete - if (_activeDownloads.TryGetValue(cacheKey, out var existingInfo) && existingInfo.IsComplete) + if (_activeDownloads.TryGetValue(cacheKey, out var existingInfo)) { - if (File.Exists(existingInfo.FilePath)) + if (existingInfo.IsComplete) { + if (File.Exists(existingInfo.FilePath)) + { + return existingInfo; + } + // Cache file was evicted: forget the stale entry and re-download + _activeDownloads.TryRemove(cacheKey, out _); + } + else if (IsUserCancelActive(existingInfo)) + { + // User stopped this cache download: don't restart it on every range + // request; playback is served through direct Telegram fetches instead return existingInfo; } - // Cache file was evicted: forget the stale entry and re-download - _activeDownloads.TryRemove(cacheKey, out _); } // Check if file already exists and is complete @@ -149,6 +165,7 @@ public async Task StartOrGetDownloadAsync( if (_activeDownloads.TryGetValue(cacheKey, out existingInfo)) { if (existingInfo.IsDownloading || + IsUserCancelActive(existingInfo) || (existingInfo.IsComplete && File.Exists(existingInfo.FilePath))) { return existingInfo; @@ -229,6 +246,9 @@ private async Task DownloadInBackgroundAsync( path = info.FilePath, name = Path.GetFileName(info.FilePath), _size = info.TotalSize, + // Start the counter at the resume offset so progress and global + // speed stats only account for newly downloaded bytes + _transmitted = startOffset, channelName = _ts.getChatName(Convert.ToInt64(channelId)) }; _tis.addToDownloadList(dm); @@ -266,6 +286,16 @@ private async Task DownloadInBackgroundAsync( while (localOffset < partSize && !ct.IsCancellationRequested) { + // React to Cancel/Pause pressed in the downloads list + if (dm.state == StateTask.Canceled || dm.state == StateTask.Paused) + { + _logger.LogInformation("Background download stopped by user: {CacheKey}", info.CacheKey); + info.CancelledByUser = true; + info.CancelledAtUtc = DateTime.UtcNow; + aborted = true; + break; + } + var toDownload = (int)Math.Min(chunkSize, partSize - localOffset); try @@ -283,15 +313,22 @@ private async Task DownloadInBackgroundAsync( localOffset += chunk.Length; info.DownloadedBytes = partStartGlobal + localOffset; - dm._transmitted = info.DownloadedBytes; consecutiveErrors = 0; - // Log progress every 10% - var progress = (double)info.DownloadedBytes / info.TotalSize * 100; - if ((int)progress % 10 == 0) + try { - _logger.LogDebug("Download progress: {FileName} - {Progress:F1}%", - Path.GetFileName(info.FilePath), progress); + // Updates progress/speed and notifies the downloads UI; + // marks the task completed when the last byte is written + dm.ProgressCallback(info.DownloadedBytes, info.TotalSize); + } + catch + { + // ProgressCallback throws when Cancel/Pause was pressed + _logger.LogInformation("Background download stopped by user: {CacheKey}", info.CacheKey); + info.CancelledByUser = true; + info.CancelledAtUtc = DateTime.UtcNow; + aborted = true; + break; } } catch (Exception ex) when (ex is not OperationCanceledException) @@ -319,6 +356,12 @@ private async Task DownloadInBackgroundAsync( info.IsComplete = info.DownloadedBytes >= info.TotalSize; info.IsDownloading = false; + // Don't leave an incomplete task hanging as "Working" in the downloads list + if (!info.IsComplete && dm.state == StateTask.Working) + { + dm.Cancel(); + } + _logger.LogInformation("Background download {Status}: {FileName} ({Downloaded}/{Total} bytes)", info.IsComplete ? "complete" : "incomplete", Path.GetFileName(info.FilePath), @@ -342,6 +385,15 @@ private async Task DownloadInBackgroundAsync( } } + // A user cancel blocks automatic restarts for this long; afterwards a new + // playback of the file starts caching again + private static readonly TimeSpan USER_CANCEL_TTL = TimeSpan.FromHours(1); + + private static bool IsUserCancelActive(ProgressiveDownloadInfo info) + { + return info.CancelledByUser && DateTime.UtcNow - info.CancelledAtUtc < USER_CANCEL_TTL; + } + /// /// Ordered list of Telegram message IDs that make up a file: a single message for /// regular files, several (one per part) for split files. From e64b8ebeb0863f2370721e4f073dc641effdf0bf Mon Sep 17 00:00:00 2001 From: mateofuentespombo Date: Sun, 19 Jul 2026 18:59:06 +0200 Subject: [PATCH 09/33] feat: audio transcoding endpoint for offline downloads (MP3/AAC) - GET api/mobile/stream/tfm/{channelId}/{tfmId}/transcoded?format=mp3|aac&bitrate=N downloads the original into the streaming cache if needed, transcodes it with FFmpeg and serves the result with Range support; transcodes are cached on disk under _temp/transcoded so repeat requests are instant - GET api/mobile/stream/transcode/info reports FFmpeg availability so clients can warn the user and fall back to original downloads - Returns 501 when FFmpeg is missing; per-target locks avoid duplicate transcodes and a semaphore caps concurrent FFmpeg processes at 2 - MP3 keeps embedded cover art and tags; AAC (m4a) keeps tags --- .../Mobile/MobileStreamController.cs | 256 ++++++++++++++++++ 1 file changed, 256 insertions(+) diff --git a/TelegramDownloader/Controllers/Mobile/MobileStreamController.cs b/TelegramDownloader/Controllers/Mobile/MobileStreamController.cs index 7ee6000..03917ff 100644 --- a/TelegramDownloader/Controllers/Mobile/MobileStreamController.cs +++ b/TelegramDownloader/Controllers/Mobile/MobileStreamController.cs @@ -4,6 +4,8 @@ using TelegramDownloader.Models; using TelegramDownloader.Models.Mobile; using TelegramDownloader.Services; +using System.Collections.Concurrent; +using System.Diagnostics; using System.Web; namespace TelegramDownloader.Controllers.Mobile @@ -857,5 +859,259 @@ private static string GetMimeType(string extension) _ => "application/octet-stream" }; } + + // ============ Audio transcoding (offline downloads in MP3/AAC) ============ + + private static bool? _ffmpegAvailable; + private static readonly SemaphoreSlim _transcodeSemaphore = new(2); // Max 2 concurrent FFmpeg processes + private static readonly ConcurrentDictionary _transcodeLocks = new(); + + private static bool IsFFmpegAvailable() + { + if (_ffmpegAvailable.HasValue) return _ffmpegAvailable.Value; + try + { + using var process = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = "ffmpeg", + Arguments = "-version", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + process.Start(); + process.WaitForExit(5000); + _ffmpegAvailable = process.ExitCode == 0; + } + catch + { + _ffmpegAvailable = false; + } + return _ffmpegAvailable.Value; + } + + /// + /// Capacidades de transcodificación del servidor + /// + /// + /// Permite al cliente comprobar si el servidor tiene FFmpeg disponible antes de + /// ofrecer descargas transcodificadas. Si no lo está, el cliente debe avisar al + /// usuario y descargar en formato original. + /// + [HttpGet("transcode/info")] + [ProducesResponseType(StatusCodes.Status200OK)] + public IActionResult GetTranscodeInfo() + { + var available = IsFFmpegAvailable(); + return Ok(ApiResponse.Ok(new + { + ffmpegAvailable = available, + formats = available ? new[] { "mp3", "aac" } : Array.Empty() + })); + } + + /// + /// Descargar audio transcodificado a MP3 o AAC (para descargas offline) + /// + /// + /// Transcodifica el archivo original (p.ej. FLAC) al formato y bitrate indicados + /// usando FFmpeg y lo sirve con soporte Range. El resultado se cachea en disco, + /// por lo que peticiones posteriores son inmediatas. + /// + /// Requiere FFmpeg instalado en el servidor: si no está disponible responde + /// **501 Not Implemented** y el cliente debe descargar el original. + /// + /// La primera petición puede tardar: descarga el original de Telegram (si no está + /// cacheado) y ejecuta la transcodificación antes de responder. + /// + /// ID del canal de Telegram + /// ID del archivo en la base de datos TFM + /// Formato destino: mp3 | aac + /// Bitrate en kbps (64-320) + /// Nombre del archivo original (opcional) + [HttpGet("tfm/{channelId}/{tfmId}/transcoded")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status206PartialContent)] + [ProducesResponseType(StatusCodes.Status404NotFound)] + [ProducesResponseType(StatusCodes.Status501NotImplemented)] + public async Task StreamTranscodedAudio( + string channelId, + string tfmId, + [FromQuery] string format = "mp3", + [FromQuery] int bitrate = 192, + [FromQuery] string? fileName = null) + { + try + { + format = format.ToLowerInvariant(); + if (format != "mp3" && format != "aac") + { + return BadRequest(ApiResponse.Fail("Unsupported format. Use mp3 or aac.")); + } + bitrate = Math.Clamp(bitrate, 64, 320); + + if (!IsFFmpegAvailable()) + { + return StatusCode(StatusCodes.Status501NotImplemented, + ApiResponse.Fail("FFmpeg is not available on the server")); + } + + var dbFile = await _fs.getItemById(channelId, tfmId); + if (dbFile == null) + { + return NotFound(ApiResponse.Fail("File not found in database")); + } + + var name = fileName ?? dbFile.Name; + var cacheFileName = $"{channelId}-{(dbFile.MessageId != null ? dbFile.MessageId.ToString() : dbFile.Id)}-{name}"; + var tempPath = Path.Combine(FileService.TEMPDIR, "_temp"); + var sourcePath = Path.Combine(tempPath, cacheFileName); + var transcodedDir = Path.Combine(tempPath, "transcoded"); + Directory.CreateDirectory(transcodedDir); + + var targetExt = format == "mp3" ? "mp3" : "m4a"; + var mimeType = format == "mp3" ? "audio/mpeg" : "audio/mp4"; + var targetName = $"{Path.GetFileNameWithoutExtension(cacheFileName)}-{format}{bitrate}.{targetExt}"; + var targetPath = Path.Combine(transcodedDir, targetName); + var downloadName = $"{Path.GetFileNameWithoutExtension(name)}.{targetExt}"; + + // Cached transcode: serve immediately with Range support + if (System.IO.File.Exists(targetPath)) + { + return PhysicalFile(targetPath, mimeType, downloadName, enableRangeProcessing: true); + } + + // Per-target lock so concurrent requests don't transcode twice + var fileLock = _transcodeLocks.GetOrAdd(targetName, _ => new SemaphoreSlim(1, 1)); + await fileLock.WaitAsync(HttpContext.RequestAborted); + try + { + if (!System.IO.File.Exists(targetPath)) + { + // Ensure the original is fully cached first + if (!System.IO.File.Exists(sourcePath) || new FileInfo(sourcePath).Length < dbFile.Size) + { + _logger.LogInformation("Downloading original before transcode: {FileName}", name); + await DownloadOriginalToCache(channelId, dbFile, sourcePath); + } + + await _transcodeSemaphore.WaitAsync(HttpContext.RequestAborted); + try + { + _logger.LogInformation("Transcoding {FileName} to {Format} {Bitrate}k", name, format, bitrate); + await TranscodeAudioFile(sourcePath, targetPath, format, bitrate, HttpContext.RequestAborted); + } + finally + { + _transcodeSemaphore.Release(); + } + } + } + finally + { + fileLock.Release(); + } + + return PhysicalFile(targetPath, mimeType, downloadName, enableRangeProcessing: true); + } + catch (OperationCanceledException) + { + _logger.LogDebug("Transcoded download cancelled for {TfmId}", tfmId); + return new EmptyResult(); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error transcoding {TfmId} from channel {ChannelId}", tfmId, channelId); + return StatusCode(500, ApiResponse.Fail("Error transcoding audio")); + } + } + + // Sequential full download of the original file into the streaming cache + private async Task DownloadOriginalToCache(string channelId, BsonFileManagerModel dbFile, string path) + { + if (!dbFile.MessageId.HasValue) + { + throw new InvalidOperationException("File has no MessageId"); + } + + var message = await _ts.getMessageFile(channelId, dbFile.MessageId.Value); + if (message == null) + { + throw new InvalidOperationException("Message not found in Telegram"); + } + + using var fileStream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read); + var offset = fileStream.Length; + fileStream.Seek(0, SeekOrigin.End); + + const int chunkSize = 512 * 1024; + while (offset < dbFile.Size) + { + HttpContext.RequestAborted.ThrowIfCancellationRequested(); + var toDownload = (int)Math.Min(chunkSize, dbFile.Size - offset); + var chunk = await _ts.DownloadFileStream(message, offset, toDownload); + if (chunk.Length == 0) break; + await fileStream.WriteAsync(chunk, 0, chunk.Length, HttpContext.RequestAborted); + offset += chunk.Length; + } + await fileStream.FlushAsync(); + } + + // Run FFmpeg to a temp file, then move into place so partial results + // are never served + private async Task TranscodeAudioFile(string sourcePath, string targetPath, string format, int bitrate, CancellationToken ct) + { + var tempOut = targetPath + ".part"; + + // mp3: keep embedded cover art (optional video stream copied as attached pic) + // aac/m4a: audio only (cover copying into ipod container is less reliable) + var codecArgs = format == "mp3" + ? $"-map 0:a:0 -map 0:v? -c:v copy -disposition:v:0 attached_pic -codec:a libmp3lame -b:a {bitrate}k -id3v2_version 3 -f mp3" + : $"-vn -codec:a aac -b:a {bitrate}k -movflags +faststart -f ipod"; + + var args = $"-y -i \"{sourcePath}\" -map_metadata 0 {codecArgs} \"{tempOut}\""; + + using var process = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = "ffmpeg", + Arguments = args, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + process.Start(); + var stderrTask = process.StandardError.ReadToEndAsync(); + _ = process.StandardOutput.ReadToEndAsync(); + + try + { + await process.WaitForExitAsync(ct); + } + catch (OperationCanceledException) + { + try { process.Kill(entireProcessTree: true); } catch { /* already exited */ } + try { System.IO.File.Delete(tempOut); } catch { /* best effort */ } + throw; + } + + if (process.ExitCode != 0) + { + var stderr = await stderrTask; + try { System.IO.File.Delete(tempOut); } catch { /* best effort */ } + throw new InvalidOperationException( + $"FFmpeg exited with code {process.ExitCode}: {stderr.Substring(0, Math.Min(500, stderr.Length))}"); + } + + System.IO.File.Move(tempOut, targetPath, overwrite: true); + } } } From 05fdae6ef73050296ec8b7a02001504a091af3a4 Mon Sep 17 00:00:00 2001 From: mateofuentespombo Date: Tue, 21 Jul 2026 19:21:30 +0200 Subject: [PATCH 10/33] fix: reliable live task updates in Tasks Manager Task, download and upload tables required a full page reload to show progress because per-model event subscriptions were only wired when the component's static cached list was empty, so newly created models never got subscribed. Static lists were also shared between the Active and Queue table instances, and grid refreshes were fired from background threads without dispatcher marshaling. TransactionInfoService now owns all per-model subscriptions (hooked when a model enters any list, unhooked on remove/clear) and exposes a single aggregated TransactionsChanged event, throttled to 250ms with a guaranteed trailing raise. Tables use instance lists and subscribe only to that event, refreshing via InvokeAsync. The Tasks Manager header stats now update live as well. --- TelegramDownloader/Pages/Downloads.razor | 17 ++ .../Pages/Partials/DownloadsTable.razor | 51 ++---- .../Pages/Partials/TasksTable.razor | 48 ++---- .../Pages/Partials/UploadsTable.razor | 55 ++---- .../Services/TransactionInfoService.cs | 159 ++++++++++++++++++ 5 files changed, 212 insertions(+), 118 deletions(-) diff --git a/TelegramDownloader/Pages/Downloads.razor b/TelegramDownloader/Pages/Downloads.razor index 69b01a3..8c3ac3f 100644 --- a/TelegramDownloader/Pages/Downloads.razor +++ b/TelegramDownloader/Pages/Downloads.razor @@ -6,6 +6,8 @@ @using TelegramDownloader.Services @using Microsoft.AspNetCore.WebUtilities +@implements IDisposable + @inject TransactionInfoService tis @inject NavigationManager NavigationManager @inject ITaskPersistenceService taskPersistence @@ -182,9 +184,24 @@ protected override async Task OnInitializedAsync() { + tis.TransactionsChanged += OnTransactionsChanged; await LoadPersistedTasksCount(); } + private async void OnTransactionsChanged(object sender, System.EventArgs e) + { + try + { + await InvokeAsync(StateHasChanged); + } + catch { } + } + + public void Dispose() + { + tis.TransactionsChanged -= OnTransactionsChanged; + } + private async Task LoadPersistedTasksCount() { try diff --git a/TelegramDownloader/Pages/Partials/DownloadsTable.razor b/TelegramDownloader/Pages/Partials/DownloadsTable.razor index 527eecd..5beac62 100644 --- a/TelegramDownloader/Pages/Partials/DownloadsTable.razor +++ b/TelegramDownloader/Pages/Partials/DownloadsTable.razor @@ -94,7 +94,7 @@ @code { [Parameter] public bool isPending { get; set; } = false; - public static List ldm = new List(); + private List ldm = new List(); BlazorBootstrap.Grid grid = default!; DownloadFileInfoModal infoModal { get; set; } private bool _disposed = false; @@ -163,17 +163,16 @@ }; } - protected override async Task OnInitializedAsync() + protected override void OnInitialized() { - checkNewEventsHandler(); - tis.EventChanged += eventChangedNew; + tis.TransactionsChanged += OnTransactionsChanged; } private async Task> DownloadsDataProvider(GridDataProviderRequest request) { - await getDownloadModels(request.PageNumber - 1, request.PageSize, ldm.Count() == 0); - int totalUploads = tis.getTotalDownloads(isPending); - return await Task.FromResult(new GridDataProviderResult { Data = ldm ?? new List(), TotalCount = totalUploads });//request.ApplyTo(uploads)); + ldm = tis.GetDownloadModels(request.PageNumber - 1, request.PageSize, isPending); + int totalDownloads = tis.getTotalDownloads(isPending); + return await Task.FromResult(new GridDataProviderResult { Data = ldm ?? new List(), TotalCount = totalDownloads }); } private async Task cancel(DownloadModel dm) @@ -200,45 +199,19 @@ } - private void checkNewEventsHandler() + private async void OnTransactionsChanged(object sender, System.EventArgs e) { - foreach (DownloadModel dm in ldm) + if (_disposed || grid is null) return; + try { - if (dm.progress != 100) - dm.EventChanged += eventChanged; - else - dm.EventChanged -= eventChanged; + await InvokeAsync(() => grid.RefreshDataAsync()); } - - } - - private async Task getDownloadModels(int pageNumber, int pageSize, bool mustCallEnventHandler = false) - { - ldm = tis.GetDownloadModels(pageNumber, pageSize, isPending); - if (mustCallEnventHandler) - checkNewEventsHandler(); - } - - void eventChanged(object sender, DownloadEventArgs e) - { - if (_disposed) return; - grid.RefreshDataAsync(); - } - - void eventChangedNew(object sender, System.EventArgs e) - { - if (_disposed) return; - checkNewEventsHandler(); - grid.RefreshDataAsync(); + catch { } } public void Dispose() { _disposed = true; - tis.EventChanged -= eventChangedNew; - foreach (DownloadModel dm in ldm) - { - dm.EventChanged -= eventChanged; - } + tis.TransactionsChanged -= OnTransactionsChanged; } } diff --git a/TelegramDownloader/Pages/Partials/TasksTable.razor b/TelegramDownloader/Pages/Partials/TasksTable.razor index abc636d..e1fc27f 100644 --- a/TelegramDownloader/Pages/Partials/TasksTable.razor +++ b/TelegramDownloader/Pages/Partials/TasksTable.razor @@ -97,7 +97,7 @@ @code { BlazorBootstrap.Grid grid = default!; - public static List lpt = new List(); + private List lpt = new List(); TaskInfoModal infoModal { get; set; } private bool _disposed = false; @@ -157,46 +157,26 @@ }; } - protected override async Task OnInitializedAsync() + protected override void OnInitialized() { - checkNewEventsHandler(); - tis.EventChanged += eventChangedNew; + tis.TransactionsChanged += OnTransactionsChanged; } private async Task> TasksDataProvider(GridDataProviderRequest request) { - await getPendingTasksModels(request.PageNumber - 1, request.PageSize, lpt.Count() == 0); - int totalUploads = tis.getTotalTasks(); - return await Task.FromResult(new GridDataProviderResult { Data = lpt ?? new List(), TotalCount = totalUploads });//request.ApplyTo(uploads)); + lpt = tis.getInfoDownloadTaksModel(request.PageNumber - 1, request.PageSize); + int totalTasks = tis.getTotalTasks(); + return await Task.FromResult(new GridDataProviderResult { Data = lpt ?? new List(), TotalCount = totalTasks }); } - private async Task getPendingTasksModels(int pageNumber, int pageSize, bool mustCallEnventHandler = false) + private async void OnTransactionsChanged(object sender, System.EventArgs e) { - lpt = tis.getInfoDownloadTaksModel(pageNumber, pageSize); - if (mustCallEnventHandler) - checkNewEventsHandler(); - } - - private void checkNewEventsHandler() - { - foreach (InfoDownloadTaksModel pt in lpt) + if (_disposed || grid is null) return; + try { - pt.EventChanged -= eventChangedPendingTask; - pt.EventChanged += eventChangedPendingTask; + await InvokeAsync(() => grid.RefreshDataAsync()); } - } - - void eventChangedPendingTask(object sender, InfoTaskEventArgs e) - { - if (_disposed) return; - grid.RefreshDataAsync(); - } - - void eventChangedNew(object sender, System.EventArgs e) - { - if (_disposed) return; - checkNewEventsHandler(); - grid.RefreshDataAsync(); + catch { } } private async Task cancel(InfoDownloadTaksModel idt) @@ -221,10 +201,6 @@ public void Dispose() { _disposed = true; - tis.EventChanged -= eventChangedNew; - foreach (InfoDownloadTaksModel pt in lpt) - { - pt.EventChanged -= eventChangedPendingTask; - } + tis.TransactionsChanged -= OnTransactionsChanged; } } diff --git a/TelegramDownloader/Pages/Partials/UploadsTable.razor b/TelegramDownloader/Pages/Partials/UploadsTable.razor index 65f5f12..bba70b9 100644 --- a/TelegramDownloader/Pages/Partials/UploadsTable.razor +++ b/TelegramDownloader/Pages/Partials/UploadsTable.razor @@ -104,7 +104,7 @@ [Parameter] public EventCallback OnUploadModalClose { get; set; } - public static List lum = new List(); + private List lum = new List(); BlazorBootstrap.Grid grid = default!; UploadFileInfoModal infoModal { get; set; } private bool _disposed = false; @@ -199,10 +199,9 @@ } } - protected override async Task OnInitializedAsync() + protected override void OnInitialized() { - checkNewEventsHandler(); - tis.EventChanged += eventChangedNew; + tis.TransactionsChanged += OnTransactionsChanged; } private async Task cancel(UploadModel um) @@ -231,54 +230,24 @@ private async Task> UploadsDataProvider(GridDataProviderRequest request) { - await getUploadModels(request.PageNumber - 1, request.PageSize, lum.Count() == 0); + lum = tis.GetUploadModels(request.PageNumber - 1, request.PageSize, isPending); int totalUploads = tis.getTotalUploads(isPending); return await Task.FromResult(new GridDataProviderResult { Data = lum ?? new List(), TotalCount = totalUploads }); } - private async Task getUploadModels(int pageNumber, int pageSize, bool mustCallEnventHandler = false) + private async void OnTransactionsChanged(object sender, System.EventArgs e) { - lum = tis.GetUploadModels(pageNumber, pageSize, isPending); - if (mustCallEnventHandler) - checkNewEventsHandler(); - } - - private void checkNewEventsHandler() - { - if (lum != null) - foreach (UploadModel um in lum) - { - if (um.progress != 100) - um.EventChanged += eventChangedUpload; - else - um.EventChanged -= eventChangedUpload; - } - - } - - void eventChangedUpload(object sender, UploadEventArgs e) - { - if (_disposed) return; - grid.RefreshDataAsync(); - } - - void eventChangedNew(object sender, System.EventArgs e) - { - if (_disposed) return; - checkNewEventsHandler(); - grid.RefreshDataAsync(); + if (_disposed || grid is null) return; + try + { + await InvokeAsync(() => grid.RefreshDataAsync()); + } + catch { } } public void Dispose() { _disposed = true; - tis.EventChanged -= eventChangedNew; - if (lum != null) - { - foreach (UploadModel um in lum) - { - um.EventChanged -= eventChangedUpload; - } - } + tis.TransactionsChanged -= OnTransactionsChanged; } } diff --git a/TelegramDownloader/Services/TransactionInfoService.cs b/TelegramDownloader/Services/TransactionInfoService.cs index 0cc1600..89e4def 100644 --- a/TelegramDownloader/Services/TransactionInfoService.cs +++ b/TelegramDownloader/Services/TransactionInfoService.cs @@ -14,6 +14,13 @@ public class TransactionInfoService public bool isPauseDownloads = false; public event EventHandler EventChanged; + /// + /// Aggregated, throttled event raised whenever any transaction changes: + /// list membership (add/remove/clear) or per-model progress/state. + /// UI components should subscribe to this single event instead of + /// subscribing to each model individually. + /// + public event EventHandler TransactionsChanged; public event EventHandler TaskEventChanged; public event EventHandler HistorykEventChanged; public event EventHandler NewSpeedHistoryPoint; @@ -43,10 +50,111 @@ public class TransactionInfoService private Timer aTimer; private readonly ILogger _logger; + // Throttling for TransactionsChanged: progress callbacks fire per network + // chunk, so raise at most once per interval with a guaranteed trailing raise. + private static readonly TimeSpan NotifyThrottle = TimeSpan.FromMilliseconds(250); + private readonly object _notifyLock = new object(); + private DateTime _lastNotifyUtc = DateTime.MinValue; + private bool _trailingNotifyScheduled = false; + private System.Threading.Timer _trailingNotifyTimer; + public TransactionInfoService(ILogger logger) { _logger = logger; } + + /// + /// Raises , coalescing bursts so + /// subscribers refresh at most once per . + /// The last change in a burst is always delivered (trailing raise). + /// + public void NotifyTransactionsChanged() + { + bool raiseNow = false; + lock (_notifyLock) + { + DateTime now = DateTime.UtcNow; + if (now - _lastNotifyUtc >= NotifyThrottle) + { + _lastNotifyUtc = now; + raiseNow = true; + } + else if (!_trailingNotifyScheduled) + { + _trailingNotifyScheduled = true; + TimeSpan delay = NotifyThrottle - (now - _lastNotifyUtc); + if (delay < TimeSpan.Zero) + delay = TimeSpan.Zero; + if (_trailingNotifyTimer == null) + _trailingNotifyTimer = new System.Threading.Timer(_ => RaiseTrailingNotify(), null, delay, Timeout.InfiniteTimeSpan); + else + _trailingNotifyTimer.Change(delay, Timeout.InfiniteTimeSpan); + } + } + if (raiseNow) + SafeRaiseTransactionsChanged(); + } + + private void RaiseTrailingNotify() + { + lock (_notifyLock) + { + _trailingNotifyScheduled = false; + _lastNotifyUtc = DateTime.UtcNow; + } + SafeRaiseTransactionsChanged(); + } + + private void SafeRaiseTransactionsChanged() + { + try + { + TransactionsChanged?.Invoke(this, EventArgs.Empty); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error notifying TransactionsChanged subscribers"); + } + } + + // The service owns the per-model subscriptions so UI components do not + // have to track which models are wired. Hooking is idempotent. + private void hookModel(DownloadModel dm) + { + dm.EventChanged -= onDownloadModelChanged; + dm.EventChanged += onDownloadModelChanged; + } + + private void unhookModel(DownloadModel dm) + { + dm.EventChanged -= onDownloadModelChanged; + } + + private void hookModel(UploadModel um) + { + um.EventChanged -= onUploadModelChanged; + um.EventChanged += onUploadModelChanged; + } + + private void unhookModel(UploadModel um) + { + um.EventChanged -= onUploadModelChanged; + } + + private void hookModel(InfoDownloadTaksModel idt) + { + idt.EventChanged -= onInfoTaskModelChanged; + idt.EventChanged += onInfoTaskModelChanged; + } + + private void unhookModel(InfoDownloadTaksModel idt) + { + idt.EventChanged -= onInfoTaskModelChanged; + } + + private void onDownloadModelChanged(object sender, DownloadEventArgs e) => NotifyTransactionsChanged(); + private void onUploadModelChanged(object sender, UploadEventArgs e) => NotifyTransactionsChanged(); + private void onInfoTaskModelChanged(object sender, InfoTaskEventArgs e) => NotifyTransactionsChanged(); public bool isWorking() { if (!(isDownloading() || isUploading())) @@ -250,8 +358,10 @@ public void addToDownloadList(DownloadModel downloadModel) downloadModel.name, downloadModel._size / (1024.0 * 1024.0)); downloadModels.Insert(0, downloadModel); PendingDownloadMutex.ReleaseMutex(); + hookModel(downloadModel); EventChanged?.Invoke(this, new EventArgs()); TaskEventChanged?.Invoke(null, EventArgs.Empty); + NotifyTransactionsChanged(); } public void addToPendingDownloadList(DownloadModel downloadModel, bool atFirst = false, bool chekDownloads = true) @@ -275,6 +385,8 @@ public void addToPendingDownloadList(DownloadModel downloadModel, bool atFirst = else pendingDownloadModels.Add(downloadModel); PendingDownloadMutex.ReleaseMutex(); + hookModel(downloadModel); + NotifyTransactionsChanged(); if (chekDownloads) CheckPendingDownloads(); } @@ -291,6 +403,7 @@ public void PauseDownloads() } PendingDownloadMutex.ReleaseMutex(); TaskEventChanged?.Invoke(null, EventArgs.Empty); + NotifyTransactionsChanged(); } public void PlayDownloads() @@ -315,6 +428,7 @@ public void StopDownloads() pendingDownloadModels.Clear(); PendingDownloadMutex.ReleaseMutex(); TaskEventChanged?.Invoke(null, EventArgs.Empty); + NotifyTransactionsChanged(); } public async Task CheckPendingDownloads() @@ -328,12 +442,14 @@ public async Task CheckPendingDownloads() DownloadModel dm = pendingDownloadModels.FirstOrDefault(); downloadModels.Insert(0, dm); pendingDownloadModels.Remove(dm); + hookModel(dm); startTimer(); dm.RetryCallback(); } PendingDownloadMutex.ReleaseMutex(); EventChanged?.Invoke(this, new EventArgs()); TaskEventChanged?.Invoke(null, EventArgs.Empty); + NotifyTransactionsChanged(); } public async Task CheckPendingUploadInfoTasks() @@ -344,12 +460,14 @@ public async Task CheckPendingUploadInfoTasks() { InfoDownloadTaksModel idt = infoDownloadTaksModel.Where(x => x.state == StateTask.Pending).OrderBy(x => x.creationDate).FirstOrDefault(); idt.state = StateTask.Working; + hookModel(idt); startTimer(); idt.RetryCallback(); } PendingUploadInfoTaskMutex.ReleaseMutex(); EventChanged?.Invoke(this, new EventArgs()); TaskEventChanged?.Invoke(null, EventArgs.Empty); + NotifyTransactionsChanged(); } public void addToUploadList(UploadModel uploadModel) @@ -365,15 +483,19 @@ public void addToUploadList(UploadModel uploadModel) _logger.LogInformation("Adding to upload list - Name: {Name}, Size: {SizeMB:F2}MB", uploadModel.name, uploadModel._size / (1024.0 * 1024.0)); uploadModels.Insert(0, uploadModel); + hookModel(uploadModel); EventChanged?.Invoke(this, new EventArgs()); TaskEventChanged?.Invoke(null, EventArgs.Empty); + NotifyTransactionsChanged(); } public void deleteUploadInList(UploadModel uploadModel) { uploadModels.Remove(uploadModel); + unhookModel(uploadModel); EventChanged?.Invoke(this, new EventArgs()); TaskEventChanged?.Invoke(null, EventArgs.Empty); + NotifyTransactionsChanged(); } public void addToInfoDownloadTaskList(InfoDownloadTaksModel infoDownloadModel) @@ -381,6 +503,8 @@ public void addToInfoDownloadTaskList(InfoDownloadTaksModel infoDownloadModel) PendingUploadInfoTaskMutex.WaitOne(); infoDownloadTaksModel.Add(infoDownloadModel); PendingUploadInfoTaskMutex.ReleaseMutex(); + hookModel(infoDownloadModel); + NotifyTransactionsChanged(); CheckPendingUploadInfoTasks(); } @@ -430,8 +554,10 @@ public void addToPendingUploadList(UploadModel uploadModel) pendingUploadModels.Add(uploadModel); PendingUploadMutex.ReleaseMutex(); + hookModel(uploadModel); EventChanged?.Invoke(this, new EventArgs()); TaskEventChanged?.Invoke(null, EventArgs.Empty); + NotifyTransactionsChanged(); } public void deletePendingUploadInList(UploadModel uploadModel) @@ -439,8 +565,11 @@ public void deletePendingUploadInList(UploadModel uploadModel) PendingUploadMutex.WaitOne(); pendingUploadModels.Remove(uploadModel); PendingUploadMutex.ReleaseMutex(); + if (!uploadModels.Contains(uploadModel)) + unhookModel(uploadModel); EventChanged?.Invoke(this, new EventArgs()); TaskEventChanged?.Invoke(null, EventArgs.Empty); + NotifyTransactionsChanged(); } public void deleteDownloadInList(DownloadModel downloadModel) @@ -448,8 +577,13 @@ public void deleteDownloadInList(DownloadModel downloadModel) PendingDownloadMutex.WaitOne(); downloadModels.Remove(downloadModel); PendingDownloadMutex.ReleaseMutex(); + // A paused download is removed from the active list but stays in the + // pending list, so keep it hooked in that case. + if (!pendingDownloadModels.Contains(downloadModel)) + unhookModel(downloadModel); EventChanged?.Invoke(this, new EventArgs()); TaskEventChanged?.Invoke(null, EventArgs.Empty); + NotifyTransactionsChanged(); } public void deletePendingDownloadInList(DownloadModel downloadModel) @@ -457,28 +591,39 @@ public void deletePendingDownloadInList(DownloadModel downloadModel) PendingDownloadMutex.WaitOne(); pendingDownloadModels.Remove(downloadModel); PendingDownloadMutex.ReleaseMutex(); + if (!downloadModels.Contains(downloadModel)) + unhookModel(downloadModel); EventChanged?.Invoke(this, new EventArgs()); TaskEventChanged?.Invoke(null, EventArgs.Empty); + NotifyTransactionsChanged(); } public void ClearPendingDownloads() { _logger.LogInformation("Clearing all pending downloads - Count: {Count}", pendingDownloadModels.Count); PendingDownloadMutex.WaitOne(); + List removed = pendingDownloadModels.ToList(); pendingDownloadModels.Clear(); PendingDownloadMutex.ReleaseMutex(); + foreach (DownloadModel dm in removed.Where(x => !downloadModels.Contains(x))) + unhookModel(dm); EventChanged?.Invoke(this, new EventArgs()); TaskEventChanged?.Invoke(null, EventArgs.Empty); + NotifyTransactionsChanged(); } public void ClearPendingUploads() { _logger.LogInformation("Clearing all pending uploads - Count: {Count}", pendingUploadModels.Count); PendingUploadMutex.WaitOne(); + List removed = pendingUploadModels.ToList(); pendingUploadModels.Clear(); PendingUploadMutex.ReleaseMutex(); + foreach (UploadModel um in removed.Where(x => !uploadModels.Contains(x))) + unhookModel(um); EventChanged?.Invoke(this, new EventArgs()); TaskEventChanged?.Invoke(null, EventArgs.Empty); + NotifyTransactionsChanged(); } public void deleteInfoDownloadTaskFromList(InfoDownloadTaksModel idt) @@ -486,8 +631,10 @@ public void deleteInfoDownloadTaskFromList(InfoDownloadTaksModel idt) PendingUploadInfoTaskMutex.WaitOne(); infoDownloadTaksModel.Remove(idt); PendingUploadInfoTaskMutex.ReleaseMutex(); + unhookModel(idt); EventChanged?.Invoke(this, new EventArgs()); TaskEventChanged?.Invoke(null, EventArgs.Empty); + NotifyTransactionsChanged(); } public List getInfoDownloadTaksModel(int pageNumber, int pageSize) @@ -504,22 +651,34 @@ public int getTotalTasks() public void clearUploadCompleted() { + List removed = uploadModels.Where(x => x.state != StateTask.Working).ToList(); uploadModels.RemoveAll(x => x.state != StateTask.Working); + foreach (UploadModel um in removed) + unhookModel(um); EventChanged?.Invoke(this, new EventArgs()); + NotifyTransactionsChanged(); } public void clearDownloadCompleted() { PendingDownloadMutex.WaitOne(); + List removed = downloadModels.Where(x => x.state != StateTask.Working).ToList(); downloadModels.RemoveAll(x => x.state != StateTask.Working); PendingDownloadMutex.ReleaseMutex(); + foreach (DownloadModel dm in removed.Where(x => !pendingDownloadModels.Contains(x))) + unhookModel(dm); EventChanged?.Invoke(this, new EventArgs()); + NotifyTransactionsChanged(); } public void clearTasksCompleted() { + List removed = infoDownloadTaksModel.Where(x => x.state != StateTask.Working).ToList(); infoDownloadTaksModel.RemoveAll(x => x.state != StateTask.Working); + foreach (InfoDownloadTaksModel idt in removed) + unhookModel(idt); EventChanged?.Invoke(this, new EventArgs()); + NotifyTransactionsChanged(); } From 893269c3ea15a4fdf589b673f8cdbd591fca8337 Mon Sep 17 00:00:00 2001 From: mateofuentespombo Date: Tue, 21 Jul 2026 19:51:58 +0200 Subject: [PATCH 11/33] feat: configurable parallel chunk transfers for faster downloads/uploads Transfers were capped at ~5-7 MB/s because WTelegramClient requests 512KB file parts with only 2 requests in flight (its default since v4.x), making throughput latency-bound at roughly 1MB per round-trip to Telegram's data center. Add a ParallelTransfers setting (default 4, range 1-16) applied to the main client and, before each document download, to the media-DC client that WTelegram resolves internally - secondary DC clients do not inherit the main client's setting, so it must be propagated per instance. The applied value is tracked per client and adjusted by delta, which stays correct even while parts are in flight. Changes take effect on the next transfer without restarting. --- TelegramDownloader/Data/TelegramService.cs | 66 ++++++++++++++++++++++ TelegramDownloader/Models/GeneralConfig.cs | 11 ++++ TelegramDownloader/Pages/Config.razor | 15 +++++ 3 files changed, 92 insertions(+) diff --git a/TelegramDownloader/Data/TelegramService.cs b/TelegramDownloader/Data/TelegramService.cs index 3f20d01..d42e5dd 100644 --- a/TelegramDownloader/Data/TelegramService.cs +++ b/TelegramDownloader/Data/TelegramService.cs @@ -161,6 +161,7 @@ private void createDownloadFolder() private void newClient() { client = new WTelegram.Client(Convert.ToInt32(GeneralConfigStatic.tlconfig?.api_id ?? Environment.GetEnvironmentVariable("api_id")), GeneralConfigStatic.tlconfig?.hash_id ?? Environment.GetEnvironmentVariable("hash_id"), UserService.USERDATAFOLDER + "/WTelegram.session"); + ApplyConfiguredParallelTransfers(client); if (GeneralConfigStatic.config.ShouldShowLogInTerminal) { // WTelegram.Helpers.Log = (lvl, str) => Console.WriteLine($"{DateTime.Now:yyyy-MM-dd HH:mm:ss} [{"TDIWE!"[lvl]}] {str}"); @@ -171,6 +172,67 @@ private void newClient() } + // WTelegramClient requests file parts through a per-client semaphore that + // defaults to 2 parts in flight, capping transfers at ~1MB per round-trip. + // Tracks the value applied to each client instance (main client and the + // media-DC clients WTelegram creates internally, which do NOT inherit the + // main client's ParallelTransfers). + private static readonly ConditionalWeakTable> appliedParallelTransfers = new(); + private const int WTELEGRAM_DEFAULT_PARALLEL_TRANSFERS = 2; + + public static int GetConfiguredParallelTransfers() + { + return Math.Clamp(GeneralConfigStatic.config?.ParallelTransfers ?? 4, 1, 16); + } + + private static void ApplyConfiguredParallelTransfers(WTelegram.Client c) + { + if (c == null) + return; + int desired = GetConfiguredParallelTransfers(); + int delta; + lock (appliedParallelTransfers) + { + StrongBox applied = appliedParallelTransfers.GetOrCreateValue(c); + if (applied.Value == 0) + applied.Value = WTELEGRAM_DEFAULT_PARALLEL_TRANSFERS; + delta = desired - applied.Value; + if (delta == 0) + return; + applied.Value = desired; + } + try + { + // The ParallelTransfers setter adjusts the semaphore relative to its + // CURRENT count, which is lower while parts are in flight. Applying + // our delta on top of the current value keeps the configured maximum + // correct even if a transfer is running on this client. + c.ParallelTransfers = c.ParallelTransfers + delta; + } + catch (Exception) + { + // Never let a tuning failure break a transfer. + } + } + + /// + /// Resolves the client instance that WTelegram's DownloadFileAsync will use + /// for the given file DC (dc_id == 0 means the main client) and applies the + /// configured chunk parallelism to it before the transfer starts. + /// + private async Task PrepareTransferClientAsync(int dcId) + { + try + { + WTelegram.Client c = dcId == 0 ? client : await client.GetClientForDC(-dcId, true); + ApplyConfiguredParallelTransfers(c); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Could not prepare transfer client for DC {DcId}", dcId); + } + } + public async Task CallQrGenerator(Action func, CancellationToken ct, bool logoutFirst = false) { return await client.LoginWithQRCode(func, logoutFirst: logoutFirst, ct: ct); @@ -701,6 +763,7 @@ public async Task uploadFile(string chatId, Stream file, string fileNam try { + ApplyConfiguredParallelTransfers(client); var inputFile = await client.UploadFileAsync(file, fileName, um.ProgressCallback); var result = await client.SendMediaAsync(peer, caption ?? fileName, inputFile, mimeType); _logger.LogInformation("File upload completed - FileName: {FileName}, MessageId: {MessageId}", fileName, result.id); @@ -1205,6 +1268,7 @@ public async Task DownloadFileAndReturn(ChatMessages message, Stream ms model.name = filename; _logger.LogInformation("Starting document download - FileName: {FileName}, Size: {SizeMB:F2}MB", filename, document.size / (1024.0 * 1024.0)); MemoryStream dest = new MemoryStream(); + await PrepareTransferClientAsync(document.dc_id); await client.DownloadFileAsync(document, ms ?? dest, (PhotoSizeBase)null, model.ProgressCallback); _logger.LogInformation("Document download completed - FileName: {FileName}", filename); return ms ?? dest; @@ -1253,6 +1317,7 @@ public async Task DownloadFileAndReturnWithOffset(ChatMessages message, if (offset == 0) { MemoryStream dest = new MemoryStream(); + await PrepareTransferClientAsync(document.dc_id); await client.DownloadFileAsync(document, ms ?? dest, (PhotoSizeBase)null, model.ProgressCallback); _logger.LogInformation("Document download completed - FileName: {FileName}", filename); return ms ?? dest; @@ -1365,6 +1430,7 @@ public async Task DownloadFile(ChatMessages message, string fileName = n _tis.addToDownloadList(model); _logger.LogInformation("Starting file download to disk - FileName: {FileName}, Size: {SizeMB:F2}MB", filename, document.size / (1024.0 * 1024.0)); using var dest = new FileStream($"{Path.Combine(folder != null ? folder : Path.Combine(Environment.CurrentDirectory, "local", "temp"), filename)}", FileMode.Create, FileAccess.Write); + await PrepareTransferClientAsync(document.dc_id); await client.DownloadFileAsync(document, dest, (PhotoSizeBase)null, model.ProgressCallback); _logger.LogInformation("File download to disk completed - FileName: {FileName}", filename); } diff --git a/TelegramDownloader/Models/GeneralConfig.cs b/TelegramDownloader/Models/GeneralConfig.cs index d51d90d..b6a606d 100644 --- a/TelegramDownloader/Models/GeneralConfig.cs +++ b/TelegramDownloader/Models/GeneralConfig.cs @@ -178,6 +178,17 @@ public StreamingMode GetEffectiveStreamingMode() /// public int MemorySplitSizeGB { get; set; } = 2; + // Transfer Speed Settings + /// + /// Number of 512KB file chunks requested in parallel per transfer (1-16). + /// WTelegramClient's default of 2 caps throughput at roughly 1MB per + /// round-trip to Telegram's data center (~5-7 MB/s on typical latency). + /// Higher values remove that latency bottleneck; the server-side speed + /// limit for non-Premium accounts still applies. Takes effect on the + /// next transfer, no restart needed. + /// + public int ParallelTransfers { get; set; } = 4; + } public class TLConfig diff --git a/TelegramDownloader/Pages/Config.razor b/TelegramDownloader/Pages/Config.razor index c7f88f0..2665831 100644 --- a/TelegramDownloader/Pages/Config.razor +++ b/TelegramDownloader/Pages/Config.razor @@ -230,6 +230,21 @@ +
+
+
+ + Parallel Chunk Transfers +
+
+ Number of 512KB chunks requested in parallel per transfer (1-16). Higher values improve download/upload speed by removing the latency bottleneck; Premium accounts benefit the most. Applied on the next transfer. +
+
+
+ +
+
+
From 11181c1eb911fa31897cc699768b4da738b792aa Mon Sep 17 00:00:00 2001 From: mateofuentespombo Date: Tue, 21 Jul 2026 22:23:57 +0200 Subject: [PATCH 12/33] feat: multi-connection downloads to bypass per-connection speed limit Telegram enforces its throughput limit per MTProto connection (~5-6 MB/s), so pipelining more chunks on one connection cannot go faster; pushing harder only triggers FLOOD_PREMIUM_WAIT penalties. Official clients reach 50+ MB/s by opening several sessions to the file DC and splitting the file between them. Add an experimental multi-connection download mode (off by default): a per-DC pool of extra authorized clients, bootstrapped once from the main client via auth.exportAuthorization/importAuthorization, persisted as session files and reused across restarts. Files >=32MB are split in 4MB blocks served concurrently by 2-8 connections (configurable), each part written at its absolute offset via RandomAccess. Progress reports the contiguous completed prefix so persistence keeps a safe resume offset, while speed accounting counts every received part. Any failure falls back transparently to the standard sequential download. --- TelegramDownloader/Data/TelegramService.cs | 267 ++++++++++++++++++++- TelegramDownloader/Models/DownloadModel.cs | 40 +++ TelegramDownloader/Models/GeneralConfig.cs | 19 ++ TelegramDownloader/Pages/Config.razor | 36 +++ 4 files changed, 360 insertions(+), 2 deletions(-) diff --git a/TelegramDownloader/Data/TelegramService.cs b/TelegramDownloader/Data/TelegramService.cs index d42e5dd..0711cbd 100644 --- a/TelegramDownloader/Data/TelegramService.cs +++ b/TelegramDownloader/Data/TelegramService.cs @@ -233,6 +233,263 @@ private async Task PrepareTransferClientAsync(int dcId) } } + #region Multi-connection downloads + + // Telegram enforces its throughput limit PER CONNECTION (~5-6 MB/s), so a + // single MTProto connection cannot go faster regardless of how many chunks + // are pipelined. Official clients reach high speeds by opening several + // sessions to the file's DC and splitting the file between them. This pool + // holds extra authorized clients per DC (bootstrapped once from the main + // client via auth.exportAuthorization/importAuthorization, persisted as + // session files and reused across restarts). + private class DcDownloadPool + { + public readonly SemaphoreSlim initLock = new SemaphoreSlim(1, 1); + public readonly List clients = new List(); + public bool bootstrapFailed = false; + } + + private static readonly Dictionary downloadPools = new Dictionary(); + private const int MULTICONN_PART_SIZE = 1024 * 1024; // upload.getFile max limit per request + private const int MULTICONN_BLOCK_SIZE = 4 * 1024 * 1024; // work unit assigned to a connection + private const long MULTICONN_MIN_FILE_SIZE = 32L * 1024 * 1024; + + public static int GetConfiguredDownloadConnections() + { + return Math.Clamp(GeneralConfigStatic.config?.DownloadConnections ?? 4, 2, 8); + } + + private static bool ShouldUseMultiConnection(TL.Document document) + { + GeneralConfig cfg = GeneralConfigStatic.config; + return cfg != null && cfg.EnableMultiConnectionDownloads && document.size >= MULTICONN_MIN_FILE_SIZE; + } + + private async Task> GetDownloadPoolAsync(int dcId, int count) + { + DcDownloadPool pool; + lock (downloadPools) + { + if (!downloadPools.TryGetValue(dcId, out pool)) + downloadPools[dcId] = pool = new DcDownloadPool(); + } + if (pool.bootstrapFailed) + return new List(); + await pool.initLock.WaitAsync(); + try + { + pool.clients.RemoveAll(c => + { + if (!c.Disconnected) return false; + try { c.Dispose(); } catch { } + return true; + }); + while (pool.clients.Count < count) + { + WTelegram.Client pc = await CreateDownloadPoolClientAsync(dcId, pool.clients.Count); + if (pc == null) + { + // Do not retry the bootstrap on every download if the DC + // refuses it (e.g. exportAuthorization not allowed). + if (pool.clients.Count == 0) + pool.bootstrapFailed = true; + break; + } + pool.clients.Add(pc); + } + return pool.clients.Take(count).ToList(); + } + finally + { + pool.initLock.Release(); + } + } + + private async Task CreateDownloadPoolClientAsync(int dcId, int index) + { + string sessionPath = Path.Combine(UserService.USERDATAFOLDER, $"WTelegram_dl_dc{dcId}_{index}.session"); + try + { + TL.Config tlConfig = await client.Help_GetConfig(); + DcOption dc = tlConfig.dc_options + .Where(x => x.id == dcId && (x.flags & (DcOption.Flags.ipv6 | DcOption.Flags.cdn | DcOption.Flags.tcpo_only)) == 0) + .OrderBy(x => (x.flags & DcOption.Flags.media_only) == 0 ? 0 : 1) + .FirstOrDefault(); + if (dc == null) + { + _logger.LogWarning("No suitable address found for DC {Dc} - multi-connection download unavailable", dcId); + return null; + } + string apiId = GeneralConfigStatic.tlconfig?.api_id ?? Environment.GetEnvironmentVariable("api_id"); + string apiHash = GeneralConfigStatic.tlconfig?.hash_id ?? Environment.GetEnvironmentVariable("hash_id"); + WTelegram.Client pc = new WTelegram.Client(what => what switch + { + "api_id" => apiId, + "api_hash" => apiHash, + "session_pathname" => sessionPath, + "server_address" => $"{dc.ip_address}:{dc.port}", + "device_model" => "TFM parallel download", + _ => null + }); + try + { + await pc.ConnectAsync(); + bool authorized = false; + try + { + await pc.Users_GetUsers(InputUser.Self); + authorized = true; + } + catch (RpcException) + { + // Fresh session, or a previously created one revoked from + // the account's device list: (re)import the authorization. + } + if (!authorized) + { + Auth_ExportedAuthorization exported = await client.Auth_ExportAuthorization(dcId); + await pc.Auth_ImportAuthorization(exported.id, exported.bytes); + await pc.Users_GetUsers(InputUser.Self); + } + ApplyConfiguredParallelTransfers(pc); + _logger.LogInformation("Download pool client {Index} ready for DC {Dc}", index, dcId); + return pc; + } + catch + { + try { pc.Dispose(); } catch { } + throw; + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Could not create download pool client {Index} for DC {Dc} - falling back to single-connection downloads", index, dcId); + return null; + } + } + + /// + /// Downloads a document by splitting it in blocks served concurrently by + /// several pool connections, writing each part at its absolute offset. + /// Returns false (leaving the destination empty) when the pool is not + /// available or the transfer failed in a recoverable way, so the caller + /// can fall back to the standard sequential download. + /// + private async Task TryMultiConnectionDownloadAsync(TL.Document document, FileStream dest, DownloadModel model) + { + long size = document.size; + List pool; + try + { + pool = await GetDownloadPoolAsync(document.dc_id, GetConfiguredDownloadConnections()); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Download pool unavailable for DC {Dc}", document.dc_id); + return false; + } + if (pool.Count < 2) + return false; + + var location = new InputDocumentFileLocation + { + id = document.id, + access_hash = document.access_hash, + file_reference = document.file_reference, + thumb_size = "" + }; + + _logger.LogInformation("Multi-connection download - FileName: {Name}, Size: {SizeMB:F2}MB, Connections: {Connections}", + model.name, size / (1024.0 * 1024.0), pool.Count); + + long blockCount = (size + MULTICONN_BLOCK_SIZE - 1) / MULTICONN_BLOCK_SIZE; + bool[] blockDone = new bool[blockCount]; + long confirmedBlocks = 0; + long nextBlock = -1; + object progressLock = new object(); + using CancellationTokenSource cts = new CancellationTokenSource(); + dest.SetLength(size); + var handle = dest.SafeFileHandle; + + void ReportPart(long block, int received, bool blockCompleted) + { + long confirmed; + lock (progressLock) + { + if (blockCompleted) + { + blockDone[block] = true; + while (confirmedBlocks < blockCount && blockDone[confirmedBlocks]) + confirmedBlocks++; + } + confirmed = Math.Min(size, confirmedBlocks * (long)MULTICONN_BLOCK_SIZE); + } + // Throws when the task gets canceled or paused, stopping the workers. + model.ReportParallelProgress(confirmed, received, size); + } + + async Task Worker(WTelegram.Client pc) + { + while (!cts.IsCancellationRequested) + { + long block = Interlocked.Increment(ref nextBlock); + if (block >= blockCount) + return; + long offset = block * (long)MULTICONN_BLOCK_SIZE; + long end = Math.Min(size, offset + MULTICONN_BLOCK_SIZE); + while (offset < end) + { + cts.Token.ThrowIfCancellationRequested(); + Upload_FileBase resp = null; + for (int attempt = 1; ; attempt++) + { + try + { + resp = await pc.Upload_GetFile(location, offset, limit: MULTICONN_PART_SIZE); + break; + } + catch (Exception) when (attempt < 3 && !cts.IsCancellationRequested) + { + await Task.Delay(1000 * attempt); + } + } + if (resp is not Upload_File part) + throw new InvalidOperationException($"Unexpected {resp?.GetType().Name} from Upload_GetFile (CDN-served files are not supported)"); + if (part.bytes.Length == 0) + throw new InvalidOperationException($"Empty chunk at offset {offset}"); + RandomAccess.Write(handle, part.bytes, offset); + offset += part.bytes.Length; + ReportPart(block, part.bytes.Length, offset >= end); + } + } + } + + async Task GuardedWorker(WTelegram.Client pc) + { + try { await Worker(pc); } + catch { cts.Cancel(); throw; } + } + + try + { + await Task.WhenAll(pool.Select(pc => Task.Run(() => GuardedWorker(pc)))); + await dest.FlushAsync(); + return true; + } + catch (Exception ex) + { + if (model.state == StateTask.Canceled || model.state == StateTask.Paused) + throw; + _logger.LogWarning(ex, "Multi-connection download failed, falling back to sequential - FileName: {Name}", model.name); + dest.SetLength(0); + dest.Position = 0; + model._transmitted = 0; + return false; + } + } + + #endregion + public async Task CallQrGenerator(Action func, CancellationToken ct, bool logoutFirst = false) { return await client.LoginWithQRCode(func, logoutFirst: logoutFirst, ct: ct); @@ -1430,8 +1687,14 @@ public async Task DownloadFile(ChatMessages message, string fileName = n _tis.addToDownloadList(model); _logger.LogInformation("Starting file download to disk - FileName: {FileName}, Size: {SizeMB:F2}MB", filename, document.size / (1024.0 * 1024.0)); using var dest = new FileStream($"{Path.Combine(folder != null ? folder : Path.Combine(Environment.CurrentDirectory, "local", "temp"), filename)}", FileMode.Create, FileAccess.Write); - await PrepareTransferClientAsync(document.dc_id); - await client.DownloadFileAsync(document, dest, (PhotoSizeBase)null, model.ProgressCallback); + bool multiConnDone = false; + if (ShouldUseMultiConnection(document)) + multiConnDone = await TryMultiConnectionDownloadAsync(document, dest, model); + if (!multiConnDone) + { + await PrepareTransferClientAsync(document.dc_id); + await client.DownloadFileAsync(document, dest, (PhotoSizeBase)null, model.ProgressCallback); + } _logger.LogInformation("File download to disk completed - FileName: {FileName}", filename); } else if (message.message.media is MessageMediaPhoto { photo: Photo photo }) diff --git a/TelegramDownloader/Models/DownloadModel.cs b/TelegramDownloader/Models/DownloadModel.cs index 5a3d848..36cc668 100644 --- a/TelegramDownloader/Models/DownloadModel.cs +++ b/TelegramDownloader/Models/DownloadModel.cs @@ -214,6 +214,46 @@ public void ProgressCallback(long transmitted, long totalSize) mutex.ReleaseMutex(); } + /// + /// Progress reporting for multi-connection downloads, where parts arrive + /// out of order. is the contiguous + /// number of bytes completed from the start of the file (the only safe + /// resume offset for persistence), while is + /// the size of the part just received, used for live speed accounting. + /// Throws like ProgressCallback when the task is canceled or paused. + /// + public void ReportParallelProgress(long confirmedPrefix, int chunkBytes, long totalSize) + { + if (state == StateTask.Canceled) + throw new Exception($"Canceled {name}"); + if (state == StateTask.Paused) + { + state = StateTask.Working; + tis.deleteDownloadInList(this); + throw new Exception($"Paused {name}"); + } + tis.addDownloadBytes(chunkBytes); + mutex.WaitOne(); + _transmitted = confirmedPrefix; + _sizeString = HelperService.SizeSuffix(totalSize); + _transmittedString = HelperService.SizeSuffix(confirmedPrefix); + progress = Convert.ToInt32(confirmedPrefix * 100 / totalSize); + EventChanged?.Invoke(this, new DownloadEventArgs()); + + OnProgressPersist?.Invoke(_transmitted, progress, state); + + if (confirmedPrefix == totalSize) + { + endnDate = DateTime.Now; + state = StateTask.Completed; + EventStatechanged?.Invoke(this, EventArgs.Empty); + NotificationModel nm = new NotificationModel(); + nm.sendEvent(new Notification($"Download {name} completed", "Download Completed", NotificationTypes.Success)); + tis.CheckPendingDownloads(); + } + mutex.ReleaseMutex(); + } + public void Cancel() { mutex.WaitOne(); diff --git a/TelegramDownloader/Models/GeneralConfig.cs b/TelegramDownloader/Models/GeneralConfig.cs index b6a606d..83e78b7 100644 --- a/TelegramDownloader/Models/GeneralConfig.cs +++ b/TelegramDownloader/Models/GeneralConfig.cs @@ -189,6 +189,25 @@ public StreamingMode GetEffectiveStreamingMode() /// public int ParallelTransfers { get; set; } = 4; + // Multi-connection download settings + /// + /// EXPERIMENTAL: download large files using several parallel MTProto + /// connections, the same technique Telegram Desktop uses to reach high + /// speeds. Telegram limits throughput per connection (~5-6 MB/s), so a + /// single connection cannot go faster no matter how many chunks are in + /// flight; multiple connections each get their own allowance. Enabling + /// this creates up to DownloadConnections extra sessions on the account + /// (visible in Telegram's device list); they are created once, stored + /// next to the main session file and reused across restarts. + /// + public bool EnableMultiConnectionDownloads { get; set; } = false; + + /// + /// Number of parallel connections used per file download (2-8). + /// Only used when EnableMultiConnectionDownloads is true. + /// + public int DownloadConnections { get; set; } = 4; + } public class TLConfig diff --git a/TelegramDownloader/Pages/Config.razor b/TelegramDownloader/Pages/Config.razor index 2665831..07ecc15 100644 --- a/TelegramDownloader/Pages/Config.razor +++ b/TelegramDownloader/Pages/Config.razor @@ -245,6 +245,42 @@
+
+
+
+ + Multi-Connection Downloads +
+
+ Download large files (>32MB) using several parallel connections, like Telegram Desktop does. + Telegram limits speed per connection (~5-6 MB/s), so this is the way to go faster. + Creates extra sessions on your account (visible in Telegram's device list as "TFM parallel download"), created once and reused. + Experimental +
+
+
+ +
+
+ + @if (Model!.EnableMultiConnectionDownloads) + { +
+
+
+ + Download Connections +
+
+ Number of parallel connections used per file download (2-8) +
+
+
+ +
+
+ } +
From 1c4a832ceadb864d369214a1ad99d2b9a8086800 Mon Sep 17 00:00:00 2001 From: mateofuentespombo Date: Tue, 21 Jul 2026 23:21:06 +0200 Subject: [PATCH 13/33] fix: enable multi-connection mode on the file-manager download path Multi-connection downloads were only hooked into DownloadFile, but file-manager download tasks go through DownloadFileNow, which calls DownloadFileAndReturn with a FileStream target, so the feature never triggered on the most common path. Hook DownloadFileAndReturn (and the offset==0 branch of DownloadFileAndReturnWithOffset) when the destination is a FileStream, which supports the positional writes multi-connection mode requires. Memory and non-seekable targets keep the sequential path. --- TelegramDownloader/Data/TelegramService.cs | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/TelegramDownloader/Data/TelegramService.cs b/TelegramDownloader/Data/TelegramService.cs index 0711cbd..ad3a908 100644 --- a/TelegramDownloader/Data/TelegramService.cs +++ b/TelegramDownloader/Data/TelegramService.cs @@ -1525,8 +1525,16 @@ public async Task DownloadFileAndReturn(ChatMessages message, Stream ms model.name = filename; _logger.LogInformation("Starting document download - FileName: {FileName}, Size: {SizeMB:F2}MB", filename, document.size / (1024.0 * 1024.0)); MemoryStream dest = new MemoryStream(); - await PrepareTransferClientAsync(document.dc_id); - await client.DownloadFileAsync(document, ms ?? dest, (PhotoSizeBase)null, model.ProgressCallback); + // File-manager download tasks land here with a FileStream target, + // which supports positional writes for multi-connection mode. + bool multiConnDone = false; + if (ms is FileStream fileDest && ShouldUseMultiConnection(document)) + multiConnDone = await TryMultiConnectionDownloadAsync(document, fileDest, model); + if (!multiConnDone) + { + await PrepareTransferClientAsync(document.dc_id); + await client.DownloadFileAsync(document, ms ?? dest, (PhotoSizeBase)null, model.ProgressCallback); + } _logger.LogInformation("Document download completed - FileName: {FileName}", filename); return ms ?? dest; } @@ -1574,8 +1582,14 @@ public async Task DownloadFileAndReturnWithOffset(ChatMessages message, if (offset == 0) { MemoryStream dest = new MemoryStream(); - await PrepareTransferClientAsync(document.dc_id); - await client.DownloadFileAsync(document, ms ?? dest, (PhotoSizeBase)null, model.ProgressCallback); + bool multiConnDone = false; + if (ms is FileStream fileDest && ShouldUseMultiConnection(document)) + multiConnDone = await TryMultiConnectionDownloadAsync(document, fileDest, model); + if (!multiConnDone) + { + await PrepareTransferClientAsync(document.dc_id); + await client.DownloadFileAsync(document, ms ?? dest, (PhotoSizeBase)null, model.ProgressCallback); + } _logger.LogInformation("Document download completed - FileName: {FileName}", filename); return ms ?? dest; } From 51ec46bc6784fc06c87c87667c33e880d9bf74c2 Mon Sep 17 00:00:00 2001 From: mateofuentespombo Date: Tue, 21 Jul 2026 23:49:21 +0200 Subject: [PATCH 14/33] fix: bootstrap download pool via neighbor DC to avoid DC_ID_INVALID Telegram rejects auth.exportAuthorization towards the DC the caller is already connected to, so the pool bootstrap failed with DC_ID_INVALID whenever the file lived on the account home DC (the common case) and downloads always fell back to a single connection. Home the pool clients on a neighbor DC instead: exporting from the main client to the neighbor is always cross-DC, and from there the pool client can export towards any file DC, including the account home DC - both hops are cross-DC and accepted. This also makes the pool global rather than per-DC (one set of sessions serves every DC through per-file-DC transfer clients resolved with GetClientForDC), and the bootstrap failure log now includes the error message. --- TelegramDownloader/Data/TelegramService.cs | 121 ++++++++++++++------- 1 file changed, 82 insertions(+), 39 deletions(-) diff --git a/TelegramDownloader/Data/TelegramService.cs b/TelegramDownloader/Data/TelegramService.cs index ad3a908..89e3e1e 100644 --- a/TelegramDownloader/Data/TelegramService.cs +++ b/TelegramDownloader/Data/TelegramService.cs @@ -238,18 +238,23 @@ private async Task PrepareTransferClientAsync(int dcId) // Telegram enforces its throughput limit PER CONNECTION (~5-6 MB/s), so a // single MTProto connection cannot go faster regardless of how many chunks // are pipelined. Official clients reach high speeds by opening several - // sessions to the file's DC and splitting the file between them. This pool - // holds extra authorized clients per DC (bootstrapped once from the main - // client via auth.exportAuthorization/importAuthorization, persisted as - // session files and reused across restarts). - private class DcDownloadPool + // sessions to the file's DC and splitting the file between them. + // + // This pool holds extra authorized account sessions (persisted as session + // files and reused across restarts). The server rejects + // auth.exportAuthorization towards the DC the caller is already on + // (DC_ID_INVALID), so pool clients are homed on a NEIGHBOR DC different + // from the account's home DC: main -> neighbor is always a cross-DC + // export, and from the neighbor the pool client can then export towards + // any file DC (including the account's home DC) - both hops cross-DC. + private class DownloadPool { public readonly SemaphoreSlim initLock = new SemaphoreSlim(1, 1); public readonly List clients = new List(); public bool bootstrapFailed = false; } - private static readonly Dictionary downloadPools = new Dictionary(); + private static readonly DownloadPool downloadPool = new DownloadPool(); private const int MULTICONN_PART_SIZE = 1024 * 1024; // upload.getFile max limit per request private const int MULTICONN_BLOCK_SIZE = 4 * 1024 * 1024; // work unit assigned to a connection private const long MULTICONN_MIN_FILE_SIZE = 32L * 1024 * 1024; @@ -265,59 +270,57 @@ private static bool ShouldUseMultiConnection(TL.Document document) return cfg != null && cfg.EnableMultiConnectionDownloads && document.size >= MULTICONN_MIN_FILE_SIZE; } - private async Task> GetDownloadPoolAsync(int dcId, int count) + private async Task> GetDownloadPoolAsync(int count) { - DcDownloadPool pool; - lock (downloadPools) - { - if (!downloadPools.TryGetValue(dcId, out pool)) - downloadPools[dcId] = pool = new DcDownloadPool(); - } - if (pool.bootstrapFailed) + if (downloadPool.bootstrapFailed) return new List(); - await pool.initLock.WaitAsync(); + await downloadPool.initLock.WaitAsync(); try { - pool.clients.RemoveAll(c => + downloadPool.clients.RemoveAll(c => { if (!c.Disconnected) return false; try { c.Dispose(); } catch { } return true; }); - while (pool.clients.Count < count) + while (downloadPool.clients.Count < count) { - WTelegram.Client pc = await CreateDownloadPoolClientAsync(dcId, pool.clients.Count); + WTelegram.Client pc = await CreateDownloadPoolClientAsync(downloadPool.clients.Count); if (pc == null) { - // Do not retry the bootstrap on every download if the DC - // refuses it (e.g. exportAuthorization not allowed). - if (pool.clients.Count == 0) - pool.bootstrapFailed = true; + // Do not retry the bootstrap on every download if the + // server refuses it. + if (downloadPool.clients.Count == 0) + downloadPool.bootstrapFailed = true; break; } - pool.clients.Add(pc); + downloadPool.clients.Add(pc); } - return pool.clients.Take(count).ToList(); + return downloadPool.clients.Take(count).ToList(); } finally { - pool.initLock.Release(); + downloadPool.initLock.Release(); } } - private async Task CreateDownloadPoolClientAsync(int dcId, int index) + private async Task CreateDownloadPoolClientAsync(int index) { - string sessionPath = Path.Combine(UserService.USERDATAFOLDER, $"WTelegram_dl_dc{dcId}_{index}.session"); + string sessionPath = Path.Combine(UserService.USERDATAFOLDER, $"WTelegram_dl_{index}.session"); try { + // Home the pool client on a DC different from the account's home + // DC, because exporting an authorization towards the caller's own + // DC is rejected with DC_ID_INVALID. TL.Config tlConfig = await client.Help_GetConfig(); + int mainHomeDc = tlConfig.this_dc; DcOption dc = tlConfig.dc_options - .Where(x => x.id == dcId && (x.flags & (DcOption.Flags.ipv6 | DcOption.Flags.cdn | DcOption.Flags.tcpo_only)) == 0) - .OrderBy(x => (x.flags & DcOption.Flags.media_only) == 0 ? 0 : 1) + .Where(x => x.id != mainHomeDc && (x.flags & (DcOption.Flags.ipv6 | DcOption.Flags.cdn | DcOption.Flags.tcpo_only | DcOption.Flags.media_only)) == 0) + .OrderBy(x => x.id) .FirstOrDefault(); if (dc == null) { - _logger.LogWarning("No suitable address found for DC {Dc} - multi-connection download unavailable", dcId); + _logger.LogWarning("No suitable neighbor DC found (home DC {Dc}) - multi-connection download unavailable", mainHomeDc); return null; } string apiId = GeneralConfigStatic.tlconfig?.api_id ?? Environment.GetEnvironmentVariable("api_id"); @@ -347,12 +350,11 @@ private static bool ShouldUseMultiConnection(TL.Document document) } if (!authorized) { - Auth_ExportedAuthorization exported = await client.Auth_ExportAuthorization(dcId); + Auth_ExportedAuthorization exported = await client.Auth_ExportAuthorization(dc.id); await pc.Auth_ImportAuthorization(exported.id, exported.bytes); await pc.Users_GetUsers(InputUser.Self); } - ApplyConfiguredParallelTransfers(pc); - _logger.LogInformation("Download pool client {Index} ready for DC {Dc}", index, dcId); + _logger.LogInformation("Download pool client {Index} ready (homed on DC {Dc})", index, dc.id); return pc; } catch @@ -363,11 +365,41 @@ private static bool ShouldUseMultiConnection(TL.Document document) } catch (Exception ex) { - _logger.LogWarning(ex, "Could not create download pool client {Index} for DC {Dc} - falling back to single-connection downloads", index, dcId); + _logger.LogWarning(ex, "Could not create download pool client {Index}: {Error} - falling back to single-connection downloads", index, ex.Message); return null; } } + /// + /// Returns a client of connected to the given DC, + /// importing an authorization for it on first use. Both the pool client's + /// home and the file DC are covered: when they match, the owner itself is + /// returned; otherwise the export is cross-DC (owner's home is never the + /// account's home DC by construction) and therefore accepted. + /// + private async Task GetAuthorizedTransferClientAsync(WTelegram.Client owner, int dcId) + { + WTelegram.Client transfer = await owner.GetClientForDC(dcId, true); + if (transfer != owner) + { + bool authorized = false; + try + { + await transfer.Users_GetUsers(InputUser.Self); + authorized = true; + } + catch (RpcException) + { + } + if (!authorized) + { + Auth_ExportedAuthorization exported = await owner.Auth_ExportAuthorization(dcId); + await transfer.Auth_ImportAuthorization(exported.id, exported.bytes); + } + } + return transfer; + } + /// /// Downloads a document by splitting it in blocks served concurrently by /// several pool connections, writing each part at its absolute offset. @@ -378,17 +410,28 @@ private static bool ShouldUseMultiConnection(TL.Document document) private async Task TryMultiConnectionDownloadAsync(TL.Document document, FileStream dest, DownloadModel model) { long size = document.size; - List pool; + List transfers = new List(); try { - pool = await GetDownloadPoolAsync(document.dc_id, GetConfiguredDownloadConnections()); + List pool = await GetDownloadPoolAsync(GetConfiguredDownloadConnections()); + foreach (WTelegram.Client owner in pool) + { + try + { + transfers.Add(await GetAuthorizedTransferClientAsync(owner, document.dc_id)); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Pool connection unavailable for DC {Dc}: {Error}", document.dc_id, ex.Message); + } + } } catch (Exception ex) { _logger.LogWarning(ex, "Download pool unavailable for DC {Dc}", document.dc_id); return false; } - if (pool.Count < 2) + if (transfers.Count < 2) return false; var location = new InputDocumentFileLocation @@ -400,7 +443,7 @@ private async Task TryMultiConnectionDownloadAsync(TL.Document document, F }; _logger.LogInformation("Multi-connection download - FileName: {Name}, Size: {SizeMB:F2}MB, Connections: {Connections}", - model.name, size / (1024.0 * 1024.0), pool.Count); + model.name, size / (1024.0 * 1024.0), transfers.Count); long blockCount = (size + MULTICONN_BLOCK_SIZE - 1) / MULTICONN_BLOCK_SIZE; bool[] blockDone = new bool[blockCount]; @@ -472,7 +515,7 @@ async Task GuardedWorker(WTelegram.Client pc) try { - await Task.WhenAll(pool.Select(pc => Task.Run(() => GuardedWorker(pc)))); + await Task.WhenAll(transfers.Select(pc => Task.Run(() => GuardedWorker(pc)))); await dest.FlushAsync(); return true; } From 39e2b5202d58d781ab65b8415e5abd48c92539df Mon Sep 17 00:00:00 2001 From: mateofuentespombo Date: Wed, 22 Jul 2026 00:34:14 +0200 Subject: [PATCH 15/33] fix: finalize pool client login and drop probe RPCs after import The pool bootstrap verified the imported authorization with users.getUsers, which answers AUTH_KEY_UNREGISTERED on an imported session even though the import succeeded and file requests work, so the bootstrap always aborted and downloads fell back to a single connection. Import the exported authorization as the first call on the fresh client, finalize it client-side with LoginAlreadyDone (recording the UserId in the session), and make no verification RPC - workers verify by use and fall back on error. With the UserId recorded, WTelegram's GetClientForDC now handles the per-file-DC authorization automatically, replacing the manual transfer-client export/import logic; persisted sessions skip the import on later runs via the recorded UserId. --- TelegramDownloader/Data/TelegramService.cs | 58 ++++++++-------------- 1 file changed, 20 insertions(+), 38 deletions(-) diff --git a/TelegramDownloader/Data/TelegramService.cs b/TelegramDownloader/Data/TelegramService.cs index 89e3e1e..c04f91a 100644 --- a/TelegramDownloader/Data/TelegramService.cs +++ b/TelegramDownloader/Data/TelegramService.cs @@ -337,22 +337,21 @@ private static bool ShouldUseMultiConnection(TL.Document document) try { await pc.ConnectAsync(); - bool authorized = false; - try - { - await pc.Users_GetUsers(InputUser.Self); - authorized = true; - } - catch (RpcException) - { - // Fresh session, or a previously created one revoked from - // the account's device list: (re)import the authorization. - } - if (!authorized) + if (pc.UserId == 0) { + // Fresh session: import the account authorization exported + // by the main client (cross-DC, so it is accepted), then + // finalize the login client-side with LoginAlreadyDone so + // the session records the UserId. An imported session is + // not a fully "logged" one - probe RPCs like + // users.getUsers can answer AUTH_KEY_UNREGISTERED even + // though file requests work - so no verification call is + // made here: workers verify by use and fall back on error. + // With the UserId recorded, WTelegram's GetClientForDC + // handles the per-file-DC authorization automatically. Auth_ExportedAuthorization exported = await client.Auth_ExportAuthorization(dc.id); - await pc.Auth_ImportAuthorization(exported.id, exported.bytes); - await pc.Users_GetUsers(InputUser.Self); + Auth_AuthorizationBase auth = await pc.Auth_ImportAuthorization(exported.id, exported.bytes); + pc.LoginAlreadyDone(auth); } _logger.LogInformation("Download pool client {Index} ready (homed on DC {Dc})", index, dc.id); return pc; @@ -371,33 +370,16 @@ private static bool ShouldUseMultiConnection(TL.Document document) } /// - /// Returns a client of connected to the given DC, - /// importing an authorization for it on first use. Both the pool client's - /// home and the file DC are covered: when they match, the owner itself is - /// returned; otherwise the export is cross-DC (owner's home is never the - /// account's home DC by construction) and therefore accepted. + /// Returns a client of connected to the given DC. + /// Because the pool client's session records a UserId (LoginAlreadyDone), + /// WTelegram's GetClientForDC exports/imports the authorization towards + /// the file DC automatically when needed; the export is cross-DC (the + /// owner's home is never the account's home DC by construction), so it is + /// accepted even when the file lives on the account's home DC. /// private async Task GetAuthorizedTransferClientAsync(WTelegram.Client owner, int dcId) { - WTelegram.Client transfer = await owner.GetClientForDC(dcId, true); - if (transfer != owner) - { - bool authorized = false; - try - { - await transfer.Users_GetUsers(InputUser.Self); - authorized = true; - } - catch (RpcException) - { - } - if (!authorized) - { - Auth_ExportedAuthorization exported = await owner.Auth_ExportAuthorization(dcId); - await transfer.Auth_ImportAuthorization(exported.id, exported.bytes); - } - } - return transfer; + return await owner.GetClientForDC(dcId, true); } /// From 1f87e1358cba9255a8d7620fe343a47de7da3462 Mon Sep 17 00:00:00 2001 From: mateofuentespombo Date: Wed, 22 Jul 2026 01:08:34 +0200 Subject: [PATCH 16/33] fix: tolerate re-import on an already-authorized pool session key Session files persisted by earlier runs (where the import succeeded but the login was never finalized client-side) hold a key that already carries the account authorization; re-importing onto it is rejected by the server with AUTH_BYTES_INVALID and the bootstrap aborted. Treat AUTH_BYTES_INVALID on import as already-authorized: finalize the login client-side with the user id returned by exportAuthorization and let the workers verify the session by use, falling back to the sequential download if it turns out to be broken. --- TelegramDownloader/Data/TelegramService.cs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/TelegramDownloader/Data/TelegramService.cs b/TelegramDownloader/Data/TelegramService.cs index c04f91a..011fde9 100644 --- a/TelegramDownloader/Data/TelegramService.cs +++ b/TelegramDownloader/Data/TelegramService.cs @@ -350,8 +350,21 @@ private static bool ShouldUseMultiConnection(TL.Document document) // With the UserId recorded, WTelegram's GetClientForDC // handles the per-file-DC authorization automatically. Auth_ExportedAuthorization exported = await client.Auth_ExportAuthorization(dc.id); - Auth_AuthorizationBase auth = await pc.Auth_ImportAuthorization(exported.id, exported.bytes); - pc.LoginAlreadyDone(auth); + Auth_AuthorizationBase auth = null; + try + { + auth = await pc.Auth_ImportAuthorization(exported.id, exported.bytes); + } + catch (RpcException rex) when (rex.Message == "AUTH_BYTES_INVALID") + { + // Session persisted by an earlier run whose import + // succeeded but whose login was never finalized: the + // key already holds the account authorization and + // re-importing onto it is rejected. Record the login + // client-side and let the workers verify by use. + _logger.LogInformation("Download pool client {Index}: key already authorized on a previous run", index); + } + pc.LoginAlreadyDone(auth ?? new Auth_Authorization { user = new User { id = exported.id } }); } _logger.LogInformation("Download pool client {Index} ready (homed on DC {Dc})", index, dc.id); return pc; From d9e673a7371f0ff5594a5386301ef27e53a98ac4 Mon Sep 17 00:00:00 2001 From: mateofuentespombo Date: Wed, 22 Jul 2026 01:40:16 +0200 Subject: [PATCH 17/33] feat: pool clients clone the main session instead of importing auth The exportAuthorization/importAuthorization bootstrap turned out to be a dead end: an imported session is a limited authorization - file requests work, but probe RPCs and re-exports answer AUTH_KEY_UNREGISTERED - so the pool client could never authorize its per-file-DC transfer connections (GetClientForDC's automatic export failed with AUTH_KEY_UNREGISTERED) and downloads always fell back. Clone the main session file instead: each pool client loads a copy of WTelegram.session, sharing the fully-authorized main auth key, while WTelegram assigns a fresh transient MTProto session id per client instance. The server sees extra connections of the existing authorization - the exact model official clients use for parallel downloads. Telegram's throughput limit is per connection/session, not per auth key, so each clone gets its own allowance. No authorizations are created, nothing appears in the device list, and cross-DC files keep working because the clones are fully logged sessions. --- TelegramDownloader/Data/TelegramService.cs | 98 +++++++++------------- TelegramDownloader/Models/GeneralConfig.cs | 7 +- TelegramDownloader/Pages/Config.razor | 2 +- 3 files changed, 45 insertions(+), 62 deletions(-) diff --git a/TelegramDownloader/Data/TelegramService.cs b/TelegramDownloader/Data/TelegramService.cs index 011fde9..19ad2dd 100644 --- a/TelegramDownloader/Data/TelegramService.cs +++ b/TelegramDownloader/Data/TelegramService.cs @@ -240,13 +240,16 @@ private async Task PrepareTransferClientAsync(int dcId) // are pipelined. Official clients reach high speeds by opening several // sessions to the file's DC and splitting the file between them. // - // This pool holds extra authorized account sessions (persisted as session - // files and reused across restarts). The server rejects - // auth.exportAuthorization towards the DC the caller is already on - // (DC_ID_INVALID), so pool clients are homed on a NEIGHBOR DC different - // from the account's home DC: main -> neighbor is always a cross-DC - // export, and from the neighbor the pool client can then export towards - // any file DC (including the account's home DC) - both hops cross-DC. + // This pool holds CLONES of the main session: each pool client loads a + // copy of the main session file, sharing its fully-authorized auth key, + // while WTelegram assigns a fresh MTProto session id per client instance + // (the id is transient, never persisted). The server sees them as extra + // connections of the existing authorization - the exact model official + // clients use for parallel downloads - so no new authorization is + // created, nothing shows up in the account's device list, and the + // auth.exportAuthorization/importAuthorization route (whose imported + // sessions turned out to be limited: file requests work but probe RPCs + // and re-exports answer AUTH_KEY_UNREGISTERED) is not needed at all. private class DownloadPool { public readonly SemaphoreSlim initLock = new SemaphoreSlim(1, 1); @@ -306,23 +309,16 @@ private static bool ShouldUseMultiConnection(TL.Document document) private async Task CreateDownloadPoolClientAsync(int index) { + string mainSessionPath = UserService.USERDATAFOLDER + "/WTelegram.session"; string sessionPath = Path.Combine(UserService.USERDATAFOLDER, $"WTelegram_dl_{index}.session"); try { - // Home the pool client on a DC different from the account's home - // DC, because exporting an authorization towards the caller's own - // DC is rejected with DC_ID_INVALID. - TL.Config tlConfig = await client.Help_GetConfig(); - int mainHomeDc = tlConfig.this_dc; - DcOption dc = tlConfig.dc_options - .Where(x => x.id != mainHomeDc && (x.flags & (DcOption.Flags.ipv6 | DcOption.Flags.cdn | DcOption.Flags.tcpo_only | DcOption.Flags.media_only)) == 0) - .OrderBy(x => x.id) - .FirstOrDefault(); - if (dc == null) - { - _logger.LogWarning("No suitable neighbor DC found (home DC {Dc}) - multi-connection download unavailable", mainHomeDc); - return null; - } + // Clone the main session file. Always copy fresh so leftovers from + // older bootstrap strategies (or a re-login of the main account) + // are overwritten. The file is encrypted with a key derived from + // the same api_hash, so the clone can read it as its own. + CopySessionWithRetry(mainSessionPath, sessionPath); + string apiId = GeneralConfigStatic.tlconfig?.api_id ?? Environment.GetEnvironmentVariable("api_id"); string apiHash = GeneralConfigStatic.tlconfig?.hash_id ?? Environment.GetEnvironmentVariable("hash_id"); WTelegram.Client pc = new WTelegram.Client(what => what switch @@ -330,43 +326,14 @@ private static bool ShouldUseMultiConnection(TL.Document document) "api_id" => apiId, "api_hash" => apiHash, "session_pathname" => sessionPath, - "server_address" => $"{dc.ip_address}:{dc.port}", - "device_model" => "TFM parallel download", _ => null }); try { await pc.ConnectAsync(); if (pc.UserId == 0) - { - // Fresh session: import the account authorization exported - // by the main client (cross-DC, so it is accepted), then - // finalize the login client-side with LoginAlreadyDone so - // the session records the UserId. An imported session is - // not a fully "logged" one - probe RPCs like - // users.getUsers can answer AUTH_KEY_UNREGISTERED even - // though file requests work - so no verification call is - // made here: workers verify by use and fall back on error. - // With the UserId recorded, WTelegram's GetClientForDC - // handles the per-file-DC authorization automatically. - Auth_ExportedAuthorization exported = await client.Auth_ExportAuthorization(dc.id); - Auth_AuthorizationBase auth = null; - try - { - auth = await pc.Auth_ImportAuthorization(exported.id, exported.bytes); - } - catch (RpcException rex) when (rex.Message == "AUTH_BYTES_INVALID") - { - // Session persisted by an earlier run whose import - // succeeded but whose login was never finalized: the - // key already holds the account authorization and - // re-importing onto it is rejected. Record the login - // client-side and let the workers verify by use. - _logger.LogInformation("Download pool client {Index}: key already authorized on a previous run", index); - } - pc.LoginAlreadyDone(auth ?? new Auth_Authorization { user = new User { id = exported.id } }); - } - _logger.LogInformation("Download pool client {Index} ready (homed on DC {Dc})", index, dc.id); + throw new InvalidOperationException("Cloned session has no logged-in user"); + _logger.LogInformation("Download pool client {Index} ready (cloned session, user {UserId})", index, pc.UserId); return pc; } catch @@ -382,13 +349,30 @@ private static bool ShouldUseMultiConnection(TL.Document document) } } + private static void CopySessionWithRetry(string source, string destination) + { + // The main client rewrites its session file on saves; retry briefly in + // case the copy races with a write. + for (int attempt = 1; ; attempt++) + { + try + { + File.Copy(source, destination, overwrite: true); + return; + } + catch (IOException) when (attempt < 3) + { + Thread.Sleep(200 * attempt); + } + } + } + /// /// Returns a client of connected to the given DC. - /// Because the pool client's session records a UserId (LoginAlreadyDone), - /// WTelegram's GetClientForDC exports/imports the authorization towards - /// the file DC automatically when needed; the export is cross-DC (the - /// owner's home is never the account's home DC by construction), so it is - /// accepted even when the file lives on the account's home DC. + /// The clone's home DC is the account's home DC, so files there are served + /// by the clone's own connection; for files on other DCs, the clone is a + /// fully logged session and WTelegram's GetClientForDC handles the + /// cross-DC authorization automatically. /// private async Task GetAuthorizedTransferClientAsync(WTelegram.Client owner, int dcId) { diff --git a/TelegramDownloader/Models/GeneralConfig.cs b/TelegramDownloader/Models/GeneralConfig.cs index 83e78b7..a35270b 100644 --- a/TelegramDownloader/Models/GeneralConfig.cs +++ b/TelegramDownloader/Models/GeneralConfig.cs @@ -195,10 +195,9 @@ public StreamingMode GetEffectiveStreamingMode() /// connections, the same technique Telegram Desktop uses to reach high /// speeds. Telegram limits throughput per connection (~5-6 MB/s), so a /// single connection cannot go faster no matter how many chunks are in - /// flight; multiple connections each get their own allowance. Enabling - /// this creates up to DownloadConnections extra sessions on the account - /// (visible in Telegram's device list); they are created once, stored - /// next to the main session file and reused across restarts. + /// flight; multiple connections each get their own allowance. The extra + /// connections are clones of the main session (same authorization, own + /// connection), so no new device entries are created on the account. /// public bool EnableMultiConnectionDownloads { get; set; } = false; diff --git a/TelegramDownloader/Pages/Config.razor b/TelegramDownloader/Pages/Config.razor index 07ecc15..6d92b1b 100644 --- a/TelegramDownloader/Pages/Config.razor +++ b/TelegramDownloader/Pages/Config.razor @@ -254,7 +254,7 @@
Download large files (>32MB) using several parallel connections, like Telegram Desktop does. Telegram limits speed per connection (~5-6 MB/s), so this is the way to go faster. - Creates extra sessions on your account (visible in Telegram's device list as "TFM parallel download"), created once and reused. + The extra connections share your existing session authorization, so nothing new appears in your Telegram device list. Experimental
From 09f60c919010a7951764344c8c0007a593c4f124 Mon Sep 17 00:00:00 2001 From: mateofuentespombo Date: Wed, 22 Jul 2026 09:21:05 +0200 Subject: [PATCH 18/33] fix: clone the session despite the live file lock The main client keeps WTelegram.session open (often exclusively), so File.Copy failed with a sharing violation and the pool bootstrap always fell back to single-connection downloads. Snapshot the session file at client creation, before WTelegram opens and locks it, and make the clone routine try a share-friendly stream copy of the live file first, falling back to the startup snapshot. Session state staleness is harmless: the auth key never changes and server salts are renegotiated automatically. --- TelegramDownloader/Data/TelegramService.cs | 39 ++++++++++++++++++---- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/TelegramDownloader/Data/TelegramService.cs b/TelegramDownloader/Data/TelegramService.cs index 19ad2dd..e0feb5f 100644 --- a/TelegramDownloader/Data/TelegramService.cs +++ b/TelegramDownloader/Data/TelegramService.cs @@ -160,6 +160,19 @@ private void createDownloadFolder() private void newClient() { + // Snapshot the session file BEFORE the client opens (and locks) it, + // so the multi-connection download pool can clone the session even + // while the live file is held exclusively by this client. + try + { + string mainSession = UserService.USERDATAFOLDER + "/WTelegram.session"; + if (File.Exists(mainSession)) + File.Copy(mainSession, mainSession + ".snapshot", overwrite: true); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Could not snapshot the session file"); + } client = new WTelegram.Client(Convert.ToInt32(GeneralConfigStatic.tlconfig?.api_id ?? Environment.GetEnvironmentVariable("api_id")), GeneralConfigStatic.tlconfig?.hash_id ?? Environment.GetEnvironmentVariable("hash_id"), UserService.USERDATAFOLDER + "/WTelegram.session"); ApplyConfiguredParallelTransfers(client); if (GeneralConfigStatic.config.ShouldShowLogInTerminal) @@ -351,20 +364,34 @@ private static bool ShouldUseMultiConnection(TL.Document document) private static void CopySessionWithRetry(string source, string destination) { - // The main client rewrites its session file on saves; retry briefly in - // case the copy races with a write. - for (int attempt = 1; ; attempt++) + // The main client keeps its session file open (often exclusively), so + // File.Copy cannot read it. Try a share-friendly stream copy of the + // live file first, then fall back to the snapshot taken at startup + // before the client opened the file. Session state staleness is fine: + // the auth key never changes and salts are renegotiated automatically. + IOException lastError = null; + for (int attempt = 1; attempt <= 3; attempt++) { try { - File.Copy(source, destination, overwrite: true); + using (FileStream src = new FileStream(source, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete)) + using (FileStream dst = new FileStream(destination, FileMode.Create, FileAccess.Write)) + src.CopyTo(dst); return; } - catch (IOException) when (attempt < 3) + catch (IOException ex) { - Thread.Sleep(200 * attempt); + lastError = ex; + Thread.Sleep(150 * attempt); } } + string snapshot = source + ".snapshot"; + if (File.Exists(snapshot)) + { + File.Copy(snapshot, destination, overwrite: true); + return; + } + throw lastError; } /// From 895097bcc32a5c31dd3cd8be5b01d563f9787e8e Mon Sep 17 00:00:00 2001 From: Mateo Date: Wed, 22 Jul 2026 12:27:37 +0200 Subject: [PATCH 19/33] feat: pipeline file parts within each download connection (#107) * feat: pipeline file parts within each download connection Workers requested their block's 1MB parts sequentially, paying a full round-trip of dead time per part and capping each connection around 3-4 MB/s regardless of its server-side allowance - the multi-connection download barely improved on a single connection (~10 MB/s with dips). Request all parts of a block concurrently on the block's connection, keeping each connection's pipe full (the same pipelining official clients use), and log a completion summary with the effective average speed to make future tuning measurable. Part writes stay positional and block completion still gates the contiguous confirmed prefix used for progress and resume persistence. * feat: make multi-connection tuning configurable with documented defaults Expose the remaining hardcoded transfer knobs in General Config: chunk size (snapped to Telegram's allowed 128/256/512/1024 KB values, 512 being WTelegramClient's own default), block size per connection (which determines the requests in flight per connection) and the minimum file size for multi-connection mode. Values are captured once per download so a config change cannot desynchronize offsets mid-transfer. The Config page now states the library default, app default and recommended value for each transfer setting, so the stock WTelegramClient behavior (2 parallel chunks, 512KB parts, single connection) can be restored by configuration alone. --- TelegramDownloader/Data/TelegramService.cs | 128 ++++++++++++++------- TelegramDownloader/Models/GeneralConfig.cs | 27 +++++ TelegramDownloader/Pages/Config.razor | 68 ++++++++++- 3 files changed, 179 insertions(+), 44 deletions(-) diff --git a/TelegramDownloader/Data/TelegramService.cs b/TelegramDownloader/Data/TelegramService.cs index e0feb5f..c35f282 100644 --- a/TelegramDownloader/Data/TelegramService.cs +++ b/TelegramDownloader/Data/TelegramService.cs @@ -271,19 +271,42 @@ private class DownloadPool } private static readonly DownloadPool downloadPool = new DownloadPool(); - private const int MULTICONN_PART_SIZE = 1024 * 1024; // upload.getFile max limit per request - private const int MULTICONN_BLOCK_SIZE = 4 * 1024 * 1024; // work unit assigned to a connection - private const long MULTICONN_MIN_FILE_SIZE = 32L * 1024 * 1024; public static int GetConfiguredDownloadConnections() { return Math.Clamp(GeneralConfigStatic.config?.DownloadConnections ?? 4, 2, 8); } + /// + /// Chunk size for upload.getFile: Telegram only accepts limits that are + /// divisible by 4KB and divide 1MB evenly, so the configured value is + /// snapped to 128/256/512/1024 KB. 512 is WTelegramClient's own default; + /// 1024 (app default) halves the number of round-trips. + /// + private static int GetConfiguredPartSize() + { + int kb = GeneralConfigStatic.config?.MultiConnectionPartSizeKB ?? 1024; + if (kb >= 1024) return 1024 * 1024; + if (kb >= 512) return 512 * 1024; + if (kb >= 256) return 256 * 1024; + return 128 * 1024; + } + + private static int GetConfiguredBlockSize(int partSize) + { + int mb = Math.Clamp(GeneralConfigStatic.config?.MultiConnectionBlockSizeMB ?? 4, 1, 16); + return Math.Max(mb * 1024 * 1024, partSize); + } + + private static long GetConfiguredMinFileSize() + { + return Math.Max(1, GeneralConfigStatic.config?.MultiConnectionMinFileSizeMB ?? 32) * 1024L * 1024L; + } + private static bool ShouldUseMultiConnection(TL.Document document) { GeneralConfig cfg = GeneralConfigStatic.config; - return cfg != null && cfg.EnableMultiConnectionDownloads && document.size >= MULTICONN_MIN_FILE_SIZE; + return cfg != null && cfg.EnableMultiConnectionDownloads && document.size >= GetConfiguredMinFileSize(); } private async Task> GetDownloadPoolAsync(int count) @@ -448,10 +471,15 @@ private async Task TryMultiConnectionDownloadAsync(TL.Document document, F thumb_size = "" }; - _logger.LogInformation("Multi-connection download - FileName: {Name}, Size: {SizeMB:F2}MB, Connections: {Connections}", - model.name, size / (1024.0 * 1024.0), transfers.Count); + // Capture the tuning values once so a config change mid-download + // cannot desynchronize offsets. + int partSize = GetConfiguredPartSize(); + int blockSize = GetConfiguredBlockSize(partSize); - long blockCount = (size + MULTICONN_BLOCK_SIZE - 1) / MULTICONN_BLOCK_SIZE; + _logger.LogInformation("Multi-connection download - FileName: {Name}, Size: {SizeMB:F2}MB, Connections: {Connections}, Part: {PartKB}KB, Block: {BlockMB}MB", + model.name, size / (1024.0 * 1024.0), transfers.Count, partSize / 1024, blockSize / (1024.0 * 1024.0)); + + long blockCount = (size + blockSize - 1) / blockSize; bool[] blockDone = new bool[blockCount]; long confirmedBlocks = 0; long nextBlock = -1; @@ -459,22 +487,52 @@ private async Task TryMultiConnectionDownloadAsync(TL.Document document, F using CancellationTokenSource cts = new CancellationTokenSource(); dest.SetLength(size); var handle = dest.SafeFileHandle; + DateTime started = DateTime.Now; - void ReportPart(long block, int received, bool blockCompleted) + void ReportBytes(int received) + { + long confirmed; + lock (progressLock) + confirmed = Math.Min(size, confirmedBlocks * (long)blockSize); + // Throws when the task gets canceled or paused, stopping the workers. + model.ReportParallelProgress(confirmed, received, size); + } + + void ReportBlockDone(long block) { long confirmed; lock (progressLock) { - if (blockCompleted) + blockDone[block] = true; + while (confirmedBlocks < blockCount && blockDone[confirmedBlocks]) + confirmedBlocks++; + confirmed = Math.Min(size, confirmedBlocks * (long)blockSize); + } + model.ReportParallelProgress(confirmed, 0, size); + } + + async Task DownloadPart(WTelegram.Client pc, long offset) + { + int expected = (int)Math.Min(partSize, size - offset); + for (int attempt = 1; ; attempt++) + { + cts.Token.ThrowIfCancellationRequested(); + try { - blockDone[block] = true; - while (confirmedBlocks < blockCount && blockDone[confirmedBlocks]) - confirmedBlocks++; + Upload_FileBase resp = await pc.Upload_GetFile(location, offset, limit: partSize); + if (resp is not Upload_File part) + throw new InvalidOperationException($"Unexpected {resp?.GetType().Name} from Upload_GetFile (CDN-served files are not supported)"); + if (part.bytes.Length < expected) + throw new InvalidOperationException($"Short chunk at offset {offset}: {part.bytes.Length} < {expected}"); + RandomAccess.Write(handle, part.bytes.AsSpan(0, expected), offset); + ReportBytes(expected); + return; + } + catch (Exception) when (attempt < 3 && !cts.IsCancellationRequested) + { + await Task.Delay(1000 * attempt); } - confirmed = Math.Min(size, confirmedBlocks * (long)MULTICONN_BLOCK_SIZE); } - // Throws when the task gets canceled or paused, stopping the workers. - model.ReportParallelProgress(confirmed, received, size); } async Task Worker(WTelegram.Client pc) @@ -484,32 +542,17 @@ async Task Worker(WTelegram.Client pc) long block = Interlocked.Increment(ref nextBlock); if (block >= blockCount) return; - long offset = block * (long)MULTICONN_BLOCK_SIZE; - long end = Math.Min(size, offset + MULTICONN_BLOCK_SIZE); - while (offset < end) - { - cts.Token.ThrowIfCancellationRequested(); - Upload_FileBase resp = null; - for (int attempt = 1; ; attempt++) - { - try - { - resp = await pc.Upload_GetFile(location, offset, limit: MULTICONN_PART_SIZE); - break; - } - catch (Exception) when (attempt < 3 && !cts.IsCancellationRequested) - { - await Task.Delay(1000 * attempt); - } - } - if (resp is not Upload_File part) - throw new InvalidOperationException($"Unexpected {resp?.GetType().Name} from Upload_GetFile (CDN-served files are not supported)"); - if (part.bytes.Length == 0) - throw new InvalidOperationException($"Empty chunk at offset {offset}"); - RandomAccess.Write(handle, part.bytes, offset); - offset += part.bytes.Length; - ReportPart(block, part.bytes.Length, offset >= end); - } + long blockStart = block * (long)blockSize; + long blockEnd = Math.Min(size, blockStart + blockSize); + // Request every part of the block concurrently on this + // connection: sequential parts pay a full round-trip of dead + // time each, capping a connection well below its server-side + // allowance. Pipelining keeps the connection's pipe full. + List parts = new List(); + for (long offset = blockStart; offset < blockEnd; offset += partSize) + parts.Add(DownloadPart(pc, offset)); + await Task.WhenAll(parts); + ReportBlockDone(block); } } @@ -523,6 +566,9 @@ async Task GuardedWorker(WTelegram.Client pc) { await Task.WhenAll(transfers.Select(pc => Task.Run(() => GuardedWorker(pc)))); await dest.FlushAsync(); + double seconds = Math.Max(0.001, (DateTime.Now - started).TotalSeconds); + _logger.LogInformation("Multi-connection download completed - FileName: {Name}, {SizeMB:F1}MB in {Seconds:F1}s = {Speed:F1} MB/s over {Connections} connections", + model.name, size / (1024.0 * 1024.0), seconds, size / (1024.0 * 1024.0) / seconds, transfers.Count); return true; } catch (Exception ex) diff --git a/TelegramDownloader/Models/GeneralConfig.cs b/TelegramDownloader/Models/GeneralConfig.cs index a35270b..9cf4911 100644 --- a/TelegramDownloader/Models/GeneralConfig.cs +++ b/TelegramDownloader/Models/GeneralConfig.cs @@ -204,9 +204,36 @@ public StreamingMode GetEffectiveStreamingMode() /// /// Number of parallel connections used per file download (2-8). /// Only used when EnableMultiConnectionDownloads is true. + /// App default: 4. Recommended: 4-8. /// public int DownloadConnections { get; set; } = 4; + /// + /// Size in KB of each file chunk requested (upload.getFile limit). + /// Telegram only allows 128, 256, 512 or 1024 (values are snapped to + /// the nearest allowed one). WTelegramClient's own default is 512; + /// app default is 1024 (fewer round-trips). + /// Only used when EnableMultiConnectionDownloads is true. + /// + public int MultiConnectionPartSizeKB { get; set; } = 1024; + + /// + /// Size in MB (1-16) of the work unit assigned to each connection. + /// All parts of a block are requested in parallel on the same + /// connection, so BlockSize / PartSize = requests in flight per + /// connection. App default: 4 (= 4 x 1MB in flight). + /// Only used when EnableMultiConnectionDownloads is true. + /// + public int MultiConnectionBlockSizeMB { get; set; } = 4; + + /// + /// Files smaller than this size in MB use the normal single-connection + /// download (the setup cost is not worth it for small files). + /// App default: 32. + /// Only used when EnableMultiConnectionDownloads is true. + /// + public int MultiConnectionMinFileSizeMB { get; set; } = 32; + } public class TLConfig diff --git a/TelegramDownloader/Pages/Config.razor b/TelegramDownloader/Pages/Config.razor index 6d92b1b..3be4e2e 100644 --- a/TelegramDownloader/Pages/Config.razor +++ b/TelegramDownloader/Pages/Config.razor @@ -237,7 +237,8 @@ Parallel Chunk Transfers
- Number of 512KB chunks requested in parallel per transfer (1-16). Higher values improve download/upload speed by removing the latency bottleneck; Premium accounts benefit the most. Applied on the next transfer. + Number of chunks requested in parallel per transfer (1-16), used by standard (single-connection) downloads and uploads. Higher values remove the latency bottleneck; Premium accounts benefit the most. Applied on the next transfer. +
WTelegramClient default: 2 · App default: 4 · Recommended: 4-8
@@ -252,10 +253,11 @@ Multi-Connection Downloads
- Download large files (>32MB) using several parallel connections, like Telegram Desktop does. + Download large files using several parallel connections, like Telegram Desktop does. Telegram limits speed per connection (~5-6 MB/s), so this is the way to go faster. The extra connections share your existing session authorization, so nothing new appears in your Telegram device list. Experimental +
Default: disabled (library behavior: one connection per download) · Recommended: enabled for large files on fast lines
@@ -272,13 +274,73 @@ Download Connections
- Number of parallel connections used per file download (2-8) + Number of parallel connections used per file download (2-8). +
App default: 4 · Recommended: 4-8 (Telegram Desktop uses up to 8)
+ +
+
+
+ + Chunk Size +
+
+ Size of each file chunk requested from Telegram. Bigger chunks mean fewer round-trips. +
WTelegramClient default: 512 KB · App default and recommended: 1024 KB +
+
+
+ + + + + + +
+
+ +
+
+
+ + Block Size +
+
+ Work unit assigned to each connection (1-16 MB). All chunks of a block are requested in parallel on the same connection, so Block ÷ Chunk = requests in flight per connection. +
App default: 4 MB (= 4 chunks of 1024 KB in flight) · Recommended: 4-8 MB +
+
+
+
+ + MB +
+
+
+ +
+
+
+ + Min. File Size +
+
+ Files smaller than this use the normal single-connection download (the setup is not worth it for small files). +
App default and recommended: 32 MB +
+
+
+
+ + MB +
+
+
}
From 7df75a691c8cdef8b6801e6e0ef94fcc40c6cc9e Mon Sep 17 00:00:00 2001 From: mateofuentespombo Date: Wed, 22 Jul 2026 12:48:57 +0200 Subject: [PATCH 20/33] feat: real sea-wave effect on the top bar download/upload buttons The water-fill buttons showed a flat colored block with a horizontal shine sweep - no actual wave at the waterline. Replace the shine with two repeating SVG wave crests that ride exactly on the fill level (the .water-wave element's top edge), drifting horizontally at different speeds and in opposite directions, overlapping the fill by 1px so no seam shows. The fill itself now uses a vertical gradient (deeper at the bottom, lighter at the surface) per variant color, wave motion is disabled under prefers-reduced-motion, and the stylesheet link gets a version query so cached browsers pick up the change. No markup changes to the buttons themselves. --- TelegramDownloader/Shared/MainLayout.razor | 2 +- .../wwwroot/css/custombuttons.css | 90 +++++++++++++++---- 2 files changed, 75 insertions(+), 17 deletions(-) diff --git a/TelegramDownloader/Shared/MainLayout.razor b/TelegramDownloader/Shared/MainLayout.razor index 291da9c..ed518b0 100644 --- a/TelegramDownloader/Shared/MainLayout.razor +++ b/TelegramDownloader/Shared/MainLayout.razor @@ -63,7 +63,7 @@ } - + diff --git a/TelegramDownloader/wwwroot/css/custombuttons.css b/TelegramDownloader/wwwroot/css/custombuttons.css index f400bdf..1807602 100644 --- a/TelegramDownloader/wwwroot/css/custombuttons.css +++ b/TelegramDownloader/wwwroot/css/custombuttons.css @@ -76,7 +76,8 @@ button.btn.water-fill-button.download:hover { box-shadow: 0 4px 12px rgba(23, 162, 184, 0.2) !important; } -/* Water fill background layer */ +/* Water fill background layer: deeper color at the bottom, lighter towards + the surface, so the liquid reads as translucent */ button.btn.water-fill-button > .water-fill { position: absolute !important; bottom: 0 !important; @@ -84,7 +85,9 @@ button.btn.water-fill-button > .water-fill { right: 0 !important; top: auto !important; width: 100% !important; - background: rgba(0, 123, 255, 0.7) !important; + background: linear-gradient(180deg, + rgba(0, 123, 255, 0.55) 0%, + rgba(0, 123, 255, 0.85) 100%) !important; transition: height 0.4s ease-out !important; pointer-events: none !important; z-index: 0 !important; @@ -92,14 +95,22 @@ button.btn.water-fill-button > .water-fill { } button.btn.water-fill-button.upload > .water-fill { - background: rgba(40, 167, 69, 0.7) !important; + background: linear-gradient(180deg, + rgba(40, 167, 69, 0.55) 0%, + rgba(40, 167, 69, 0.85) 100%) !important; } button.btn.water-fill-button.download > .water-fill { - background: rgba(23, 162, 184, 0.7) !important; + background: linear-gradient(180deg, + rgba(23, 162, 184, 0.55) 0%, + rgba(23, 162, 184, 0.85) 100%) !important; } -/* Wave effect on top of water */ +/* Waterline waves: the .water-wave element is an invisible box whose top edge + sits exactly at the fill level (inline height = progress). Two repeating + SVG wave strips ride on that edge, drifting horizontally at different + speeds and in opposite directions, which is what creates the sea-wave + motion; they overlap the fill by 1px so no seam shows */ button.btn.water-fill-button > .water-wave { position: absolute !important; bottom: 0 !important; @@ -107,23 +118,70 @@ button.btn.water-fill-button > .water-wave { right: 0 !important; top: auto !important; width: 100% !important; - background: linear-gradient(90deg, - transparent 0%, - rgba(255,255,255,0.4) 50%, - transparent 100%) !important; - background-size: 50% 100% !important; - animation: waveShine 1.5s ease-in-out infinite !important; + background: none !important; + animation: none !important; + transition: height 0.4s ease-out !important; pointer-events: none !important; z-index: 1 !important; display: block !important; + overflow: visible !important; +} + +button.btn.water-fill-button > .water-wave::before, +button.btn.water-fill-button > .water-wave::after { + content: ""; + position: absolute; + left: 0; + right: 0; + height: 8px; + background-repeat: repeat-x; + background-size: 80px 8px; + will-change: background-position-x; +} + +button.btn.water-fill-button > .water-wave::before { + top: -7px; + animation: waterWaveDrift 2.2s linear infinite; +} + +button.btn.water-fill-button > .water-wave::after { + top: -5px; + opacity: 0.5; + animation: waterWaveDrift 3.6s linear infinite reverse; +} + +/* Default (blue) wave crests */ +button.btn.water-fill-button > .water-wave::before, +button.btn.water-fill-button > .water-wave::after { + background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 80 8' preserveAspectRatio='none'%3E%3Cpath d='M0 4 Q10 0.5 20 4 T40 4 T60 4 T80 4 V8 H0 Z' fill='%23007bff' fill-opacity='0.75'/%3E%3C/svg%3E"); +} + +/* Upload (green) wave crests */ +button.btn.water-fill-button.upload > .water-wave::before, +button.btn.water-fill-button.upload > .water-wave::after { + background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 80 8' preserveAspectRatio='none'%3E%3Cpath d='M0 4 Q10 0.5 20 4 T40 4 T60 4 T80 4 V8 H0 Z' fill='%2328a745' fill-opacity='0.75'/%3E%3C/svg%3E"); +} + +/* Download (cyan) wave crests */ +button.btn.water-fill-button.download > .water-wave::before, +button.btn.water-fill-button.download > .water-wave::after { + background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 80 8' preserveAspectRatio='none'%3E%3Cpath d='M0 4 Q10 0.5 20 4 T40 4 T60 4 T80 4 V8 H0 Z' fill='%2317a2b8' fill-opacity='0.75'/%3E%3C/svg%3E"); } -@keyframes waveShine { - 0% { - background-position: -100% 0; +@keyframes waterWaveDrift { + from { + background-position-x: 0; + } + to { + background-position-x: 80px; } - 100% { - background-position: 200% 0; +} + +@media (prefers-reduced-motion: reduce) { + button.btn.water-fill-button > .water-wave::before, + button.btn.water-fill-button > .water-wave::after, + button.btn.water-fill-button .arrow { + animation: none !important; } } From 5adb1eb3dfabcd8fb99371c3854ab21a1205acc2 Mon Sep 17 00:00:00 2001 From: mateofuentespombo Date: Wed, 22 Jul 2026 16:08:38 +0200 Subject: [PATCH 21/33] feat: auth guard before rendering with return to the requested page After an app restart the Telegram client exists but the user is not loaded, so pages rendered first and only then (on first render) the layout noticed the missing user and hard-redirected to the login page, losing the URL the user was visiting; after authenticating, login always landed on /fetchdata. Resolve the session in the layouts' OnInitializedAsync, before any page content renders: the body is gated behind the check (page components are not instantiated until it completes), a silent session restore via checkAuth(null) is attempted first, and only when interactive login is really needed does the guard bounce to the login page, passing the requested URL as ?returnUrl. The login page navigates back to that URL (local paths only, to avoid open redirects) after a successful login. ConfigLayout gets the same guard; the old after-render check is removed. --- TelegramDownloader/Pages/Index.razor | 27 +++++++++- TelegramDownloader/Shared/ConfigLayout.razor | 34 ++++++++++-- TelegramDownloader/Shared/MainLayout.razor | 54 ++++++++++++++------ 3 files changed, 96 insertions(+), 19 deletions(-) diff --git a/TelegramDownloader/Pages/Index.razor b/TelegramDownloader/Pages/Index.razor index 5bffe0b..18f37a6 100644 --- a/TelegramDownloader/Pages/Index.razor +++ b/TelegramDownloader/Pages/Index.razor @@ -3,6 +3,7 @@ @implements IDisposable @using Microsoft.AspNetCore.Components +@using Microsoft.AspNetCore.WebUtilities @using QRCoder @using TelegramDownloader.Data @using TelegramDownloader.Models @@ -276,10 +277,34 @@ source.Cancel(); } } - NavManager.NavigateTo("/fetchdata"); + NavManager.NavigateTo(GetReturnUrl() ?? "/fetchdata"); } } + /// + /// Returns the page the user originally requested (passed by the layouts' + /// auth guard as ?returnUrl=...), so a successful login lands back on it. + /// Only local paths are accepted, to avoid open redirects. + /// + private string GetReturnUrl() + { + try + { + var uri = NavManager.ToAbsoluteUri(NavManager.Uri); + if (QueryHelpers.ParseQuery(uri.Query).TryGetValue("returnUrl", out var value)) + { + string url = value.ToString(); + if (url.StartsWith('/') && !url.StartsWith("//") && url != "/" && !url.StartsWith("/?")) + return url; + } + } + catch + { + // Malformed query - fall back to the default landing page + } + return null; + } + private void GenerateQR(string data) { using (QRCodeGenerator qrGenerator = new QRCodeGenerator()) diff --git a/TelegramDownloader/Shared/ConfigLayout.razor b/TelegramDownloader/Shared/ConfigLayout.razor index b332539..8857166 100644 --- a/TelegramDownloader/Shared/ConfigLayout.razor +++ b/TelegramDownloader/Shared/ConfigLayout.razor @@ -8,9 +8,34 @@ TelegramFileManager @code { - protected override void OnInitialized() + private bool authResolved { get; set; } = false; + + protected override async Task OnInitializedAsync() { - if (!ts.checkUserLogin()) NavManager.NavigateTo("/"); + // Same auth guard as MainLayout: resolve the session before rendering + // any page content, trying a silent restore first and keeping the + // requested URL when interactive login is needed. + if (!ts.checkUserLogin()) + { + string authType = null; + try + { + authType = await ts.checkAuth(null); + } + catch (Exception e) + { + Console.WriteLine(e); + } + if (authType != "ok") + { + string returnUrl = new Uri(NavManager.Uri).PathAndQuery; + NavManager.NavigateTo(string.IsNullOrEmpty(returnUrl) || returnUrl == "/" + ? "/" + : $"/?returnUrl={Uri.EscapeDataString(returnUrl)}", forceLoad: true); + return; + } + } + authResolved = true; } } @@ -39,7 +64,10 @@
- @Body + @if (authResolved) + { + @Body + }
diff --git a/TelegramDownloader/Shared/MainLayout.razor b/TelegramDownloader/Shared/MainLayout.razor index ed518b0..7c85991 100644 --- a/TelegramDownloader/Shared/MainLayout.razor +++ b/TelegramDownloader/Shared/MainLayout.razor @@ -136,7 +136,17 @@
- @Body + @if (authResolved) + { + @Body + } + else + { +
+ + Checking session... +
+ }
@@ -333,6 +343,7 @@ private VersionBadge versionBadge { get; set; } private bool active { get; set; } = true; private bool sidebarCollapsed { get; set; } = false; + private bool authResolved { get; set; } = false; private bool navMenuCollapsed { get; set; } = true; private bool showVersionModal { get; set; } = false; private static MainLayout? _instance; @@ -514,30 +525,43 @@ } } - protected override async Task OnAfterRenderAsync(bool firstRender) + protected override async Task OnInitializedAsync() { - if (firstRender) + // Check if setup is complete using async method to avoid deadlock + var status = await SetupService.GetSetupStatusAsync(); + if (status.CurrentStep != SetupStep.Complete) + { + NavManager.NavigateTo("/setup", forceLoad: true); + return; + } + + // Auth guard: resolve the session BEFORE any page content renders (the + // page body is gated on authResolved, so page components are not even + // instantiated until this completes). After an app restart the client + // exists but the user is not loaded yet, so attempt a silent session + // restore first; only bounce to the login page - keeping the requested + // URL so login can come back to it - when interactive login is needed. + if (!ts.checkUserLogin()) { + string authType = null; try { - if (!ts.checkUserLogin()) NavManager.NavigateTo("/", true); + authType = await ts.checkAuth(null); } catch (Exception e) { Console.WriteLine(e); } + if (authType != "ok") + { + string returnUrl = new Uri(NavManager.Uri).PathAndQuery; + NavManager.NavigateTo(string.IsNullOrEmpty(returnUrl) || returnUrl == "/" + ? "/" + : $"/?returnUrl={Uri.EscapeDataString(returnUrl)}", forceLoad: true); + return; + } } - } - - protected override async Task OnInitializedAsync() - { - // Check if setup is complete using async method to avoid deadlock - var status = await SetupService.GetSetupStatusAsync(); - if (status.CurrentStep != SetupStep.Complete) - { - NavManager.NavigateTo("/setup", forceLoad: true); - return; - } + authResolved = true; await checkWorkingTasks(); tis.TaskEventChanged += eventChangedWorkingTasks; From f86d1e20e8ac5053972fab412e892aa1ca290f5a Mon Sep 17 00:00:00 2001 From: mateofuentespombo Date: Wed, 22 Jul 2026 20:36:54 +0200 Subject: [PATCH 22/33] feat: QR code login option in the web login page The QR login plumbing (CallQrGenerator wrapping LoginWithQRCode, QR rendering in Index) existed but nothing invoked it, and it could not have worked from the web anyway: when the account has 2FA, LoginWithQRCode asks for the password through Config("password"), which with the convenience constructor prompts on the console - QR login only worked when driving the library from a terminal. Wire it end to end: - The main client is now built with a custom config callback that routes the "password" request to the web UI: a QrPasswordNeeded event switches the login form to the 2FA password step and the value is handed back to the waiting login task, which blocks on a TaskCompletionSource (5 minute timeout) on its background thread. - The login page offers "Log in with QR code" next to the phone step: it shows the tg://login QR (auto-refreshed by the library when each token expires), instructions and a back button; on success the shared post-login initialization runs and navigation proceeds as with the normal flow. - DoLogin's post-login tail is extracted into CompleteLogin, reused by the QR flow. - checkAuth: when a session exists but no user data file is saved (the QR case - there is no phone to save), attempt a silent session restore via DoLogin(null) instead of always bouncing to the phone step, so QR sessions survive app restarts. --- TelegramDownloader/Data/ITelegramService.cs | 1 + TelegramDownloader/Data/TelegramService.cs | 75 ++++++++++++++++- TelegramDownloader/Pages/Index.razor | 93 +++++++++++++++++++++ 3 files changed, 165 insertions(+), 4 deletions(-) diff --git a/TelegramDownloader/Data/ITelegramService.cs b/TelegramDownloader/Data/ITelegramService.cs index 5963b83..fc0b05b 100644 --- a/TelegramDownloader/Data/ITelegramService.cs +++ b/TelegramDownloader/Data/ITelegramService.cs @@ -19,6 +19,7 @@ public interface ITelegramService Task getInvitationHash(long id); Task joinChatInvitationHash(string? hash); Task CallQrGenerator(Action func, CancellationToken ct, bool logoutFirst = false); + void ProvideQrLoginPassword(string password); Task DownloadFile(ChatMessages message, string fileName = null, string folder = null, DownloadModel model = null, bool shouldAddToList = false); Task DownloadFileStream(Message message, long offset, int limit); IAsyncEnumerable DownloadFileStreamChunks(Message message, long offset, long limit, CancellationToken ct = default); diff --git a/TelegramDownloader/Data/TelegramService.cs b/TelegramDownloader/Data/TelegramService.cs index c35f282..d115e7e 100644 --- a/TelegramDownloader/Data/TelegramService.cs +++ b/TelegramDownloader/Data/TelegramService.cs @@ -173,7 +173,20 @@ private void newClient() { _logger.LogDebug(ex, "Could not snapshot the session file"); } - client = new WTelegram.Client(Convert.ToInt32(GeneralConfigStatic.tlconfig?.api_id ?? Environment.GetEnvironmentVariable("api_id")), GeneralConfigStatic.tlconfig?.hash_id ?? Environment.GetEnvironmentVariable("hash_id"), UserService.USERDATAFOLDER + "/WTelegram.session"); + string apiId = GeneralConfigStatic.tlconfig?.api_id ?? Environment.GetEnvironmentVariable("api_id"); + string apiHash = GeneralConfigStatic.tlconfig?.hash_id ?? Environment.GetEnvironmentVariable("hash_id"); + // Custom config callback instead of the convenience constructor: the + // QR login flow (LoginWithQRCode) asks for the 2FA password through + // Config("password") - with the default config that prompt would go + // to the console. Route it to the web UI instead. + client = new WTelegram.Client(what => what switch + { + "api_id" => apiId, + "api_hash" => apiHash, + "session_pathname" => UserService.USERDATAFOLDER + "/WTelegram.session", + "password" => RequestLoginPassword(), + _ => null + }); ApplyConfiguredParallelTransfers(client); if (GeneralConfigStatic.config.ShouldShowLogInTerminal) { @@ -585,9 +598,41 @@ async Task GuardedWorker(WTelegram.Client pc) #endregion + // 2FA password bridge for the QR login flow: WTelegram asks for the + // password via Config("password") on a background task; the UI is + // notified through QrPasswordNeeded and supplies the value with + // ProvideQrLoginPassword, unblocking the pending request. + private static TaskCompletionSource qrPasswordRequest; + public static event EventHandler QrPasswordNeeded; + + private static string RequestLoginPassword() + { + TaskCompletionSource tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + qrPasswordRequest = tcs; + try + { + QrPasswordNeeded?.Invoke(null, EventArgs.Empty); + } + catch (Exception) + { + } + // Blocks the QR login background task, never the UI thread. + if (!tcs.Task.Wait(TimeSpan.FromMinutes(5))) + throw new TimeoutException("2FA password was not provided in time"); + return tcs.Task.Result; + } + + public void ProvideQrLoginPassword(string password) + { + qrPasswordRequest?.TrySetResult(password); + } + public async Task CallQrGenerator(Action func, CancellationToken ct, bool logoutFirst = false) { - return await client.LoginWithQRCode(func, logoutFirst: logoutFirst, ct: ct); + User user = await client.LoginWithQRCode(func, logoutFirst: logoutFirst, ct: ct); + if (user != null) + await CompleteLogin(); + return user; } public async Task GetUser() @@ -632,6 +677,16 @@ async Task DoLogin(string loginInfo) // (add this method to your code) return "pass"; // if user has enabled 2FA default: break; } + return await CompleteLogin(); + } + + /// + /// Post-login initialization shared by the interactive login flow and the + /// QR login flow: loads chats, resolves premium limits and notifies + /// subscribers. + /// + private async Task CompleteLogin() + { await getAllChats(); SetSplitSizeGB(); if (client.User.flags.HasFlag(User.Flags.premium)) @@ -696,8 +751,20 @@ public async Task checkAuth(string number, bool isPhone = false) } else { - _logger.LogInformation("No saved user data found, requesting phone"); - return "phone"; + // No saved phone (e.g. the session was created via QR login): + // try to restore the session directly - Login(null) completes + // silently when the stored session is still authorized and + // returns the "phone" step otherwise. + try + { + _logger.LogInformation("No saved user data found, attempting session restore"); + return await DoLogin(null) ?? "phone"; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Session restore without saved user data failed, requesting phone"); + return "phone"; + } } } diff --git a/TelegramDownloader/Pages/Index.razor b/TelegramDownloader/Pages/Index.razor index 5bffe0b..32db3ae 100644 --- a/TelegramDownloader/Pages/Index.razor +++ b/TelegramDownloader/Pages/Index.razor @@ -43,6 +43,10 @@

Two-Factor Auth

break; + case "qr": +

Scan QR Code

+ + break; case "ok":

Welcome Back!

@@ -81,6 +85,15 @@ @bind-Value="Model!.value" placeholder="Enter your 2FA password" /> break; + case "qr": +
+ +
+ break; case "ok":
@@ -98,6 +111,12 @@ Disconnect } + else if (Model.type == "qr") + { + + } else { } } + @if (Model.type == "phone") + { + + }
@if (!string.IsNullOrEmpty(imageString)) @@ -248,8 +273,75 @@ } } + private bool qrMode = false; + + private async Task StartQrLogin() + { + qrMode = true; + imageString = null; + Model.type = "qr"; + source?.Cancel(); + source = new CancellationTokenSource(); + StateHasChanged(); + TelegramService.QrPasswordNeeded += OnQrPasswordNeeded; + try + { + var user = await ts.CallQrGenerator(url => { _ = InvokeAsync(() => GenerateQR(url)); }, source.Token); + if (user != null) + { + imageString = null; + Model.type = "ok"; + await InvokeAsync(StateHasChanged); + await isLogin(); + } + } + catch (OperationCanceledException) + { + // User went back to phone login or left the page + } + catch (Exception ex) + { + Console.WriteLine($"QR login failed: {ex.Message}"); + qrMode = false; + imageString = null; + Model.type = "phone"; + await InvokeAsync(StateHasChanged); + } + finally + { + TelegramService.QrPasswordNeeded -= OnQrPasswordNeeded; + } + } + + private void OnQrPasswordNeeded(object sender, System.EventArgs e) + { + // The QR was accepted on the phone but the account has 2FA: switch the + // form to the password step; Submit routes the value back to the + // pending QR login. + imageString = null; + Model.value = ""; + Model.type = "pass"; + _ = InvokeAsync(StateHasChanged); + } + + private void CancelQrLogin() + { + qrMode = false; + imageString = null; + source?.Cancel(); + Model.type = "phone"; + } + private async void Submit() { + if (qrMode && Model.type == "pass") + { + // 2FA password for a QR login: hand it to the waiting login task, + // which finishes the flow and navigates from StartQrLogin. + ts.ProvideQrLoginPassword(Model.value); + Model.value = ""; + return; + } if (Model.type == "phone") { var phone = await JSRuntime.InvokeAsync("getNumber"); @@ -308,6 +400,7 @@ public void Dispose() { + TelegramService.QrPasswordNeeded -= OnQrPasswordNeeded; if (source != null) { if (source.Token.CanBeCanceled) From e432b95a2dc5d5ec67db09e8d9cb0632743d92cf Mon Sep 17 00:00:00 2001 From: mateofuentespombo Date: Wed, 22 Jul 2026 21:23:19 +0200 Subject: [PATCH 23/33] feat: live refresh in the download/upload/task info modals The info modals held a live reference to their model but only rendered a snapshot taken when opened, so progress, state, duration and speed froze until the modal was reopened. Subscribe each modal to the aggregated throttled TransactionsChanged event while it is visible (subscribed on ShowModal, unsubscribed on HideModal and Dispose) and re-render on the UI thread, following the same pattern the transfer tables use. --- .../InfoModals/DownloadFileInfoModal.razor | 24 +++++++++++++++++++ .../Modals/InfoModals/TaskInfoModal.razor | 24 +++++++++++++++++++ .../InfoModals/UploadFileInfoModal.razor | 24 +++++++++++++++++++ 3 files changed, 72 insertions(+) diff --git a/TelegramDownloader/Pages/Modals/InfoModals/DownloadFileInfoModal.razor b/TelegramDownloader/Pages/Modals/InfoModals/DownloadFileInfoModal.razor index 767b216..58a0f4a 100644 --- a/TelegramDownloader/Pages/Modals/InfoModals/DownloadFileInfoModal.razor +++ b/TelegramDownloader/Pages/Modals/InfoModals/DownloadFileInfoModal.razor @@ -1,5 +1,8 @@ @using TelegramDownloader.Models @using TelegramDownloader.Services +@implements IDisposable + +@inject TransactionInfoService tis