From 048a61ba6218f0e93364825633679b44beb167bb Mon Sep 17 00:00:00 2001 From: Max Horn Date: Sun, 16 Aug 2026 02:26:58 +0200 Subject: [PATCH] Download: add a 'resume' option A download interrupted halfway currently starts again from zero, which for a data set of any size is the difference between a retry and an afternoon. With 'resume' set and a target given, the curl and wget methods continue the partial file. The clean-up in 'Download' skips the target for the same reason: otherwise it would throw away the very thing the next attempt is meant to continue. Methods that cannot resume decline the request rather than proceeding to avoid destroying the partial file the resuming methods need. 'resume' is deliberately left unbound when the caller does not ask for it, rather than defaulted to false. Co-Authored-By: Claude Opus 5 --- doc/download.xml | 23 ++++++++++++++++++++- lib/download.gi | 29 +++++++++++++++++++++++++-- tst/download.tst | 51 ++++++++++++++++++++++++++++++++++++++++++++++- tst/http-server.g | 27 ++++++++++++++++++++++++- 4 files changed, 125 insertions(+), 5 deletions(-) diff --git a/doc/download.xml b/doc/download.xml index cf8548c..0d2afde 100644 --- a/doc/download.xml +++ b/doc/download.xml @@ -69,7 +69,28 @@ The following components are supported. and the function writes the downloaded contents to this file; the returned record does not have a result component in this case.

- If the download fails then this file is not left behind. + If the download fails then this file is not left behind, + unless resume is set. + +resume + + If this component is bound and has the value true, + and target is given, + then a partially downloaded file is continued rather than fetched again, + and it is kept if the download fails again. +

+ Methods that cannot resume decline the request, since they would discard + the partial file that a method which can resume needs; currently the + methods based on the external programs curl and wget resume. + If none is available the download fails, and the caller can retry without + resume. +

+ It is the caller's responsibility that an existing target really is + a prefix of what url delivers; otherwise the two get concatenated. +

+ If the server does not support range requests then no data is lost: + wget fetches the file again from the start, and curl fails + and leaves the partial file alone. verifyCert diff --git a/lib/download.gi b/lib/download.gi index 4b9a6ce..af84df5 100644 --- a/lib/download.gi +++ b/lib/download.gi @@ -36,6 +36,13 @@ Add( Download_Methods, rec( download:= function( url, opt ) local res; + if IsBound( opt.resume ) and opt.resume = true and + IsBound( opt.target ) and IsString( opt.target ) then + # Declining matters: this method would truncate the partial file that a + # method which can resume needs. + return rec( success:= false, error:= "no support for resuming" ); + fi; + opt:= ShallowCopy( opt ); if not IsBound( opt.failOnError ) then opt.failOnError:= true; @@ -61,6 +68,11 @@ Add( Download_Methods, rec( return rec( success:= false, error:= "protocol is not http" ); elif IsBound( opt.maxTime ) and opt.maxTime <> 0 then return rec( success:= false, error:= "no support for given timeout" ); + elif IsBound( opt.resume ) and opt.resume = true and + IsBound( opt.target ) and IsString( opt.target ) then + # No range request, so this would overwrite the partial file that a + # method which can resume needs. + return rec( success:= false, error:= "no support for resuming" ); fi; # Split the URL after 'http://' into the authority and HTTP request target. @@ -164,6 +176,10 @@ Add( Download_Methods, rec( else args:= [ "--quiet", "-O", "-", url ]; fi; + if IsBound( opt.resume ) and opt.resume = true and + IsBound( opt.target ) and IsString( opt.target ) then + Add( args, "-c" ); + fi; if IsBound( opt.verifyCert ) and opt.verifyCert = false then Add( args, "--no-check-certificate" ); fi; @@ -173,8 +189,10 @@ Add( Download_Methods, rec( code:= Process( DirectoryCurrent(), exec, InputTextNone(), outstream, args ); CloseStream( outstream ); if code <> 0 then - # wget may have created the target file; try to remove it + # wget may have created the target file; try to remove it, unless the + # caller wants to resume from what is there if IsBound( opt.target ) and IsString( opt.target ) and + not ( IsBound( opt.resume ) and opt.resume = true ) and IsExistingFile( opt.target ) and RemoveFile( opt.target ) <> true then Error( "Download cannot remove unwanted file ", opt.target ); fi; @@ -211,6 +229,11 @@ Add( Download_Methods, rec( else Add( args, "-" ); fi; + if IsBound( opt.resume ) and opt.resume = true and + IsBound( opt.target ) and IsString( opt.target ) then + Add( args, "-C" ); + Add( args, "-" ); + fi; if IsBound( opt.maxTime ) and IsPosInt( opt.maxTime ) then Add( args, "--max-time" ); Add( args, opt.maxTime ); @@ -275,8 +298,10 @@ InstallMethod( Download, fi; # A failed method may have left a partial or bogus target file behind. # Remove it here, so that the guarantee holds for every method, - # including ones added to 'Download_Methods' from outside. + # including ones added to 'Download_Methods' from outside -- but not + # when resuming, where the partial file is the whole point. if IsBound( opt.target ) and IsString( opt.target ) and + not ( IsBound( opt.resume ) and opt.resume = true ) and IsExistingFile( opt.target ) then RemoveFile( opt.target ); fi; diff --git a/tst/download.tst b/tst/download.tst index 02856d3..afa6b4b 100644 --- a/tst/download.tst +++ b/tst/download.tst @@ -1,4 +1,4 @@ -#@local meths, i, urls, pair, url, expected, res1, good1, n, file, res2, good2, contents, r, res3, good3, bad, server, baseurl, iometh, opt, oldpref +#@local meths, i, urls, pair, url, expected, res1, good1, n, file, res2, good2, contents, r, res3, good3, bad, server, baseurl, iometh, opt, oldpref, resumers, name ############################################################################ ## #W download.tst Utils Package Thomas Breuer @@ -164,6 +164,55 @@ true gap> IsExistingFile( file ); false +## 'resume' continues a partial file rather than fetching it again. The +## test server answers a Range request with the remainder in upper case, so +## a resumed download is distinguishable from a restarted one. +gap> resumers:= Filtered( meths, r -> r.name in [ "via curl", "via wget" ] );; +gap> for r in resumers do +> FileString( file, "abcde" );; +> res1:= r.download( Concatenation( baseurl, "/resumable" ), +> rec( target:= file, resume:= true ) ); +> if res1.success <> true then +> Print( "resume failed for ", r.name, ": ", res1.error, "\n" ); +> elif StringFile( file ) <> "abcdeFGHIJKLMNOPQRST" then +> Print( "did not resume for ", r.name, ": ", StringFile( file ), "\n" ); +> fi; +> RemoveFile( file ); +> od; + +## Without 'resume' the target is replaced, not appended to. +gap> FileString( file, "abcde" );; +gap> res1:= Download( Concatenation( baseurl, "/resumable" ), +> rec( target:= file ) );; +gap> StringFile( file ); +"abcdefghijklmnopqrst" + +## Methods that cannot resume decline, rather than discarding the partial +## file that a method which can resume needs. +gap> iometh.download( Concatenation( baseurl, "/resumable" ), +> rec( target:= file, resume:= true ) ).error; +"no support for resuming" + +## With 'resume', a failed download keeps the partial file to continue from. +## Without this, the clean-up in 'Download' would throw away the very thing +## the next attempt is meant to continue. +gap> FileString( file, "abcde" );; +gap> res1:= Download( Concatenation( baseurl, "/missing" ), +> rec( target:= file, resume:= true ) );; +gap> res1.success; +false +gap> StringFile( file ); +"abcde" +gap> RemoveFile( file );; + +## 'resume' is only declined when it is actually requested: a method must +## not be skipped merely because the component is present. +gap> res1:= Download( Concatenation( baseurl, "/file" ), +> rec( target:= file, resume:= false ) );; +gap> res1.success; +true +gap> RemoveFile( file );; + ## test errors and redirects gap> res1:= Download( Concatenation( baseurl, "/missing" ) );; gap> res1.success = false; diff --git a/tst/http-server.g b/tst/http-server.g index 40a532a..ad2b0b4 100644 --- a/tst/http-server.g +++ b/tst/http-server.g @@ -4,7 +4,7 @@ ## BindGlobal( "UTILS_HandleHTTPTestRequest", function( listener, socket ) - local connection, line, parts, uri, body, status, location; + local connection, line, parts, uri, body, status, location, range, from; IO_close( listener ); connection:= IO_WrapFD( socket, IO.DefaultBufSize, IO.DefaultBufSize ); @@ -16,8 +16,12 @@ BindGlobal( "UTILS_HandleHTTPTestRequest", function( listener, socket ) fi; uri:= parts[2]; + range:= fail; repeat line:= IO_ReadLine( connection ); + if IsString( line ) and StartsWith( LowercaseString( line ), "range:" ) then + range:= line; + fi; until line = fail or line = "" or line = "\n" or line = "\r\n"; body:= "download test response\n"; @@ -32,6 +36,27 @@ BindGlobal( "UTILS_HandleHTTPTestRequest", function( listener, socket ) body:= ""; status:= "302 Found"; location:= "Location: /success\r\n"; + elif StartsWith( uri, "/resumable" ) then + # Answer a 'Range: bytes=N-' request with the remainder, but in upper + # case, so that a test can tell a resumed download from a restarted one. + body:= "abcdefghijklmnopqrst"; + if range <> fail then + from:= Int( Filtered( range, c -> c in "0123456789" ) ); + if from <> fail and 0 < from and from < Length( body ) then + IO_Write( connection, + "HTTP/1.1 206 Partial Content\r\n", + "Content-Type: text/plain\r\n", + "Content-Range: bytes ", String( from ), "-", + String( Length( body ) - 1 ), "/", String( Length( body ) ), + "\r\n", + "Content-Length: ", String( Length( body ) - from ), "\r\n", + "Connection: close\r\n\r\n", + UppercaseString( body{ [ from+1 .. Length( body ) ] } ) ); + IO_Flush( connection ); + IO_Close( connection ); + IO_exit( 0 ); + fi; + fi; fi; IO_Write( connection,