diff --git a/doc/download.xml b/doc/download.xml index cf8548c..0d2afde 100644 --- a/doc/download.xml +++ b/doc/download.xml @@ -69,7 +69,28 @@ The following components are supported. and the function writes the downloaded contents to this file; the returned record does not have a result component in this case.

- If the download fails then this file is not left behind. + If the download fails then this file is not left behind, + unless resume is set. + +resume + + If this component is bound and has the value true, + and target is given, + then a partially downloaded file is continued rather than fetched again, + and it is kept if the download fails again. +

+ Methods that cannot resume decline the request, since they would discard + the partial file that a method which can resume needs; currently the + methods based on the external programs curl and wget resume. + If none is available the download fails, and the caller can retry without + resume. +

+ It is the caller's responsibility that an existing target really is + a prefix of what url delivers; otherwise the two get concatenated. +

+ If the server does not support range requests then no data is lost: + wget fetches the file again from the start, and curl fails + and leaves the partial file alone. verifyCert diff --git a/lib/download.gi b/lib/download.gi index 4b9a6ce..af84df5 100644 --- a/lib/download.gi +++ b/lib/download.gi @@ -36,6 +36,13 @@ Add( Download_Methods, rec( download:= function( url, opt ) local res; + if IsBound( opt.resume ) and opt.resume = true and + IsBound( opt.target ) and IsString( opt.target ) then + # Declining matters: this method would truncate the partial file that a + # method which can resume needs. + return rec( success:= false, error:= "no support for resuming" ); + fi; + opt:= ShallowCopy( opt ); if not IsBound( opt.failOnError ) then opt.failOnError:= true; @@ -61,6 +68,11 @@ Add( Download_Methods, rec( return rec( success:= false, error:= "protocol is not http" ); elif IsBound( opt.maxTime ) and opt.maxTime <> 0 then return rec( success:= false, error:= "no support for given timeout" ); + elif IsBound( opt.resume ) and opt.resume = true and + IsBound( opt.target ) and IsString( opt.target ) then + # No range request, so this would overwrite the partial file that a + # method which can resume needs. + return rec( success:= false, error:= "no support for resuming" ); fi; # Split the URL after 'http://' into the authority and HTTP request target. @@ -164,6 +176,10 @@ Add( Download_Methods, rec( else args:= [ "--quiet", "-O", "-", url ]; fi; + if IsBound( opt.resume ) and opt.resume = true and + IsBound( opt.target ) and IsString( opt.target ) then + Add( args, "-c" ); + fi; if IsBound( opt.verifyCert ) and opt.verifyCert = false then Add( args, "--no-check-certificate" ); fi; @@ -173,8 +189,10 @@ Add( Download_Methods, rec( code:= Process( DirectoryCurrent(), exec, InputTextNone(), outstream, args ); CloseStream( outstream ); if code <> 0 then - # wget may have created the target file; try to remove it + # wget may have created the target file; try to remove it, unless the + # caller wants to resume from what is there if IsBound( opt.target ) and IsString( opt.target ) and + not ( IsBound( opt.resume ) and opt.resume = true ) and IsExistingFile( opt.target ) and RemoveFile( opt.target ) <> true then Error( "Download cannot remove unwanted file ", opt.target ); fi; @@ -211,6 +229,11 @@ Add( Download_Methods, rec( else Add( args, "-" ); fi; + if IsBound( opt.resume ) and opt.resume = true and + IsBound( opt.target ) and IsString( opt.target ) then + Add( args, "-C" ); + Add( args, "-" ); + fi; if IsBound( opt.maxTime ) and IsPosInt( opt.maxTime ) then Add( args, "--max-time" ); Add( args, opt.maxTime ); @@ -275,8 +298,10 @@ InstallMethod( Download, fi; # A failed method may have left a partial or bogus target file behind. # Remove it here, so that the guarantee holds for every method, - # including ones added to 'Download_Methods' from outside. + # including ones added to 'Download_Methods' from outside -- but not + # when resuming, where the partial file is the whole point. if IsBound( opt.target ) and IsString( opt.target ) and + not ( IsBound( opt.resume ) and opt.resume = true ) and IsExistingFile( opt.target ) then RemoveFile( opt.target ); fi; diff --git a/tst/download.tst b/tst/download.tst index 02856d3..afa6b4b 100644 --- a/tst/download.tst +++ b/tst/download.tst @@ -1,4 +1,4 @@ -#@local meths, i, urls, pair, url, expected, res1, good1, n, file, res2, good2, contents, r, res3, good3, bad, server, baseurl, iometh, opt, oldpref +#@local meths, i, urls, pair, url, expected, res1, good1, n, file, res2, good2, contents, r, res3, good3, bad, server, baseurl, iometh, opt, oldpref, resumers, name ############################################################################ ## #W download.tst Utils Package Thomas Breuer @@ -164,6 +164,55 @@ true gap> IsExistingFile( file ); false +## 'resume' continues a partial file rather than fetching it again. The +## test server answers a Range request with the remainder in upper case, so +## a resumed download is distinguishable from a restarted one. +gap> resumers:= Filtered( meths, r -> r.name in [ "via curl", "via wget" ] );; +gap> for r in resumers do +> FileString( file, "abcde" );; +> res1:= r.download( Concatenation( baseurl, "/resumable" ), +> rec( target:= file, resume:= true ) ); +> if res1.success <> true then +> Print( "resume failed for ", r.name, ": ", res1.error, "\n" ); +> elif StringFile( file ) <> "abcdeFGHIJKLMNOPQRST" then +> Print( "did not resume for ", r.name, ": ", StringFile( file ), "\n" ); +> fi; +> RemoveFile( file ); +> od; + +## Without 'resume' the target is replaced, not appended to. +gap> FileString( file, "abcde" );; +gap> res1:= Download( Concatenation( baseurl, "/resumable" ), +> rec( target:= file ) );; +gap> StringFile( file ); +"abcdefghijklmnopqrst" + +## Methods that cannot resume decline, rather than discarding the partial +## file that a method which can resume needs. +gap> iometh.download( Concatenation( baseurl, "/resumable" ), +> rec( target:= file, resume:= true ) ).error; +"no support for resuming" + +## With 'resume', a failed download keeps the partial file to continue from. +## Without this, the clean-up in 'Download' would throw away the very thing +## the next attempt is meant to continue. +gap> FileString( file, "abcde" );; +gap> res1:= Download( Concatenation( baseurl, "/missing" ), +> rec( target:= file, resume:= true ) );; +gap> res1.success; +false +gap> StringFile( file ); +"abcde" +gap> RemoveFile( file );; + +## 'resume' is only declined when it is actually requested: a method must +## not be skipped merely because the component is present. +gap> res1:= Download( Concatenation( baseurl, "/file" ), +> rec( target:= file, resume:= false ) );; +gap> res1.success; +true +gap> RemoveFile( file );; + ## test errors and redirects gap> res1:= Download( Concatenation( baseurl, "/missing" ) );; gap> res1.success = false; diff --git a/tst/http-server.g b/tst/http-server.g index 40a532a..ad2b0b4 100644 --- a/tst/http-server.g +++ b/tst/http-server.g @@ -4,7 +4,7 @@ ## BindGlobal( "UTILS_HandleHTTPTestRequest", function( listener, socket ) - local connection, line, parts, uri, body, status, location; + local connection, line, parts, uri, body, status, location, range, from; IO_close( listener ); connection:= IO_WrapFD( socket, IO.DefaultBufSize, IO.DefaultBufSize ); @@ -16,8 +16,12 @@ BindGlobal( "UTILS_HandleHTTPTestRequest", function( listener, socket ) fi; uri:= parts[2]; + range:= fail; repeat line:= IO_ReadLine( connection ); + if IsString( line ) and StartsWith( LowercaseString( line ), "range:" ) then + range:= line; + fi; until line = fail or line = "" or line = "\n" or line = "\r\n"; body:= "download test response\n"; @@ -32,6 +36,27 @@ BindGlobal( "UTILS_HandleHTTPTestRequest", function( listener, socket ) body:= ""; status:= "302 Found"; location:= "Location: /success\r\n"; + elif StartsWith( uri, "/resumable" ) then + # Answer a 'Range: bytes=N-' request with the remainder, but in upper + # case, so that a test can tell a resumed download from a restarted one. + body:= "abcdefghijklmnopqrst"; + if range <> fail then + from:= Int( Filtered( range, c -> c in "0123456789" ) ); + if from <> fail and 0 < from and from < Length( body ) then + IO_Write( connection, + "HTTP/1.1 206 Partial Content\r\n", + "Content-Type: text/plain\r\n", + "Content-Range: bytes ", String( from ), "-", + String( Length( body ) - 1 ), "/", String( Length( body ) ), + "\r\n", + "Content-Length: ", String( Length( body ) - from ), "\r\n", + "Connection: close\r\n\r\n", + UppercaseString( body{ [ from+1 .. Length( body ) ] } ) ); + IO_Flush( connection ); + IO_Close( connection ); + IO_exit( 0 ); + fi; + fi; fi; IO_Write( connection,