diff --git a/doc/download.xml b/doc/download.xml
index cf8548c..0d2afde 100644
--- a/doc/download.xml
+++ b/doc/download.xml
@@ -69,7 +69,28 @@ The following components are supported.
and the function writes the downloaded contents to this file;
the returned record does not have a result component in this case.
- If the download fails then this file is not left behind.
+ If the download fails then this file is not left behind,
+ unless resume is set.
+
+resume
+-
+ If this component is bound and has the value true,
+ and target is given,
+ then a partially downloaded file is continued rather than fetched again,
+ and it is kept if the download fails again.
+
+ Methods that cannot resume decline the request, since they would discard
+ the partial file that a method which can resume needs; currently the
+ methods based on the external programs curl and wget resume.
+ If none is available the download fails, and the caller can retry without
+ resume.
+
+ It is the caller's responsibility that an existing target really is
+ a prefix of what url delivers; otherwise the two get concatenated.
+
+ If the server does not support range requests then no data is lost:
+ wget fetches the file again from the start, and curl fails
+ and leaves the partial file alone.
verifyCert
-
diff --git a/lib/download.gi b/lib/download.gi
index 4b9a6ce..af84df5 100644
--- a/lib/download.gi
+++ b/lib/download.gi
@@ -36,6 +36,13 @@ Add( Download_Methods, rec(
download:= function( url, opt )
local res;
+ if IsBound( opt.resume ) and opt.resume = true and
+ IsBound( opt.target ) and IsString( opt.target ) then
+ # Declining matters: this method would truncate the partial file that a
+ # method which can resume needs.
+ return rec( success:= false, error:= "no support for resuming" );
+ fi;
+
opt:= ShallowCopy( opt );
if not IsBound( opt.failOnError ) then
opt.failOnError:= true;
@@ -61,6 +68,11 @@ Add( Download_Methods, rec(
return rec( success:= false, error:= "protocol is not http" );
elif IsBound( opt.maxTime ) and opt.maxTime <> 0 then
return rec( success:= false, error:= "no support for given timeout" );
+ elif IsBound( opt.resume ) and opt.resume = true and
+ IsBound( opt.target ) and IsString( opt.target ) then
+ # No range request, so this would overwrite the partial file that a
+ # method which can resume needs.
+ return rec( success:= false, error:= "no support for resuming" );
fi;
# Split the URL after 'http://' into the authority and HTTP request target.
@@ -164,6 +176,10 @@ Add( Download_Methods, rec(
else
args:= [ "--quiet", "-O", "-", url ];
fi;
+ if IsBound( opt.resume ) and opt.resume = true and
+ IsBound( opt.target ) and IsString( opt.target ) then
+ Add( args, "-c" );
+ fi;
if IsBound( opt.verifyCert ) and opt.verifyCert = false then
Add( args, "--no-check-certificate" );
fi;
@@ -173,8 +189,10 @@ Add( Download_Methods, rec(
code:= Process( DirectoryCurrent(), exec, InputTextNone(), outstream, args );
CloseStream( outstream );
if code <> 0 then
- # wget may have created the target file; try to remove it
+ # wget may have created the target file; try to remove it, unless the
+ # caller wants to resume from what is there
if IsBound( opt.target ) and IsString( opt.target ) and
+ not ( IsBound( opt.resume ) and opt.resume = true ) and
IsExistingFile( opt.target ) and RemoveFile( opt.target ) <> true then
Error( "Download cannot remove unwanted file ", opt.target );
fi;
@@ -211,6 +229,11 @@ Add( Download_Methods, rec(
else
Add( args, "-" );
fi;
+ if IsBound( opt.resume ) and opt.resume = true and
+ IsBound( opt.target ) and IsString( opt.target ) then
+ Add( args, "-C" );
+ Add( args, "-" );
+ fi;
if IsBound( opt.maxTime ) and IsPosInt( opt.maxTime ) then
Add( args, "--max-time" );
Add( args, opt.maxTime );
@@ -275,8 +298,10 @@ InstallMethod( Download,
fi;
# A failed method may have left a partial or bogus target file behind.
# Remove it here, so that the guarantee holds for every method,
- # including ones added to 'Download_Methods' from outside.
+ # including ones added to 'Download_Methods' from outside -- but not
+ # when resuming, where the partial file is the whole point.
if IsBound( opt.target ) and IsString( opt.target ) and
+ not ( IsBound( opt.resume ) and opt.resume = true ) and
IsExistingFile( opt.target ) then
RemoveFile( opt.target );
fi;
diff --git a/tst/download.tst b/tst/download.tst
index 02856d3..afa6b4b 100644
--- a/tst/download.tst
+++ b/tst/download.tst
@@ -1,4 +1,4 @@
-#@local meths, i, urls, pair, url, expected, res1, good1, n, file, res2, good2, contents, r, res3, good3, bad, server, baseurl, iometh, opt, oldpref
+#@local meths, i, urls, pair, url, expected, res1, good1, n, file, res2, good2, contents, r, res3, good3, bad, server, baseurl, iometh, opt, oldpref, resumers, name
############################################################################
##
#W download.tst Utils Package Thomas Breuer
@@ -164,6 +164,55 @@ true
gap> IsExistingFile( file );
false
+## 'resume' continues a partial file rather than fetching it again. The
+## test server answers a Range request with the remainder in upper case, so
+## a resumed download is distinguishable from a restarted one.
+gap> resumers:= Filtered( meths, r -> r.name in [ "via curl", "via wget" ] );;
+gap> for r in resumers do
+> FileString( file, "abcde" );;
+> res1:= r.download( Concatenation( baseurl, "/resumable" ),
+> rec( target:= file, resume:= true ) );
+> if res1.success <> true then
+> Print( "resume failed for ", r.name, ": ", res1.error, "\n" );
+> elif StringFile( file ) <> "abcdeFGHIJKLMNOPQRST" then
+> Print( "did not resume for ", r.name, ": ", StringFile( file ), "\n" );
+> fi;
+> RemoveFile( file );
+> od;
+
+## Without 'resume' the target is replaced, not appended to.
+gap> FileString( file, "abcde" );;
+gap> res1:= Download( Concatenation( baseurl, "/resumable" ),
+> rec( target:= file ) );;
+gap> StringFile( file );
+"abcdefghijklmnopqrst"
+
+## Methods that cannot resume decline, rather than discarding the partial
+## file that a method which can resume needs.
+gap> iometh.download( Concatenation( baseurl, "/resumable" ),
+> rec( target:= file, resume:= true ) ).error;
+"no support for resuming"
+
+## With 'resume', a failed download keeps the partial file to continue from.
+## Without this, the clean-up in 'Download' would throw away the very thing
+## the next attempt is meant to continue.
+gap> FileString( file, "abcde" );;
+gap> res1:= Download( Concatenation( baseurl, "/missing" ),
+> rec( target:= file, resume:= true ) );;
+gap> res1.success;
+false
+gap> StringFile( file );
+"abcde"
+gap> RemoveFile( file );;
+
+## 'resume' is only declined when it is actually requested: a method must
+## not be skipped merely because the component is present.
+gap> res1:= Download( Concatenation( baseurl, "/file" ),
+> rec( target:= file, resume:= false ) );;
+gap> res1.success;
+true
+gap> RemoveFile( file );;
+
## test errors and redirects
gap> res1:= Download( Concatenation( baseurl, "/missing" ) );;
gap> res1.success = false;
diff --git a/tst/http-server.g b/tst/http-server.g
index 40a532a..ad2b0b4 100644
--- a/tst/http-server.g
+++ b/tst/http-server.g
@@ -4,7 +4,7 @@
##
BindGlobal( "UTILS_HandleHTTPTestRequest", function( listener, socket )
- local connection, line, parts, uri, body, status, location;
+ local connection, line, parts, uri, body, status, location, range, from;
IO_close( listener );
connection:= IO_WrapFD( socket, IO.DefaultBufSize, IO.DefaultBufSize );
@@ -16,8 +16,12 @@ BindGlobal( "UTILS_HandleHTTPTestRequest", function( listener, socket )
fi;
uri:= parts[2];
+ range:= fail;
repeat
line:= IO_ReadLine( connection );
+ if IsString( line ) and StartsWith( LowercaseString( line ), "range:" ) then
+ range:= line;
+ fi;
until line = fail or line = "" or line = "\n" or line = "\r\n";
body:= "download test response\n";
@@ -32,6 +36,27 @@ BindGlobal( "UTILS_HandleHTTPTestRequest", function( listener, socket )
body:= "";
status:= "302 Found";
location:= "Location: /success\r\n";
+ elif StartsWith( uri, "/resumable" ) then
+ # Answer a 'Range: bytes=N-' request with the remainder, but in upper
+ # case, so that a test can tell a resumed download from a restarted one.
+ body:= "abcdefghijklmnopqrst";
+ if range <> fail then
+ from:= Int( Filtered( range, c -> c in "0123456789" ) );
+ if from <> fail and 0 < from and from < Length( body ) then
+ IO_Write( connection,
+ "HTTP/1.1 206 Partial Content\r\n",
+ "Content-Type: text/plain\r\n",
+ "Content-Range: bytes ", String( from ), "-",
+ String( Length( body ) - 1 ), "/", String( Length( body ) ),
+ "\r\n",
+ "Content-Length: ", String( Length( body ) - from ), "\r\n",
+ "Connection: close\r\n\r\n",
+ UppercaseString( body{ [ from+1 .. Length( body ) ] } ) );
+ IO_Flush( connection );
+ IO_Close( connection );
+ IO_exit( 0 );
+ fi;
+ fi;
fi;
IO_Write( connection,