Download: add a 'resume' option

doc/download.xml

@@ -69,7 +69,28 @@ The following components are supported.
   and the function writes the downloaded contents to this file;
   the returned record does not have a <C>result</C> component in this case.
   <P/>
-  If the download fails then this file is not left behind.
+  If the download fails then this file is not left behind,
+  unless <C>resume</C> is set.
+</Item>
+<Mark><C>resume</C></Mark>
+<Item>
+  If this component is bound and has the value <K>true</K>,
+  and <C>target</C> is given,
+  then a partially downloaded file is continued rather than fetched again,
+  and it is kept if the download fails again.
+  <P/>
+  Methods that cannot resume decline the request, since they would discard
+  the partial file that a method which can resume needs; currently the
+  methods based on the external programs <C>curl</C> and <C>wget</C> resume.
+  If none is available the download fails, and the caller can retry without
+  <C>resume</C>.
+  <P/>
+  It is the caller's responsibility that an existing <C>target</C> really is
+  a prefix of what <A>url</A> delivers; otherwise the two get concatenated.
+  <P/>
+  If the server does not support range requests then no data is lost:
+  <C>wget</C> fetches the file again from the start, and <C>curl</C> fails
+  and leaves the partial file alone.
 </Item>
 <Mark><C>verifyCert</C></Mark>
 <Item>

lib/download.gi

@@ -36,6 +36,13 @@ Add( Download_Methods, rec(
   download:= function( url, opt )
     local res;
 
+    if IsBound( opt.resume ) and opt.resume = true and
+       IsBound( opt.target ) and IsString( opt.target ) then
+      # Declining matters: this method would truncate the partial file that a
+      # method which can resume needs.
+      return rec( success:= false, error:= "no support for resuming" );
+    fi;
+
     opt:= ShallowCopy( opt );
     if not IsBound( opt.failOnError ) then
       opt.failOnError:= true;
@@ -61,6 +68,11 @@ Add( Download_Methods, rec(
       return rec( success:= false, error:= "protocol is not http" );
     elif IsBound( opt.maxTime ) and opt.maxTime <> 0 then
       return rec( success:= false, error:= "no support for given timeout" );
+    elif IsBound( opt.resume ) and opt.resume = true and
+         IsBound( opt.target ) and IsString( opt.target ) then
+      # No range request, so this would overwrite the partial file that a
+      # method which can resume needs.
+      return rec( success:= false, error:= "no support for resuming" );
     fi;
 
     # Split the URL after 'http://' into the authority and HTTP request target.
@@ -164,6 +176,10 @@ Add( Download_Methods, rec(
     else
       args:= [ "--quiet", "-O", "-", url ];
     fi;
+    if IsBound( opt.resume ) and opt.resume = true and
+       IsBound( opt.target ) and IsString( opt.target ) then
+      Add( args, "-c" );
+    fi;
     if IsBound( opt.verifyCert ) and opt.verifyCert = false then
       Add( args, "--no-check-certificate" );
     fi;
@@ -173,8 +189,10 @@ Add( Download_Methods, rec(
     code:= Process( DirectoryCurrent(), exec, InputTextNone(), outstream, args );
     CloseStream( outstream );
     if code <> 0 then
-      # wget may have created the target file; try to remove it
+      # wget may have created the target file; try to remove it, unless the
+      # caller wants to resume from what is there
       if IsBound( opt.target ) and IsString( opt.target ) and
+         not ( IsBound( opt.resume ) and opt.resume = true ) and
          IsExistingFile( opt.target ) and RemoveFile( opt.target ) <> true then
         Error( "Download cannot remove unwanted file ", opt.target );
       fi;
@@ -211,6 +229,11 @@ Add( Download_Methods, rec(
     else
       Add( args, "-" );
     fi;
+    if IsBound( opt.resume ) and opt.resume = true and
+       IsBound( opt.target ) and IsString( opt.target ) then
+      Add( args, "-C" );
+      Add( args, "-" );
+    fi;
     if IsBound( opt.maxTime ) and IsPosInt( opt.maxTime ) then
       Add( args, "--max-time" );
       Add( args, opt.maxTime );
@@ -275,8 +298,10 @@ InstallMethod( Download,
         fi;
         # A failed method may have left a partial or bogus target file behind.
         # Remove it here, so that the guarantee holds for every method,
-        # including ones added to 'Download_Methods' from outside.
+        # including ones added to 'Download_Methods' from outside -- but not
+        # when resuming, where the partial file is the whole point.
         if IsBound( opt.target ) and IsString( opt.target ) and
+           not ( IsBound( opt.resume ) and opt.resume = true ) and
            IsExistingFile( opt.target ) then
           RemoveFile( opt.target );
         fi;

tst/download.tst

@@ -1,4 +1,4 @@
-#@local meths, i, urls, pair, url, expected, res1, good1, n, file, res2, good2, contents, r, res3, good3, bad, server, baseurl, iometh, opt, oldpref
+#@local meths, i, urls, pair, url, expected, res1, good1, n, file, res2, good2, contents, r, res3, good3, bad, server, baseurl, iometh, opt, oldpref, resumers, name
 ############################################################################
 ##
 #W  download.tst               Utils Package                   Thomas Breuer
@@ -164,6 +164,55 @@ true
 gap> IsExistingFile( file );
 false
 
+##  'resume' continues a partial file rather than fetching it again.  The
+##  test server answers a Range request with the remainder in upper case, so
+##  a resumed download is distinguishable from a restarted one.
+gap> resumers:= Filtered( meths, r -> r.name in [ "via curl", "via wget" ] );;
+gap> for r in resumers do
+>      FileString( file, "abcde" );;
+>      res1:= r.download( Concatenation( baseurl, "/resumable" ),
+>                         rec( target:= file, resume:= true ) );
+>      if res1.success <> true then
+>        Print( "resume failed for ", r.name, ": ", res1.error, "\n" );
+>      elif StringFile( file ) <> "abcdeFGHIJKLMNOPQRST" then
+>        Print( "did not resume for ", r.name, ": ", StringFile( file ), "\n" );
+>      fi;
+>      RemoveFile( file );
+>    od;
+
+##  Without 'resume' the target is replaced, not appended to.
+gap> FileString( file, "abcde" );;
+gap> res1:= Download( Concatenation( baseurl, "/resumable" ),
+>                     rec( target:= file ) );;
+gap> StringFile( file );
+"abcdefghijklmnopqrst"
+
+##  Methods that cannot resume decline, rather than discarding the partial
+##  file that a method which can resume needs.
+gap> iometh.download( Concatenation( baseurl, "/resumable" ),
+>                     rec( target:= file, resume:= true ) ).error;
+"no support for resuming"
+
+##  With 'resume', a failed download keeps the partial file to continue from.
+##  Without this, the clean-up in 'Download' would throw away the very thing
+##  the next attempt is meant to continue.
+gap> FileString( file, "abcde" );;
+gap> res1:= Download( Concatenation( baseurl, "/missing" ),
+>                     rec( target:= file, resume:= true ) );;
+gap> res1.success;
+false
+gap> StringFile( file );
+"abcde"
+gap> RemoveFile( file );;
+
+##  'resume' is only declined when it is actually requested: a method must
+##  not be skipped merely because the component is present.
+gap> res1:= Download( Concatenation( baseurl, "/file" ),
+>                     rec( target:= file, resume:= false ) );;
+gap> res1.success;
+true
+gap> RemoveFile( file );;
+
 ##  test errors and redirects
 gap> res1:= Download( Concatenation( baseurl, "/missing" ) );;
 gap> res1.success = false;

tst/http-server.g

@@ -4,7 +4,7 @@
 ##
 
 BindGlobal( "UTILS_HandleHTTPTestRequest", function( listener, socket )
-  local connection, line, parts, uri, body, status, location;
+  local connection, line, parts, uri, body, status, location, range, from;
 
   IO_close( listener );
   connection:= IO_WrapFD( socket, IO.DefaultBufSize, IO.DefaultBufSize );
@@ -16,8 +16,12 @@ BindGlobal( "UTILS_HandleHTTPTestRequest", function( listener, socket )
   fi;
   uri:= parts[2];
 
+  range:= fail;
   repeat
     line:= IO_ReadLine( connection );
+    if IsString( line ) and StartsWith( LowercaseString( line ), "range:" ) then
+      range:= line;
+    fi;
   until line = fail or line = "" or line = "\n" or line = "\r\n";
 
   body:= "download test response\n";
@@ -32,6 +36,27 @@ BindGlobal( "UTILS_HandleHTTPTestRequest", function( listener, socket )
     body:= "";
     status:= "302 Found";
     location:= "Location: /success\r\n";
+  elif StartsWith( uri, "/resumable" ) then
+    # Answer a 'Range: bytes=N-' request with the remainder, but in upper
+    # case, so that a test can tell a resumed download from a restarted one.
+    body:= "abcdefghijklmnopqrst";
+    if range <> fail then
+      from:= Int( Filtered( range, c -> c in "0123456789" ) );
+      if from <> fail and 0 < from and from < Length( body ) then
+        IO_Write( connection,
+            "HTTP/1.1 206 Partial Content\r\n",
+            "Content-Type: text/plain\r\n",
+            "Content-Range: bytes ", String( from ), "-",
+                String( Length( body ) - 1 ), "/", String( Length( body ) ),
+                "\r\n",
+            "Content-Length: ", String( Length( body ) - from ), "\r\n",
+            "Connection: close\r\n\r\n",
+            UppercaseString( body{ [ from+1 .. Length( body ) ] } ) );
+        IO_Flush( connection );
+        IO_Close( connection );
+        IO_exit( 0 );
+      fi;
+    fi;
   fi;
 
   IO_Write( connection,