download: add support for gitweb snapshots

When downloading a snapshot archive from gitweb,
the filename is not part of the URL,
and adding the filename to the URL causes errors.

The gitweb API exclusively uses query parameters
instead of paths in order to execute snapshot downloads.

Add a condition to the Perl download script
that removes the filename if the relevant
query parameter matches in the URL.

Also, to reduce server load of the original sources
try the Openwrt CDN servers first for these downloads.

Even though snapshot downloads are not ideal
due to the impact on the source's server health,
they are better for download performance than using git only.
Therefore, attempting it last will reduce the impact
and thus encourage maintainers to keep the option enabled.

This change is partly inspired by a conversation linked below
about snapshot downloads and server performance issues
which led to the feature being disabled for a particular server.

Link: https://lists.gnu.org/archive/html/bug-gnulib/2024-12/msg00124.html
Signed-off-by: Michael Pratt <mcpratt@pm.me>
Link: https://github.com/openwrt/openwrt/pull/16522
Signed-off-by: Robert Marko <robimarko@gmail.com>

1 files changed, 16 insertions, 2 deletions

diff --git a/scripts/download.pl b/scripts/download.pl
index c6c9b8e56c..09dc91b04b 100755
--- a/scripts/download.pl
+++ b/scripts/download.pl
@@ -163,6 +163,7 @@ sub download
 	my $mirror = shift;
 	my $download_filename = shift;
 	my @additional_mirrors = @_;
+	my @cmd;
 
 	$mirror =~ s!/$!!;
 
@@ -209,7 +210,11 @@ sub download
 			}
 		};
 	} else {
-		my @cmd = download_cmd("$mirror/$download_filename", $download_filename, @additional_mirrors);
+		if ($mirror =~ /a=snapshot/) {
+			@cmd = download_cmd("$mirror", $download_filename, @additional_mirrors);
+		} else {
+			@cmd = download_cmd("$mirror/$download_filename", $download_filename, @additional_mirrors);
+		}
 		print STDERR "+ ".join(" ",@cmd)."\n";
 		open(FETCH_FD, '-|', @cmd) or die "Cannot launch aria2c, curl or wget.\n";
 		$hash_cmd and do {
@@ -317,14 +322,23 @@ if (-f "$target/$filename") {
 
 $download_tool = select_tool();
 
+my $mirror = shift @mirrors;
+
+# Try snapshot original source last
+if ($mirror =~ /snapshot/) {
+	push @mirrors, $mirror;
+	$mirror = shift @mirrors;
+}
+
 while (!-f "$target/$filename") {
-	my $mirror = shift @mirrors;
 	$mirror or die "No more mirrors to try - giving up.\n";
 
 	download($mirror, $url_filename, @mirrors);
 	if (!-f "$target/$filename" && $url_filename ne $filename) {
 		download($mirror, $filename, @mirrors);
 	}
+
+	$mirror = shift @mirrors;
 }
 
 $SIG{INT} = \&cleanup;