X-Git-Url: http://git.openwrt.org/?p=openwrt%2Fstaging%2Fdedeckeh.git;a=blobdiff_plain;f=scripts%2Fdownload.pl;h=bf9fe8c76190a6088940523124313d4d91256e15;hp=50ecfc696659ea63b1974259e03dddfa6b5411d1;hb=0a761fe8ef346c6f0a78097938b083e2d078b0e2;hpb=a9c7feb0f77addb23f484ac42bbe31e767e49ff7 diff --git a/scripts/download.pl b/scripts/download.pl index 50ecfc6966..bf9fe8c761 100755 --- a/scripts/download.pl +++ b/scripts/download.pl @@ -1,6 +1,7 @@ #!/usr/bin/env perl # # Copyright (C) 2006 OpenWrt.org +# Copyright (C) 2016 LEDE project # # This is free software, licensed under the GNU General Public License v2. # See /LICENSE for more information. @@ -9,22 +10,28 @@ use strict; use warnings; use File::Basename; +use File::Copy; +use Text::ParseWords; -@ARGV > 2 or die "Syntax: $0 [ ...]\n"; +@ARGV > 2 or die "Syntax: $0 [ ...]\n"; -my $target = shift @ARGV; +my $url_filename; +my $target = glob(shift @ARGV); my $filename = shift @ARGV; -my $md5sum = shift @ARGV; +my $file_hash = shift @ARGV; +$url_filename = shift @ARGV unless $ARGV[0] =~ /:\/\//; my $scriptdir = dirname($0); my @mirrors; my $ok; +$url_filename or $url_filename = $filename; + sub localmirrors { my @mlist; open LM, "$scriptdir/localmirrors" and do { while () { chomp $_; - push @mlist, $_; + push @mlist, $_ if $_; } close LM; }; @@ -39,6 +46,9 @@ sub localmirrors { close CONFIG; }; + my $mirror = $ENV{'DOWNLOAD_MIRROR'}; + $mirror and push @mlist, split(/;/, $mirror); + return @mlist; } @@ -51,70 +61,127 @@ sub which($) { return $res; } -my $md5cmd = which("md5sum"); -$md5cmd or $md5cmd = which("md5"); -$md5cmd or die 'no md5 checksum program found, please install md5 or md5sum'; -chomp $md5cmd; +sub hash_cmd() { + my $len = length($file_hash); + my $cmd; + + $len == 64 and return "mkhash sha256"; + $len == 32 and return "mkhash md5"; + return undef; +} + +sub download_cmd($) { + my $url = shift; + my $have_curl = 0; + + if (open CURL, '-|', 'curl', '--version') { + if (defined(my $line = readline CURL)) { + $have_curl = 1 if $line =~ /^curl /; + } + close CURL; + } + + return $have_curl + ? (qw(curl -f --connect-timeout 20 --retry 5 --location --insecure), shellwords($ENV{CURL_OPTIONS} || ''), $url) + : (qw(wget --tries=5 --timeout=20 --no-check-certificate --output-document=-), shellwords($ENV{WGET_OPTIONS} || ''), $url) + ; +} + +my $hash_cmd = hash_cmd(); sub download { my $mirror = shift; - my $options = $ENV{WGET_OPTIONS}; - $options or $options = ""; - - $mirror =~ s/\/$//; - if( $mirror =~ /^file:\/\// ) { - my $cache = $mirror; - $cache =~ s/file:\/\///g; - if(system("test -d $cache")) { - print STDERR "Wrong local cache directory -$cache-.\n"; + + $mirror =~ s!/$!!; + + if ($mirror =~ s!^file://!!) { + if (! -d "$mirror") { + print STDERR "Wrong local cache directory -$mirror-.\n"; cleanup(); return; } - if(! -d $target) { - system("mkdir -p $target/"); + + if (! -d "$target") { + system("mkdir", "-p", "$target/"); + } + + if (! open TMPDLS, "find $mirror -follow -name $filename 2>/dev/null |") { + print("Failed to search for $filename in $mirror\n"); + return; } - system("cp -vf $cache/$filename $target/$filename.dl") == 0 or return; - system("$md5cmd $target/$filename.dl > \"$target/$filename.md5sum\" ") == 0 or return; + + my $link; + + while (defined(my $line = readline TMPDLS)) { + chomp ($link = $line); + if ($. > 1) { + print("$. or more instances of $filename in $mirror found . Only one instance allowed.\n"); + return; + } + } + + close TMPDLS; + + if (! $link) { + print("No instances of $filename found in $mirror.\n"); + return; + } + + print("Copying $filename from $link\n"); + copy($link, "$target/$filename.dl"); + + $hash_cmd and do { + if (system("cat '$target/$filename.dl' | $hash_cmd > '$target/$filename.hash'")) { + print("Failed to generate hash for $filename\n"); + return; + } + }; } else { - open WGET, "wget -t5 --timeout=20 --no-check-certificate $options -O- \"$mirror/$filename\" |" or die "Cannot launch wget.\n"; - open MD5SUM, "| $md5cmd > \"$target/$filename.md5sum\"" or die "Cannot launch md5sum.\n"; + my @cmd = download_cmd("$mirror/$url_filename"); + print STDERR "+ ".join(" ",@cmd)."\n"; + open(FETCH_FD, '-|', @cmd) or die "Cannot launch curl or wget.\n"; + $hash_cmd and do { + open MD5SUM, "| $hash_cmd > '$target/$filename.hash'" or die "Cannot launch $hash_cmd.\n"; + }; open OUTPUT, "> $target/$filename.dl" or die "Cannot create file $target/$filename.dl: $!\n"; my $buffer; - while (read WGET, $buffer, 1048576) { - print MD5SUM $buffer; + while (read FETCH_FD, $buffer, 1048576) { + $hash_cmd and print MD5SUM $buffer; print OUTPUT $buffer; } - close MD5SUM; - close WGET; + $hash_cmd and close MD5SUM; + close FETCH_FD; close OUTPUT; - if (($? >> 8) != 0 ) { + if ($? >> 8) { print STDERR "Download failed.\n"; cleanup(); return; } } - my $sum = `cat "$target/$filename.md5sum"`; - $sum =~ /^(\w+)\s*/ or die "Could not generate md5sum\n"; - $sum = $1; + $hash_cmd and do { + my $sum = `cat "$target/$filename.hash"`; + $sum =~ /^(\w+)\s*/ or die "Could not generate file hash\n"; + $sum = $1; - if (($md5sum =~ /\w{32}/) and ($sum ne $md5sum)) { - print STDERR "MD5 sum of the downloaded file does not match (file: $sum, requested: $md5sum) - deleting download.\n"; - cleanup(); - return; - } + if ($sum ne $file_hash) { + print STDERR "Hash of the downloaded file does not match (file: $sum, requested: $file_hash) - deleting download.\n"; + cleanup(); + return; + } + }; unlink "$target/$filename"; - system("mv \"$target/$filename.dl\" \"$target/$filename\""); + system("mv", "$target/$filename.dl", "$target/$filename"); cleanup(); } sub cleanup { unlink "$target/$filename.dl"; - unlink "$target/$filename.md5sum"; + unlink "$target/$filename.hash"; } @mirrors = localmirrors(); @@ -125,43 +192,62 @@ foreach my $mirror (@ARGV) { for (1 .. 5) { push @mirrors, "http://downloads.sourceforge.net/$1"; } + } elsif ($mirror =~ /^\@APACHE\/(.+)$/) { + push @mirrors, "https://mirror.netcologne.de/apache.org/$1"; + push @mirrors, "https://mirror.aarnet.edu.au/pub/apache/$1"; + push @mirrors, "https://mirror.csclub.uwaterloo.ca/apache/$1"; + push @mirrors, "http://mirror.cogentco.com/pub/apache/$1"; + push @mirrors, "http://mirror.navercorp.com/apache/$1"; + push @mirrors, "http://ftp.jaist.ac.jp/pub/apache/$1"; + push @mirrors, "ftp://apache.cs.utah.edu/apache.org/$1"; + push @mirrors, "ftp://apache.mirrors.ovh.net/ftp.apache.org/dist/$1"; + } elsif ($mirror =~ /^\@GITHUB\/(.+)$/) { + # give github a few more tries (different mirrors) + for (1 .. 5) { + push @mirrors, "https://raw.githubusercontent.com/$1"; + } } elsif ($mirror =~ /^\@GNU\/(.+)$/) { - push @mirrors, "ftp://ftp.gnu.org/gnu/$1"; - push @mirrors, "ftp://ftp.belnet.be/mirror/ftp.gnu.org/gnu/$1"; - push @mirrors, "ftp://ftp.mirror.nl/pub/mirror/gnu/$1"; - push @mirrors, "http://mirror.switch.ch/ftp/mirror/gnu/$1"; - push @mirrors, "ftp://ftp.uu.net/archive/systems/gnu/$1"; - push @mirrors, "ftp://ftp.eu.uu.net/pub/gnu/$1"; - push @mirrors, "ftp://ftp.leo.org/pub/comp/os/unix/gnu/$1"; - push @mirrors, "ftp://ftp.digex.net/pub/gnu/$1"; + push @mirrors, "https://mirror.csclub.uwaterloo.ca/gnu/$1"; + push @mirrors, "https://mirror.netcologne.de/gnu/$1"; + push @mirrors, "http://ftp.kddilabs.jp/GNU/gnu/$1"; + push @mirrors, "http://www.nic.funet.fi/pub/gnu/gnu/$1"; + push @mirrors, "http://mirror.internode.on.net/pub/gnu/$1"; + push @mirrors, "http://mirror.navercorp.com/gnu/$1"; + push @mirrors, "ftp://mirrors.rit.edu/gnu/$1"; + push @mirrors, "ftp://download.xs4all.nl/pub/gnu/"; + } elsif ($mirror =~ /^\@SAVANNAH\/(.+)$/) { + push @mirrors, "https://mirror.netcologne.de/savannah/$1"; + push @mirrors, "https://mirror.csclub.uwaterloo.ca/nongnu/$1"; + push @mirrors, "http://ftp.acc.umu.se/mirror/gnu.org/savannah/$1"; + push @mirrors, "http://nongnu.uib.no/$1"; + push @mirrors, "http://ftp.igh.cnrs.fr/pub/nongnu/$1"; + push @mirrors, "http://public.p-knowledge.co.jp/Savannah-nongnu-mirror/$1"; + push @mirrors, "ftp://cdimage.debian.org/mirror/gnu.org/savannah/$1"; + push @mirrors, "ftp://ftp.acc.umu.se/mirror/gnu.org/savannah/$1"; } elsif ($mirror =~ /^\@KERNEL\/(.+)$/) { my @extra = ( $1 ); - if ($filename =~ /linux-\d+\.\d+\.\d+-rc/) { + if ($filename =~ /linux-\d+\.\d+(?:\.\d+)?-rc/) { push @extra, "$extra[0]/testing"; - } elsif ($filename =~ /linux-(\d+\.\d+\.\d+)/) { + } elsif ($filename =~ /linux-(\d+\.\d+(?:\.\d+)?)/) { push @extra, "$extra[0]/longterm/v$1"; } foreach my $dir (@extra) { - push @mirrors, "ftp://ftp.geo.kernel.org/pub/$dir"; - push @mirrors, "http://ftp.geo.kernel.org/pub/$dir"; - push @mirrors, "ftp://ftp.all.kernel.org/pub/$dir"; - push @mirrors, "http://ftp.all.kernel.org/pub/$dir"; - push @mirrors, "ftp://ftp.de.kernel.org/pub/$dir"; - push @mirrors, "http://ftp.de.kernel.org/pub/$dir"; - push @mirrors, "ftp://ftp.fr.kernel.org/pub/$dir"; - push @mirrors, "http://ftp.fr.kernel.org/pub/$dir"; + push @mirrors, "https://cdn.kernel.org/pub/$dir"; + push @mirrors, "https://mirror.rackspace.com/kernel.org/$dir"; + push @mirrors, "http://download.xs4all.nl/ftp.kernel.org/pub/$dir"; + push @mirrors, "http://mirrors.mit.edu/kernel/$dir"; + push @mirrors, "http://ftp.nara.wide.ad.jp/pub/kernel.org/$dir"; + push @mirrors, "http://www.ring.gr.jp/archives/linux/kernel.org/$dir"; + push @mirrors, "ftp://ftp.riken.jp/Linux/kernel.org/$dir"; + push @mirrors, "ftp://www.mirrorservice.org/sites/ftp.kernel.org/pub/$dir"; } } elsif ($mirror =~ /^\@GNOME\/(.+)$/) { - push @mirrors, "http://ftp.gnome.org/pub/GNOME/sources/$1"; - push @mirrors, "http://ftp.unina.it/pub/linux/GNOME/sources/$1"; - push @mirrors, "http://fr2.rpmfind.net/linux/gnome.org/sources/$1"; - push @mirrors, "ftp://ftp.dit.upm.es/pub/GNOME/sources/$1"; - push @mirrors, "ftp://ftp.no.gnome.org/pub/GNOME/sources/$1"; + push @mirrors, "https://mirror.csclub.uwaterloo.ca/gnome/sources/$1"; push @mirrors, "http://ftp.acc.umu.se/pub/GNOME/sources/$1"; - push @mirrors, "http://ftp.belnet.be/mirror/ftp.gnome.org/sources/$1"; - push @mirrors, "http://linorg.usp.br/gnome/sources/$1"; - push @mirrors, "http://mirror.aarnet.edu.au/pub/GNOME/sources/$1"; - push @mirrors, "http://mirrors.ibiblio.org/pub/mirrors/gnome/sources/$1"; + push @mirrors, "http://ftp.kaist.ac.kr/gnome/sources/$1"; + push @mirrors, "http://www.mirrorservice.org/sites/ftp.gnome.org/pub/GNOME/sources/$1"; + push @mirrors, "http://mirror.internode.on.net/pub/gnome/sources/$1"; + push @mirrors, "http://ftp.belnet.be/ftp.gnome.org/sources/$1"; push @mirrors, "ftp://ftp.cse.buffalo.edu/pub/Gnome/sources/$1"; push @mirrors, "ftp://ftp.nara.wide.ad.jp/pub/X11/GNOME/sources/$1"; } @@ -171,15 +257,15 @@ foreach my $mirror (@ARGV) { } #push @mirrors, 'http://mirror1.openwrt.org'; +push @mirrors, 'http://sources.lede-project.org'; push @mirrors, 'http://mirror2.openwrt.org/sources'; push @mirrors, 'http://downloads.openwrt.org/sources'; -while (!$ok) { +while (!-f "$target/$filename") { my $mirror = shift @mirrors; $mirror or die "No more mirrors to try - giving up.\n"; download($mirror); - -f "$target/$filename" and $ok = 1; } $SIG{INT} = \&cleanup;