uclient-fetch.c

   1 /*
   2  * uclient - ustream based protocol client library
   3  *
   4  * Copyright (C) 2014 Felix Fietkau <nbd@openwrt.org>
   5  *
   6  * Permission to use, copy, modify, and/or distribute this software for any
   7  * purpose with or without fee is hereby granted, provided that the above
   8  * copyright notice and this permission notice appear in all copies.
   9  *
  10  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  11  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  12  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  13  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  14  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  15  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  16  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  17  */
  18
  19 #define _GNU_SOURCE
  20 #include <sys/stat.h>
  21 #include <sys/socket.h>
  22 #include <unistd.h>
  23 #include <stdio.h>
  24 #include <dlfcn.h>
  25 #include <getopt.h>
  26 #include <fcntl.h>
  27 #include <glob.h>
  28 #include <stdint.h>
  29 #include <inttypes.h>
  30 #include <signal.h>
  31
  32 #include <libubox/blobmsg.h>
  33
  34 #include "progress.h"
  35 #include "uclient.h"
  36 #include "uclient-utils.h"
  37
  38 #ifdef __APPLE__
  39 #define LIB_EXT "dylib"
  40 #else
  41 #define LIB_EXT "so"
  42 #endif
  43
  44 static const char *user_agent = "uclient-fetch";
  45 static const char *post_data;
  46 static struct ustream_ssl_ctx *ssl_ctx;
  47 static const struct ustream_ssl_ops *ssl_ops;
  48 static int quiet = false;
  49 static bool verify = true;
  50 static bool proxy = true;
  51 static bool default_certs = false;
  52 static bool no_output;
  53 static const char *opt_output_file;
  54 static int output_fd = -1;
  55 static int error_ret;
  56 static off_t out_offset;
  57 static off_t out_bytes;
  58 static off_t out_len;
  59 static char *auth_str;
  60 static char **urls;
  61 static int n_urls;
  62 static int timeout;
  63 static bool resume, cur_resume;
  64
  65 static struct progress pmt;
  66 static struct uloop_timeout pmt_timer;
  67
  68 static int init_request(struct uclient *cl);
  69 static void request_done(struct uclient *cl);
  70
  71 static void pmt_update(struct uloop_timeout *t)
  72 {
  73         progress_update(&pmt, out_offset, out_bytes, out_len);
  74         uloop_timeout_set(t, 1000);
  75 }
  76
  77 static const char *
  78 get_proxy_url(char *url)
  79 {
  80         char prefix[16];
  81         char *sep;
  82
  83         if (!proxy)
  84                 return NULL;
  85
  86         sep = strchr(url, ':');
  87         if (!sep)
  88                 return NULL;
  89
  90         if (sep - url > 5)
  91                 return NULL;
  92
  93         memcpy(prefix, url, sep - url);
  94         strcpy(prefix + (sep - url), "_proxy");
  95         return getenv(prefix);
  96 }
  97
  98 static int open_output_file(const char *path, uint64_t resume_offset)
  99 {
 100         const char *output_file = opt_output_file;
 101         char *filename = NULL;
 102         int flags;
 103         int ret;
 104
 105         if (cur_resume)
 106                 flags = O_RDWR;
 107         else
 108                 flags = O_WRONLY | O_TRUNC;
 109
 110         if (!cur_resume && !output_file)
 111                 flags |= O_EXCL;
 112
 113         flags |= O_CREAT;
 114
 115         if (output_file) {
 116                 if (!strcmp(output_file, "-")) {
 117                         if (!quiet)
 118                                 fprintf(stderr, "Writing to stdout\n");
 119
 120                         ret = STDOUT_FILENO;
 121                         goto done;
 122                 }
 123         } else {
 124                 filename = uclient_get_url_filename(path, "index.html");
 125                 output_file = filename;
 126         }
 127
 128         if (!quiet)
 129                 fprintf(stderr, "Writing to '%s'\n", output_file);
 130         ret = open(output_file, flags, 0644);
 131         if (ret < 0)
 132                 goto free;
 133
 134         if (resume_offset &&
 135             lseek(ret, resume_offset, SEEK_SET) < 0) {
 136                 if (!quiet)
 137                         fprintf(stderr, "Failed to seek %"PRIu64" bytes in output file\n", resume_offset);
 138                 close(ret);
 139                 ret = -1;
 140                 goto free;
 141         }
 142
 143         out_offset = resume_offset;
 144         out_bytes += resume_offset;
 145 done:
 146         if (!quiet) {
 147                 progress_init(&pmt, output_file);
 148                 pmt_timer.cb = pmt_update;
 149                 pmt_timer.cb(&pmt_timer);
 150         }
 151
 152 free:
 153         free(filename);
 154         return ret;
 155 }
 156
 157 static void header_done_cb(struct uclient *cl)
 158 {
 159         enum {
 160                 H_RANGE,
 161                 H_LEN,
 162                 __H_MAX
 163         };
 164         static const struct blobmsg_policy policy[__H_MAX] = {
 165                 [H_RANGE] = { .name = "content-range", .type = BLOBMSG_TYPE_STRING },
 166                 [H_LEN] = { .name = "content-length", .type = BLOBMSG_TYPE_STRING },
 167         };
 168         struct blob_attr *tb[__H_MAX];
 169         uint64_t resume_offset = 0, resume_end, resume_size;
 170         static int retries;
 171
 172         if (retries < 10) {
 173                 int ret = uclient_http_redirect(cl);
 174                 if (ret < 0) {
 175                         if (!quiet)
 176                                 fprintf(stderr, "Failed to redirect to %s on %s\n", cl->url->location, cl->url->host);
 177                         error_ret = 8;
 178                         request_done(cl);
 179                         return;
 180                 }
 181                 if (ret > 0) {
 182                         if (!quiet)
 183                                 fprintf(stderr, "Redirected to %s on %s\n", cl->url->location, cl->url->host);
 184
 185                         retries++;
 186                         return;
 187                 }
 188         }
 189
 190         if (cl->status_code == 204 && cur_resume) {
 191                 /* Resume attempt failed, try normal download */
 192                 cur_resume = false;
 193                 init_request(cl);
 194                 return;
 195         }
 196
 197         blobmsg_parse(policy, __H_MAX, tb, blob_data(cl->meta), blob_len(cl->meta));
 198
 199         switch (cl->status_code) {
 200         case 416:
 201                 if (!quiet)
 202                         fprintf(stderr, "File download already fully retrieved; nothing to do.\n");
 203                 request_done(cl);
 204                 break;
 205         case 206:
 206                 if (!cur_resume) {
 207                         if (!quiet)
 208                                 fprintf(stderr, "Error: Partial content received, full content requested\n");
 209                         error_ret = 8;
 210                         request_done(cl);
 211                         break;
 212                 }
 213
 214                 if (!tb[H_RANGE]) {
 215                         if (!quiet)
 216                                 fprintf(stderr, "Content-Range header is missing\n");
 217                         error_ret = 8;
 218                         break;
 219                 }
 220
 221                 if (sscanf(blobmsg_get_string(tb[H_RANGE]),
 222                            "bytes %"PRIu64"-%"PRIu64"/%"PRIu64,
 223                            &resume_offset, &resume_end, &resume_size) != 3) {
 224                         if (!quiet)
 225                                 fprintf(stderr, "Content-Range header is invalid\n");
 226                         error_ret = 8;
 227                         break;
 228                 }
 229         case 204:
 230         case 200:
 231                 if (no_output)
 232                         break;
 233
 234                 if (tb[H_LEN])
 235                         out_len = strtoul(blobmsg_get_string(tb[H_LEN]), NULL, 10);
 236
 237                 output_fd = open_output_file(cl->url->location, resume_offset);
 238                 if (output_fd < 0) {
 239                         if (!quiet)
 240                                 perror("Cannot open output file");
 241                         error_ret = 3;
 242                         request_done(cl);
 243                 }
 244                 break;
 245
 246         default:
 247                 if (!quiet)
 248                         fprintf(stderr, "HTTP error %d\n", cl->status_code);
 249                 request_done(cl);
 250                 error_ret = 8;
 251                 break;
 252         }
 253 }
 254
 255 static void read_data_cb(struct uclient *cl)
 256 {
 257         char buf[256];
 258         ssize_t n;
 259         int len;
 260
 261         if (!no_output && output_fd < 0)
 262                 return;
 263
 264         while (1) {
 265                 len = uclient_read(cl, buf, sizeof(buf));
 266                 if (len <= 0)
 267                         return;
 268
 269                 out_bytes += len;
 270                 if (!no_output) {
 271                         n = write(output_fd, buf, len);
 272                         if (n < 0)
 273                                 return;
 274                 }
 275         }
 276 }
 277
 278 static void msg_connecting(struct uclient *cl)
 279 {
 280         char addr[INET6_ADDRSTRLEN];
 281         int port;
 282
 283         if (quiet)
 284                 return;
 285
 286         uclient_get_addr(addr, &port, &cl->remote_addr);
 287         fprintf(stderr, "Connecting to %s:%d\n", addr, port);
 288 }
 289
 290 static void check_resume_offset(struct uclient *cl)
 291 {
 292         char range_str[64];
 293         struct stat st;
 294         char *file;
 295         int ret;
 296
 297         file = uclient_get_url_filename(cl->url->location, "index.html");
 298         if (!file)
 299                 return;
 300
 301         ret = stat(file, &st);
 302         free(file);
 303         if (ret)
 304                 return;
 305
 306         if (!st.st_size)
 307                 return;
 308
 309         snprintf(range_str, sizeof(range_str), "bytes=%"PRIu64"-", (uint64_t) st.st_size);
 310         uclient_http_set_header(cl, "Range", range_str);
 311 }
 312
 313 static int init_request(struct uclient *cl)
 314 {
 315         int rc;
 316
 317         out_offset = 0;
 318         out_bytes = 0;
 319         out_len = 0;
 320         uclient_http_set_ssl_ctx(cl, ssl_ops, ssl_ctx, verify);
 321
 322         if (timeout)
 323                 cl->timeout_msecs = timeout * 1000;
 324
 325         rc = uclient_connect(cl);
 326         if (rc)
 327                 return rc;
 328
 329         msg_connecting(cl);
 330
 331         rc = uclient_http_set_request_type(cl, post_data ? "POST" : "GET");
 332         if (rc)
 333                 return rc;
 334
 335         uclient_http_reset_headers(cl);
 336         uclient_http_set_header(cl, "User-Agent", user_agent);
 337         if (cur_resume)
 338                 check_resume_offset(cl);
 339
 340         if (post_data) {
 341                 uclient_http_set_header(cl, "Content-Type", "application/x-www-form-urlencoded");
 342                 uclient_write(cl, post_data, strlen(post_data));
 343         }
 344
 345         rc = uclient_request(cl);
 346         if (rc)
 347                 return rc;
 348
 349         return 0;
 350 }
 351
 352 static void request_done(struct uclient *cl)
 353 {
 354         const char *proxy_url;
 355
 356         if (n_urls) {
 357                 proxy_url = get_proxy_url(*urls);
 358                 if (proxy_url) {
 359                         uclient_set_url(cl, proxy_url, NULL);
 360                         uclient_set_proxy_url(cl, *urls, auth_str);
 361                 } else {
 362                         uclient_set_url(cl, *urls, auth_str);
 363                 }
 364                 n_urls--;
 365                 cur_resume = resume;
 366                 error_ret = init_request(cl);
 367                 if (error_ret == 0)
 368                         return;
 369         }
 370
 371         if (output_fd >= 0 && !opt_output_file) {
 372                 close(output_fd);
 373                 output_fd = -1;
 374         }
 375         uclient_disconnect(cl);
 376         uloop_end();
 377 }
 378
 379
 380 static void eof_cb(struct uclient *cl)
 381 {
 382         if (!quiet) {
 383                 pmt_update(&pmt_timer);
 384                 uloop_timeout_cancel(&pmt_timer);
 385                 fprintf(stderr, "\n");
 386         }
 387
 388         if (!cl->data_eof) {
 389                 if (!quiet)
 390                         fprintf(stderr, "Connection reset prematurely\n");
 391                 error_ret = 4;
 392         } else if (!quiet) {
 393                 fprintf(stderr, "Download completed (%"PRIu64" bytes)\n", (uint64_t) out_bytes);
 394         }
 395         request_done(cl);
 396 }
 397
 398 static void handle_uclient_error(struct uclient *cl, int code)
 399 {
 400         const char *type = "Unknown error";
 401         bool ignore = false;
 402
 403         switch(code) {
 404         case UCLIENT_ERROR_CONNECT:
 405                 type = "Connection failed";
 406                 error_ret = 4;
 407                 break;
 408         case UCLIENT_ERROR_TIMEDOUT:
 409                 type = "Connection timed out";
 410                 error_ret = 4;
 411                 break;
 412         case UCLIENT_ERROR_SSL_INVALID_CERT:
 413                 type = "Invalid SSL certificate";
 414                 ignore = !verify;
 415                 error_ret = 5;
 416                 break;
 417         case UCLIENT_ERROR_SSL_CN_MISMATCH:
 418                 type = "Server hostname does not match SSL certificate";
 419                 ignore = !verify;
 420                 error_ret = 5;
 421                 break;
 422         default:
 423                 error_ret = 1;
 424                 break;
 425         }
 426
 427         if (!quiet)
 428                 fprintf(stderr, "Connection error: %s%s\n", type, ignore ? " (ignored)" : "");
 429
 430         if (ignore)
 431                 error_ret = 0;
 432         else
 433                 request_done(cl);
 434 }
 435
 436 static const struct uclient_cb cb = {
 437         .header_done = header_done_cb,
 438         .data_read = read_data_cb,
 439         .data_eof = eof_cb,
 440         .error = handle_uclient_error,
 441 };
 442
 443 static int usage(const char *progname)
 444 {
 445         fprintf(stderr,
 446                 "Usage: %s [options] <URL>\n"
 447                 "Options:\n"
 448                 "       -4                              Use IPv4 only\n"
 449                 "       -6                              Use IPv6 only\n"
 450                 "       -q                              Turn off status messages\n"
 451                 "       -O <file>                       Redirect output to file (use \"-\" for stdout)\n"
 452                 "       -P <dir>                        Set directory for output files\n"
 453                 "       --user=<user>                   HTTP authentication username\n"
 454                 "       --password=<password>           HTTP authentication password\n"
 455                 "       --user-agent|-U <str>           Set HTTP user agent\n"
 456                 "       --post-data=STRING              use the POST method; send STRING as the data\n"
 457                 "       --spider|-s                     Spider mode - only check file existence\n"
 458                 "       --timeout=N|-T N                Set connect/request timeout to N seconds\n"
 459                 "       --proxy=on|off|-Y on|off        Enable/disable env var configured proxy\n"
 460                 "\n"
 461                 "HTTPS options:\n"
 462                 "       --ca-certificate=<cert>         Load CA certificates from file <cert>\n"
 463                 "       --no-check-certificate          don't validate the server's certificate\n"
 464                 "\n", progname);
 465         return 1;
 466 }
 467
 468 static void init_ca_cert(void)
 469 {
 470         glob_t gl;
 471         int i;
 472
 473         glob("/etc/ssl/certs/*.crt", 0, NULL, &gl);
 474         for (i = 0; i < gl.gl_pathc; i++)
 475                 ssl_ops->context_add_ca_crt_file(ssl_ctx, gl.gl_pathv[i]);
 476 }
 477
 478 static void init_ustream_ssl(void)
 479 {
 480         void *dlh;
 481
 482         dlh = dlopen("libustream-ssl." LIB_EXT, RTLD_LAZY | RTLD_LOCAL);
 483         if (!dlh)
 484                 return;
 485
 486         ssl_ops = dlsym(dlh, "ustream_ssl_ops");
 487         if (!ssl_ops)
 488                 return;
 489
 490         ssl_ctx = ssl_ops->context_new(false);
 491 }
 492
 493 static int no_ssl(const char *progname)
 494 {
 495         fprintf(stderr,
 496                 "%s: SSL support not available, please install one of the "
 497                 "libustream-ssl-* libraries as well as the ca-bundle and "
 498                 "ca-certificates packages.\n",
 499                 progname);
 500
 501         return 1;
 502 }
 503
 504 enum {
 505         L_NO_CHECK_CERTIFICATE,
 506         L_CA_CERTIFICATE,
 507         L_USER,
 508         L_PASSWORD,
 509         L_USER_AGENT,
 510         L_POST_DATA,
 511         L_SPIDER,
 512         L_TIMEOUT,
 513         L_CONTINUE,
 514         L_PROXY,
 515         L_NO_PROXY,
 516         L_QUIET,
 517 };
 518
 519 static const struct option longopts[] = {
 520         [L_NO_CHECK_CERTIFICATE] = { "no-check-certificate", no_argument },
 521         [L_CA_CERTIFICATE] = { "ca-certificate", required_argument },
 522         [L_USER] = { "user", required_argument },
 523         [L_PASSWORD] = { "password", required_argument },
 524         [L_USER_AGENT] = { "user-agent", required_argument },
 525         [L_POST_DATA] = { "post-data", required_argument },
 526         [L_SPIDER] = { "spider", no_argument },
 527         [L_TIMEOUT] = { "timeout", required_argument },
 528         [L_CONTINUE] = { "continue", no_argument },
 529         [L_PROXY] = { "proxy", required_argument },
 530         [L_NO_PROXY] = { "no-proxy", no_argument },
 531         [L_QUIET] = { "quiet", no_argument },
 532         {}
 533 };
 534
 535
 536
 537 int main(int argc, char **argv)
 538 {
 539         const char *progname = argv[0];
 540         const char *proxy_url;
 541         char *username = NULL;
 542         char *password = NULL;
 543         struct uclient *cl;
 544         int longopt_idx = 0;
 545         bool has_cert = false;
 546         int i, ch;
 547         int rc;
 548         int af = -1;
 549
 550         signal(SIGPIPE, SIG_IGN);
 551         init_ustream_ssl();
 552
 553         while ((ch = getopt_long(argc, argv, "46cO:P:qsT:U:Y:", longopts, &longopt_idx)) != -1) {
 554                 switch(ch) {
 555                 case 0:
 556                         switch (longopt_idx) {
 557                         case L_NO_CHECK_CERTIFICATE:
 558                                 verify = false;
 559                                 break;
 560                         case L_CA_CERTIFICATE:
 561                                 has_cert = true;
 562                                 if (ssl_ctx)
 563                                         ssl_ops->context_add_ca_crt_file(ssl_ctx, optarg);
 564                                 break;
 565                         case L_USER:
 566                                 if (!strlen(optarg))
 567                                         break;
 568                                 username = strdup(optarg);
 569                                 memset(optarg, '*', strlen(optarg));
 570                                 break;
 571                         case L_PASSWORD:
 572                                 if (!strlen(optarg))
 573                                         break;
 574                                 password = strdup(optarg);
 575                                 memset(optarg, '*', strlen(optarg));
 576                                 break;
 577                         case L_USER_AGENT:
 578                                 user_agent = optarg;
 579                                 break;
 580                         case L_POST_DATA:
 581                                 post_data = optarg;
 582                                 break;
 583                         case L_SPIDER:
 584                                 no_output = true;
 585                                 break;
 586                         case L_TIMEOUT:
 587                                 timeout = atoi(optarg);
 588                                 break;
 589                         case L_CONTINUE:
 590                                 resume = true;
 591                                 break;
 592                         case L_PROXY:
 593                                 if (strcmp(optarg, "on") != 0)
 594                                         proxy = false;
 595                                 break;
 596                         case L_NO_PROXY:
 597                                 proxy = false;
 598                                 break;
 599                         case L_QUIET:
 600                                 quiet = true;
 601                                 break;
 602                         default:
 603                                 return usage(progname);
 604                         }
 605                         break;
 606                 case '4':
 607                         af = AF_INET;
 608                         break;
 609                 case '6':
 610                         af = AF_INET6;
 611                         break;
 612                 case 'c':
 613                         resume = true;
 614                         break;
 615                 case 'U':
 616                         user_agent = optarg;
 617                         break;
 618                 case 'O':
 619                         opt_output_file = optarg;
 620                         break;
 621                 case 'P':
 622                         if (chdir(optarg)) {
 623                                 if (!quiet)
 624                                         perror("Change output directory");
 625                                 exit(1);
 626                         }
 627                         break;
 628                 case 'q':
 629                         quiet = true;
 630                         break;
 631                 case 's':
 632                         no_output = true;
 633                         break;
 634                 case 'T':
 635                         timeout = atoi(optarg);
 636                         break;
 637                 case 'Y':
 638                         if (strcmp(optarg, "on") != 0)
 639                                 proxy = false;
 640                         break;
 641                 default:
 642                         return usage(progname);
 643                 }
 644         }
 645
 646         argv += optind;
 647         argc -= optind;
 648
 649         if (verify && !has_cert)
 650                 default_certs = true;
 651
 652         if (argc < 1)
 653                 return usage(progname);
 654
 655         if (!ssl_ctx) {
 656                 for (i = 0; i < argc; i++) {
 657                         if (!strncmp(argv[i], "https", 5))
 658                                 return no_ssl(progname);
 659                 }
 660         }
 661
 662         urls = argv + 1;
 663         n_urls = argc - 1;
 664
 665         uloop_init();
 666
 667         if (username) {
 668                 if (password) {
 669                         rc = asprintf(&auth_str, "%s:%s", username, password);
 670                         if (rc < 0)
 671                                 return rc;
 672                 } else
 673                         auth_str = username;
 674         }
 675
 676         if (!quiet)
 677                 fprintf(stderr, "Downloading '%s'\n", argv[0]);
 678
 679         proxy_url = get_proxy_url(argv[0]);
 680         if (proxy_url) {
 681                 cl = uclient_new(proxy_url, auth_str, &cb);
 682                 if (cl)
 683                     uclient_set_proxy_url(cl, argv[0], NULL);
 684         } else {
 685                 cl = uclient_new(argv[0], auth_str, &cb);
 686         }
 687         if (!cl) {
 688                 fprintf(stderr, "Failed to allocate uclient context\n");
 689                 return 1;
 690         }
 691         if (af >= 0)
 692             uclient_http_set_address_family(cl, af);
 693
 694         if (ssl_ctx && default_certs)
 695                 init_ca_cert();
 696
 697         cur_resume = resume;
 698         rc = init_request(cl);
 699         if (!rc) {
 700                 /* no error received, we can enter main loop */
 701                 uloop_run();
 702         } else {
 703                 fprintf(stderr, "Failed to establish connection\n");
 704                 error_ret = 4;
 705         }
 706
 707         uloop_done();
 708
 709         uclient_free(cl);
 710
 711         if (output_fd >= 0 && output_fd != STDOUT_FILENO)
 712                 close(output_fd);
 713
 714         if (ssl_ctx)
 715                 ssl_ops->context_free(ssl_ctx);
 716
 717         return error_ret;
 718 }