1 commit a5a7d3a297b836387b0ac677383bdddaf2ac3598
2 Author: Wayne Davison <wayned@samba.org>
3 Date: Sun May 1 16:32:45 2016 -0700
5 Add --checksum-choice option to choose the checksum algorithms.
7 diff --git a/authenticate.c b/authenticate.c
8 index 5f125de..d60ee20 100644
11 @@ -71,7 +71,7 @@ static void gen_challenge(const char *addr, char *challenge)
12 SIVAL(input, 20, tv.tv_usec);
13 SIVAL(input, 24, getpid());
17 sum_update(input, sizeof input);
18 len = sum_end(digest);
20 @@ -85,7 +85,7 @@ static void generate_hash(const char *in, const char *challenge, char *out)
21 char buf[MAX_DIGEST_LEN];
26 sum_update(in, strlen(in));
27 sum_update(challenge, strlen(challenge));
29 diff --git a/checksum.c b/checksum.c
30 index bac775d..8b38833 100644
34 extern int checksum_seed;
35 extern int protocol_version;
36 extern int proper_seed_order;
37 +extern char *checksum_choice;
40 +#define CSUM_ARCHAIC 1
41 +#define CSUM_MD4_BUSTED 2
42 +#define CSUM_MD4_OLD 3
46 +int xfersum_type = 0; /* used for the file transfer checksums */
47 +int checksum_type = 0; /* used for the pre-transfer (--checksum) checksums */
49 +/* Returns 1 if --whole-file must be enabled. */
50 +int parse_checksum_choice(void)
52 + char *cp = checksum_choice ? strchr(checksum_choice, ',') : NULL;
54 + xfersum_type = parse_csum_name(checksum_choice, cp - checksum_choice);
55 + checksum_type = parse_csum_name(cp+1, -1);
57 + xfersum_type = checksum_type = parse_csum_name(checksum_choice, -1);
58 + return xfersum_type == CSUM_NONE;
61 +int parse_csum_name(const char *name, int len)
63 + if (len < 0 && name)
66 + if (!name || (len == 4 && strncasecmp(name, "auto", 4) == 0)) {
67 + if (protocol_version >= 30)
69 + if (protocol_version >= 27)
70 + return CSUM_MD4_OLD;
71 + if (protocol_version >= 21)
72 + return CSUM_MD4_BUSTED;
73 + return CSUM_ARCHAIC;
75 + if (len == 3 && strncasecmp(name, "md4", 3) == 0)
77 + if (len == 3 && strncasecmp(name, "md5", 3) == 0)
79 + if (len == 4 && strncasecmp(name, "none", 4) == 0)
82 + rprintf(FERROR, "unknown checksum name: %s\n", name);
83 + exit_cleanup(RERR_UNSUPPORTED);
86 +int csum_len_for_type(int cst)
95 + case CSUM_MD4_BUSTED:
96 + return MD4_DIGEST_LEN;
98 + return MD5_DIGEST_LEN;
103 +int canonical_checksum(int csum_type)
105 + return csum_type >= CSUM_MD4 ? 1 : 0;
109 a simple 32 bit checksum that can be upadted from either end
110 @@ -47,12 +117,12 @@ uint32 get_checksum1(char *buf1, int32 len)
111 return (s1 & 0xffff) + (s2 << 16);
115 void get_checksum2(char *buf, int32 len, char *sum)
119 - if (protocol_version >= 30) {
120 + switch (xfersum_type) {
124 if (proper_seed_order) {
125 @@ -69,7 +139,11 @@ void get_checksum2(char *buf, int32 len, char *sum)
128 md5_result(&m, (uchar *)sum);
134 + case CSUM_MD4_BUSTED: {
138 @@ -100,10 +174,12 @@ void get_checksum2(char *buf, int32 len, char *sum)
139 * are multiples of 64. This is fixed by calling mdfour_update()
140 * even when there are no more bytes.
142 - if (len - i > 0 || protocol_version >= 27)
143 + if (len - i > 0 || xfersum_type != CSUM_MD4_BUSTED)
144 mdfour_update(&m, (uchar *)(buf1+i), len-i);
146 mdfour_result(&m, (uchar *)sum);
152 @@ -123,7 +199,8 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)
154 buf = map_file(fd, len, MAX_MAP_SIZE, CSUM_CHUNK);
156 - if (protocol_version >= 30) {
157 + switch (checksum_type) {
161 for (i = 0; i + CSUM_CHUNK <= len; i += CSUM_CHUNK) {
162 @@ -136,7 +213,10 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)
163 md5_update(&m, (uchar *)map_ptr(buf, i, remainder), remainder);
165 md5_result(&m, (uchar *)sum);
170 + case CSUM_MD4_BUSTED:
173 for (i = 0; i + CSUM_CHUNK <= len; i += CSUM_CHUNK) {
174 @@ -149,10 +229,14 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)
175 * are multiples of 64. This is fixed by calling mdfour_update()
176 * even when there are no more bytes. */
177 remainder = (int32)(len - i);
178 - if (remainder > 0 || protocol_version >= 27)
179 + if (remainder > 0 || checksum_type != CSUM_MD4_BUSTED)
180 mdfour_update(&m, (uchar *)map_ptr(buf, i, remainder), remainder);
182 mdfour_result(&m, (uchar *)sum);
185 + rprintf(FERROR, "invalid checksum-choice for the --checksum option (%d)\n", checksum_type);
186 + exit_cleanup(RERR_UNSUPPORTED);
190 @@ -161,18 +245,33 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)
192 static int32 sumresidue;
193 static md_context md;
194 +static int cursum_type;
196 -void sum_init(int seed)
197 +void sum_init(int csum_type, int seed)
201 - if (protocol_version >= 30)
203 + csum_type = parse_csum_name(NULL, 0);
204 + cursum_type = csum_type;
206 + switch (csum_type) {
216 + case CSUM_MD4_BUSTED:
227 @@ -186,13 +285,17 @@ void sum_init(int seed)
229 void sum_update(const char *p, int32 len)
231 - if (protocol_version >= 30) {
232 + switch (cursum_type) {
234 md5_update(&md, (uchar *)p, len);
239 + case CSUM_MD4_BUSTED:
240 if (len + sumresidue < CSUM_CHUNK) {
241 memcpy(md.buffer + sumresidue, p, len);
248 @@ -212,20 +315,32 @@ void sum_update(const char *p, int32 len)
251 memcpy(md.buffer, p, sumresidue);
258 int sum_end(char *sum)
260 - if (protocol_version >= 30) {
261 + switch (cursum_type) {
263 md5_result(&md, (uchar *)sum);
264 - return MD5_DIGEST_LEN;
266 - if (sumresidue || protocol_version >= 27)
270 + mdfour_update(&md, (uchar *)md.buffer, sumresidue);
271 + mdfour_result(&md, (uchar *)sum);
273 + case CSUM_MD4_BUSTED:
275 mdfour_update(&md, (uchar *)md.buffer, sumresidue);
277 mdfour_result(&md, (uchar *)sum);
279 - return MD4_DIGEST_LEN;
286 + return csum_len_for_type(cursum_type);
288 diff --git a/compat.c b/compat.c
289 index c792312..505cb7f 100644
292 @@ -338,4 +338,6 @@ void setup_protocol(int f_out,int f_in)
294 checksum_seed = read_int(f_in);
299 diff --git a/flist.c b/flist.c
300 index c1e48b3..acb95f7 100644
303 @@ -33,6 +33,7 @@ extern int am_sender;
304 extern int am_generator;
305 extern int inc_recurse;
306 extern int always_checksum;
307 +extern int checksum_type;
308 extern int module_id;
309 extern int ignore_errors;
310 extern int numeric_ids;
311 @@ -137,9 +138,8 @@ void init_flist(void)
312 rprintf(FINFO, "FILE_STRUCT_LEN=%d, EXTRA_LEN=%d\n",
313 (int)FILE_STRUCT_LEN, (int)EXTRA_LEN);
315 - checksum_len = protocol_version < 21 ? 2
316 - : protocol_version < 30 ? MD4_DIGEST_LEN
318 + parse_checksum_choice(); /* Sets checksum_type && xfersum_type */
319 + checksum_len = csum_len_for_type(checksum_type);
322 static int show_filelist_p(void)
323 diff --git a/log.c b/log.c
324 index 24256de..f7da1e5 100644
327 @@ -31,12 +31,13 @@ extern int am_generator;
328 extern int local_server;
330 extern int module_id;
331 -extern int checksum_len;
332 extern int allow_8bit_chars;
333 extern int protocol_version;
334 extern int always_checksum;
335 extern int preserve_times;
336 extern int msgs2stderr;
337 +extern int xfersum_type;
338 +extern int checksum_type;
339 extern int stdout_format_has_i;
340 extern int stdout_format_has_o_or_i;
341 extern int logfile_format_has_i;
342 @@ -46,6 +47,7 @@ extern int64 total_data_written;
343 extern int64 total_data_read;
344 extern mode_t orig_umask;
345 extern char *auth_user;
346 +extern char *checksum_choice;
347 extern char *stdout_format;
348 extern char *logfile_format;
349 extern char *logfile_name;
350 @@ -669,13 +671,15 @@ static void log_formatted(enum logcode code, const char *format, const char *op,
354 - if (protocol_version >= 30
355 - && (iflags & ITEM_TRANSFER
356 - || (always_checksum && S_ISREG(file->mode)))) {
357 - const char *sum = iflags & ITEM_TRANSFER
358 - ? sender_file_sum : F_SUM(file);
359 - n = sum_as_hex(sum);
362 + if (S_ISREG(file->mode)) {
363 + if (always_checksum && canonical_checksum(checksum_type))
364 + n = sum_as_hex(checksum_type, F_SUM(file));
365 + else if (iflags & ITEM_TRANSFER && canonical_checksum(xfersum_type))
366 + n = sum_as_hex(xfersum_type, sender_file_sum);
369 + int checksum_len = csum_len_for_type(always_checksum ? checksum_type : xfersum_type);
370 memset(buf2, ' ', checksum_len*2);
371 buf2[checksum_len*2] = '\0';
373 diff --git a/main.c b/main.c
374 index 3132aa9..3908ccf 100644
377 @@ -1595,8 +1595,6 @@ int main(int argc,char *argv[])
378 * that implement getcwd that way "pwd" can't be found after chroot. */
379 change_dir(NULL, CD_NORMAL);
383 if ((write_batch || read_batch) && !am_server) {
385 write_batch_shell_file(orig_argc, orig_argv, argc);
386 diff --git a/match.c b/match.c
387 index b15f2eb..ff10310 100644
392 extern int checksum_seed;
393 extern int append_mode;
394 -extern int checksum_len;
395 +extern int xfersum_type;
397 int updating_basis_file;
398 char sender_file_sum[MAX_DIGEST_LEN];
399 @@ -360,13 +360,15 @@ static void hash_search(int f,struct sum_struct *s,
401 void match_sums(int f, struct sum_struct *s, struct map_struct *buf, OFF_T len)
411 - sum_init(checksum_seed);
412 + sum_init(xfersum_type, checksum_seed);
414 if (append_mode > 0) {
415 if (append_mode == 2) {
416 @@ -407,8 +409,7 @@ void match_sums(int f, struct sum_struct *s, struct map_struct *buf, OFF_T len)
417 matched(f, s, buf, len, -1);
420 - if (sum_end(sender_file_sum) != checksum_len)
421 - overflow_exit("checksum_len"); /* Impossible... */
422 + checksum_len = sum_end(sender_file_sum);
424 /* If we had a read error, send a bad checksum. We use all bits
425 * off as long as the checksum doesn't happen to be that, in
426 diff --git a/options.c b/options.c
427 index 4a5cdc8..308443b 100644
430 @@ -182,6 +182,7 @@ char *dest_option = NULL;
431 static int remote_option_alloc = 0;
432 int remote_option_cnt = 0;
433 const char **remote_options = NULL;
434 +const char *checksum_choice = NULL;
438 @@ -721,6 +722,7 @@ void usage(enum logcode F)
440 rprintf(F," -n, --dry-run perform a trial run with no changes made\n");
441 rprintf(F," -W, --whole-file copy files whole (without delta-xfer algorithm)\n");
442 + rprintf(F," --checksum-choice=STR choose the checksum algorithms\n");
443 rprintf(F," -x, --one-file-system don't cross filesystem boundaries\n");
444 rprintf(F," -B, --block-size=SIZE force a fixed checksum block-size\n");
445 rprintf(F," -e, --rsh=COMMAND specify the remote shell to use\n");
446 @@ -953,6 +955,7 @@ static struct poptOption long_options[] = {
447 {"cvs-exclude", 'C', POPT_ARG_NONE, &cvs_exclude, 0, 0, 0 },
448 {"whole-file", 'W', POPT_ARG_VAL, &whole_file, 1, 0, 0 },
449 {"no-whole-file", 0, POPT_ARG_VAL, &whole_file, 0, 0, 0 },
450 + {"checksum-choice", 0, POPT_ARG_STRING, &checksum_choice, 0, 0, 0 },
451 {"no-W", 0, POPT_ARG_VAL, &whole_file, 0, 0, 0 },
452 {"checksum", 'c', POPT_ARG_VAL, &always_checksum, 1, 0, 0 },
453 {"no-checksum", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
454 @@ -1814,6 +1817,15 @@ int parse_arguments(int *argc_p, const char ***argv_p)
458 + if (checksum_choice && strcmp(checksum_choice, "auto") != 0 && strcmp(checksum_choice, "auto,auto") != 0) {
459 + /* Call this early to verify the args and figure out if we need to force
460 + * --whole-file. Note that the parse function will get called again later,
461 + * just in case an "auto" choice needs to know the protocol_version. */
462 + if (parse_checksum_choice())
465 + checksum_choice = NULL;
467 if (human_readable > 1 && argc == 2 && !am_server) {
468 /* Allow the old meaning of 'h' (--help) on its own. */
470 @@ -2597,6 +2609,12 @@ void server_options(char **args, int *argc_p)
474 + if (checksum_choice) {
475 + if (asprintf(&arg, "--checksum-choice=%s", checksum_choice) < 0)
481 if (max_delete > 0) {
482 if (asprintf(&arg, "--max-delete=%d", max_delete) < 0)
483 diff --git a/receiver.c b/receiver.c
484 index 4ea4c09..f9b97dd 100644
487 @@ -48,11 +48,11 @@ extern int append_mode;
488 extern int sparse_files;
489 extern int preallocate_files;
490 extern int keep_partial;
491 -extern int checksum_len;
492 extern int checksum_seed;
494 extern int allowed_lull;
495 extern int delay_updates;
496 +extern int xfersum_type;
497 extern mode_t orig_umask;
498 extern struct stats stats;
500 @@ -234,6 +234,7 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
501 static char file_sum1[MAX_DIGEST_LEN];
502 struct map_struct *mapbuf;
503 struct sum_struct sum;
508 @@ -269,7 +270,7 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
512 - sum_init(checksum_seed);
513 + sum_init(xfersum_type, checksum_seed);
515 if (append_mode > 0) {
517 @@ -393,8 +394,7 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
518 exit_cleanup(RERR_FILEIO);
521 - if (sum_end(file_sum1) != checksum_len)
522 - overflow_exit("checksum_len"); /* Impossible... */
523 + checksum_len = sum_end(file_sum1);
527 diff --git a/rsync.yo b/rsync.yo
528 index 8971828..0ec5e55 100644
531 @@ -380,6 +380,7 @@ to the detailed description below for a complete description. verb(
532 --preallocate allocate dest files before writing
533 -n, --dry-run perform a trial run with no changes made
534 -W, --whole-file copy files whole (w/o delta-xfer algorithm)
535 + --checksum-choice=STR choose the checksum algorithms
536 -x, --one-file-system don't cross filesystem boundaries
537 -B, --block-size=SIZE force a fixed checksum block-size
538 -e, --rsh=COMMAND specify the remote shell to use
539 @@ -1280,14 +1281,27 @@ the "bytes sent", "bytes received", "literal data", and "matched data"
540 statistics are too small, and the "speedup" value is equivalent to a run
541 where no file transfers were needed.
543 -dit(bf(-W, --whole-file)) With this option rsync's delta-transfer algorithm
544 -is not used and the whole file is sent as-is instead. The transfer may be
545 +dit(bf(-W, --whole-file)) This option disables rsync's delta-transfer algorithm,
546 +which causes all transferred files to be sent whole. The transfer may be
547 faster if this option is used when the bandwidth between the source and
548 destination machines is higher than the bandwidth to disk (especially when the
549 "disk" is actually a networked filesystem). This is the default when both
550 the source and destination are specified as local paths, but only if no
551 batch-writing option is in effect.
553 +dit(bf(--checksum-choice=STR)) This option overrides the checksum algoriths.
554 +If one algorithm name is specified, it is used for both the transfer checksums
555 +and (assuming bf(--checksum) is specifed) the pre-transfer checksumming. If two
556 +comma-separated names are supplied, the first name affects the transfer
557 +checksums, and the second name affects the pre-transfer checksumming.
559 +The algorithm choices are "auto", "md4", "md5", and "none". If "none" is
560 +specified for the first name, the bf(--whole-file) option is forced on and no
561 +checksum verification is performed on the transferred data. If "none" is
562 +specified for the second name, the bf(--checksum) option cannot be used. The
563 +"auto" option is the default, where rsync bases its algorithm choice on the
564 +protocol version (for backward compatibility with older rsync versions).
566 dit(bf(-x, --one-file-system)) This tells rsync to avoid crossing a
567 filesystem boundary when recursing. This does not limit the user's ability
568 to specify items to copy from multiple filesystems, just rsync's recursion
569 diff --git a/rsyncd.conf.yo b/rsyncd.conf.yo
570 index 1813354..64156ae 100644
573 @@ -656,7 +656,7 @@ quote(itemization(
574 it() %b the number of bytes actually transferred
575 it() %B the permission bits of the file (e.g. rwxrwxrwt)
576 it() %c the total size of the block checksums received for the basis file (only when sending)
577 - it() %C the full-file MD5 checksum if bf(--checksum) is enabled or a file was transferred (only for protocol 30 or above).
578 + it() %C the full-file checksum if it is known for the file. For older rsync protocols/versions, the checksum was salted, and is thus not a useful value (and is not displayed when that is the case). For the checksum to output for a file, either the bf(--checksum) option must be in-effect or the file must have been transferred without a salted checksum being used. See the bf(--checksum-choice) option for a way to choose the algorithm.
579 it() %f the filename (long form on sender; no trailing "/")
580 it() %G the gid of the file (decimal) or "DEFAULT"
581 it() %h the remote host name (only available for a daemon)
582 diff --git a/t_stub.c b/t_stub.c
583 index 6002250..26951a6 100644
586 @@ -25,7 +25,6 @@ int modify_window = 0;
587 int preallocate_files = 0;
588 int protect_args = 0;
590 -int checksum_len = 0;
591 int relative_paths = 0;
592 int module_dirlen = 0;
593 int preserve_acls = 0;
594 @@ -97,3 +96,8 @@ filter_rule_list daemon_filter_list;
599 + int csum_len_for_type(int cst)
601 + return cst ? 16 : 1;
603 diff --git a/util2.c b/util2.c
604 index cc368af..a892e51 100644
611 -extern int checksum_len;
614 * Sleep for a specified number of milliseconds.
616 @@ -79,10 +77,11 @@ void *_realloc_array(void *ptr, unsigned int size, size_t num)
617 return realloc(ptr, size * num);
620 -const char *sum_as_hex(const char *sum)
621 +const char *sum_as_hex(int csum_type, const char *sum)
623 static char buf[MAX_DIGEST_LEN*2+1];
625 + int checksum_len = csum_len_for_type(csum_type);
626 char *c = buf + checksum_len*2;
628 assert(c - buf < (int)sizeof buf);
629 diff --git a/xattrs.c b/xattrs.c
630 index 57833e5..6a77a0b 100644
633 @@ -258,7 +258,7 @@ static int rsync_xal_get(const char *fname, item_list *xalp)
634 if (datum_len > MAX_FULL_DATUM) {
635 /* For large datums, we store a flag and a checksum. */
636 name_offset = 1 + MAX_DIGEST_LEN;
637 - sum_init(checksum_seed);
638 + sum_init(-1, checksum_seed);
639 sum_update(ptr, datum_len);
642 @@ -821,7 +821,7 @@ static int rsync_xal_set(const char *fname, item_list *xalp,
646 - sum_init(checksum_seed);
647 + sum_init(-1, checksum_seed);
648 sum_update(ptr, len);
650 if (memcmp(sum, rxas[i].datum + 1, MAX_DIGEST_LEN) != 0) {