diff --git a/Makefile.in b/Makefile.in index 716171db1..570929913 100644 --- a/Makefile.in +++ b/Makefile.in @@ -45,7 +45,7 @@ LIBOBJ=lib/wildmatch.o lib/compat.o lib/snprintf.o lib/mdfour.o lib/md5.o \ zlib_OBJS=zlib/deflate.o zlib/inffast.o zlib/inflate.o zlib/inftrees.o \ zlib/trees.o zlib/zutil.o zlib/adler32.o zlib/compress.o zlib/crc32.o OBJS1_NO_MAIN=flist.o rsync.o generator.o receiver.o cleanup.o sender.o exclude.o \ - util1.o util2.o checksum.o match.o syscall.o log.o backup.o delete.o + util1.o util2.o checksum.o match.o syscall.o log.o backup.o delete.o ltfs.o OBJS1=$(OBJS1_NO_MAIN) main.o OBJS2=options.o io.o compat.o hlink.o token.o uidlist.o socket.o hashtable.o \ usage.o fileio.o batch.o clientname.o chmod.o acls.o xattrs.o @@ -55,18 +55,26 @@ popt_OBJS= popt/popt.o popt/poptconfig.o \ popt/popthelp.o popt/poptparse.o popt/poptint.o OBJS=$(OBJS1) $(OBJS2) $(OBJS3) $(DAEMON_OBJ) $(LIBOBJ) @BUILD_ZLIB@ @BUILD_POPT@ +# rsync_noltfs: identical to rsync but options.c recompiled with +# -DLTFS_SERVER_REJECT_TEST, which forces the server-side --ltfs rejection +# path active regardless of SUPPORT_LTFS. Used by the ltfs-server-reject test. +OBJS_NOLTFS=$(OBJS1) options_noltfs.o io.o compat.o hlink.o token.o uidlist.o \ + socket.o hashtable.o usage.o fileio.o batch.o clientname.o chmod.o acls.o \ + xattrs.o $(OBJS3) $(DAEMON_OBJ) $(LIBOBJ) @BUILD_ZLIB@ @BUILD_POPT@ + TLS_OBJ = tls.o syscall.o util2.o t_stub.o lib/compat.o lib/snprintf.o lib/permstring.o lib/sysxattrs.o @BUILD_POPT@ # Programs we must have to run the test cases CHECK_PROGS = rsync$(EXEEXT) tls$(EXEEXT) getgroups$(EXEEXT) getfsdev$(EXEEXT) \ testrun$(EXEEXT) trimslash$(EXEEXT) t_unsafe$(EXEEXT) t_chmod_secure$(EXEEXT) \ - t_rename_secure$(EXEEXT) t_symlink_secure$(EXEEXT) t_secure_relpath$(EXEEXT) t_acl$(EXEEXT) t_hashtable_overflow$(EXEEXT) t_iwildmatch$(EXEEXT) t_clean_fname$(EXEEXT) t_safe_arg$(EXEEXT) wildtest$(EXEEXT) simdtest$(EXEEXT) + t_rename_secure$(EXEEXT) t_symlink_secure$(EXEEXT) t_secure_relpath$(EXEEXT) t_acl$(EXEEXT) t_hashtable_overflow$(EXEEXT) t_iwildmatch$(EXEEXT) t_clean_fname$(EXEEXT) t_safe_arg$(EXEEXT) wildtest$(EXEEXT) simdtest$(EXEEXT) \ + @LTFS_CHECK_PROGS@ CHECK_SYMLINKS = testsuite/chown-fake_test.py testsuite/devices-fake_test.py \ testsuite/xattrs-hlink_test.py testsuite/exclude-lsh_test.py # Objects for CHECK_PROGS to clean -CHECK_OBJS=tls.o testrun.o getgroups.o getfsdev.o t_stub.o t_unsafe.o t_chmod_secure.o t_rename_secure.o t_symlink_secure.o t_secure_relpath.o t_acl.o t_hashtable_overflow.o t_iwildmatch.o t_clean_fname.o t_safe_arg.o trimslash.o wildtest.o +CHECK_OBJS=tls.o testrun.o getgroups.o getfsdev.o t_stub.o t_unsafe.o t_chmod_secure.o t_rename_secure.o t_symlink_secure.o t_secure_relpath.o t_acl.o t_hashtable_overflow.o t_iwildmatch.o t_clean_fname.o t_safe_arg.o trimslash.o wildtest.o @LTFS_CHECK_OBJS@ # Compile-only feature-shape checks. CHECK_COMPILE_OBJS=syscall-no-at-fdcwd.o @@ -273,8 +281,7 @@ aclocal.m4: $(srcdir)/m4/*.m4 configure.sh config.h.in: configure.ac aclocal.m4 @if test -f configure.sh; then cp -p configure.sh configure.sh.old; else touch configure.sh.old; fi @if test -f config.h.in; then cp -p config.h.in config.h.in.old; else touch config.h.in.old; fi - autoconf -o configure.sh - autoheader && touch config.h.in + $(srcdir)/autogen.sh @if diff configure.sh configure.sh.old >/dev/null 2>&1; then \ echo "configure.sh is unchanged."; \ rm configure.sh.old; \ @@ -443,15 +450,15 @@ check-progs: all $(CHECK_PROGS) $(CHECK_COMPILE_OBJS) $(CHECK_SYMLINKS) .PHONY: check check: all $(CHECK_PROGS) $(CHECK_COMPILE_OBJS) $(CHECK_SYMLINKS) - "$(srcdir)/runtests.py" --rsync-bin="`pwd`/rsync$(EXEEXT)" -j $(CHECK_J) + "$(srcdir)/runtests.py" --rsync-bin="`pwd`/rsync$(EXEEXT)" -j $(CHECK_J) @LTFS_RUNTESTS_ARGS@ .PHONY: check29 check29: all $(CHECK_PROGS) $(CHECK_COMPILE_OBJS) $(CHECK_SYMLINKS) - "$(srcdir)/runtests.py" --rsync-bin="`pwd`/rsync$(EXEEXT)" -j $(CHECK_J) --protocol=29 + "$(srcdir)/runtests.py" --rsync-bin="`pwd`/rsync$(EXEEXT)" -j $(CHECK_J) --protocol=29 @LTFS_RUNTESTS_ARGS@ .PHONY: check30 check30: all $(CHECK_PROGS) $(CHECK_COMPILE_OBJS) $(CHECK_SYMLINKS) - "$(srcdir)/runtests.py" --rsync-bin="`pwd`/rsync$(EXEEXT)" -j $(CHECK_J) --protocol=30 + "$(srcdir)/runtests.py" --rsync-bin="`pwd`/rsync$(EXEEXT)" -j $(CHECK_J) --protocol=30 @LTFS_RUNTESTS_ARGS@ # Whole-suite gcov coverage report (HTML, with branch + decision coverage). # Requires a build configured with --enable-coverage and the `gcovr` tool @@ -479,7 +486,7 @@ coverage: all $(CHECK_PROGS) $(CHECK_SYMLINKS) chmod a+rwx "$$d"; \ setfacl -m 'd:u::rwx,d:g::rwx,d:o::rwx' "$$d" 2>/dev/null || true; \ done - @rc=0; "$(srcdir)/runtests.py" --rsync-bin="`pwd`/rsync$(EXEEXT)" -j $(COVERAGE_J) $(COVERAGE_RUNFLAGS) || rc=$$?; \ + @rc=0; "$(srcdir)/runtests.py" --rsync-bin="`pwd`/rsync$(EXEEXT)" -j $(COVERAGE_J) $(COVERAGE_RUNFLAGS) @LTFS_RUNTESTS_ARGS@ || rc=$$?; \ rm -rf $(COVERAGE_DIR) && mkdir -p $(COVERAGE_DIR); \ gcovr --root $(srcdir) $(COVERAGE_EXCLUDE) --decisions --print-summary \ --gcov-ignore-parse-errors=negative_hits.warn_once_per_file \ @@ -519,7 +526,7 @@ coverage-all: all $(CHECK_PROGS) $(CHECK_SYMLINKS) @rc=0; \ for cfg in '' '--protocol=30' '--protocol=29' '--use-tcp'; do \ echo "===== coverage-all: runtests.py $$cfg ====="; \ - "$(srcdir)/runtests.py" --rsync-bin="`pwd`/rsync$(EXEEXT)" -j $(COVERAGE_J) $$cfg || rc=$$?; \ + "$(srcdir)/runtests.py" --rsync-bin="`pwd`/rsync$(EXEEXT)" -j $(COVERAGE_J) $$cfg @LTFS_RUNTESTS_ARGS@ || rc=$$?; \ done; \ rm -rf coverage-all && mkdir -p coverage-all; \ gcovr --root $(srcdir) $(COVERAGE_EXCLUDE) --decisions --print-summary \ @@ -550,6 +557,12 @@ simdtest$(EXEEXT): simd-checksum-x86_64.cpp $(HEADERS) touch $@; \ fi +options_noltfs.o: $(srcdir)/options.c $(HEADERS) + $(CC) -I. -I$(srcdir) $(CFLAGS) $(CPPFLAGS) -DLTFS_SERVER_REJECT_TEST -c -o $@ $(srcdir)/options.c + +rsync_noltfs$(EXEEXT): $(OBJS_NOLTFS) + $(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(OBJS_NOLTFS) $(LIBS) + testsuite/chown-fake_test.py: ln -s chown_test.py $(srcdir)/testsuite/chown-fake_test.py @@ -568,7 +581,7 @@ testsuite/exclude-lsh_test.py: .PHONY: installcheck installcheck: $(CHECK_PROGS) $(CHECK_SYMLINKS) - "$(srcdir)/runtests.py" --rsync-bin="$(bindir)/rsync$(EXEEXT)" --srcdir="$(srcdir)" --tooldir="`pwd`" -j $(CHECK_J) + "$(srcdir)/runtests.py" --rsync-bin="$(bindir)/rsync$(EXEEXT)" --srcdir="$(srcdir)" --tooldir="`pwd`" -j $(CHECK_J) @LTFS_RUNTESTS_ARGS@ # TODO: Add 'dist' target; need to know which files will be included diff --git a/autogen.sh b/autogen.sh new file mode 100755 index 000000000..f13aa6edc --- /dev/null +++ b/autogen.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# Regenerate the autoconf build files from configure.ac. +# Run this after "make distclean" or a fresh clone before ./configure. +set -e +aclocal -I m4 +autoconf -o configure.sh +autoheader && touch config.h.in diff --git a/compat.c b/compat.c index ba1b0c949..680cfe4c2 100644 --- a/compat.c +++ b/compat.c @@ -45,6 +45,7 @@ extern int preserve_uid; extern int preserve_gid; extern int preserve_atimes; extern int preserve_crtimes; +extern int ltfs_mode; extern int preserve_acls; extern int preserve_xattrs; extern int xfer_flags_as_varint; @@ -87,7 +88,7 @@ struct name_num_item *xattr_sum_nni; int xattr_sum_len = 0; /* These index values are for the file-list's extra-attribute array. */ -int pathname_ndx, depth_ndx, atimes_ndx, crtimes_ndx, uid_ndx, gid_ndx, acls_ndx, xattrs_ndx, unsort_ndx; +int pathname_ndx, depth_ndx, atimes_ndx, crtimes_ndx, startblock_ndx, uid_ndx, gid_ndx, acls_ndx, xattrs_ndx, unsort_ndx; int receiver_symlink_times = 0; /* receiver can set the time on a symlink */ int sender_symlink_iconv = 0; /* sender should convert symlink content */ @@ -584,6 +585,8 @@ void setup_protocol(int f_out,int f_in) atimes_ndx = (file_extra_cnt += EXTRA64_CNT); if (preserve_crtimes) crtimes_ndx = (file_extra_cnt += EXTRA64_CNT); + if (ltfs_mode) + startblock_ndx = (file_extra_cnt += EXTRA64_CNT); if (am_sender) /* This is most likely in the file_extras64 union as well. */ pathname_ndx = (file_extra_cnt += PTR_EXTRA_CNT); else diff --git a/configure.ac b/configure.ac index 8e4b74505..db1dc1aee 100644 --- a/configure.ac +++ b/configure.ac @@ -1547,6 +1547,50 @@ if test x"$samba_cv_HAVE_POSIX_ACLS" = x"yes"; then esac fi +# check for LTFS tape-ordering support +AC_MSG_CHECKING(whether to support LTFS tape ordering) +AC_ARG_ENABLE(ltfs, + AS_HELP_STRING([--disable-ltfs],[disable LTFS tape-order awareness (--ltfs option)]), + [], [enable_ltfs=yes]) +AH_TEMPLATE([SUPPORT_LTFS], +[Define to 1 to add support for LTFS tape-order-aware transfers]) +if test x"$enable_ltfs" = x"no"; then + AC_MSG_RESULT(no) +elif test x"$enable_xattr_support" = x"no"; then + AC_MSG_RESULT([no (requires xattr support)]) +else + case $host_os in + *cygwin*) + AC_MSG_RESULT([no (Cygwin)]) + ;; + *) + AC_MSG_RESULT(yes) + AC_DEFINE(SUPPORT_LTFS, 1) + ;; + esac +fi +if test x"$enable_ltfs" != x"no" && test x"$enable_xattr_support" != x"no"; then + case $host_os in + *cygwin*) + LTFS_CHECK_PROGS='' + LTFS_CHECK_OBJS='' + LTFS_RUNTESTS_ARGS='--exclude=ltfs-server-reject,ltfs-client-compat' + ;; + *) + LTFS_CHECK_PROGS='rsync_noltfs$(EXEEXT)' + LTFS_CHECK_OBJS='options_noltfs.o' + LTFS_RUNTESTS_ARGS='' + ;; + esac +else + LTFS_CHECK_PROGS='' + LTFS_CHECK_OBJS='' + LTFS_RUNTESTS_ARGS='--exclude=ltfs-server-reject,ltfs-client-compat' +fi +AC_SUBST(LTFS_CHECK_PROGS) +AC_SUBST(LTFS_CHECK_OBJS) +AC_SUBST(LTFS_RUNTESTS_ARGS) + if test x"$enable_acl_support" = x"no" || test x"$enable_xattr_support" = x"no" || test x"$enable_iconv" = x"no"; then AC_MSG_CHECKING([whether $CC supports -Wno-unused-parameter]) OLD_CFLAGS="$CFLAGS" diff --git a/flist.c b/flist.c index 9276c65fc..01876c024 100644 --- a/flist.c +++ b/flist.c @@ -62,6 +62,7 @@ extern int missing_args; extern int eol_nulls; extern int atimes_ndx; extern int crtimes_ndx; +extern int startblock_ndx; extern int relative_paths; extern int implied_dirs; extern int ignore_perishable; @@ -685,6 +686,8 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file, if (crtimes_ndx && !(xflags & XMIT_CRTIME_EQ_MTIME)) write_varlong(f, crtime, 4); #endif + if (startblock_ndx) + write_varlong(f, F_STARTBLOCK(file), 3); if (!(xflags & XMIT_SAME_MODE)) write_int(f, to_wire_mode(mode)); if (atimes_ndx && !S_ISDIR(mode) && !(xflags & XMIT_SAME_ATIME)) @@ -780,6 +783,7 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x #ifdef SUPPORT_CRTIMES static time_t crtime; #endif + static int64 startblock; static mode_t mode; #ifdef SUPPORT_HARD_LINKS static int64 dev; @@ -964,6 +968,8 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x #endif } #endif + if (startblock_ndx) + startblock = read_varlong(f, 3); if (!(xflags & XMIT_SAME_MODE)) { mode = from_wire_mode(read_int(f)); /* Reject modes whose type bits are not one of the standard @@ -1215,6 +1221,8 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x if (crtimes_ndx) F_CRTIME(file) = crtime; #endif + if (startblock_ndx) + F_STARTBLOCK(file) = startblock; if (unsort_ndx) F_NDX(file) = flist->used + flist->ndx_start; @@ -1679,6 +1687,12 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, if (crtimes_ndx) F_CRTIME(file) = get_create_time(fname, &st); #endif + if (startblock_ndx) { + int64 blk = am_sender && S_ISREG(file->mode) ? ltfs_startblock(fname) : -1; + /* Unknown sorts first; keep it non-negative so write_varlong + * stays compact (a real LTFS data block is well past block 0). */ + F_STARTBLOCK(file) = blk < 0 ? 0 : blk; + } if (basename != thisname) file->dirname = lastdir; diff --git a/generator.c b/generator.c index 7e5ad60a3..039c5f836 100644 --- a/generator.c +++ b/generator.c @@ -33,6 +33,7 @@ extern int operator_path_resolve; extern int am_server; extern int am_daemon; extern int inc_recurse; +extern int ltfs_mode; extern int relative_paths; extern int implied_dirs; extern int keep_dirlinks; @@ -2716,6 +2717,7 @@ void check_for_finished_files(int itemizing, enum logcode code, int check_redo) void generate_files(int f_out, const char *local_name) { int i, ndx, next_loopchk = 0; + int *ltfs_order = NULL; char fbuf[MAXPATHLEN]; int itemizing; enum logcode code; @@ -2800,8 +2802,15 @@ void generate_files(int f_out, const char *local_name) change_local_filter_dir(fbuf, strlen(fbuf), F_DEPTH(fp)); } } + /* For an LTFS source, read the files in physical tape order + * (by start block) rather than name order, so the drive makes + * one forward streaming pass instead of seeking back and forth. + * ltfs_order maps the natural sweep position to a sorted[] index; + * a NULL result falls back to the natural low..high order. */ + ltfs_order = ltfs_mode ? ltfs_build_order(cur_flist) : NULL; for (i = cur_flist->low; i <= cur_flist->high; i++) { - struct file_struct *file = cur_flist->sorted[i]; + int si = ltfs_order ? ltfs_order[i - cur_flist->low] : i; + struct file_struct *file = cur_flist->sorted[si]; if (!F_IS_ACTIVE(file)) continue; @@ -2809,7 +2818,7 @@ void generate_files(int f_out, const char *local_name) if (unsort_ndx) ndx = F_NDX(file); else - ndx = i + cur_flist->ndx_start; + ndx = si + cur_flist->ndx_start; if (solo_file) strlcpy(fbuf, solo_file, sizeof fbuf); @@ -2828,6 +2837,9 @@ void generate_files(int f_out, const char *local_name) } } + if (ltfs_order) + free(ltfs_order); + if (!inc_recurse) { write_ndx(f_out, NDX_DONE); break; diff --git a/ltfs.c b/ltfs.c new file mode 100644 index 000000000..4f302f972 --- /dev/null +++ b/ltfs.c @@ -0,0 +1,136 @@ +/* + * LTFS (Linear Tape File System) awareness for rsync. + * + * Copyright (C) 2026 Wayne Davison & contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, visit the http://fsf.org website. + * + * --------------------------------------------------------------------------- + * + * On an LTFS-mounted tape every file's metadata -- name, size, mtime, and the + * physical block where its data begins -- lives in the tape index, which is + * served from memory once the cartridge is mounted. Reading that metadata is + * therefore cheap, so building rsync's file list from an LTFS source is fast. + * + * Reading file *content*, on the other hand, requires physically positioning + * the tape. rsync's normal name-sorted traversal bears no relation to the + * order in which bytes are laid down on the medium, so a restore that opens + * files in name order makes the drive seek back and forth ("shoe-shining"), + * turning a single forward streaming pass into hours of repositioning. + * + * This module lets the generator drive the data-read phase in physical + * (start-block) order instead. LTFS exposes each file's starting block as a + * virtual extended attribute; we read it during generation and hand the + * generator a permutation of the file list sorted by that block, so the tape + * streams forward in one pass. Entries whose start block is unknown + * (directories, symlinks, files not on tape) sort first, in their original + * order, which conveniently front-loads directory creation before the bulk + * data read begins. + */ + +#include "rsync.h" +#ifdef SUPPORT_XATTRS +#include "lib/sysxattrs.h" +#endif + +extern int ltfs_mode; +extern int startblock_ndx; + +/* Return the LTFS starting block of fname, or -1 if it cannot be determined. + * The attribute value is an ASCII decimal block number. */ +int64 ltfs_startblock(const char *fname) +{ +#ifdef SUPPORT_XATTRS + /* The virtual xattr names under which LTFS publishes a file's starting + * data block. The bare "ltfs.*" name is what an LTFS FUSE mount + * presents; the "user.ltfs.*" alias lets the feature be exercised on an + * ordinary filesystem (e.g. by the test suite) where only the "user." + * namespace is writable. */ + static const char *startblock_attrs[] = { + "ltfs.startblock", + "user.ltfs.startblock", + }; + char buf[32]; + unsigned int i; + + for (i = 0; i < sizeof startblock_attrs / sizeof startblock_attrs[0]; i++) { + ssize_t len = sys_lgetxattr(fname, startblock_attrs[i], buf, sizeof buf - 1); + if (len > 0) { + char *end; + int64 blk; + buf[len] = '\0'; + blk = (int64)strtoll(buf, &end, 10); + if (end != buf && blk >= 0) + return blk; + } + } +#else + (void)fname; +#endif + return -1; +} + +struct ltfs_ent { + int64 startblock; + int idx; /* index into flist->sorted[] */ +}; + +static int ltfs_ent_cmp(const void *a, const void *b) +{ + const struct ltfs_ent *ea = a, *eb = b; + + if (ea->startblock != eb->startblock) + return ea->startblock < eb->startblock ? -1 : 1; + /* Stable tie-break so unknown-block entries (all -1, e.g. directories) + * keep their original parent-before-child name ordering. */ + return ea->idx < eb->idx ? -1 : ea->idx > eb->idx ? 1 : 0; +} + +/* Build a tape-physical read order for the active range of flist. Returns a + * malloc'd array of (flist->high - flist->low + 1) entries, each an index into + * flist->sorted[], ordered by ascending LTFS start block. The caller iterates + * the returned array in place of the natural low..high sweep. Returns NULL + * (caller falls back to natural order) if ltfs_mode is off, no start-block + * metadata was negotiated, or the range is empty. */ +int *ltfs_build_order(struct file_list *flist) +{ + struct ltfs_ent *ents; + int *order; + int n, j, count; + + if (!ltfs_mode || !startblock_ndx || flist->high < flist->low) + return NULL; + + n = flist->high - flist->low + 1; + ents = new_array(struct ltfs_ent, n); + order = new_array(int, n); + + for (j = 0, count = 0; j < n; j++) { + struct file_struct *file = flist->sorted[flist->low + j]; + ents[count].idx = flist->low + j; + if (F_IS_ACTIVE(file) && S_ISREG(file->mode)) + ents[count].startblock = F_STARTBLOCK(file); + else + ents[count].startblock = -1; + count++; + } + + qsort(ents, count, sizeof ents[0], ltfs_ent_cmp); + + for (j = 0; j < count; j++) + order[j] = ents[j].idx; + + free(ents); + return order; +} diff --git a/options.c b/options.c index 0efd02eb0..69c3baded 100644 --- a/options.c +++ b/options.c @@ -117,6 +117,7 @@ int human_readable = 1; int recurse = 0; int mkpath_dest_arg = 0; int allow_inc_recurse = 1; +int ltfs_mode = 0; int xfer_dirs = -1; int am_daemon = 0; /* Set after a successful per-module chroot ("use chroot = yes") in @@ -603,7 +604,7 @@ enum {OPT_SERVER = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM, OPT_NO_D, OPT_APPEND, OPT_NO_ICONV, OPT_INFO, OPT_DEBUG, OPT_BLOCK_SIZE, OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, OPT_BWLIMIT, OPT_STDERR, OPT_OLD_COMPRESS, OPT_NEW_COMPRESS, OPT_NO_COMPRESS, OPT_OLD_ARGS, - OPT_STOP_AFTER, OPT_STOP_AT, + OPT_STOP_AFTER, OPT_STOP_AT, OPT_LTFS, OPT_REFUSED_BASE = 9000}; static struct poptOption long_options[] = { @@ -632,6 +633,8 @@ static struct poptOption long_options[] = { {"no-r", 0, POPT_ARG_VAL, &recurse, 0, 0, 0 }, {"inc-recursive", 0, POPT_ARG_VAL, &allow_inc_recurse, 1, 0, 0 }, {"no-inc-recursive", 0, POPT_ARG_VAL, &allow_inc_recurse, 0, 0, 0 }, + {"ltfs", 0, POPT_ARG_NONE, 0, OPT_LTFS, 0, 0 }, + {"no-ltfs", 0, POPT_ARG_VAL, <fs_mode, 0, 0, 0 }, {"i-r", 0, POPT_ARG_VAL, &allow_inc_recurse, 1, 0, 0 }, {"no-i-r", 0, POPT_ARG_VAL, &allow_inc_recurse, 0, 0, 0 }, {"dirs", 'd', POPT_ARG_VAL, &xfer_dirs, 2, 0, 0 }, @@ -1676,6 +1679,14 @@ int parse_arguments(int *argc_p, const char ***argv_p) preserve_devices = preserve_specials = 0; break; + case OPT_LTFS: + ltfs_mode = 1; + /* Imply -t (like --archive does) so the index quick-check can + * skip unchanged files on a later run. Processed here in option + * order so a subsequent --no-times can still override it. */ + preserve_mtimes = 1; + break; + case 'h': human_readable++; break; @@ -2539,6 +2550,37 @@ int parse_arguments(int *argc_p, const char ***argv_p) bwlimit_writemax = 512; } +#if !defined(SUPPORT_LTFS) || defined(LTFS_SERVER_REJECT_TEST) + if (ltfs_mode && (am_server || am_daemon)) { + snprintf(err_buf, sizeof err_buf, + "--ltfs is not supported on this server\n"); + goto cleanup; + } +#endif + if (ltfs_mode) { + /* A delta read would only re-read the source file we must + * stream off the tape anyway, so force whole-file. */ + if (whole_file < 0) + whole_file = 1; + /* We need the complete file list before we can order the read + * by physical block, so incremental recursion is incompatible. */ + allow_inc_recurse = 0; + /* --checksum would read every byte of every file off the tape + * just to decide what to transfer, defeating the whole point. */ + if (always_checksum) { + snprintf(err_buf, sizeof err_buf, + "--checksum cannot be used with --ltfs (it would read the entire tape)\n"); + goto cleanup; + } + /* --ltfs implies -t (see OPT_LTFS); a 0 here means the user added an + * explicit --no-times afterward. We honor it, but warn: without + * preserved mtimes the index quick-check can't skip unchanged files, + * so every run re-reads the whole tape. */ + if (!preserve_mtimes && !am_server) + rprintf(FWARNING, + "--ltfs with --no-times: unchanged files cannot be skipped by mtime, so every run re-reads the tape.\n"); + } + if (append_mode) { if (whole_file > 0) { snprintf(err_buf, sizeof err_buf, @@ -2931,6 +2973,11 @@ void server_options(char **args, int *argc_p) } else if (preserve_specials) args[ac++] = "--specials"; + /* The sender reads the start-block metadata and both sides must agree + * on the file-list extra layout, so tell the server side about --ltfs. */ + if (ltfs_mode) + args[ac++] = "--ltfs"; + /* The server side doesn't use our log-format, but in certain * circumstances they need to know a little about the option. */ if (stdout_format && am_sender) { diff --git a/rsync.1.md b/rsync.1.md index ad686b646..59f7a4c07 100644 --- a/rsync.1.md +++ b/rsync.1.md @@ -544,6 +544,7 @@ has its own detailed description later in this manpage. --inc-recursive, --i-r enable incremental recursion --no-inc-recursive disable incremental recursion --no-i-r same as --no-inc-recursive +--ltfs read an LTFS-tape source in physical block order --relative, -R use relative path names --no-implied-dirs don't send implied directories with --relative --backup, -b make backups (see --suffix & --backup-dir) @@ -1036,6 +1037,42 @@ sign) if you want the local shell to expand it. before it begins to transfer files. See [`--inc-recursive`](#opt) for more info. +0. `--ltfs` + + Optimize reading from a source that lives on an LTFS (Linear Tape File + System) volume. On tape, a file's metadata (name, size, modify time, and + the block where its data begins) is held in the volume index and is cheap + to read, but reading file *content* requires physically positioning the + tape. rsync's normal name-sorted order bears no relation to the physical + layout, so a restore seeks back and forth ("shoe-shining") and can take + many times longer than a single streaming pass. + + With `--ltfs`, rsync reads each file's starting block from the + `ltfs.startblock` virtual extended attribute and drives the transfer in + ascending block order, so the drive makes one forward pass. Files whose + start block is unknown (directories, symlinks, anything not on tape) are + handled first, which conveniently creates the destination directory tree + before the bulk data read begins. + + Because the whole point is to avoid re-reading tape data, this option + implies [`--whole-file`](#opt) (a delta transfer would re-read the source + file anyway) and forces [`--no-inc-recursive`](#opt) (the complete file + list is needed before the read order can be chosen). It also refuses + [`--checksum`](#opt), which would read every byte of every file off the + tape just to decide what to transfer. + + The fast index metadata still drives the normal quick check (size & modify + time), so unchanged files are skipped without touching their data. For + that to work across runs the destination must keep the source mtimes, so + `--ltfs` also implies [`--times`](#opt) (like [`--archive`](#opt) does); a + later `--no-times` overrides it but triggers a warning, since without + preserved mtimes every run re-reads the whole tape. + + This option only affects reading the source; writing a transfer *onto* an + LTFS volume is not currently optimized. It requires a build with extended + attribute support and the start-block ordering only takes effect when the + source files expose the `ltfs.startblock` attribute. + 0. `--relative`, `-R` Use relative paths. This means that the full path names specified on the diff --git a/rsync.h b/rsync.h index b15aa1af6..870b4dede 100644 --- a/rsync.h +++ b/rsync.h @@ -628,6 +628,8 @@ typedef unsigned int size_t; #define SUPPORT_CRTIMES 1 #endif +/* SUPPORT_LTFS is defined by configure (requires SUPPORT_XATTRS). */ + /* Find a variable that is either exactly 32-bits or longer. * If some code depends on 32-bit truncation, it will need to * take special action in a "#if SIZEOF_INT32 > 4" section. */ @@ -857,6 +859,7 @@ extern int file_extra_cnt; extern int inc_recurse; extern int atimes_ndx; extern int crtimes_ndx; +extern int startblock_ndx; extern int pathname_ndx; extern int depth_ndx; extern int uid_ndx; @@ -921,6 +924,7 @@ extern int file_sum_extra_cnt; #define F_NDX(f) REQ_EXTRA(f, unsort_ndx)->num #define F_ATIME(f) REQ_EXTRA64(f, atimes_ndx)->num #define F_CRTIME(f) REQ_EXTRA64(f, crtimes_ndx)->num +#define F_STARTBLOCK(f) REQ_EXTRA64(f, startblock_ndx)->num /* These items are per-entry optional: */ #define F_HL_GNUM(f) OPT_EXTRA(f, START_BUMP(f))->num /* non-dirs */ diff --git a/testsuite/ltfs-client-compat_test.py b/testsuite/ltfs-client-compat_test.py new file mode 100644 index 000000000..1821f6908 --- /dev/null +++ b/testsuite/ltfs-client-compat_test.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +# Test that a client built without SUPPORT_LTFS can still perform normal +# transfers to and from an LTFS-capable server. +# +# The LTFS code path on the server is only activated when --ltfs is passed. +# A no-LTFS client connecting without that flag must be fully compatible: +# the server must not allocate LTFS file-list extras, must not attempt to +# send startblock data, and must complete the transfer correctly. +# +# rsync_noltfs is an otherwise-identical rsync binary compiled without +# SUPPORT_XATTRS (which also suppresses SUPPORT_LTFS). It acts as the +# client here; the daemon runs the normal LTFS-capable build. + +import subprocess + +from rsyncfns import ( + FROMDIR, TODIR, TOOLDIR, + build_rsyncd_conf, checkit, makepath, rsync_argv, + start_test_daemon, test_fail, +) + +DAEMON_PORT = 12900 + +noltfs_bin = str(TOOLDIR / 'rsync_noltfs') + +# --- setup ------------------------------------------------------------------- + +makepath(FROMDIR, TODIR) +files = { + 'alpha.txt': 'content alpha\n', + 'bravo.txt': 'content bravo\n', + 'sub/gamma.txt': 'content gamma\n', +} +for rel, content in files.items(): + f = FROMDIR / rel + f.parent.mkdir(parents=True, exist_ok=True) + f.write_text(content) + +# Daemon runs the LTFS-capable binary (default). +conf = build_rsyncd_conf() +url = start_test_daemon(conf, DAEMON_PORT) + +# --- 1. push: no-LTFS client → LTFS-capable server -------------------------- + +res = subprocess.run( + [noltfs_bin, '-rt', f'{FROMDIR}/', f'{url}test-to/'], + capture_output=True, text=True, +) +if res.returncode != 0: + test_fail( + f"no-LTFS client push to LTFS-capable server failed " + f"(exit {res.returncode}):\n{res.stderr}" + ) + +checkit(['-rt', f'{FROMDIR}/', f'{TODIR}/'], FROMDIR, TODIR) + +# --- 2. pull: no-LTFS client ← LTFS-capable server -------------------------- + +import tempfile +from pathlib import Path +with tempfile.TemporaryDirectory() as tmp: + dst = Path(tmp) / 'pull-dst' + dst.mkdir() + res = subprocess.run( + [noltfs_bin, '-rt', f'{url}test-to/', f'{dst}/'], + capture_output=True, text=True, + ) + if res.returncode != 0: + test_fail( + f"no-LTFS client pull from LTFS-capable server failed " + f"(exit {res.returncode}):\n{res.stderr}" + ) + checkit(['-rt', f'{TODIR}/', f'{dst}/'], TODIR, dst) + +print("ltfs-client-compat: no-LTFS client completed push and pull " + "against LTFS-capable server without errors") diff --git a/testsuite/ltfs-daemon_test.py b/testsuite/ltfs-daemon_test.py new file mode 100644 index 000000000..abdad9ec4 --- /dev/null +++ b/testsuite/ltfs-daemon_test.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +# Test --ltfs tape-ordering end-to-end over a daemon pipe connection. +# +# Both sides of the connection are the same binary (SUPPORT_LTFS on both). +# This covers what ltfs-local_test.py cannot: startblock_ndx must be +# allocated identically on both sides of the pipe, startblock values must +# survive the file-list wire format, and ltfs_build_order() must run in the +# daemon-side generator and produce an ordering that is visible in the +# itemized output returned to the client. +# +# Files are assigned user.ltfs.startblock values in reverse name order. +# In a daemon push the generator runs on the daemon (receiver) side; it +# reorders requests so the client sender serves files by ascending startblock. +# The -i output echoed back to the client reflects that generator order. + +import os +import re + +from rsyncfns import ( + FROMDIR, SCRATCHDIR, TODIR, TOOLDIR, + build_rsyncd_conf, makepath, run_rsync, start_test_daemon, + test_fail, test_skipped, +) + +DAEMON_PORT = 12898 + +# --- skip guards (same as ltfs-local_test.py) -------------------------------- + +vv = run_rsync('-VV', check=True, capture_output=True) +if '"ltfs": true' not in vv.stdout: + test_skipped("rsync built without LTFS support") + +if not hasattr(os, 'setxattr'): + test_skipped("os.setxattr not available on this platform") + +makepath(FROMDIR) +probe = FROMDIR / '.xattr-probe' +probe.write_text('x') +try: + os.setxattr(str(probe), 'user.ltfs.startblock', b'0') +except OSError as e: + test_skipped(f"scratch filesystem does not support user xattrs: {e}") +probe.unlink() + + +def set_block(path, block): + os.setxattr(str(path), 'user.ltfs.startblock', str(block).encode()) + + +# --- daemon setup ------------------------------------------------------------ + +makepath(TODIR) +conf = build_rsyncd_conf() +url = start_test_daemon(conf, DAEMON_PORT) + + +# --- 1. round-trip integrity over daemon pipe -------------------------------- + +flat = {'alpha': 500, 'bravo': 400, 'charlie': 300, 'delta': 200, 'echo': 100} +for name, blk in flat.items(): + f = FROMDIR / f'{name}.dat' + f.write_text(f'content of {name}\n') + set_block(f, blk) + +run_rsync('-r', '--ltfs', f'{FROMDIR}/', f'{url}test-to/', check=True) + + +# --- 2. physical read order preserved over the pipe -------------------------- +# Files in a subdirectory tree with startblocks in reverse name order; push to +# daemon and capture -i output to verify ascending startblock ordering. + +src2 = SCRATCHDIR / 'from2' +makepath(src2 / 'asub', src2 / 'zsub') + +layout = { + 'zsub/low.dat': 150, + 'zsub/mid.dat': 300, + 'top.dat': 600, + 'asub/high.dat': 900, +} +for rel, blk in layout.items(): + f = src2 / rel + f.write_text(f'block {blk}\n') + set_block(f, blk) + +res = run_rsync('-r', '-i', '--ltfs', f'{src2}/', f'{url}test-to/ordering/', + check=True, capture_output=True) + +got = re.findall(r'^[<>ch.][fdLDS]\S*\s+(\S+\.dat)$', res.stdout, re.MULTILINE) +expected = [rel for rel, _ in sorted(layout.items(), key=lambda kv: kv[1])] +if got != expected: + test_fail( + f"--ltfs read order over daemon pipe was {got!r}, " + f"expected tape order {expected!r}" + ) + +print("ltfs-daemon: round-trip integrity and tape ordering verified " + "over daemon pipe") diff --git a/testsuite/ltfs-local_test.py b/testsuite/ltfs-local_test.py new file mode 100644 index 000000000..7e4f254ba --- /dev/null +++ b/testsuite/ltfs-local_test.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +# Test rsync's --ltfs mode: LOCAL copy behaviour and tape-aware read ordering. +# This test exercises only local-to-local file copying; it does NOT test +# forwarding --ltfs to a remote server. +# +# On a real LTFS mount each file's starting data block is published as the +# "ltfs.startblock" virtual xattr; --ltfs reads files in ascending start-block +# order so the tape streams forward in one pass instead of seeking back and +# forth in name order. We can't mount a tape here, so we stand in the +# "user.ltfs.startblock" alias (which the feature also honors) on an ordinary +# filesystem and assign blocks that run opposite to name order. The generator +# processes files in its read order and -i emits one itemized line per file in +# that same order, so the itemized output is our observable proxy for the +# physical read schedule. + +import datetime +import os +import re + +from rsyncfns import ( + FROMDIR, SCRATCHDIR, TODIR, + checkit, makepath, run_rsync, test_fail, test_skipped, +) + + +# The feature needs a build with LTFS support... +vv = run_rsync('-VV', check=True, capture_output=True) +if '"ltfs": true' not in vv.stdout: + test_skipped("Rsync is configured without LTFS support") + +# ...and a scratch filesystem that lets us set a user.* xattr to stand in for +# the tape's ltfs.startblock attribute. +if not hasattr(os, 'setxattr'): + test_skipped("os.setxattr not available on this platform") +makepath(FROMDIR) +probe = FROMDIR / '.xattr-probe' +probe.write_text('x') +try: + os.setxattr(str(probe), 'user.ltfs.startblock', b'0') +except OSError as e: + test_skipped(f"scratch filesystem does not support user xattrs: {e}") +probe.unlink() + + +def set_block(path, block): + os.setxattr(str(path), 'user.ltfs.startblock', str(block).encode()) + + +# --- 1. round-trip integrity ----------------------------------------------- +# Five files whose start blocks run opposite to name order. +flat = {'alpha': 500, 'bravo': 400, 'charlie': 300, 'delta': 200, 'echo': 100} +for name, blk in flat.items(): + f = FROMDIR / f'{name}.dat' + f.write_text(f'content of {name}\n') + set_block(f, blk) + +# --ltfs must produce a byte-identical destination tree. +checkit(['-r', '--ltfs', f'{FROMDIR}/', f'{TODIR}/'], FROMDIR, TODIR) + + +# --- 2. physical read order, across subdirectories ------------------------- +# The itemized output (one line per file, in generator processing order) must +# come out in ascending start-block order regardless of name/directory order, +# and directories (no start block) must be handled before the bulk data read. +src2 = SCRATCHDIR / 'from2' +dst2 = SCRATCHDIR / 'to2' +makepath(src2 / 'asub', src2 / 'zsub') + +layout = { + 'zsub/low.dat': 150, + 'zsub/mid.dat': 300, + 'top.dat': 600, + 'asub/high.dat': 900, +} +for rel, blk in layout.items(): + f = src2 / rel + f.write_text(f'block {blk}\n') + set_block(f, blk) + +res = run_rsync('-r', '-i', '--ltfs', f'{src2}/', f'{dst2}/', + check=True, capture_output=True) + +# Pull the per-file itemized lines (a leading ">f"/"cf"/etc. transfer code) +# in the order rsync emitted them. +got = re.findall(r'^[<>ch.][fdLDS]\S*\s+(\S+\.dat)$', res.stdout, re.MULTILINE) +expected = [rel for rel, _ in sorted(layout.items(), key=lambda kv: kv[1])] +if got != expected: + test_fail(f"--ltfs read order was {got}, expected tape order {expected}") + +# And the content must still be correct. +checkit(['-r', '--ltfs', f'{src2}/', f'{dst2}/'], src2, dst2) + + +# --- 3. --checksum is refused ---------------------------------------------- +# It would read every byte of every file off the tape just to decide what to +# transfer, defeating the point, so it must error rather than be honored. +res = run_rsync('-r', '--ltfs', '--checksum', f'{FROMDIR}/', f'{TODIR}/', + check=False, capture_output=True) +if res.returncode == 0: + test_fail("--ltfs --checksum was accepted; expected a usage error") +if 'checksum' not in (res.stderr + res.stdout): + test_fail(f"--ltfs --checksum gave an unexpected error: {res.stderr!r}") + + +# --- 4. --ltfs implies -t so the index quick-check can skip files ----------- +# Pin a known old mtime on the source; --ltfs (no explicit -t) must preserve +# it on the destination, so an immediate re-run finds nothing to transfer. +old = datetime.datetime(2008, 1, 1, 12, 0, 0).timestamp() +src4 = SCRATCHDIR / 'from4' +dst4 = SCRATCHDIR / 'to4' +makepath(src4) +f4 = src4 / 'pinned.dat' +f4.write_text('pinned\n') +set_block(f4, 100) +os.utime(f4, (old, old)) + +run_rsync('-r', '--ltfs', f'{src4}/', f'{dst4}/', check=True) +if abs((dst4 / 'pinned.dat').stat().st_mtime - old) > 1: + test_fail("--ltfs did not preserve mtime (expected an implied -t)") +res = run_rsync('-r', '-i', '--ltfs', f'{src4}/', f'{dst4}/', + check=True, capture_output=True) +if 'pinned.dat' in res.stdout: + test_fail(f"--ltfs re-transferred an unchanged file: {res.stdout!r}") + +# An explicit --no-times defeats that, so it must warn (but still run). +res = run_rsync('-r', '--ltfs', '--no-times', f'{src4}/', f'{dst4}/', + check=True, capture_output=True) +if 'ltfs' not in (res.stderr + res.stdout).lower(): + test_fail(f"--ltfs --no-times did not warn: {res.stderr!r}") diff --git a/testsuite/ltfs-server-reject_test.py b/testsuite/ltfs-server-reject_test.py new file mode 100644 index 000000000..aa7ed8001 --- /dev/null +++ b/testsuite/ltfs-server-reject_test.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# Test that a server built without SUPPORT_LTFS rejects --ltfs with a clear +# error instead of silently ignoring it or corrupting the transfer. +# +# rsync_noltfs is an otherwise-identical rsync binary with only options.c +# recompiled without SUPPORT_XATTRS, which suppresses SUPPORT_LTFS and +# activates the server-side abort guard in options.c. The connection uses a +# local pipe (RSYNC_CONNECT_PROG) so no TCP socket or real LTFS volume is +# needed, and the test runs on all platforms regardless of xattr support. + +import subprocess + +from rsyncfns import ( + FROMDIR, TODIR, TOOLDIR, + build_rsyncd_conf, makepath, rsync_argv, + start_test_daemon, test_fail, +) + +DAEMON_PORT = 12896 + +noltfs_bin = str(TOOLDIR / 'rsync_noltfs') + +makepath(FROMDIR, TODIR) +(FROMDIR / 'probe.txt').write_text('hello from the ltfs-server-reject test\n') + +conf = build_rsyncd_conf() +url = start_test_daemon(conf, DAEMON_PORT, rsync_cmd=noltfs_bin) + +res = subprocess.run( + rsync_argv('-r', '--ltfs', f'{FROMDIR}/', f'{url}test-to/'), + capture_output=True, text=True, +) + +if res.returncode == 0: + test_fail( + "--ltfs succeeded against a no-LTFS server; expected a non-zero exit" + ) + +combined = res.stderr + res.stdout +if '--ltfs is not supported on this server' not in combined: + test_fail( + f"--ltfs against a no-LTFS server exited {res.returncode} but did " + f"not produce the expected error message.\n" + f"stdout: {res.stdout!r}\nstderr: {res.stderr!r}" + ) + +print("ltfs-server-reject: no-LTFS server correctly refused --ltfs " + "with a clear error message") diff --git a/testsuite/skiplist/cygwin.txt b/testsuite/skiplist/cygwin.txt index 7cdbac732..5cc0e6fc9 100644 --- a/testsuite/skiplist/cygwin.txt +++ b/testsuite/skiplist/cygwin.txt @@ -41,6 +41,9 @@ filter-merge-symlink insecure-links-admin-optout link-dest-symlink-enotsup # the ENOTSUP hard-link hook is an LD_PRELOAD, Linux-only log-file-symlink +ltfs-client-compat +ltfs-daemon +ltfs-local msg-io-timeout-overflow nondaemon-symlink-race nonroot-restrictive-perms diff --git a/testsuite/skiplist/macos.txt b/testsuite/skiplist/macos.txt index 3ec3c6b04..2e771c576 100644 --- a/testsuite/skiplist/macos.txt +++ b/testsuite/skiplist/macos.txt @@ -18,6 +18,9 @@ daemon-max-alloc-zero dir-sgid fake-super-acl-xattr link-dest-symlink-enotsup # the ENOTSUP hard-link hook is an LD_PRELOAD, Linux-only +ltfs-client-compat +ltfs-daemon +ltfs-local open-noatime partial-protected-regular-retry-linux preallocate diff --git a/usage.c b/usage.c index de586839c..460915102 100644 --- a/usage.c +++ b/usage.c @@ -129,6 +129,11 @@ static void print_info_flags(enum logcode f) #endif "xattrs", +#ifndef SUPPORT_LTFS + "no " +#endif + "ltfs", + #ifdef RSYNC_USE_SECLUDED_ARGS "default " #else