Initial LTFS order awarness functionality

Makefile.in

@@ -45,7 +45,7 @@ LIBOBJ=lib/wildmatch.o lib/compat.o lib/snprintf.o lib/mdfour.o lib/md5.o \
 zlib_OBJS=zlib/deflate.o zlib/inffast.o zlib/inflate.o zlib/inftrees.o \
 	zlib/trees.o zlib/zutil.o zlib/adler32.o zlib/compress.o zlib/crc32.o
 OBJS1_NO_MAIN=flist.o rsync.o generator.o receiver.o cleanup.o sender.o exclude.o \
-	util1.o util2.o checksum.o match.o syscall.o log.o backup.o delete.o
+	util1.o util2.o checksum.o match.o syscall.o log.o backup.o delete.o ltfs.o
 OBJS1=$(OBJS1_NO_MAIN) main.o
 OBJS2=options.o io.o compat.o hlink.o token.o uidlist.o socket.o hashtable.o \
 	usage.o fileio.o batch.o clientname.o chmod.o acls.o xattrs.o
@@ -55,18 +55,26 @@ popt_OBJS= popt/popt.o  popt/poptconfig.o \
 	popt/popthelp.o popt/poptparse.o popt/poptint.o
 OBJS=$(OBJS1) $(OBJS2) $(OBJS3) $(DAEMON_OBJ) $(LIBOBJ) @BUILD_ZLIB@ @BUILD_POPT@
 
+# rsync_noltfs: identical to rsync but options.c recompiled with
+# -DLTFS_SERVER_REJECT_TEST, which forces the server-side --ltfs rejection
+# path active regardless of SUPPORT_LTFS.  Used by the ltfs-server-reject test.
+OBJS_NOLTFS=$(OBJS1) options_noltfs.o io.o compat.o hlink.o token.o uidlist.o \
+	socket.o hashtable.o usage.o fileio.o batch.o clientname.o chmod.o acls.o \
+	xattrs.o $(OBJS3) $(DAEMON_OBJ) $(LIBOBJ) @BUILD_ZLIB@ @BUILD_POPT@
+
 TLS_OBJ = tls.o syscall.o util2.o t_stub.o lib/compat.o lib/snprintf.o lib/permstring.o lib/sysxattrs.o @BUILD_POPT@
 
 # Programs we must have to run the test cases
 CHECK_PROGS = rsync$(EXEEXT) tls$(EXEEXT) getgroups$(EXEEXT) getfsdev$(EXEEXT) \
 	testrun$(EXEEXT) trimslash$(EXEEXT) t_unsafe$(EXEEXT) t_chmod_secure$(EXEEXT) \
-	t_rename_secure$(EXEEXT) t_symlink_secure$(EXEEXT) t_secure_relpath$(EXEEXT) t_acl$(EXEEXT) t_hashtable_overflow$(EXEEXT) t_iwildmatch$(EXEEXT) t_clean_fname$(EXEEXT) t_safe_arg$(EXEEXT) wildtest$(EXEEXT) simdtest$(EXEEXT)
+	t_rename_secure$(EXEEXT) t_symlink_secure$(EXEEXT) t_secure_relpath$(EXEEXT) t_acl$(EXEEXT) t_hashtable_overflow$(EXEEXT) t_iwildmatch$(EXEEXT) t_clean_fname$(EXEEXT) t_safe_arg$(EXEEXT) wildtest$(EXEEXT) simdtest$(EXEEXT) \
+	@LTFS_CHECK_PROGS@
 
 CHECK_SYMLINKS = testsuite/chown-fake_test.py testsuite/devices-fake_test.py \
 	testsuite/xattrs-hlink_test.py testsuite/exclude-lsh_test.py
 
 # Objects for CHECK_PROGS to clean
-CHECK_OBJS=tls.o testrun.o getgroups.o getfsdev.o t_stub.o t_unsafe.o t_chmod_secure.o t_rename_secure.o t_symlink_secure.o t_secure_relpath.o t_acl.o t_hashtable_overflow.o t_iwildmatch.o t_clean_fname.o t_safe_arg.o trimslash.o wildtest.o
+CHECK_OBJS=tls.o testrun.o getgroups.o getfsdev.o t_stub.o t_unsafe.o t_chmod_secure.o t_rename_secure.o t_symlink_secure.o t_secure_relpath.o t_acl.o t_hashtable_overflow.o t_iwildmatch.o t_clean_fname.o t_safe_arg.o trimslash.o wildtest.o @LTFS_CHECK_OBJS@
 # Compile-only feature-shape checks.
 CHECK_COMPILE_OBJS=syscall-no-at-fdcwd.o
 
@@ -273,8 +281,7 @@ aclocal.m4: $(srcdir)/m4/*.m4
 configure.sh config.h.in: configure.ac aclocal.m4
 	@if test -f configure.sh; then cp -p configure.sh configure.sh.old; else touch configure.sh.old; fi
 	@if test -f config.h.in; then cp -p config.h.in config.h.in.old; else touch config.h.in.old; fi
-	autoconf -o configure.sh
-	autoheader && touch config.h.in
+	$(srcdir)/autogen.sh
 	@if diff configure.sh configure.sh.old >/dev/null 2>&1; then \
 	    echo "configure.sh is unchanged."; \
 	    rm configure.sh.old; \
@@ -443,15 +450,15 @@ check-progs: all $(CHECK_PROGS) $(CHECK_COMPILE_OBJS) $(CHECK_SYMLINKS)
 
 .PHONY: check
 check: all $(CHECK_PROGS) $(CHECK_COMPILE_OBJS) $(CHECK_SYMLINKS)
-	"$(srcdir)/runtests.py" --rsync-bin="`pwd`/rsync$(EXEEXT)" -j $(CHECK_J)
+	"$(srcdir)/runtests.py" --rsync-bin="`pwd`/rsync$(EXEEXT)" -j $(CHECK_J) @LTFS_RUNTESTS_ARGS@
 
 .PHONY: check29
 check29: all $(CHECK_PROGS) $(CHECK_COMPILE_OBJS) $(CHECK_SYMLINKS)
-	"$(srcdir)/runtests.py" --rsync-bin="`pwd`/rsync$(EXEEXT)" -j $(CHECK_J) --protocol=29
+	"$(srcdir)/runtests.py" --rsync-bin="`pwd`/rsync$(EXEEXT)" -j $(CHECK_J) --protocol=29 @LTFS_RUNTESTS_ARGS@
 
 .PHONY: check30
 check30: all $(CHECK_PROGS) $(CHECK_COMPILE_OBJS) $(CHECK_SYMLINKS)
-	"$(srcdir)/runtests.py" --rsync-bin="`pwd`/rsync$(EXEEXT)" -j $(CHECK_J) --protocol=30
+	"$(srcdir)/runtests.py" --rsync-bin="`pwd`/rsync$(EXEEXT)" -j $(CHECK_J) --protocol=30 @LTFS_RUNTESTS_ARGS@
 
 # Whole-suite gcov coverage report (HTML, with branch + decision coverage).
 # Requires a build configured with --enable-coverage and the `gcovr` tool
@@ -479,7 +486,7 @@ coverage: all $(CHECK_PROGS) $(CHECK_SYMLINKS)
 	    chmod a+rwx "$$d"; \
 	    setfacl -m 'd:u::rwx,d:g::rwx,d:o::rwx' "$$d" 2>/dev/null || true; \
 	  done
-	@rc=0; "$(srcdir)/runtests.py" --rsync-bin="`pwd`/rsync$(EXEEXT)" -j $(COVERAGE_J) $(COVERAGE_RUNFLAGS) || rc=$$?; \
+	@rc=0; "$(srcdir)/runtests.py" --rsync-bin="`pwd`/rsync$(EXEEXT)" -j $(COVERAGE_J) $(COVERAGE_RUNFLAGS) @LTFS_RUNTESTS_ARGS@ || rc=$$?; \
 	  rm -rf $(COVERAGE_DIR) && mkdir -p $(COVERAGE_DIR); \
 	  gcovr --root $(srcdir) $(COVERAGE_EXCLUDE) --decisions --print-summary \
 	      --gcov-ignore-parse-errors=negative_hits.warn_once_per_file \
@@ -519,7 +526,7 @@ coverage-all: all $(CHECK_PROGS) $(CHECK_SYMLINKS)
 	@rc=0; \
 	  for cfg in '' '--protocol=30' '--protocol=29' '--use-tcp'; do \
 	    echo "===== coverage-all: runtests.py $$cfg ====="; \
-	    "$(srcdir)/runtests.py" --rsync-bin="`pwd`/rsync$(EXEEXT)" -j $(COVERAGE_J) $$cfg || rc=$$?; \
+	    "$(srcdir)/runtests.py" --rsync-bin="`pwd`/rsync$(EXEEXT)" -j $(COVERAGE_J) $$cfg @LTFS_RUNTESTS_ARGS@ || rc=$$?; \
 	  done; \
 	  rm -rf coverage-all && mkdir -p coverage-all; \
 	  gcovr --root $(srcdir) $(COVERAGE_EXCLUDE) --decisions --print-summary \
@@ -550,6 +557,12 @@ simdtest$(EXEEXT): simd-checksum-x86_64.cpp $(HEADERS)
 	    touch $@; \
 	fi
 
+options_noltfs.o: $(srcdir)/options.c $(HEADERS)
+	$(CC) -I. -I$(srcdir) $(CFLAGS) $(CPPFLAGS) -DLTFS_SERVER_REJECT_TEST -c -o $@ $(srcdir)/options.c
+
+rsync_noltfs$(EXEEXT): $(OBJS_NOLTFS)
+	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(OBJS_NOLTFS) $(LIBS)
+
 testsuite/chown-fake_test.py:
 	ln -s chown_test.py $(srcdir)/testsuite/chown-fake_test.py
 
@@ -568,7 +581,7 @@ testsuite/exclude-lsh_test.py:
 
 .PHONY: installcheck
 installcheck: $(CHECK_PROGS) $(CHECK_SYMLINKS)
-	"$(srcdir)/runtests.py" --rsync-bin="$(bindir)/rsync$(EXEEXT)" --srcdir="$(srcdir)" --tooldir="`pwd`" -j $(CHECK_J)
+	"$(srcdir)/runtests.py" --rsync-bin="$(bindir)/rsync$(EXEEXT)" --srcdir="$(srcdir)" --tooldir="`pwd`" -j $(CHECK_J) @LTFS_RUNTESTS_ARGS@
 
 # TODO: Add 'dist' target; need to know which files will be included
 

autogen.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+# Regenerate the autoconf build files from configure.ac.
+# Run this after "make distclean" or a fresh clone before ./configure.
+set -e
+aclocal -I m4
+autoconf -o configure.sh
+autoheader && touch config.h.in

compat.c

@@ -45,6 +45,7 @@ extern int preserve_uid;
 extern int preserve_gid;
 extern int preserve_atimes;
 extern int preserve_crtimes;
+extern int ltfs_mode;
 extern int preserve_acls;
 extern int preserve_xattrs;
 extern int xfer_flags_as_varint;
@@ -87,7 +88,7 @@ struct name_num_item *xattr_sum_nni;
 int xattr_sum_len = 0;
 
 /* These index values are for the file-list's extra-attribute array. */
-int pathname_ndx, depth_ndx, atimes_ndx, crtimes_ndx, uid_ndx, gid_ndx, acls_ndx, xattrs_ndx, unsort_ndx;
+int pathname_ndx, depth_ndx, atimes_ndx, crtimes_ndx, startblock_ndx, uid_ndx, gid_ndx, acls_ndx, xattrs_ndx, unsort_ndx;
 
 int receiver_symlink_times = 0; /* receiver can set the time on a symlink */
 int sender_symlink_iconv = 0;	/* sender should convert symlink content */
@@ -584,6 +585,8 @@ void setup_protocol(int f_out,int f_in)
 		atimes_ndx = (file_extra_cnt += EXTRA64_CNT);
 	if (preserve_crtimes)
 		crtimes_ndx = (file_extra_cnt += EXTRA64_CNT);
+	if (ltfs_mode)
+		startblock_ndx = (file_extra_cnt += EXTRA64_CNT);
 	if (am_sender) /* This is most likely in the file_extras64 union as well. */
 		pathname_ndx = (file_extra_cnt += PTR_EXTRA_CNT);
 	else

configure.ac

@@ -1547,6 +1547,50 @@ if test x"$samba_cv_HAVE_POSIX_ACLS" = x"yes"; then
     esac
 fi
 
+# check for LTFS tape-ordering support
+AC_MSG_CHECKING(whether to support LTFS tape ordering)
+AC_ARG_ENABLE(ltfs,
+    AS_HELP_STRING([--disable-ltfs],[disable LTFS tape-order awareness (--ltfs option)]),
+    [], [enable_ltfs=yes])
+AH_TEMPLATE([SUPPORT_LTFS],
+[Define to 1 to add support for LTFS tape-order-aware transfers])
+if test x"$enable_ltfs" = x"no"; then
+    AC_MSG_RESULT(no)
+elif test x"$enable_xattr_support" = x"no"; then
+    AC_MSG_RESULT([no (requires xattr support)])
+else
+    case $host_os in
+        *cygwin*)
+            AC_MSG_RESULT([no (Cygwin)])
+            ;;
+        *)
+            AC_MSG_RESULT(yes)
+            AC_DEFINE(SUPPORT_LTFS, 1)
+            ;;
+    esac
+fi
+if test x"$enable_ltfs" != x"no" && test x"$enable_xattr_support" != x"no"; then
+    case $host_os in
+        *cygwin*)
+            LTFS_CHECK_PROGS=''
+            LTFS_CHECK_OBJS=''
+            LTFS_RUNTESTS_ARGS='--exclude=ltfs-server-reject,ltfs-client-compat'
+            ;;
+        *)
+            LTFS_CHECK_PROGS='rsync_noltfs$(EXEEXT)'
+            LTFS_CHECK_OBJS='options_noltfs.o'
+            LTFS_RUNTESTS_ARGS=''
+            ;;
+    esac
+else
+    LTFS_CHECK_PROGS=''
+    LTFS_CHECK_OBJS=''
+    LTFS_RUNTESTS_ARGS='--exclude=ltfs-server-reject,ltfs-client-compat'
+fi
+AC_SUBST(LTFS_CHECK_PROGS)
+AC_SUBST(LTFS_CHECK_OBJS)
+AC_SUBST(LTFS_RUNTESTS_ARGS)
+
 if test x"$enable_acl_support" = x"no" || test x"$enable_xattr_support" = x"no" || test x"$enable_iconv" = x"no"; then
     AC_MSG_CHECKING([whether $CC supports -Wno-unused-parameter])
     OLD_CFLAGS="$CFLAGS"

flist.c

@@ -62,6 +62,7 @@ extern int missing_args;
 extern int eol_nulls;
 extern int atimes_ndx;
 extern int crtimes_ndx;
+extern int startblock_ndx;
 extern int relative_paths;
 extern int implied_dirs;
 extern int ignore_perishable;
@@ -685,6 +686,8 @@ static void send_file_entry(int f, const char *fname, struct file_struct *file,
 	if (crtimes_ndx && !(xflags & XMIT_CRTIME_EQ_MTIME))
 		write_varlong(f, crtime, 4);
 #endif
+	if (startblock_ndx)
+		write_varlong(f, F_STARTBLOCK(file), 3);
 	if (!(xflags & XMIT_SAME_MODE))
 		write_int(f, to_wire_mode(mode));
 	if (atimes_ndx && !S_ISDIR(mode) && !(xflags & XMIT_SAME_ATIME))
@@ -780,6 +783,7 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x
 #ifdef SUPPORT_CRTIMES
 	static time_t crtime;
 #endif
+	static int64 startblock;
 	static mode_t mode;
 #ifdef SUPPORT_HARD_LINKS
 	static int64 dev;
@@ -964,6 +968,8 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x
 #endif
 	}
 #endif
+	if (startblock_ndx)
+		startblock = read_varlong(f, 3);
 	if (!(xflags & XMIT_SAME_MODE)) {
 		mode = from_wire_mode(read_int(f));
 		/* Reject modes whose type bits are not one of the standard
@@ -1215,6 +1221,8 @@ static struct file_struct *recv_file_entry(int f, struct file_list *flist, int x
 	if (crtimes_ndx)
 		F_CRTIME(file) = crtime;
 #endif
+	if (startblock_ndx)
+		F_STARTBLOCK(file) = startblock;
 	if (unsort_ndx)
 		F_NDX(file) = flist->used + flist->ndx_start;
 
@@ -1679,6 +1687,12 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
 	if (crtimes_ndx)
 		F_CRTIME(file) = get_create_time(fname, &st);
 #endif
+	if (startblock_ndx) {
+		int64 blk = am_sender && S_ISREG(file->mode) ? ltfs_startblock(fname) : -1;
+		/* Unknown sorts first; keep it non-negative so write_varlong
+		 * stays compact (a real LTFS data block is well past block 0). */
+		F_STARTBLOCK(file) = blk < 0 ? 0 : blk;
+	}
 
 	if (basename != thisname)
 		file->dirname = lastdir;

generator.c

@@ -33,6 +33,7 @@ extern int operator_path_resolve;
 extern int am_server;
 extern int am_daemon;
 extern int inc_recurse;
+extern int ltfs_mode;
 extern int relative_paths;
 extern int implied_dirs;
 extern int keep_dirlinks;
@@ -2716,6 +2717,7 @@ void check_for_finished_files(int itemizing, enum logcode code, int check_redo)
 void generate_files(int f_out, const char *local_name)
 {
 	int i, ndx, next_loopchk = 0;
+	int *ltfs_order = NULL;
 	char fbuf[MAXPATHLEN];
 	int itemizing;
 	enum logcode code;
@@ -2800,16 +2802,23 @@ void generate_files(int f_out, const char *local_name)
 					change_local_filter_dir(fbuf, strlen(fbuf), F_DEPTH(fp));
 			}
 		}
+		/* For an LTFS source, read the files in physical tape order
+		 * (by start block) rather than name order, so the drive makes
+		 * one forward streaming pass instead of seeking back and forth.
+		 * ltfs_order maps the natural sweep position to a sorted[] index;
+		 * a NULL result falls back to the natural low..high order. */
+		ltfs_order = ltfs_mode ? ltfs_build_order(cur_flist) : NULL;
 		for (i = cur_flist->low; i <= cur_flist->high; i++) {
-			struct file_struct *file = cur_flist->sorted[i];
+			int si = ltfs_order ? ltfs_order[i - cur_flist->low] : i;
+			struct file_struct *file = cur_flist->sorted[si];
 
 			if (!F_IS_ACTIVE(file))
 				continue;
 
 			if (unsort_ndx)
 				ndx = F_NDX(file);
 			else
-				ndx = i + cur_flist->ndx_start;
+				ndx = si + cur_flist->ndx_start;
 
 			if (solo_file)
 				strlcpy(fbuf, solo_file, sizeof fbuf);
@@ -2828,6 +2837,9 @@ void generate_files(int f_out, const char *local_name)
 			}
 		}
 
+		if (ltfs_order)
+			free(ltfs_order);
+
 		if (!inc_recurse) {
 			write_ndx(f_out, NDX_DONE);
 			break;