import/export-tar: --tar-format=PAX (1.4-maint)

src/borg/archive.py

@@ -40,7 +40,7 @@
 from .helpers import safe_encode, safe_decode, make_path_safe, remove_surrogates
 from .helpers import StableDict
 from .helpers import bin_to_hex
-from .helpers import safe_ns
+from .helpers import safe_ns, pax_time_to_ns
 from .helpers import ellipsis_truncate, ProgressIndicatorPercent, log_multi
 from .helpers import os_open, flags_normal, flags_dir
 from .helpers import os_stat
@@ -1636,6 +1636,28 @@ def process_file(self, *, path, parent_fd, name, st, cache, flags=flags_normal,
                 return status
 
 
+def tar_acl_to_borg(acl):
+    """Convert a POSIX ACL text from a tar PAX header (SCHILY.acl.*) to borg's ACL format.
+
+    Borg separates entries by newlines and appends the numeric uid/gid as a 4th field to
+    named user/group entries (user:name:perms:uid), see acl_get on Linux and FreeBSD.
+    star appends it too, but separates entries by commas. GNU tar separates entries by
+    newlines and does not append the numeric id, so we look it up locally (like borg create
+    does), falling back to the name.
+    """
+    entries = []
+    for entry in acl.replace(',', '\n').split('\n'):
+        entry = entry.split('#', 1)[0].strip()  # remove comments
+        if not entry:
+            continue
+        fields = entry.split(':')
+        if len(fields) == 3 and fields[1] and fields[0] in ('user', 'group'):
+            name = fields[1]
+            fields.append(str(user2uid(name, name) if fields[0] == 'user' else group2gid(name, name)))
+        entries.append(':'.join(fields))
+    return '\n'.join(entries).encode('utf-8', errors='surrogateescape')
+
+
 class TarfileObjectProcessors:
     def __init__(self, *, cache, key,
                  add_item, process_file_chunks,
@@ -1659,6 +1681,32 @@ def create_helper(self, tarinfo, status=None, type=None):
         item = Item(path=make_path_safe(normalized_path), mode=tarinfo.mode | type,
                     uid=tarinfo.uid, gid=tarinfo.gid, user=tarinfo.uname or None, group=tarinfo.gname or None,
                     mtime=safe_ns(int(tarinfo.mtime * 1000**3)))
+        ph = tarinfo.pax_headers
+        if ph:
+            # the tarfile module only gives us float timestamps, parse the original strings for full precision.
+            for name in 'atime', 'ctime', 'mtime':
+                if name in ph:
+                    ns = pax_time_to_ns(ph[name])
+                    if ns is not None:
+                        setattr(item, name, ns)
+            xattrs = StableDict()
+            for key, value in ph.items():
+                if key.startswith(SCHILY_XATTR):
+                    key = key.removeprefix(SCHILY_XATTR)
+                    if key.startswith('system.posix_acl_'):
+                        # like borg create, we store the POSIX ACLs separately, not as xattrs.
+                        continue
+                    # the tarfile code gives us str keys and str values,
+                    # but we need bytes keys and bytes values (or None for an empty value, like xattr.get_all).
+                    bkey = key.encode('utf-8', errors='surrogateescape')
+                    bvalue = value.encode('utf-8', errors='surrogateescape')
+                    xattrs[bkey] = bvalue or None
+                elif key == SCHILY_ACL_ACCESS:
+                    item.acl_access = tar_acl_to_borg(value)
+                elif key == SCHILY_ACL_DEFAULT:
+                    item.acl_default = tar_acl_to_borg(value)
+            if xattrs:
+                item.xattrs = xattrs
         yield item, status
         # if we get here, "with"-block worked ok without error/exception, the item was processed ok...
         self.add_item(item, stats=self.stats)

src/borg/archiver.py

@@ -85,6 +85,7 @@
     from .helpers import sig_int, ignore_sigint
     from .helpers import iter_separated
     from .helpers import get_tar_filter
+    from .helpers import ns_to_pax_time
     from .helpers import ignore_invalid_archive_tam
     from .helpers.parseformat import BorgJsonEncoder, safe_decode
     from .nanorst import rst_to_terminal
@@ -248,6 +249,40 @@ def __call__(self, parser, namespace, values, option_string=None):
         setattr(namespace, self.dest, values)
 
 
+def item_to_paxheaders(item):
+    """Transform (parts of) a Borg *item* into a pax_headers dict."""
+    # PAX format
+    # ----------
+    # When using the PAX (POSIX) format, we can support some things that aren't possible
+    # with classic tar formats, including GNU tar, such as:
+    # - atime, ctime, mtime with ns precision (DONE)
+    # - xattrs, POSIX ACLs (DONE)
+    # - various additions supported by GNU tar in POSIX mode (TODO)
+    #
+    ph = {}
+    # note: for mtime this is a bit redundant as it is already done by tarfile module,
+    #       but it only has a float, so we do it in our way to have exact ns precision.
+    for name in 'atime', 'ctime', 'mtime':
+        if hasattr(item, name):
+            ns = getattr(item, name)
+            ph[name] = ns_to_pax_time(ns)
+    if hasattr(item, 'xattrs'):
+        for bkey, bvalue in item.xattrs.items():
+            # we have bytes key and bytes value (or None for an empty value), but the tarfile code
+            # expects str key and str value.
+            key = SCHILY_XATTR + bkey.decode('utf-8', errors='surrogateescape')
+            value = (bvalue or b'').decode('utf-8', errors='surrogateescape')
+            ph[key] = value
+    # Add POSIX access and default ACL if present
+    acl_access = item.get('acl_access')
+    if acl_access is not None:
+        ph[SCHILY_ACL_ACCESS] = acl_access.decode('utf-8', errors='surrogateescape')
+    acl_default = item.get('acl_default')
+    if acl_default is not None:
+        ph[SCHILY_ACL_DEFAULT] = acl_default.decode('utf-8', errors='surrogateescape')
+    return ph
+
+
 class Archiver:
 
     def __init__(self, lock_wait=None, prog=None):
@@ -1054,7 +1089,8 @@ def peek_and_store_hardlink_masters(item, matched):
 
         # The | (pipe) symbol instructs tarfile to use a streaming mode of operation
         # where it never seeks on the passed fileobj.
-        tar = tarfile.open(fileobj=tarstream, mode='w|', format=tarfile.GNU_FORMAT)
+        tar_format = dict(GNU=tarfile.GNU_FORMAT, PAX=tarfile.PAX_FORMAT)[args.tar_format]
+        tar = tarfile.open(fileobj=tarstream, mode='w|', format=tar_format)
 
         if progress:
             pi = ProgressIndicatorPercent(msg='%5.1f%% Processing: %s', step=0.1, msgid='extract')
@@ -1085,13 +1121,6 @@ def item_to_tarinfo(item, original_path):
             the file contents, if any, and is None otherwise. When *tarinfo* is None, the *item*
             cannot be represented as a TarInfo object and should be skipped.
             """
-
-            # If we would use the PAX (POSIX) format (which we currently don't),
-            # we can support most things that aren't possible with classic tar
-            # formats, including GNU tar, such as:
-            # atime, ctime, possibly Linux capabilities (security.* xattrs)
-            # and various additions supported by GNU tar in POSIX mode.
-
             stream = None
             tarinfo = tarfile.TarInfo()
             tarinfo.name = item.path
@@ -1159,6 +1188,8 @@ def item_to_tarinfo(item, original_path):
                 item.path = os.sep.join(orig_path.split(os.sep)[strip_components:])
             tarinfo, stream = item_to_tarinfo(item, orig_path)
             if tarinfo:
+                if args.tar_format == 'PAX':
+                    tarinfo.pax_headers = item_to_paxheaders(item)
                 if output_list:
                     logging.getLogger('borg.output.list').info(remove_surrogates(orig_path))
                 tar.addfile(tarinfo, stream)
@@ -4448,12 +4479,17 @@ def diff_sort_spec_validator(s):
         read the uncompressed tar stream from stdin and write a compressed/filtered
         tar stream to stdout.
 
-        The generated tarball uses the GNU tar format.
+        Depending on the ``--tar-format`` option, these formats are created:
 
-        export-tar is a lossy conversion:
-        BSD flags, ACLs, extended attributes (xattrs), atime and ctime are not exported.
-        Timestamp resolution is limited to whole seconds, not the nanosecond resolution
-        otherwise supported by Borg.
+        +--------------+---------------------------+----------------------------+
+        | --tar-format | Specification             | Metadata                   |
+        +--------------+---------------------------+----------------------------+
+        | PAX          | POSIX.1-2001 (pax) format | GNU + atime/ctime/mtime ns |
+        |              |                           | + xattrs, POSIX ACLs       |
+        +--------------+---------------------------+----------------------------+
+        | GNU          | GNU tar format            | mtime s, no atime/ctime,   |
+        |              |                           | no ACLs/xattrs/bsdflags    |
+        +--------------+---------------------------+----------------------------+
 
         A ``--sparse`` option (as found in ``borg extract``) is not supported.
 
@@ -4476,6 +4512,9 @@ def diff_sort_spec_validator(s):
                                help='filter program to pipe data through')
         subparser.add_argument('--list', dest='output_list', action='store_true',
                                help='output verbose list of items (files, dirs, ...)')
+        subparser.add_argument('--tar-format', metavar='FMT', dest='tar_format', default='GNU',
+                               choices=('PAX', 'GNU'), action=Highlander,
+                               help='select tar format: PAX or GNU (default: GNU)')
         subparser.add_argument('location', metavar='ARCHIVE',
                                type=location_validator(archive=True),
                                help='archive to export')
@@ -5619,15 +5658,17 @@ def diff_sort_spec_validator(s):
         Most documentation of ``borg create`` applies. Note that this command does not
         support excluding files.
 
-        import-tar is a lossy conversion:
-        BSD flags, ACLs, extended attributes (xattrs), atime and ctime are not exported.
-        Timestamp resolution is limited to whole seconds, not the nanosecond resolution
-        otherwise supported by Borg.
-
         A ``--sparse`` option (as found in borg create) is not supported.
 
-        import-tar reads POSIX.1-1988 (ustar), POSIX.1-2001 (pax), GNU tar, UNIX V7 tar
-        and SunOS tar with extended attributes.
+        About tar formats and metadata conservation or loss, please see ``borg export-tar``.
+
+        import-tar reads these tar formats:
+
+        - PAX: POSIX.1-2001
+        - GNU: GNU tar
+        - POSIX.1-1988 (ustar)
+        - UNIX V7 tar
+        - SunOS tar with extended attributes
 
         To import multiple tarballs into a single archive, they can be simply
         concatenated (e.g. using "cat") into a single file, and imported with an

src/borg/constants.py

@@ -92,6 +92,11 @@
 FILES_CACHE_MODE_UI_DEFAULT = 'ctime,size,inode'  # default for "borg create" command (CLI UI)
 FILES_CACHE_MODE_DISABLED = 'd'  # Most Borg commands do not use the files cache at all (disable).
 
+# tar related
+SCHILY_XATTR = 'SCHILY.xattr.'  # xattr key prefix in tar PAX headers
+SCHILY_ACL_ACCESS = 'SCHILY.acl.access'  # POSIX access ACL in tar PAX headers
+SCHILY_ACL_DEFAULT = 'SCHILY.acl.default'  # POSIX default ACL in tar PAX headers
+
 # return codes returned by borg command
 EXIT_SUCCESS = 0  # everything done, no problems
 EXIT_WARNING = 1  # reached normal end of operation, but there were issues (generic warning)

src/borg/helpers/time.py

@@ -1,6 +1,7 @@
 import os
 import time
 from datetime import datetime, timezone
+from decimal import Decimal, InvalidOperation
 
 from ..constants import ISO_FORMAT, ISO_FORMAT_NO_USECS
 
@@ -89,6 +90,21 @@ def safe_ns(ts):
         return MAX_NS
 
 
+def ns_to_pax_time(ns):
+    """Format a nanoseconds timestamp as an exact decimal seconds string for a tar PAX header."""
+    sign = '-' if ns < 0 else ''
+    s, ns = divmod(abs(ns), 1000000000)
+    return f'{sign}{s}.{ns:09d}'
+
+
+def pax_time_to_ns(value):
+    """Parse a tar PAX header timestamp (decimal seconds string) into nanoseconds, return None if invalid."""
+    try:
+        return safe_ns(int(Decimal(value).scaleb(9)))
+    except (InvalidOperation, ValueError, OverflowError):
+        return None
+
+
 def safe_timestamp(item_timestamp_ns):
     t_ns = safe_ns(item_timestamp_ns)
     return datetime.fromtimestamp(t_ns / 1e9)

src/borg/testsuite/archive.py

@@ -1,4 +1,5 @@
 import json
+import os
 from collections import OrderedDict
 from datetime import datetime, timezone
 from io import StringIO
@@ -9,11 +10,12 @@
 from . import BaseTestCase
 from ..crypto.key import PlaintextKey
 from ..archive import Archive, CacheChunkBuffer, RobustUnpacker, valid_msgpacked_dict, ITEM_KEYS, Statistics
-from ..archive import BackupOSError, backup_io, backup_io_iter, get_item_uid_gid
+from ..archive import BackupOSError, backup_io, backup_io_iter, get_item_uid_gid, tar_acl_to_borg
 from ..helpers import Manifest
 from ..helpers import msgpack
 from ..item import Item, ArchiveItem
 from ..platform import uid2user, gid2group
+from ..platformflags import is_win32
 
 
 @pytest.fixture()
@@ -368,3 +370,31 @@ def test_get_item_uid_gid():
     # because item uid/gid seems valid, do not use the given uid/gid defaults
     assert uid == 9
     assert gid == 10
+
+
+@pytest.mark.parametrize('acl, expected', [
+    # GNU tar: newline separated, no numeric id for named entries
+    ('user::rw-\nuser:{user}:rw-\ngroup::r--\nmask::rw-\nother::r--\n',
+     'user::rw-\nuser:{user}:rw-:{uid}\ngroup::r--\nmask::rw-\nother::r--'),
+    # star: comma separated, numeric id appended
+    ('user::rw-,user:root:rw-:0,group::r--,mask::rw-,other::r--',
+     'user::rw-\nuser:root:rw-:0\ngroup::r--\nmask::rw-\nother::r--'),
+    # borg export-tar: newline separated, numeric id appended
+    ('user::rw-\nuser:root:rw-:0\ngroup::r--\nmask::rw-\nother::r--',
+     'user::rw-\nuser:root:rw-:0\ngroup::r--\nmask::rw-\nother::r--'),
+    # unknown names fall back to the name (no name lookups on Windows)
+    pytest.param('group:nosuchgroup-borgtest:r--', 'group:nosuchgroup-borgtest:r--:nosuchgroup-borgtest',
+                 marks=pytest.mark.skipif(is_win32, reason='no name lookups on Windows')),
+    # comments get removed
+    ('user:{user}:r--\t#effective:r--\n', 'user:{user}:r--:{uid}'),
+    ('', ''),
+])
+def test_tar_acl_to_borg(acl, expected):
+    # the name lookups need an existing user, e.g. Haiku has no "root" user.
+    try:
+        uid = os.getuid()  # UNIX only
+    except AttributeError:
+        uid = 0
+    user = uid2user(uid)
+    acl, expected = acl.format(user=user, uid=uid), expected.format(user=user, uid=uid)
+    assert tar_acl_to_borg(acl) == expected.encode()