Skip to content

Commit 8d7569b

Browse files
Merge pull request #9966 from mr-raj12/check-soft-interrupt-7893
check: handle Ctrl-C at safe boundaries (#7893)
2 parents e60f7f5 + 992fc79 commit 8d7569b

4 files changed

Lines changed: 211 additions & 20 deletions

File tree

‎src/borg/archive.py‎

Lines changed: 48 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
from .helpers import BackupOSError, BackupPermissionError, BackupFileNotFoundError, BackupIOError
3333
from .helpers import HardLinkManager
3434
from .helpers import ChunkIteratorFileWrapper, open_item
35-
from .helpers import Error, IntegrityError, set_ec
35+
from .helpers import Error, IntegrityError, set_ec, sig_int
3636
from .platform import uid2user, user2uid, gid2group, group2gid, get_birthtime_ns
3737
from .helpers import parse_timestamp, archive_ts_now, CompressionSpec
3838
from .helpers import OutputTimestamp, format_timedelta, format_file_size, file_status, FileSize
@@ -1935,12 +1935,28 @@ def check(
19351935
rebuild_manifest = True
19361936
if rebuild_manifest:
19371937
self.manifest = self.rebuild_manifest()
1938-
if find_lost_archives:
1938+
# On Ctrl-C, skip any scan not yet started; a scan already running stops at its own boundary.
1939+
if find_lost_archives and not sig_int:
19391940
self.rebuild_archives_directory()
1940-
self.rebuild_archives(
1941-
match=match, first=first, last=last, sort_by=sort_by, older=older, oldest=oldest, newer=newer, newest=newest
1942-
)
1941+
if not sig_int:
1942+
self.rebuild_archives(
1943+
match=match,
1944+
first=first,
1945+
last=last,
1946+
sort_by=sort_by,
1947+
older=older,
1948+
oldest=oldest,
1949+
newer=newer,
1950+
newest=newest,
1951+
)
1952+
# finish() writes the manifest and a consistent chunk index; run it on Ctrl-C too (#9850).
19431953
self.finish()
1954+
if sig_int:
1955+
if self.error_found:
1956+
logger.error("Archive consistency check interrupted, problems found so far.")
1957+
else:
1958+
logger.info("Archive consistency check interrupted, no problems found so far.")
1959+
raise Error("Got Ctrl-C / SIGINT.")
19441960
if self.error_found:
19451961
logger.error("Archive consistency check complete, problems found.")
19461962
else:
@@ -1988,12 +2004,16 @@ def verify_data(self):
19882004
logger.info("Starting cryptographic data integrity verification...")
19892005
chunks_count = len(self.chunks)
19902006
errors = 0
2007+
verified = 0 # chunks actually verified
19912008
defect_chunks = []
19922009
pi = ProgressIndicatorPercent(
19932010
total=chunks_count, msg="Verifying data %6.2f%%", step=0.01, msgid="check.verify_data"
19942011
)
19952012
for chunk_id, _ in self.chunks.iteritems():
2013+
if sig_int:
2014+
break
19962015
pi.show()
2016+
verified += 1
19972017
try:
19982018
encrypted_data = self.repository.get(chunk_id)
19992019
except (Repository.ObjectNotFound, IntegrityErrorBase) as err:
@@ -2049,11 +2069,20 @@ def verify_data(self):
20492069
for defect_chunk in defect_chunks:
20502070
logger.debug("chunk %s is defect.", bin_to_hex(defect_chunk))
20512071
log = logger.error if errors else logger.info
2052-
log(
2053-
"Finished cryptographic data integrity verification, verified %d chunks with %d integrity errors.",
2054-
chunks_count,
2055-
errors,
2056-
)
2072+
if sig_int:
2073+
log(
2074+
"Interrupted cryptographic data integrity verification, "
2075+
"verified %d of %d chunks with %d integrity errors.",
2076+
verified,
2077+
chunks_count,
2078+
errors,
2079+
)
2080+
else:
2081+
log(
2082+
"Finished cryptographic data integrity verification, verified %d chunks with %d integrity errors.",
2083+
verified,
2084+
errors,
2085+
)
20572086

20582087
def rebuild_manifest(self):
20592088
"""Rebuild the manifest object."""
@@ -2089,6 +2118,8 @@ def valid_archive(obj):
20892118
msgid="check.rebuild_archives_directory",
20902119
)
20912120
for chunk_id, _ in self.chunks.iteritems():
2121+
if sig_int:
2122+
break
20922123
pi.show()
20932124
cdata = self.repository.get(chunk_id, read_data=False) # only get metadata
20942125
try:
@@ -2134,7 +2165,10 @@ def valid_archive(obj):
21342165
logger.warning(f"Would create archives directory entry for {name} {archive_id_hex}.")
21352166

21362167
pi.finish()
2137-
logger.info("Rebuilding missing archives directory entries completed.")
2168+
if sig_int:
2169+
logger.info("Rebuilding missing archives directory entries interrupted.")
2170+
else:
2171+
logger.info("Rebuilding missing archives directory entries completed.")
21382172

21392173
def rebuild_archives(
21402174
self, first=0, last=0, sort_by="", match=None, older=None, newer=None, oldest=None, newest=None
@@ -2332,6 +2366,9 @@ def valid_item(obj):
23322366
# badly damaged repo does not throw away everything it already found.
23332367
try:
23342368
for i, info in enumerate(archive_infos):
2369+
if sig_int:
2370+
# Break only between archives, as --repair rewrites each archive as a whole.
2371+
break
23352372
pi.show(i)
23362373
archive_id, archive_id_hex = info.id, bin_to_hex(info.id)
23372374
try:

‎src/borg/archiver/check_cmd.py‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
from ._common import with_repository, Highlander
44
from ..archive import ArchiveChecker
55
from ..constants import * # NOQA
6-
from ..helpers import set_ec, EXIT_WARNING, CancelledByUser, CommandError, IntegrityError
7-
from ..helpers import relative_time_marker_validator, yes, ArchiveFormatter
6+
from ..helpers import set_ec, EXIT_WARNING, CancelledByUser, CommandError, Error, IntegrityError
7+
from ..helpers import relative_time_marker_validator, yes, ArchiveFormatter, sig_int
88
from ..helpers.argparsing import ArgumentParser
99
from ..helpers.time import archive_ts_now, calculate_relative_offset
1010

@@ -80,6 +80,8 @@ def do_check(self, args, repository):
8080
if not args.archives_only:
8181
if not repository.check(repair=args.repair, max_duration=args.max_duration, max_age=max_age):
8282
set_ec(EXIT_WARNING)
83+
if sig_int: # repository check interrupted; skip the archive check
84+
raise Error("Got Ctrl-C / SIGINT.")
8385
if not args.repo_only and not archive_checker.check(
8486
repository,
8587
verify_data=args.verify_data,
@@ -186,6 +188,20 @@ def build_parser_check(self, subparsers, common_parser, mid_common_parser):
186188
formatted by giving a custom format using ``--format`` (see the ``borg repo-list``
187189
description for more details about the format string).
188190
191+
If the ``borg check`` process receives a SIGINT signal (Ctrl-C), it stops at the
192+
next safe boundary, leaving the repository and its chunk index in a consistent state.
193+
The repository check stops after the current pack; ``--verify-data`` and
194+
``--find-lost-archives`` stop after the current chunk; a ``--repair`` archive check
195+
stops between whole archives. Results recorded before the interrupt are kept, so a later
196+
check does not re-verify those packs until they are due again. With ``--repair``, an
197+
interrupted archive check may leave some archives already repaired and others not yet
198+
processed, so run ``borg check --repair`` again to finish.
199+
200+
During a ``--repair`` run, the archive check first rebuilds the chunk index from the
201+
packs, and, if the key must be recovered, scans chunks for it. These phases do not yet
202+
respond to SIGINT, so on a large repository a Ctrl-C during them may appear to have no
203+
effect until they finish.
204+
189205
About repair mode
190206
+++++++++++++++++
191207

‎src/borg/repository.py‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1071,6 +1071,9 @@ def recorded_ts(info):
10711071
pack_infos.sort(key=recorded_ts)
10721072
pack_pi = ProgressIndicatorPercent(total=len(pack_infos), msg="Checking packs %3.0f%%", msgid="check.packs")
10731073
for info in pack_infos:
1074+
if sig_int: # on Ctrl-C, stop; tracker.prune() below persists the records past the loop
1075+
logger.info(f"Interrupted repository check, {pack_files} packs checked so far.")
1076+
break
10741077
self._lock_refresh()
10751078
pack_pi.show(increase=1) # advance for skipped packs too, so the bar tracks packs/, not work done
10761079
pack_id = hex_to_bin(info.name)
@@ -1124,12 +1127,15 @@ def recorded_ts(info):
11241127
logger.error(f"Corrupt pack: {bin_to_hex(pack_id)}")
11251128
# fail if this run found errors, or any pack is recorded corrupt.
11261129
problems = objs_errors != 0 or bool(corrupt_ids)
1130+
# On Ctrl-C the check stopped early, so the summary only covers the packs seen so far.
1131+
done, so_far = ("Interrupted", " so far") if sig_int else ("Finished", "")
11271132
if not problems:
1128-
logger.info(f"Finished {mode} repository check, no problems found.")
1133+
logger.info(f"{done} {mode} repository check, no problems found{so_far}.")
11291134
elif repair:
1130-
logger.error(f"Finished {mode} repository check, errors found (repository repair not implemented).")
1135+
logger.error(f"{done} {mode} repository check, errors found{so_far} (repository repair not implemented).")
11311136
else:
1132-
logger.error(f"Finished {mode} repository check, errors found.")
1137+
logger.error(f"{done} {mode} repository check, errors found{so_far}.")
1138+
# True means the checked objects were clean; --repair returns True so the caller proceeds to fix them.
11331139
return not problems or repair
11341140

11351141
def list(self, limit=None, marker=None):

‎src/borg/testsuite/archiver/check_cmd_test.py‎

Lines changed: 136 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,11 @@
66

77
import pytest
88

9-
from ...archive import ChunkBuffer, ArchiveChecker
9+
from ...archive import ArchiveChecker, ChunkBuffer
1010
from ...constants import * # NOQA
11-
from ...helpers import bin_to_hex, msgpack, CommandError, IntegrityError
12-
from ...manifest import Manifest
13-
from ...repository import Repository
11+
from ...helpers import bin_to_hex, msgpack, CommandError, Error, IntegrityError, sig_int
12+
from ...manifest import Archives, Manifest
13+
from ...repository import PackTracker, Repository
1414
from ..repository_test import fchunk, corrupt_chunk_on_disk
1515
from . import (
1616
cmd,
@@ -75,6 +75,138 @@ def test_check_usage(archivers, request):
7575
assert "archive2" in output
7676

7777

78+
def test_check_soft_interrupt(archivers, request, monkeypatch):
79+
"""A mid-run Ctrl-C stops both check phases at a safe boundary (#7893): the repository check persists
80+
its checked packs for a later partial check to resume, and the archive check runs finish() and then
81+
raises. The check is read-only, so a normal check still passes afterwards."""
82+
archiver = request.getfixturevalue(archivers)
83+
check_cmd_setup(archiver) # produces many packs
84+
85+
# repository check: interrupt after the first pack.
86+
with Repository(archiver.repository_path, exclusive=True) as repository:
87+
orig_hash = repository.store.hash
88+
pack_checks = []
89+
90+
def hash_then_interrupt(key):
91+
result = orig_hash(key)
92+
if key.startswith("packs/"): # count pack checks, not the index files hashed first
93+
pack_checks.append(key)
94+
if len(pack_checks) == 1: # one Ctrl-C after the first pack is checked
95+
sig_int._sig_int_triggered = True
96+
return result
97+
98+
monkeypatch.setattr(repository.store, "hash", hash_then_interrupt)
99+
try:
100+
repository.check()
101+
finally:
102+
sig_int._sig_int_triggered = False
103+
assert len(PackTracker.load(repository.store)) == 1 # the pack checked before the break persisted
104+
105+
# a partial check resumes from the saved record (the one pack checked before the interrupt).
106+
output = cmd(archiver, "check", "-v", "--repository-only", "--max-duration=600", exit_code=0)
107+
assert "1 pack check results on record" in output
108+
109+
# archive check: interrupt verify_data after 3 chunks.
110+
with Repository(archiver.repository_path, exclusive=True) as repository:
111+
orig_get = repository.get
112+
get_calls = 0
113+
114+
def get_then_interrupt(*args, **kwargs):
115+
nonlocal get_calls
116+
get_calls += 1
117+
if get_calls == 3: # trip mid-loop, after 3 chunks
118+
sig_int._sig_int_triggered = True
119+
return orig_get(*args, **kwargs)
120+
121+
monkeypatch.setattr(repository, "get", get_then_interrupt)
122+
try:
123+
with pytest.raises(Error, match="Got Ctrl-C"):
124+
ArchiveChecker().check(repository, verify_data=True, sort_by="ts", format="{archive} {time} {id}")
125+
finally:
126+
sig_int._sig_int_triggered = False
127+
# verify_data breaks at the chunk it interrupted on, and the skipped scans issue no more get()s.
128+
assert get_calls == 3
129+
130+
# nothing changed, so a normal check passes.
131+
cmd(archiver, "check", exit_code=0)
132+
133+
134+
def test_check_repair_soft_interrupt(archivers, request, monkeypatch):
135+
"""A Ctrl-C after the first archive of a --repair archive check stops at the archive boundary, runs
136+
finish() (dropping the chunk index, writing the manifest), then raises. No archive is lost, and a
137+
second --repair finishes the job so a following check reports the repository consistent."""
138+
archiver = request.getfixturevalue(archivers)
139+
check_cmd_setup(archiver) # two archives
140+
141+
orig_create = Archives.create
142+
143+
def create_then_interrupt(self, *args, **kwargs):
144+
orig_create(self, *args, **kwargs)
145+
sig_int._sig_int_triggered = True # one Ctrl-C after the first archive was rebuilt
146+
147+
monkeypatch.setattr(Archives, "create", create_then_interrupt)
148+
try:
149+
with Repository(archiver.repository_path, exclusive=True) as repository:
150+
with pytest.raises(Error, match="Got Ctrl-C"):
151+
ArchiveChecker().check(repository, repair=True, sort_by="ts", format="{archive} {time} {id}")
152+
finally:
153+
sig_int._sig_int_triggered = False # reset the global flag for the following tests
154+
# restore the real method; monkeypatch.undo() would also drop the autouse env (BORG_TESTONLY_WEAKEN_KDF).
155+
monkeypatch.setattr(Archives, "create", orig_create)
156+
157+
# both archives survive the interrupt between archives.
158+
output = cmd(archiver, "repo-list", exit_code=0)
159+
assert "archive1" in output
160+
assert "archive2" in output
161+
162+
# a second --repair finishes the job; a plain check then finds no problems.
163+
cmd(archiver, "check", "--repair", exit_code=0)
164+
cmd(archiver, "check", exit_code=0)
165+
166+
167+
def test_check_interrupt_skips_archive_check(archivers, request, monkeypatch):
168+
"""A Ctrl-C during the repository check makes a full `borg check` skip the archive check. do_check
169+
raises at the sig_int guard, which sits before the archive_checker.check() call, so the raise itself
170+
is the skip. Exercises the do_check path (the other soft-interrupt tests call check() directly)."""
171+
archiver = request.getfixturevalue(archivers)
172+
if archiver.EXE: # a class-level monkeypatch cannot reach the borg.exe subprocess
173+
pytest.skip("in-process store patch does not apply to the binary")
174+
check_cmd_setup(archiver) # produces many packs
175+
176+
from borgstore.store import Store
177+
178+
orig_hash = Store.hash
179+
pack_checks = []
180+
181+
def hash_then_interrupt(self, key):
182+
result = orig_hash(self, key)
183+
if key.startswith("packs/"): # count pack checks, not the index files hashed first
184+
pack_checks.append(key)
185+
if len(pack_checks) == 1: # one Ctrl-C after the first pack is checked
186+
sig_int._sig_int_triggered = True
187+
return result
188+
189+
# spy on the archive check: "Got Ctrl-C" is also raised inside ArchiveChecker.check(), so matching the
190+
# message alone would not prove the skip. Recording that check() never runs is the load-bearing assertion.
191+
orig_check = ArchiveChecker.check
192+
archive_check_ran = False
193+
194+
def spy_check(self, *args, **kwargs):
195+
nonlocal archive_check_ran
196+
archive_check_ran = True
197+
return orig_check(self, *args, **kwargs)
198+
199+
monkeypatch.setattr(Store, "hash", hash_then_interrupt)
200+
monkeypatch.setattr(ArchiveChecker, "check", spy_check)
201+
try:
202+
# exec_cmd calls Archiver.run() directly; only main() maps Error to an exit code, so it propagates.
203+
with pytest.raises(Error, match="Got Ctrl-C"):
204+
cmd(archiver, "check", "-v")
205+
finally:
206+
sig_int._sig_int_triggered = False
207+
assert archive_check_ran is False # do_check raised at the sig_int guard, before archive_checker.check()
208+
209+
78210
def test_check_max_age(archivers, request):
79211
archiver = request.getfixturevalue(archivers)
80212
check_cmd_setup(archiver)

0 commit comments

Comments
 (0)