diff --git a/Jenkinsfile b/Jenkinsfile index cfa4963f841..bda1ebb0c6b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -993,12 +993,17 @@ pipeline { agent { label params.CI_NLT_1_LABEL } + environment { + // Temporary, for this branch only: shorten stall detection while + // hunting an intermittent hang. The deliberate wedge that proved + // the watchdog is gone, so a failure here now means a real one. + NLT_FI_STALL_SECS = '120' + } steps { // NLT memchecks the valgrind-tagged build, not the shared -race one. unstash 'opt-daos-valgrind' job_step_update( - unitTest(timeout_time: 60 * cachedCommitPragma(pragma: 'NLT-repeat', - def_val: '1').toInteger(), + unitTest(timeout_time: 360, inst_repos: daosRepos(), test_script: 'ci/unit/test_nlt.sh' + ' --system-ram-reserved 4' + diff --git a/ci/unit/test_nlt.sh b/ci/unit/test_nlt.sh index cd42c7e002f..37b8889e21b 100755 --- a/ci/unit/test_nlt.sh +++ b/ci/unit/test_nlt.sh @@ -22,4 +22,6 @@ rsync -rlpt -z -e "ssh $SSH_KEY_ARGS" .build_vars* opt-daos-install.tar utils re ssh -T "$SSH_KEY_ARGS" jenkins@"$NODE" \ "DAOS_HTTPS_PROXY=\"${DAOS_HTTPS_PROXY:-}\" \ DAOS_NO_PROXY=\"${DAOS_NO_PROXY:-}\" \ + NLT_FI_STALL_SECS=\"${NLT_FI_STALL_SECS:-}\" \ + NLT_FI_WEDGE_AT=\"${NLT_FI_WEDGE_AT:-}\" \ bash -s -- $*" < "$mydir/test_nlt_node.sh" diff --git a/ci/unit/test_nlt_node.sh b/ci/unit/test_nlt_node.sh index 77099b5bd2c..79d298d59d2 100755 --- a/ci/unit/test_nlt_node.sh +++ b/ci/unit/test_nlt_node.sh @@ -51,7 +51,11 @@ mkdir -p nlt_logs sudo mount -t tmpfs tmpfs nlt_logs sudo chown jenkins:jenkins nlt_logs +# Unbuffered so the console shows exactly where a hang occurs. TMPDIR="$(pwd)/nlt_logs" \ + PYTHONUNBUFFERED=1 \ + NLT_FI_STALL_SECS="${NLT_FI_STALL_SECS:-}" \ + NLT_FI_WEDGE_AT="${NLT_FI_WEDGE_AT:-}" \ HTTPS_PROXY="${DAOS_HTTPS_PROXY:-}" \ NO_PROXY="${DAOS_NO_PROXY:-}" \ exec ./utils/node_local_test.py "$@" diff --git a/utils/node_local_test.py b/utils/node_local_test.py index 7d2e3191ab2..e82a6958b21 100755 --- a/utils/node_local_test.py +++ b/utils/node_local_test.py @@ -20,6 +20,7 @@ import argparse import copy import errno +import faulthandler import functools import importlib import json @@ -125,10 +126,47 @@ def __init__(self, json_file, args): os.makedirs(self.tmp_dir) self._compress_procs = [] + self._cleaned = False def __del__(self): - self.flush_bz2() - os.rmdir(self.dfuse_parent_dir) + self.cleanup() + + def cleanup(self, timeout=60): + """Flush compression and remove the working directory, giving up if it blocks. + + A directory holding a wedged FUSE mount cannot be removed, and the attempt blocks + in the kernel, so do this somewhere it can be abandoned rather than on the path + the process needs to take to exit. + """ + if self._cleaned: + return + self._cleaned = True + + def _work(): + self.flush_bz2() + try: + os.rmdir(self.dfuse_parent_dir) + except OSError: + pass + + try: + worker = threading.Thread(target=_work, daemon=True) + worker.start() + worker.join(timeout) + except RuntimeError: + # Too late in shutdown to start a thread; skip rather than risk blocking. + return + if worker.is_alive(): + # Nothing under a wedged mount can be removed while it holds requests nobody + # will answer, and the attempt does not time out on its own. Fail those + # requests and give the cleanup one more chance to finish. + print(f'Cleanup blocked after {timeout}s; aborting backed-up FUSE connections', + flush=True) + for line in _abort_fuse_connections(): + print(line, flush=True) + worker.join(30) + if worker.is_alive(): + print('Cleanup still blocked; abandoning it', flush=True) def set_wf(self, wf): """Set the WarningsFactory object""" @@ -1433,7 +1471,9 @@ def start(self, v_hint=None, use_oopt=False): if not self.cores: # Use a lower default thread-count for NLT due to running tests in parallel. - cmd.extend(['--thread-count', '4']) + # Overridable: a sweep running more concurrent clients than dfuse has threads + # can starve it of threads entirely. + cmd.extend(['--thread-count', os.environ.get('NLT_DFUSE_THREAD_COUNT', '4')]) if not self.caching: cmd.append('--disable-caching') @@ -1519,7 +1559,9 @@ def stop(self, ignore_einval=False): print('Stopping fuse') if self.container: - self.run_query(use_json=True) + # Bounded: this queries the very mount that may be the thing that is wedged, + # and an unbounded query here strands teardown until the CI stage timeout. + self.run_query(use_json=True, timeout=120) ret = umount(self.dir) if ret: umount(self.dir, background=True) @@ -1606,10 +1648,10 @@ def il_cmd(self, cmd, check_read=True, check_write=True, check_fstat=True): assert ret.returncode == 0, ret return ret - def run_query(self, use_json=False, quiet=False): + def run_query(self, use_json=False, quiet=False, timeout=None): """Run filesystem query""" rc = run_daos_cmd(self.conf, ['filesystem', 'query', self.dir], - use_json=use_json, log_check=quiet, valgrind=quiet) + use_json=use_json, log_check=quiet, valgrind=quiet, timeout=timeout) print(rc) return rc @@ -1730,12 +1772,14 @@ def run_daos_cmd(conf, log_check=True, ignore_busy=False, use_json=False, - cwd=None): + cwd=None, + timeout=None): """Run a DAOS command Run a command, returning what subprocess.run() would. - Enable logging, and valgrind for the command. + Enable logging, and valgrind for the command. A timeout, where given, kills the + command rather than blocking forever on an unresponsive filesystem. """ dcr = DaosCmdReturn() valgrind_hdl = ValgrindHelper(conf) @@ -1777,8 +1821,13 @@ def run_daos_cmd(conf, cmd_env['DAOS_AGENT_DRPC_DIR'] = conf.agent_dir - rc = subprocess.run(exec_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=cmd_env, check=False, cwd=cwd) + try: + rc = subprocess.run(exec_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=cmd_env, check=False, cwd=cwd, timeout=timeout) + except subprocess.TimeoutExpired as error: + print(f'Timeout after {timeout}s running {" ".join(cmd)}') + rc = subprocess.CompletedProcess(exec_cmd, (-signal.SIGKILL), error.stdout or b'', + error.stderr or b'') if rc.stderr != b'': print('Stderr from command') @@ -1807,7 +1856,16 @@ def run_daos_cmd(conf, conf.valgrind_errors = True rc.returncode = 0 if use_json: - rc.json = json.loads(rc.stdout.decode('utf-8')) + try: + rc.json = json.loads(rc.stdout.decode('utf-8')) + except json.JSONDecodeError: + # A command killed on timeout has no output to parse; report that rather than + # raising a decode error that hides why the command produced nothing. + if rc.returncode == -signal.SIGKILL: + print(f'No JSON output from timed-out command: {" ".join(cmd)}') + rc.json = None + else: + raise dcr.rc = rc return dcr @@ -5616,6 +5674,276 @@ def test_pydaos_kv_obj_class(server, conf): # +def _run_diag(cmd, timeout=60): + """Run a diagnostic command returning its output, never raising or blocking. + + The command gets its own process group so a wedged tool can be killed outright, and + one that survives SIGKILL (uninterruptible in the kernel) is abandoned rather than + waited on -- a diagnostic must never become the reason a run hangs. + """ + try: + # pylint: disable-next=consider-using-with + proc = subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, start_new_session=True) + except OSError as err: + return f'<{cmd[0]} failed: {err}>' + try: + return proc.communicate(timeout=timeout)[0].decode('utf-8', errors='replace').rstrip() + except subprocess.TimeoutExpired: + pass + try: + os.killpg(proc.pid, signal.SIGKILL) + except OSError: + pass + try: + text = proc.communicate(timeout=15)[0].decode('utf-8', errors='replace').rstrip() + except subprocess.TimeoutExpired: + return f'<{cmd[0]} exceeded {timeout}s and could not be reaped>' + return f'{text}\n<{cmd[0]} exceeded {timeout}s>' + + +def _proc_field(pid, field): + """Return one /proc//status field, or '' if unreadable""" + try: + with open(f'/proc/{pid}/status', encoding='utf-8') as pfile: + for line in pfile: + if line.startswith(f'{field}:'): + return line.split(':', 1)[1].strip() + except OSError: + pass + return '' + + +def _proc_state(pid): + """Return blocked system-call state for a pid from /proc""" + out = [] + for name in ('wchan', 'syscall'): + try: + with open(f'/proc/{pid}/{name}', encoding='utf-8', errors='replace') as pfile: + out.append(f'{name}={pfile.read().strip()}') + except OSError as err: + out.append(f'{name}=') + for field in ('State', 'Threads'): + value = _proc_field(pid, field) + if value: + out.append(f'{field}: {value}') + return ' '.join(out) + + +def _proc_threads(pid): + """Report every thread of a process from /proc. + + Debuggers attach in a way that cannot stop a thread that is in uninterruptible + sleep -- exactly the state a wedged process is in -- so they time out and return + nothing at the one moment the information is wanted. /proc always answers, and its + kernel stacks name the function a thread is blocked in. + """ + lines = [] + try: + tids = sorted(os.listdir(f'/proc/{pid}/task'), key=int) + except OSError as err: + return f'' + for tid in tids: + base = f'/proc/{pid}/task/{tid}' + fields = [] + for name in ('comm', 'wchan'): + try: + with open(f'{base}/{name}', encoding='utf-8', errors='replace') as tfile: + fields.append(f'{name}={tfile.read().strip()}') + except OSError: + fields.append(f'{name}=?') + fields.append(f'state={_proc_field(pid, "State") if tid == str(pid) else ""}'.rstrip('=')) + try: + with open(f'{base}/stat', encoding='utf-8') as tfile: + fields.append(f'state={tfile.read().split(") ", 1)[1].split()[0]}') + except (OSError, IndexError): + pass + lines.append(f' TID {tid}: ' + ' '.join(f for f in fields if f)) + kstack = _run_diag(['sudo', 'cat', f'{base}/stack'], timeout=15) + if kstack and not kstack.startswith('<') and 'denied' not in kstack: + for kline in kstack.splitlines()[:12]: + lines.append(f' {kline.strip()}') + return '\n'.join(lines) + + +def _gdb_backtrace(pid, timeout): + """Best-effort gdb stack trace that never leaves its target stopped. + + Uninterruptible and already-traced targets are skipped: gdb cannot attach to them + and trying costs the whole timeout for nothing. + """ + state = _proc_field(pid, 'State') + if state.startswith('D'): + return f'' + if state.startswith('t'): + return f'' + # Keep this brief: /proc has already reported every thread, and neither tool can + # read a process whose threads are wedged, so a generous timeout here buys nothing + # and costs the dump its budget. + out = _run_diag(['eu-stack', '-p', str(pid)], timeout=min(timeout, 15)) + # A gdb killed mid-attach leaves its target stopped; put it back or the target is + # frozen for good and a later SIGKILL lands only once something detaches. + if _proc_field(pid, 'State').startswith('t'): + try: + os.kill(pid, signal.SIGCONT) + out += f'\n' + except OSError as err: + out += f'\n' + return out + + +def dump_fi_stall(active, log_dir=None, deadline_secs=600): + """Dump diagnostics for wedged fault-injection children. + + Output goes to stdout so it survives a discarded build tree, and to a file so it + survives a hostage stdout. The whole dump is bounded: partial evidence delivered + on time beats complete evidence that never arrives. + """ + give_up_at = time.monotonic() + deadline_secs + dump_file = None + if log_dir: + try: + # pylint: disable-next=consider-using-with + dump_file = open(join(log_dir, 'fi_stall_dump.txt'), 'w', encoding='utf-8') + except OSError: + dump_file = None + + def emit(text): + # File first: it is local and cannot be held hostage by whatever is consuming + # stdout, and a dump that only reaches stdout is lost when that consumer stalls. + if dump_file: + try: + dump_file.write(f'{text}\n') + dump_file.flush() + except OSError: + pass + print(text, flush=True) + + def remaining(): + return give_up_at - time.monotonic() + + try: + emit(f'\n===== NLT FI STALL DETECTED {time.strftime("%Y-%m-%d %H:%M:%S")} =====') + emit('--- node_local_test.py thread stacks:') + # The traceback dumper needs a real file descriptor, so it cannot be given the + # stdout wrapper this script installs to capture per-thread output; fall back to + # the real stdout underneath it. Never let this step take the dump down with it. + for stream in (dump_file, sys.__stdout__): + if stream is None or not hasattr(stream, 'fileno'): + continue + try: + faulthandler.dump_traceback(file=stream, all_threads=True) + stream.flush() + except Exception as err: # pylint: disable=broad-except + emit(f'') + + # /proc first: it cannot block, so this evidence is never lost to the deadline. + for child in active: + pid = child.pid() + emit(f'--- stalled child: loc={child.loc} pid={pid} elapsed={child.elapsed():.0f}s') + emit(_proc_state(pid)) + emit(_proc_threads(pid)) + daemons = _run_diag(['pgrep', '-a', 'daos_engine|daos_agent|dfuse'], timeout=30) + emit(f'--- daemons:\n{daemons}') + daemon_pids = [int(line.split()[0]) for line in daemons.splitlines() + if line and line.split()[0].isdigit()] + for pid in daemon_pids: + emit(f'--- daemon state pid={pid}: {_proc_state(pid)}') + emit(_proc_threads(pid)) + emit('--- process tree:') + emit(_run_diag(['ps', 'auxwwf'], timeout=60)) + + # Daemons before children: a wedged dfuse or engine explains stalled clients, + # and the clients are the ones gdb is least likely to attach to. + for pid in daemon_pids + [c.pid() for c in active]: + if remaining() <= 0: + emit(f'--- gdb pass abandoned, {deadline_secs}s dump deadline reached') + break + emit(f'--- gdb backtrace pid={pid}:') + emit(_gdb_backtrace(pid, timeout=int(min(120, max(20, remaining()))))) + emit('===== NLT FI STALL DUMP COMPLETE =====') + # A broken dump must announce itself and must never become the reason the run dies: + # an earlier version aborted silently here and left no trace of why. + except Exception as err: # pylint: disable=broad-except + emit(f'===== NLT FI STALL DUMP ABORTED: {err!r} =====') + traceback.print_exc(file=sys.stdout) + sys.stdout.flush() + finally: + if dump_file: + dump_file.close() + + +def _abort_fuse_connections(): + """Abort every backed-up FUSE connection, returning what was done. + + A process blocked in an unanswered FUSE request is in uninterruptible sleep and cannot + be killed -- SIGKILL is recorded but not delivered until the request completes, which + for a wedged mount is never. Aborting the connection fails the outstanding requests + with EIO, which is the only thing that lets those processes die. + """ + done = [] + try: + conns = sorted(os.listdir('/sys/fs/fuse/connections')) + except OSError as err: + return [f''] + for conn in conns: + waiting = '' + try: + with open(f'/sys/fs/fuse/connections/{conn}/waiting', encoding='utf-8') as wfile: + waiting = wfile.read().strip() + except OSError: + continue + if waiting in ('', '0'): + continue + path = f'/sys/fs/fuse/connections/{conn}/abort' + res = _run_diag(['sudo', 'sh', '-c', f'echo 1 > {path}'], timeout=30) + done.append(f'aborted fuse connection {conn} (waiting={waiting}): {res or "ok"}') + return done or [''] + + +def handle_fi_stall(active, log_dir=None): + """Dump diagnostics then clear the wedged children so the run fails fast. + + Children left in an unanswered FUSE request cannot be killed, so a plain SIGKILL is + not enough to recover: the sweep would find them still running, stall again, and + repeat until the CI stage timed out -- which is the outcome this whole mechanism + exists to avoid. + + Returns whether a mount had to be aborted, which leaves it unusable for anything + that follows. + """ + dump_fi_stall(active, log_dir=log_dir) + aborted_mount = False + for child in active: + child.hang_kill() + + def _survivors(seconds): + deadline = time.monotonic() + seconds + alive = list(active) + while alive and time.monotonic() < deadline: + alive = [c for c in alive if not c.is_dead()] + if alive: + time.sleep(1) + return alive + + survivors = _survivors(30) + if survivors: + stuck_on_fuse = [c for c in survivors + if 'request_wait_answer' in _proc_state(c.pid())] + if stuck_on_fuse: + print(f'{len(stuck_on_fuse)} child(ren) unkillable in a FUSE wait; ' + f'aborting the connection to release them', flush=True) + for line in _abort_fuse_connections(): + print(line, flush=True) + aborted_mount = True + survivors = _survivors(60) + + for child in survivors: + print(f'WARNING: pid {child.pid()} (loc {child.loc}) survived SIGKILL: ' + f'{_proc_state(child.pid())}', flush=True) + return aborted_mount + + class AllocFailTestRun(): """Class to run a fault injection command with a single fault""" @@ -5631,6 +5959,9 @@ def __init__(self, aft, cmd, env, loc, cwd): self.dir_handle = None self.stdout = None self.returncode = None + # Set when the stall watchdog killed this run. + self.was_killed = False + self._start_time = None # Set this to disable memory leak checking if the command outputs a DER_BUSY message. This # is to allow tests to leak memory if there are errors during shutdown. @@ -5715,6 +6046,24 @@ def start(self): stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self._start_time = time.monotonic() + + def pid(self): + """Return the pid of the command""" + return self._sp.pid + + def elapsed(self): + """Return seconds since the command started""" + return time.monotonic() - self._start_time + + def hang_kill(self): + """Kill a wedged command; result checks are skipped on reap""" + self.was_killed = True + self._sp.kill() + + def is_dead(self): + """Return whether the command has exited, without blocking""" + return self._sp.poll() is not None def has_finished(self): """Check if the command has completed""" @@ -5724,15 +6073,47 @@ def has_finished(self): rc = self._sp.poll() if rc is None: return False - self._post(rc) + self._reap(rc) return True - def wait(self): + def wait(self, timeout=None): """Wait for the command to complete""" if self.returncode is not None: return - self._post(self._sp.wait()) + if timeout is not None: + try: + self._reap(self._sp.wait(timeout=timeout)) + return + except subprocess.TimeoutExpired: + # Dump and kill rather than hanging until the stage timeout. + handle_fi_stall([self], log_dir=self._aft.log_dir) + + self._reap(self._sp.wait()) + + def _reap(self, rc): + """Process a completed command, bypassing result checks for watchdog kills""" + if self.was_killed: + self._post_killed(rc) + else: + self._post(rc) + + def _post_killed(self, rc): + """Reap after a watchdog kill. + + Grandchildren may hold the pipes open so use a bounded read. A shell-style + positive returncode keeps the generic signal handling in launch() from + scheduling a valgrind re-run, and fault_injected is set so the sweep + continues past the killed location. + """ + print() + self.returncode = 128 - rc if rc < 0 else rc + try: + self.stdout, self._stderr = self._sp.communicate(timeout=30) + except subprocess.TimeoutExpired: + self.stdout = b'' + self._stderr = b'' + self.fault_injected = True def _post(self, rc): """Helper function, called once after command is complete. @@ -5891,6 +6272,10 @@ class AllocFailTest(): # pylint: disable=too-few-public-methods """Class to describe fault injection command""" + # Set once the deliberate stall-watchdog wedge has been used, so it fires a single + # time per run rather than in every sweep. + wedge_done = False + def __init__(self, conf, desc, cmd): self.conf = conf self.cmd = cmd @@ -5929,7 +6314,7 @@ def launch(self): def _prep(self): rc = self._run_cmd(None) - rc.wait() + rc.wait(timeout=600) self.expected_stdout = rc.stdout assert not rc.fault_injected @@ -5965,6 +6350,11 @@ def _prep(self): max_load_avg = 100 + # Iterations normally complete in ~1s; if none complete for this long + # the run is wedged and would spin silently until the stage timeout. + stall_secs = int(os.environ.get('NLT_FI_STALL_SECS', '300')) + last_progress = time.monotonic() + # Now run all iterations in parallel up to max_child. Iterations will be launched # in order but may not finish in order, rather they are processed in the order they # finish. After each repetition completes then check for re-launch new processes @@ -6005,6 +6395,7 @@ def _prep(self): if not ret.has_finished(): continue active.remove(ret) + last_progress = time.monotonic() print() print(ret) if ret.returncode < 0: @@ -6016,6 +6407,15 @@ def _prep(self): finished = True break + if active and time.monotonic() - last_progress > stall_secs: + if handle_fi_stall(active, log_dir=self.log_dir): + # Releasing the wedged children required destroying the mount they + # were using, so the remaining iterations have nothing to run against. + print('Mount was aborted to clear the stall; ending this sweep') + finished = True + fatal_errors = True + last_progress = time.monotonic() + print(f'Completed, fid {fid}') print(f'Max in flight {max_count}/{max_child}') if to_rerun: @@ -6024,7 +6424,7 @@ def _prep(self): for fid in to_rerun: rerun = self._run_cmd(fid, valgrind=True) print(rerun) - rerun.wait() + rerun.wait(timeout=1800) return fatal_errors @@ -6068,6 +6468,18 @@ def _run_cmd(self, loc, valgrind=False): cmd_env['DD_STDERR'] = 'CRIT' cmd[0] = join(self.conf['PREFIX'], 'bin', 'daos') + # Deliberately wedge one iteration, so the stall watchdog can be exercised on demand + # rather than only when a real hang happens to occur. Without this the reporting + # path is only ever tested by the rare failure it exists to report on. Once per + # run, not once per sweep: every sweep counts iterations from the same base, so + # matching on the number alone wedges each of them in turn. + wedge_at = os.environ.get('NLT_FI_WEDGE_AT') + if wedge_at and loc == int(wedge_at) and not AllocFailTest.wedge_done: + AllocFailTest.wedge_done = True + print(f'Wedging iteration {loc} of {self.description} deliberately ' + f'to exercise the stall watchdog') + cmd = ['sleep', '3600'] + aftf = AllocFailTestRun(self, cmd, cmd_env, loc, cwd) if valgrind: aftf.valgrind_hdl = ValgrindHelper(self.conf, logid=f'fi_{self.description}_{loc}') @@ -6805,13 +7217,19 @@ def run(wf, args): print(fs) if fs.returncode == 0: run_fi = True + elif fi_test or fi_test_dfuse: + # Explicitly requested modes must not silently skip their tests. + print("Unable to detect fault injection feature - cannot run requested FI tests") + sys.exit(1) else: print("Unable to detect fault injection feature - skipping FI testing") if run_fi: args.server_debug = 'INFO' args.memcheck = 'no' - args.dfuse_debug = 'WARN' + # dfuse serves every request the sweeps make, so it is the prime suspect when a + # sweep wedges -- but at WARN it logs nothing at all. Allow raising it. + args.dfuse_debug = os.environ.get('NLT_FI_DFUSE_DEBUG', 'WARN') with DaosServer(conf, test_class='no-debug', wf=wf_server, fatal_errors=fatal_errors) as server: if fi_test: @@ -6880,11 +7298,26 @@ def run(wf, args): wf_server.close() close_log_test(conf) + conf.cleanup() print(f'Total time in log analysis: {conf.log_timer.total:.2f} seconds') print(f'Total time in log compression: {conf.compress_timer.total:.2f} seconds') return fatal_errors +def _exit_now(code): + """Terminate without running interpreter shutdown. + + Cleanup handlers here touch the filesystem: waiting on compression, removing the + working directory, stopping a dfuse instance. None of that can complete against a + wedged mount -- the calls block in the kernel -- so the process outlives its own exit + and is killed only when CI gives up on the whole stage. Everything worth keeping has + been written by this point. + """ + sys.stdout.flush() + sys.stderr.flush() + os._exit(code) # pylint: disable=protected-access + + def _positive_int(value): """argparse type that rejects values below 1.""" ivalue = int(value) @@ -6988,7 +7421,8 @@ def main(): if fatal_errors.errors: print("Significant errors encountered") - sys.exit(1) + _exit_now(1) + _exit_now(0) if __name__ == '__main__':