From 4860b9a1e01e87ac9916dd9344693274a52121af Mon Sep 17 00:00:00 2001 From: Caglar Demir Date: Fri, 7 Aug 2026 16:53:13 +0200 Subject: [PATCH] Fix NCESTrainer DataLoader worker-pool deadlock under coverage NCESTrainer.train() creates a fresh DataLoader(num_workers=...) worker pool per architecture in a loop over self.synthesizer.model. The existing os.cpu_count()-based cap on num_workers only engaged when cpu_count <= num_workers, so on any machine with more cores than the default of 8 it silently did nothing, leaving num_workers at 8. Reproduced locally: coverage run -m pytest against test_nces_trainer.py hung indefinitely with num_workers=8 (confirmed via /proc//wchan showing genuine futex/socket waits, not just slow computation); num_workers of 1/2/4 all completed reliably in under a minute. This matches CI's multi-hour hangs on PR #607 (issue #610). Cap num_workers unconditionally to min(requested, cpu_count-1, 4) in both NCES.train() and NCESTrainer.__init__ (defense in depth for direct NCESTrainer usage). Fixes #610. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_018iKbU9gaut8SBBPnXcZhZA --- CHANGELOG.md | 7 +++++++ ontolearn/learners/nces.py | 8 ++++++-- ontolearn/nces_trainer.py | 6 +++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09a7bcf6..7dd811cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,13 @@ date) and use that section as the basis for the GitHub release notes. by default, matching the other learners. (#585) ### Fixed +- `NCESTrainer.train()` created a fresh `DataLoader(num_workers=...)` worker + pool per architecture in a loop; the existing `os.cpu_count()`-based cap on + `num_workers` only engaged when `cpu_count <= num_workers`, so it silently + did nothing on machines with more cores than the default of 8, and repeated + worker-pool creation at that count could deadlock under `coverage run` + (`NCES`/`NCES2`/`ROCES` training, e.g. `test_nces_trainer.py`). Now capped + unconditionally to `min(requested, cpu_count-1, 4)`. (#610) - `ConceptAbstractSyntaxTreeBuilder._fix_mid_tokens_errors`/`_postprocess_tail_fix`/ `_enforce` repaired malformed token sequences via `random.choice(...)`, making parser output non-deterministic across runs on identical input; replaced with diff --git a/ontolearn/learners/nces.py b/ontolearn/learners/nces.py index 6600a5de..68e96c97 100644 --- a/ontolearn/learners/nces.py +++ b/ontolearn/learners/nces.py @@ -431,8 +431,12 @@ def train(self, data: Iterable[List[Tuple]]=None, epochs=50, batch_size=64, max_ refinement_expressivity=0.2, refs_sample_size=50, learning_rate=1e-4, tmax=20, eta_min=1e-5, clip_value=5.0, num_workers=8, save_model=True, storage_path=None, optimizer='Adam', record_runtime=True, example_sizes=None, shuffle_examples=False): - if os.cpu_count() <= num_workers: - num_workers = max(0,os.cpu_count()-1) + # Cap unconditionally: on machines with many cores this previously stayed + # uncapped at the requested value, and repeatedly spawning that many + # DataLoader worker processes (one fresh pool per architecture in + # NCESTrainer.train()) can deadlock under `coverage run` regardless of + # how many cores are available (see issue #610). + num_workers = max(0, min(num_workers, (os.cpu_count() or 1) - 1, 4)) if storage_path is None: currentDateAndTime = datetime.now() storage_path = f'NCES-Experiment-{currentDateAndTime.strftime("%H-%M-%S")}' diff --git a/ontolearn/nces_trainer.py b/ontolearn/nces_trainer.py index 1c7528ec..12f6ab20 100644 --- a/ontolearn/nces_trainer.py +++ b/ontolearn/nces_trainer.py @@ -60,7 +60,11 @@ def __init__(self, synthesizer, epochs=300, batch_size=128, learning_rate=1e-4, self.tmax = tmax self.eta_min = eta_min self.clip_value = clip_value - self.num_workers = num_workers + # Cap unconditionally, not just on low-core machines: train() spawns a + # fresh DataLoader worker pool per architecture in a loop, and doing so + # repeatedly with a large num_workers can deadlock under `coverage run` + # regardless of how many cores are available (see issue #610). + self.num_workers = max(0, min(num_workers, (os.cpu_count() or 1) - 1, 4)) self.storage_path = storage_path @staticmethod