diff --git a/CHANGELOG.md b/CHANGELOG.md index 09a7bcf6..7dd811cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,13 @@ date) and use that section as the basis for the GitHub release notes. by default, matching the other learners. (#585) ### Fixed +- `NCESTrainer.train()` created a fresh `DataLoader(num_workers=...)` worker + pool per architecture in a loop; the existing `os.cpu_count()`-based cap on + `num_workers` only engaged when `cpu_count <= num_workers`, so it silently + did nothing on machines with more cores than the default of 8, and repeated + worker-pool creation at that count could deadlock under `coverage run` + (`NCES`/`NCES2`/`ROCES` training, e.g. `test_nces_trainer.py`). Now capped + unconditionally to `min(requested, cpu_count-1, 4)`. (#610) - `ConceptAbstractSyntaxTreeBuilder._fix_mid_tokens_errors`/`_postprocess_tail_fix`/ `_enforce` repaired malformed token sequences via `random.choice(...)`, making parser output non-deterministic across runs on identical input; replaced with diff --git a/ontolearn/learners/nces.py b/ontolearn/learners/nces.py index 6600a5de..68e96c97 100644 --- a/ontolearn/learners/nces.py +++ b/ontolearn/learners/nces.py @@ -431,8 +431,12 @@ def train(self, data: Iterable[List[Tuple]]=None, epochs=50, batch_size=64, max_ refinement_expressivity=0.2, refs_sample_size=50, learning_rate=1e-4, tmax=20, eta_min=1e-5, clip_value=5.0, num_workers=8, save_model=True, storage_path=None, optimizer='Adam', record_runtime=True, example_sizes=None, shuffle_examples=False): - if os.cpu_count() <= num_workers: - num_workers = max(0,os.cpu_count()-1) + # Cap unconditionally: on machines with many cores this previously stayed + # uncapped at the requested value, and repeatedly spawning that many + # DataLoader worker processes (one fresh pool per architecture in + # NCESTrainer.train()) can deadlock under `coverage run` regardless of + # how many cores are available (see issue #610). + num_workers = max(0, min(num_workers, (os.cpu_count() or 1) - 1, 4)) if storage_path is None: currentDateAndTime = datetime.now() storage_path = f'NCES-Experiment-{currentDateAndTime.strftime("%H-%M-%S")}' diff --git a/ontolearn/nces_trainer.py b/ontolearn/nces_trainer.py index 1c7528ec..12f6ab20 100644 --- a/ontolearn/nces_trainer.py +++ b/ontolearn/nces_trainer.py @@ -60,7 +60,11 @@ def __init__(self, synthesizer, epochs=300, batch_size=128, learning_rate=1e-4, self.tmax = tmax self.eta_min = eta_min self.clip_value = clip_value - self.num_workers = num_workers + # Cap unconditionally, not just on low-core machines: train() spawns a + # fresh DataLoader worker pool per architecture in a loop, and doing so + # repeatedly with a large num_workers can deadlock under `coverage run` + # regardless of how many cores are available (see issue #610). + self.num_workers = max(0, min(num_workers, (os.cpu_count() or 1) - 1, 4)) self.storage_path = storage_path @staticmethod