From 9b148c78e3d822fadb618be61673dd1329c64c26 Mon Sep 17 00:00:00 2001 From: Leonhard Reichenbach Date: Thu, 23 Oct 2025 18:54:44 +0200 Subject: [PATCH 1/7] upstream SequenceLoader --- python/k4FWCore/SequenceLoader.py | 158 ++++++++++++++++++++++++++++++ python/k4FWCore/__init__.py | 1 + 2 files changed, 159 insertions(+) create mode 100644 python/k4FWCore/SequenceLoader.py diff --git a/python/k4FWCore/SequenceLoader.py b/python/k4FWCore/SequenceLoader.py new file mode 100644 index 000000000..b72da5cac --- /dev/null +++ b/python/k4FWCore/SequenceLoader.py @@ -0,0 +1,158 @@ +# +# Copyright (c) 2014-2024 Key4hep-Project. +# +# This file is part of Key4hep. +# See https://key4hep.github.io/key4hep-doc/ for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +from io import TextIOWrapper +from typing import Union, Optional, Dict, Any +import importlib.util +from importlib.machinery import SourceFileLoader + + +def import_from( + filename: Union[str, os.PathLike], + module_name: Optional[str] = None, + global_vars: Optional[Dict[str, Any]] = None, +) -> Any: + """Dynamically imports a module from the specified file path. + + This function imports a module from a given filename, with the option to + specify the module's name and inject global variables into the module before + it is returned. If `module_name` is not provided, the filename is used as + the module name after replacing '.' with '_'. Global variables can be passed + as a dictionary to `global_vars`, which will be injected into the module's + namespace. + + Args: + filename (str): The path to the file from which to import the module. + module_name (Optional[str]): The name to assign to the module. Defaults + to None, in which case the filename is used as the module name. + global_vars (Optional[Dict[str, Any]]): A dictionary of global variables + to inject into the module's namespace. Defaults to None. + + Returns: + Any: The imported module with the specified modifications. + + Raises: + FileNotFoundError: If the specified file does not exist. + ImportError: If there is an error during the import process. + + """ + filename = os.path.abspath(filename) + if not os.path.exists(filename): + raise FileNotFoundError(f"No such file: '{filename}'") + + module_name = module_name or os.path.basename(filename).replace(".", "_") + loader = SourceFileLoader(module_name, filename) + spec = importlib.util.spec_from_loader(loader.name, loader) + module = importlib.util.module_from_spec(spec) + if global_vars: + module.__dict__.update(global_vars) + loader.exec_module(module) + return module + + +def load_file(opt_file: Union[TextIOWrapper, str, os.PathLike]) -> None: + """Loads and executes the content of a given file in the current interpreter session. + + This function takes a file object or a path to a file, reads its content, + and then executes it as Python code within the global scope of the current + interpreter session. If `opt_file` is a file handle it will not be closed. + + Args: + opt_file (Union[TextIOWrapper, str, os.PathLike]): A file object or a + path to the file that + contains Python code + to be executed. + + Raises: + FileNotFoundError: If `opt_file` is a path and no file exists at that path. + IOError: If there's an error opening or reading the file. + SyntaxError: If there's a syntax error in the code being executed. + Exception: Any exception raised by the executed code will be propagated. + + """ + if isinstance(opt_file, (str, os.PathLike)): + with open(opt_file, "r") as file: + code = compile(file.read(), file.name, "exec") + else: + code = compile(opt_file.read(), opt_file.name, "exec") + + exec(code, globals()) + + +class SequenceLoader: + """A class for loading algorithm sequences onto a list of algorithms + + It dynamically loads algorithms from Python files based on the given + sequence names. In the import process it will look for a Sequence of + algorithms which might have configuration constants that depend on some + global calibration configuration. These constants are provided during the + import of a sequence, such that the imported python files do not need to + define all of them. + """ + + def __init__(self, alg_list: list, global_vars: Optional[Dict[str, Any]] = None) -> None: + """Initialize the SequenceLoader + + This initializes a SequenceLoader with the list of algorithms to which + dynamically loaded algorithms should be appended to. It optionally takes + some global calibration constants that should be injected during import + of the sequence files + + Args: + alg_list (List): A list to store loaded sequence algorithms. + global_vars (Optional[Dict[str, Any]]): A dictionary of global + variables for the sequences. Defaults to None. The keys in this + dictionary will be the available variables in the imported + module and the values will be the values of these variables. + """ + self.alg_list = alg_list + self.global_vars = global_vars + + def load(self, sequence: str) -> None: + """Loads a sequence algorithm from a specified Python file and appends + it to the algorithm list + + The method constructs the filename from the sequence parameter name and + imports the sequence from the imported module. + + Args: + sequence (str): The name of the sequence to load. The sequence name + should correspond to a Python file and class name following the + pattern `{sequence}.py` and `{sequence}Sequence`, respectively. + + Examples: + >>> alg_list = [] + >>> seq_loader = SequenceLoader(alg_list) + >>> seq_loader.load("Tracking/TrackingDigi") + + This will import the file `Tracking/TrackingDigi.py` and add the + sequence of algorithms that is defined in `TrackingDigiSequence` in + that file to the alg_list + """ + filename = f"{sequence}.py" + seq_name = f"{sequence.split('/')[-1]}Sequence" + + seq_module = import_from( + filename, + global_vars=self.global_vars, + ) + + seq = getattr(seq_module, seq_name) + self.alg_list.extend(seq) diff --git a/python/k4FWCore/__init__.py b/python/k4FWCore/__init__.py index a3db4cf67..2e17582c2 100644 --- a/python/k4FWCore/__init__.py +++ b/python/k4FWCore/__init__.py @@ -18,3 +18,4 @@ # from .ApplicationMgr import ApplicationMgr from .IOSvc import IOSvc +from .SequenceLoader import SequenceLoader From 091e83bfd9ee61a4a53ee5be53f39307f8bc151a Mon Sep 17 00:00:00 2001 From: Leonhard Reichenbach Date: Mon, 27 Oct 2025 08:24:19 +0100 Subject: [PATCH 2/7] remove unused `load_file` from SequenceLoader --- python/k4FWCore/SequenceLoader.py | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/python/k4FWCore/SequenceLoader.py b/python/k4FWCore/SequenceLoader.py index b72da5cac..09e39c474 100644 --- a/python/k4FWCore/SequenceLoader.py +++ b/python/k4FWCore/SequenceLoader.py @@ -67,35 +67,6 @@ def import_from( return module -def load_file(opt_file: Union[TextIOWrapper, str, os.PathLike]) -> None: - """Loads and executes the content of a given file in the current interpreter session. - - This function takes a file object or a path to a file, reads its content, - and then executes it as Python code within the global scope of the current - interpreter session. If `opt_file` is a file handle it will not be closed. - - Args: - opt_file (Union[TextIOWrapper, str, os.PathLike]): A file object or a - path to the file that - contains Python code - to be executed. - - Raises: - FileNotFoundError: If `opt_file` is a path and no file exists at that path. - IOError: If there's an error opening or reading the file. - SyntaxError: If there's a syntax error in the code being executed. - Exception: Any exception raised by the executed code will be propagated. - - """ - if isinstance(opt_file, (str, os.PathLike)): - with open(opt_file, "r") as file: - code = compile(file.read(), file.name, "exec") - else: - code = compile(opt_file.read(), opt_file.name, "exec") - - exec(code, globals()) - - class SequenceLoader: """A class for loading algorithm sequences onto a list of algorithms From 4c05a268b674f1a4df0ae28c32b5e68ba93beeb6 Mon Sep 17 00:00:00 2001 From: Leonhard Reichenbach Date: Mon, 27 Oct 2025 16:47:20 +0100 Subject: [PATCH 3/7] move import_from and cleanup imports --- python/k4FWCore/SequenceLoader.py | 50 ++----------------------------- python/k4FWCore/utils.py | 45 +++++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 49 deletions(-) diff --git a/python/k4FWCore/SequenceLoader.py b/python/k4FWCore/SequenceLoader.py index 09e39c474..80b204b5d 100644 --- a/python/k4FWCore/SequenceLoader.py +++ b/python/k4FWCore/SequenceLoader.py @@ -17,54 +17,8 @@ # limitations under the License. # -import os -from io import TextIOWrapper -from typing import Union, Optional, Dict, Any -import importlib.util -from importlib.machinery import SourceFileLoader - - -def import_from( - filename: Union[str, os.PathLike], - module_name: Optional[str] = None, - global_vars: Optional[Dict[str, Any]] = None, -) -> Any: - """Dynamically imports a module from the specified file path. - - This function imports a module from a given filename, with the option to - specify the module's name and inject global variables into the module before - it is returned. If `module_name` is not provided, the filename is used as - the module name after replacing '.' with '_'. Global variables can be passed - as a dictionary to `global_vars`, which will be injected into the module's - namespace. - - Args: - filename (str): The path to the file from which to import the module. - module_name (Optional[str]): The name to assign to the module. Defaults - to None, in which case the filename is used as the module name. - global_vars (Optional[Dict[str, Any]]): A dictionary of global variables - to inject into the module's namespace. Defaults to None. - - Returns: - Any: The imported module with the specified modifications. - - Raises: - FileNotFoundError: If the specified file does not exist. - ImportError: If there is an error during the import process. - - """ - filename = os.path.abspath(filename) - if not os.path.exists(filename): - raise FileNotFoundError(f"No such file: '{filename}'") - - module_name = module_name or os.path.basename(filename).replace(".", "_") - loader = SourceFileLoader(module_name, filename) - spec = importlib.util.spec_from_loader(loader.name, loader) - module = importlib.util.module_from_spec(spec) - if global_vars: - module.__dict__.update(global_vars) - loader.exec_module(module) - return module +from typing import Optional, Dict, Any +from k4FWCore.utils import import_from class SequenceLoader: diff --git a/python/k4FWCore/utils.py b/python/k4FWCore/utils.py index 5b743563b..3fcb3db1b 100644 --- a/python/k4FWCore/utils.py +++ b/python/k4FWCore/utils.py @@ -21,7 +21,7 @@ import re import logging import sys -from typing import Union +from typing import Union, Optional, Dict, Any from importlib.machinery import SourceFileLoader import importlib.util from pathlib import Path @@ -127,3 +127,46 @@ def get_logger() -> logging.Logger: _logger.handlers = [handler] return _logger + + +def import_from( + filename: Union[str, os.PathLike], + module_name: Optional[str] = None, + global_vars: Optional[Dict[str, Any]] = None, +) -> Any: + """Dynamically imports a module from the specified file path. + + This function imports a module from a given filename, with the option to + specify the module's name and inject global variables into the module before + it is returned. If `module_name` is not provided, the filename is used as + the module name after replacing '.' with '_'. Global variables can be passed + as a dictionary to `global_vars`, which will be injected into the module's + namespace. + + Args: + filename (str): The path to the file from which to import the module. + module_name (Optional[str]): The name to assign to the module. Defaults + to None, in which case the filename is used as the module name. + global_vars (Optional[Dict[str, Any]]): A dictionary of global variables + to inject into the module's namespace. Defaults to None. + + Returns: + Any: The imported module with the specified modifications. + + Raises: + FileNotFoundError: If the specified file does not exist. + ImportError: If there is an error during the import process. + + """ + filename = os.path.abspath(filename) + if not os.path.exists(filename): + raise FileNotFoundError(f"No such file: '{filename}'") + + module_name = module_name or os.path.basename(filename).replace(".", "_") + loader = SourceFileLoader(module_name, filename) + spec = importlib.util.spec_from_loader(loader.name, loader) + module = importlib.util.module_from_spec(spec) + if global_vars: + module.__dict__.update(global_vars) + loader.exec_module(module) + return module From 6fd444ded547f90a54d55c96e539e97a985e9a19 Mon Sep 17 00:00:00 2001 From: Leonhard Reichenbach Date: Tue, 28 Oct 2025 09:23:04 +0100 Subject: [PATCH 4/7] added test --- test/k4FWCoreTest/CMakeLists.txt | 1 + test/k4FWCoreTest/options/ExampleSequence.py | 28 +++++++++++ .../options/ExampleSequenceLoader.py | 49 +++++++++++++++++++ 3 files changed, 78 insertions(+) create mode 100644 test/k4FWCoreTest/options/ExampleSequence.py create mode 100644 test/k4FWCoreTest/options/ExampleSequenceLoader.py diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt index eea9c4f0e..7f23bc791 100644 --- a/test/k4FWCoreTest/CMakeLists.txt +++ b/test/k4FWCoreTest/CMakeLists.txt @@ -227,6 +227,7 @@ add_test_fwcore(FunctionalProducerRNTuple options/ExampleFunctionalProducer.py - add_test_fwcore(FunctionalFileRNTuple options/ExampleFunctionalFile.py --IOSvc.OutputType RNTuple --IOSvc.Input functional_producer_rntuple.root --IOSvc.Output functional_producer_rntuple_file.root PROPERTIES FIXTURES_REQUIRED FunctionalRNTupleFile ADD_TO_CHECK_FILES) add_test_fwcore(FunctionalTTreeToRNTuple options/ExampleFunctionalTTreeToRNTuple.py PROPERTIES FIXTURES_REQUIRED ProducerFile ADD_TO_CHECK_FILES) add_test_fwcore(GaudiFunctional options/ExampleGaudiFunctional.py PROPERTIES FIXTURES_REQUIRED ProducerFile ADD_TO_CHECK_FILES) +add_test_fwcore(SequenceLoader options/ExampleSequenceLoader.py) add_test_fwcore(ReadLimitedInputsIOSvc options/ExampleIOSvcLimitInputCollections.py PROPERTIES FIXTURES_REQUIRED ExampleEventDataFile ADD_TO_CHECK_FILES) add_test_fwcore(ReadLimitedInputsAllEventsIOSvc options/ExampleIOSvcLimitInputCollections.py --IOSvc.Output "functional_limited_input_all_events.root" -n -1 PROPERTIES FIXTURES_REQUIRED ExampleEventDataFile ADD_TO_CHECK_FILES) diff --git a/test/k4FWCoreTest/options/ExampleSequence.py b/test/k4FWCoreTest/options/ExampleSequence.py new file mode 100644 index 000000000..59a659a10 --- /dev/null +++ b/test/k4FWCoreTest/options/ExampleSequence.py @@ -0,0 +1,28 @@ +# +# Copyright (c) 2014-2024 Key4hep-Project. +# +# This file is part of Key4hep. +# See https://key4hep.github.io/key4hep-doc/ for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This is an example reading from a file and using a producer to create new +# data + +from Configurables import ExampleGaudiFunctionalProducer + + +gaudi_producer = ExampleGaudiFunctionalProducer("GaudiProducer", OutputCollectionName="Output") + +ExampleSequenceSequence = [gaudi_producer] \ No newline at end of file diff --git a/test/k4FWCoreTest/options/ExampleSequenceLoader.py b/test/k4FWCoreTest/options/ExampleSequenceLoader.py new file mode 100644 index 000000000..b020f6624 --- /dev/null +++ b/test/k4FWCoreTest/options/ExampleSequenceLoader.py @@ -0,0 +1,49 @@ +# +# Copyright (c) 2014-2024 Key4hep-Project. +# +# This file is part of Key4hep. +# See https://key4hep.github.io/key4hep-doc/ for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This is an example reading from a file and using a producer to create new +# data + +from Gaudi.Configuration import INFO +from Configurables import ExampleFunctionalTransformer +from Configurables import EventDataSvc +from k4FWCore import ApplicationMgr, IOSvc, SequenceLoader +from pathlib import Path + +svc = IOSvc("IOSvc") +svc.Input = "functional_producer.root" +svc.Output = "gaudi_functional.root" + +algList = [] +sequenceLoader = SequenceLoader(algList) + +sequenceLoader.load(f"{Path(__file__).parent}/ExampleSequence") + +transformer = ExampleFunctionalTransformer( + "Transformer", InputCollection="MCParticles", OutputCollection="NewMCParticles" +) +algList.append(transformer) + +mgr = ApplicationMgr( + TopAlg=algList, + EvtSel="NONE", + EvtMax=-1, + ExtSvc=[EventDataSvc("EventDataSvc")], + OutputLevel=INFO, +) From 6f3a324b8ab97fc53310cdf3221edb20524e5958 Mon Sep 17 00:00:00 2001 From: Thomas Madlener Date: Thu, 30 Oct 2025 16:55:39 +0100 Subject: [PATCH 5/7] Clarify the usage of an absolute path --- test/k4FWCoreTest/options/ExampleSequence.py | 2 +- test/k4FWCoreTest/options/ExampleSequenceLoader.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/test/k4FWCoreTest/options/ExampleSequence.py b/test/k4FWCoreTest/options/ExampleSequence.py index 59a659a10..a8784fa59 100644 --- a/test/k4FWCoreTest/options/ExampleSequence.py +++ b/test/k4FWCoreTest/options/ExampleSequence.py @@ -25,4 +25,4 @@ gaudi_producer = ExampleGaudiFunctionalProducer("GaudiProducer", OutputCollectionName="Output") -ExampleSequenceSequence = [gaudi_producer] \ No newline at end of file +ExampleSequenceSequence = [gaudi_producer] diff --git a/test/k4FWCoreTest/options/ExampleSequenceLoader.py b/test/k4FWCoreTest/options/ExampleSequenceLoader.py index b020f6624..8d9e4b8d4 100644 --- a/test/k4FWCoreTest/options/ExampleSequenceLoader.py +++ b/test/k4FWCoreTest/options/ExampleSequenceLoader.py @@ -33,6 +33,8 @@ algList = [] sequenceLoader = SequenceLoader(algList) +# Use an absolute path here to be independent of the working directory in which +# the tests run sequenceLoader.load(f"{Path(__file__).parent}/ExampleSequence") transformer = ExampleFunctionalTransformer( From 86575b36b86b8b36ec734e60f137303e058f3ad0 Mon Sep 17 00:00:00 2001 From: Thomas Madlener Date: Thu, 30 Oct 2025 16:55:57 +0100 Subject: [PATCH 6/7] Add a generic version and call that from the non-generic one --- python/k4FWCore/SequenceLoader.py | 51 ++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/python/k4FWCore/SequenceLoader.py b/python/k4FWCore/SequenceLoader.py index 80b204b5d..bf9989df5 100644 --- a/python/k4FWCore/SequenceLoader.py +++ b/python/k4FWCore/SequenceLoader.py @@ -17,7 +17,8 @@ # limitations under the License. # -from typing import Optional, Dict, Any +import os +from typing import Optional, Dict, Any, Union from k4FWCore.utils import import_from @@ -50,17 +51,51 @@ def __init__(self, alg_list: list, global_vars: Optional[Dict[str, Any]] = None) self.alg_list = alg_list self.global_vars = global_vars + def load_from(self, module_path: Union[str, os.PathLike], sequence_name: str) -> None: + """Load a sequence of algorithms from a specified Python file and append + it to the algorithm list + + Args: + module_path (Union[str, os.PathLike]): The path to the python module + (file) from which to load the sequence. The path is interpreted + to be relative to the execution directory of the process from + which this method is called unless an absolute path is passed. + sequence_name (str): The name of the sequence to load from the + specified python module + + Examples: + >>> alg_list = [] + >>> seq_loader = SequenceLoader(alg_list) + >>> seq_loader.load_from("Tracking/TrackingDigi.py", + "TrackingDigiSequence") + + This will import the file `Tracking/TrackingDigi.py` and add the + sequence of algorithms that is defined in `TrackingDigiSequence` in + that file to the alg_list + + """ + seq_module = import_from( + module_path, + global_vars=self.global_vars, + ) + + seq = getattr(seq_module, sequence_name) + self.alg_list.extend(seq) + def load(self, sequence: str) -> None: """Loads a sequence algorithm from a specified Python file and appends it to the algorithm list - The method constructs the filename from the sequence parameter name and - imports the sequence from the imported module. + This is a convenience overload for load_from that constructs the + filename from the sequence parameter name and imports the sequence from + the imported module. Args: sequence (str): The name of the sequence to load. The sequence name should correspond to a Python file and class name following the pattern `{sequence}.py` and `{sequence}Sequence`, respectively. + The sequence is interpreted to be relative to the path from + which the process is launched, unless it's an absolute path. Examples: >>> alg_list = [] @@ -70,14 +105,8 @@ def load(self, sequence: str) -> None: This will import the file `Tracking/TrackingDigi.py` and add the sequence of algorithms that is defined in `TrackingDigiSequence` in that file to the alg_list + """ filename = f"{sequence}.py" seq_name = f"{sequence.split('/')[-1]}Sequence" - - seq_module = import_from( - filename, - global_vars=self.global_vars, - ) - - seq = getattr(seq_module, seq_name) - self.alg_list.extend(seq) + return self.load_from(filename, seq_name) From 85f8fb3c3ce0e6ba94193e6e6b30438c1c531f62 Mon Sep 17 00:00:00 2001 From: Thomas Madlener Date: Thu, 30 Oct 2025 17:10:03 +0100 Subject: [PATCH 7/7] Add basic logging to make debugging a bit easier --- python/k4FWCore/SequenceLoader.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python/k4FWCore/SequenceLoader.py b/python/k4FWCore/SequenceLoader.py index bf9989df5..0375be769 100644 --- a/python/k4FWCore/SequenceLoader.py +++ b/python/k4FWCore/SequenceLoader.py @@ -19,7 +19,9 @@ import os from typing import Optional, Dict, Any, Union -from k4FWCore.utils import import_from +from k4FWCore.utils import import_from, get_logger + +logger = get_logger() class SequenceLoader: @@ -48,6 +50,7 @@ def __init__(self, alg_list: list, global_vars: Optional[Dict[str, Any]] = None) dictionary will be the available variables in the imported module and the values will be the values of these variables. """ + logger.info(f"Creating SequenceLoader with {len(alg_list)} algorithms already defined") self.alg_list = alg_list self.global_vars = global_vars @@ -74,12 +77,14 @@ def load_from(self, module_path: Union[str, os.PathLike], sequence_name: str) -> that file to the alg_list """ + logger.info(f"Loading '{sequence_name} from '{module_path}'") seq_module = import_from( module_path, global_vars=self.global_vars, ) seq = getattr(seq_module, sequence_name) + logger.debug(f"Adding {len(seq)} algorithms contained in '{sequence_name}'") self.alg_list.extend(seq) def load(self, sequence: str) -> None: