diff --git a/maestrowf/__init__.py b/maestrowf/__init__.py index f0d1aa45d..cdc4bb564 100644 --- a/maestrowf/__init__.py +++ b/maestrowf/__init__.py @@ -41,15 +41,24 @@ try: # Python 2.7+ from logging import NullHandler except ImportError: + class NullHandler(logging.Handler): """Null logging handler for Python 3+.""" def emit(self, record): - """Override so that logging outputs nothing.""" + """Override so that logging outputs nothing. + + Args: + record: + + Returns: + + """ pass + LOGGER = logging.getLogger(__name__) LOGGER.addHandler(NullHandler()) __version_info__ = ("1", "1", "9dev0") -__version__ = '.'.join(__version_info__) +__version__ = ".".join(__version_info__) diff --git a/maestrowf/abstracts/__init__.py b/maestrowf/abstracts/__init__.py index c920262ff..065523964 100644 --- a/maestrowf/abstracts/__init__.py +++ b/maestrowf/abstracts/__init__.py @@ -46,8 +46,16 @@ from maestrowf.abstracts.specification import Specification -__all__ = ("abstractclassmethod", "Dependency", "Graph", "PickleInterface", - "Singleton", "Source", "Specification", "Substitution") +__all__ = ( + "abstractclassmethod", + "Dependency", + "Graph", + "PickleInterface", + "Singleton", + "Source", + "Specification", + "Substitution", +) LOGGER = logging.getLogger(__name__) @@ -57,44 +65,56 @@ class PickleInterface: @classmethod def unpickle(cls, path): - """ - Load a pickled instance from a pickle file. + """Load a pickled instance from a pickle file. + + Args: + path: Path to a pickle file containing a class instance. + + Returns: - :param path: Path to a pickle file containing a class instance. """ - with open(path, 'rb') as pkl: + with open(path, "rb") as pkl: obj = dill.load(pkl) if not isinstance(obj, cls): - msg = "Object loaded from {path} is of type {type}. Expected an" \ - " object of type '{cls}.'".format(path=path, type=type(obj), - cls=type(cls)) + msg = ( + "Object loaded from {path} is of type {type}. Expected an" + " object of type '{cls}.'".format( + path=path, type=type(obj), cls=type(cls) + ) + ) LOGGER.error(msg) raise TypeError(msg) return obj def pickle(self, path): - """ - Generate a pickle file of of a class instance. + """Generate a pickle file of of a class instance. + + Args: + path: The path to write the pickle to. + + Returns: - :param path: The path to write the pickle to. """ - with open(path, 'wb') as pkl: + with open(path, "wb") as pkl: dill.dump(self, pkl) class _Singleton(type): + """ """ + _instances = {} def __call__(cls, *args, **kwargs): if cls not in cls._instances: - cls._instances[cls] = super(_Singleton, cls).__call__(*args, - **kwargs) + cls._instances[cls] = super(_Singleton, cls).__call__( + *args, **kwargs + ) return cls._instances[cls] -class Singleton(_Singleton('SingletonMeta', (object,), {})): +class Singleton(_Singleton("SingletonMeta", (object,), {})): """Single type to allow for classes to be typed as a singleton.""" pass diff --git a/maestrowf/abstracts/abstractclassmethod.py b/maestrowf/abstracts/abstractclassmethod.py index 389702813..08404d5c1 100644 --- a/maestrowf/abstracts/abstractclassmethod.py +++ b/maestrowf/abstracts/abstractclassmethod.py @@ -34,7 +34,13 @@ class abstractclassmethod(classmethod): """Python 2.7 does not include built in @abstractclassmethod so we create our own class that extends @classmethod and then attaches a __isabstractmethod variable to the callable function. See ref: - https://stackoverflow.com/a/11218474""" + https://stackoverflow.com/a/11218474 + + Args: + + Returns: + + """ __isabstractmethod__ = True diff --git a/maestrowf/abstracts/containers/__init__.py b/maestrowf/abstracts/containers/__init__.py index 0dfdc524c..fa4e9601a 100644 --- a/maestrowf/abstracts/containers/__init__.py +++ b/maestrowf/abstracts/containers/__init__.py @@ -38,14 +38,16 @@ def __init__(self): self._info = {} def get(self, key, default=None): - """ - Get information by key in a record. + """Get information by key in a record. + + Args: + key: The key to look up in a Record's stored information. + default: The default value to return if the key is not found + (Default: None). - :param key: The key to look up in a Record's stored information. - :param default: The default value to return if the key is not found - (Default: None). - :returns: The information labeled by parameter key. Default if key does - not exist. + Returns: + The information labeled by parameter key. Default if key does + not exist. """ return self._info.get(key, default) diff --git a/maestrowf/abstracts/enums/__init__.py b/maestrowf/abstracts/enums/__init__.py index af33cb5f8..b00f20ce9 100644 --- a/maestrowf/abstracts/enums/__init__.py +++ b/maestrowf/abstracts/enums/__init__.py @@ -34,16 +34,22 @@ class SubmissionCode(Enum): + """ """ + OK = 0 ERROR = 1 class CancelCode(Enum): + """ """ + OK = 0 ERROR = 1 class JobStatusCode(Enum): + """ """ + OK = 0 NOJOBS = 1 ERROR = 2 @@ -71,7 +77,8 @@ class State(Enum): class StudyStatus(Enum): """Workflow status enumeration""" - FINISHED = 0 # The Study has finished successfully, all steps ran - RUNNING = 1 # The Study is currently running - FAILURE = 2 # The Study has finished, but 1 or more steps failed + + FINISHED = 0 # The Study has finished successfully, all steps ran + RUNNING = 1 # The Study is currently running + FAILURE = 2 # The Study has finished, but 1 or more steps failed CANCELLED = 3 # The Study has finished, but was cancelled diff --git a/maestrowf/abstracts/envobject.py b/maestrowf/abstracts/envobject.py index 172b2ffde..873c9a746 100644 --- a/maestrowf/abstracts/envobject.py +++ b/maestrowf/abstracts/envobject.py @@ -38,8 +38,7 @@ @six.add_metaclass(ABCMeta) class EnvObject: - """ - An abstract class representing objects that exist in a study's environment. + """An abstract class representing objects that exist in a study's environment. The EnvObject is meant to be used to represent entities in the larger environment that affect the execution of a study (and therefore jobs). @@ -47,12 +46,16 @@ class EnvObject: dependencies, code dependencies, variables, aliases, etc. The only method we require is _verify, to allow users to verify that they've provided the minimal information for the object to be valid. + + Args: + + Returns: + """ @abstractmethod def _verify(self): - """ - Verify that the object is valid. + """Verify that the object is valid. Subclasses that inherit from the EnvObject abstract class are expected to provide a method for asserting that the contents contained within @@ -60,14 +63,21 @@ def _verify(self): expected member variables are populated to asserting specific values of members, etc. - :returns: True if the EnvObject is verified, False otherwise. + Args: + + Returns: + True if the EnvObject is verified, False otherwise. + """ def _verification(self, error): - """ - A wrapper method for verifying for using a custom error message. + """A wrapper method for verifying for using a custom error message. + + Args: + error: String containing a custom error message. + + Returns: - :param error: String containing a custom error message. """ if not self._verify(): LOGGER.exception(error) @@ -80,22 +90,24 @@ class Substitution(EnvObject): @abstractmethod def substitute(self, data): - """ - Perform a replacement of some substring into data. + """Perform a replacement of some substring into data. The method takes the input string data and performs a replacement. This API is used to represent concepts such as variables or parameters that would want to be replaced within the string data. - :param data: A string to perform a replacement on. - :returns: A string equal to the original string data with substitutions - made (if any were performed). + Args: + data: A string to perform a replacement on. + + Returns: + A string equal to the original string data with substitutions + made (if any were performed). + """ class Source(EnvObject): - """ - Abstract class representing classes that alter environment sourcing. + """Abstract class representing classes that alter environment sourcing. WARNING: The API for this class is still in development. The Source environment class is meant to provide a way to programmatically @@ -104,20 +116,28 @@ class Source(EnvObject): * Exporting of shell/environment variables (using 'export') * Setting of an environment package with the 'use' command + + Args: + + Returns: + """ @abstractmethod def apply(self, data): - """ - Apply the Source to some string data. + """Apply the Source to some string data. Subclasses of Source should use this method in order to apply an environment altering change. The 'data' parameter should be a string representing a command to apply Source to or a list of other commands that Source should be included with. - :param data: A string representing a command or set of other sources. - :returns: A string with the Source applied. + Args: + data: A string representing a command or set of other sources. + + Returns: + A string with the Source applied. + """ # NOTE: This functionality has not been settled yet. The use of this # class or this design may not be the best for applying script sources @@ -126,8 +146,7 @@ def apply(self, data): @six.add_metaclass(ABCMeta) class Dependency(Substitution): - """ - Abstract object representing a dependency. + """Abstract object representing a dependency. The Dependency base class is intended to be used to capture external items the workflow is dependent on. These items include (but are not limited to): @@ -140,17 +159,26 @@ class Dependency(Substitution): The goal of this base class is to make it so that this package is able to pull external dependencies in a consistent manner. + + Args: + + Returns: + """ @abstractmethod def acquire(self, substitutions=None): - """ - Acquire the dependency as specfied by the class instance. + """Acquire the dependency as specfied by the class instance. Subclasses that implement this interface should raise exceptions during acquisition should they be unable to retrieve their specified dependency. It is assumed that if acquiring throws an exception that the study cannot proceed forward. - :param substitutions: List of Substitution objects that can be applied. + Args: + substitutions: List of Substitution objects that can be applied. + (Default value = None) + + Returns: + """ diff --git a/maestrowf/abstracts/graph.py b/maestrowf/abstracts/graph.py index c8e0dd83c..37a12cfb8 100644 --- a/maestrowf/abstracts/graph.py +++ b/maestrowf/abstracts/graph.py @@ -44,27 +44,36 @@ class Graph: @abstractmethod def add_node(self, name, obj): - """ - Method to add a node to the graph. + """Method to add a node to the graph. + + Args: + name: String identifier of the node. + obj: An object representing the value of the node. + + Returns: - :param name: String identifier of the node. - :param obj: An object representing the value of the node. """ @abstractmethod def add_edge(self, src, dest): - """ - Add the edge (src, dest) to the graph. + """Add the edge (src, dest) to the graph. + + Args: + src: Source vertex name. + dest: Destination vertex name. + + Returns: - :param src: Source vertex name. - :param dest: Destination vertex name. """ @abstractmethod def remove_edge(self, src, dest): - """ - Remove edge (src, dest) from the graph. + """Remove edge (src, dest) from the graph. + + Args: + src: Source vertex name. + dest: Destination vertex name. + + Returns: - :param src: Source vertex name. - :param dest: Destination vertex name. """ diff --git a/maestrowf/abstracts/interfaces/__init__.py b/maestrowf/abstracts/interfaces/__init__.py index e94906a6c..c32df325e 100644 --- a/maestrowf/abstracts/interfaces/__init__.py +++ b/maestrowf/abstracts/interfaces/__init__.py @@ -31,8 +31,9 @@ Abstract classes for handling interfacing with various services. """ -from maestrowf.abstracts.interfaces.schedulerscriptadapter import \ - SchedulerScriptAdapter +from maestrowf.abstracts.interfaces.schedulerscriptadapter import ( + SchedulerScriptAdapter, +) from maestrowf.abstracts.interfaces.scriptadapter import ScriptAdapter diff --git a/maestrowf/abstracts/interfaces/flux.py b/maestrowf/abstracts/interfaces/flux.py index 77ab2248c..7263188c5 100644 --- a/maestrowf/abstracts/interfaces/flux.py +++ b/maestrowf/abstracts/interfaces/flux.py @@ -1,78 +1,107 @@ -from abc import ABC, abstractclassmethod, abstractmethod, \ - abstractstaticmethod +from abc import ABC, abstractclassmethod, abstractmethod, abstractstaticmethod class FluxInterface(ABC): + """ """ @abstractclassmethod def get_statuses(cls, joblist): - """ - Return the statuses from a given Flux handle and joblist. + """Return the statuses from a given Flux handle and joblist. + + Args: + joblist: A list of jobs to check the status of. + + Returns: + A dictionary of job identifiers to statuses. - :param joblist: A list of jobs to check the status of. - :return: A dictionary of job identifiers to statuses. """ @abstractstaticmethod def state(state): - """ - Map a scheduler specific job state to a Study.State enum. + """Map a scheduler specific job state to a Study.State enum. + + Args: + adapter: Instance of a FluxAdapter + state: A string of the state returned by Flux + + Returns: + The mapped Study.State enumeration - :param adapter: Instance of a FluxAdapter - :param state: A string of the state returned by Flux - :return: The mapped Study.State enumeration """ @abstractclassmethod def parallelize(cls, procs, nodes=None, **kwargs): - """ - Create a parallelized Flux command for launching. + """Create a parallelized Flux command for launching. + + Args: + procs: Number of processors to use. + nodes: Number of nodes the parallel call will span. + (Default value = None) + kwargs: Extra keyword arguments. + **kwargs: + + Returns: + A string of a Flux MPI command. - :param procs: Number of processors to use. - :param nodes: Number of nodes the parallel call will span. - :param kwargs: Extra keyword arguments. - :return: A string of a Flux MPI command. """ @abstractclassmethod def submit( - cls, nodes, procs, cores_per_task, path, cwd, walltime, - npgus=0, job_name=None, force_broker=False + cls, + nodes, + procs, + cores_per_task, + path, + cwd, + walltime, + npgus=0, + job_name=None, + force_broker=False, ): - """ - Submit a job using this Flux interface's submit API. - - :param nodes: The number of nodes to request on submission. - :param procs: The number of cores to request on submission. - :param cores_per_task: The number of cores per MPI task. - :param path: Path to the script to be submitted. - :param cwd: Path to the workspace to execute the script in. - :param walltime: HH:MM:SS formatted time string for job duration. - :param ngpus: The number of GPUs to request on submission. - :param job_name: A name string to assign the submitted job. - :param force_broker: Forces the script to run under a Flux sub-broker. - :return: A string representing the jobid returned by Flux submit. - :return: An integer of the return code submission returned. - :return: SubmissionCode enumeration that reflects result of submission. + """Submit a job using this Flux interface's submit API. + + Args: + nodes: The number of nodes to request on submission. + procs: The number of cores to request on submission. + cores_per_task: The number of cores per MPI task. + path: Path to the script to be submitted. + cwd: Path to the workspace to execute the script in. + walltime: HH:MM:SS formatted time string for job duration. + ngpus: The number of GPUs to request on submission. + job_name: A name string to assign the submitted job. + (Default value = None) + force_broker: Forces the script to run under a Flux sub-broker. + (Default value = False) + npgus: (Default value = 0) + + Returns: + A string representing the jobid returned by Flux submit. + """ @abstractclassmethod def cancel(cls, joblist): - """ - Cancel a job using this Flux interface's cancellation API. + """Cancel a job using this Flux interface's cancellation API. + + Args: + joblist: A list of job identifiers to cancel. + + Returns: + CancelCode enumeration that reflects result of cancellation. - :param joblist: A list of job identifiers to cancel. - :return: CancelCode enumeration that reflects result of cancellation. """ @property @abstractmethod def key(self): """ - Return the key name for a ScriptAdapter. - This is used to register the adapter in the ScriptAdapterFactory - and when writing the workflow specification. + Args: + + Returns: + This is used to register the adapter in the ScriptAdapterFactory + and when writing the workflow specification. + + :return: A string of the name of a FluxInterface class. - :return: A string of the name of a FluxInterface class. """ diff --git a/maestrowf/abstracts/interfaces/schedulerscriptadapter.py b/maestrowf/abstracts/interfaces/schedulerscriptadapter.py index 91496b7a2..cdc60e0a6 100644 --- a/maestrowf/abstracts/interfaces/schedulerscriptadapter.py +++ b/maestrowf/abstracts/interfaces/schedulerscriptadapter.py @@ -1,4 +1,3 @@ - ############################################################################### # Copyright (c) 2017, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory @@ -41,21 +40,24 @@ @six.add_metaclass(ABCMeta) class SchedulerScriptAdapter(ScriptAdapter): - """ - Abstract class representing the interface for scheduling scripts. + """Abstract class representing the interface for scheduling scripts. This class handles both the construction of scripts (as required by the ScriptAdapter base class) but also includes the necessary methods for constructing parallel commands. The adapter will substitute parallelized commands but also defines how to schedule and check job status. + + Args: + + Returns: + """ # The var tag to look for to replace for parallelized commands. launcher_var = "$(LAUNCHER)" # Allocation regex and compilation # Keeping this one here for legacy. - launcher_regex = re.compile( - re.escape(launcher_var) + r"\[(?P.*)\]") + launcher_regex = re.compile(re.escape(launcher_var) + r"\[(?P.*)\]") # We can have multiple requested submission properties. # Legacy allocation of nodes and procs. @@ -83,50 +85,66 @@ def __init__(self, **kwargs): self._batch = {} def add_batch_parameter(self, name, value): - """ - Add a parameter to the ScriptAdapter instance. + """Add a parameter to the ScriptAdapter instance. + + Args: + name: String name of the parameter that's being added. + value: Value associated with the parameter name (should have a + str method). + + Returns: - :param name: String name of the parameter that's being added. - :param value: Value associated with the parameter name (should have a - str method). """ self._batch[name] = value @abstractmethod def get_header(self, step): - """ - Generate the header present at the top of execution scripts. + """Generate the header present at the top of execution scripts. + + Args: + step: A StudyStep instance. + + Returns: + A string of the header based on internal batch parameters and + the parameter step. - :param step: A StudyStep instance. - :returns: A string of the header based on internal batch parameters and - the parameter step. """ pass @abstractmethod def get_parallelize_command(self, procs, nodes, **kwargs): - """ - Generate the parallelization segment of the command line. + """Generate the parallelization segment of the command line. + + Args: + procs: Number of processors to allocate to the parallel call. + nodes: Number of nodes to allocate to the parallel call + (default = 1). + **kwargs: + + Returns: + A string of the parallelize command configured using nodes + and procs. - :param procs: Number of processors to allocate to the parallel call. - :param nodes: Number of nodes to allocate to the parallel call - (default = 1). - :returns: A string of the parallelize command configured using nodes - and procs. """ pass def _substitute_parallel_command(self, step_cmd, **kwargs): - """ - Substitute parallelized segments into a specified command. + """Substitute parallelized segments into a specified command. + + Args: + step_cmd: Command string to parallelize. + nodes: Total number of requested nodes. + procs: Total number of requested processors. + **kwargs: + + Returns: + The new command with all allocations substituted. - :param step_cmd: Command string to parallelize. - :param nodes: Total number of requested nodes. - :param procs: Total number of requested processors. - :returns: The new command with all allocations substituted. """ - err_msg = "{} attempting to allocate {} {} for a parallel call with" \ - " a maximum allocation of {}" + err_msg = ( + "{} attempting to allocate {} {} for a parallel call with" + " a maximum allocation of {}" + ) nodes = kwargs.get("nodes") procs = kwargs.get("procs") @@ -139,9 +157,9 @@ def _substitute_parallel_command(self, step_cmd, **kwargs): alloc_search = list(re.finditer(self.launcher_regex, step_cmd)) if alloc_search: # If we find that launcher nomenclature. - total_nodes = 0 # Total nodes we've allocated so far. - total_procs = 0 # Total processors we've allocated so far. - cmd = step_cmd # The step command we'll substitute into. + total_nodes = 0 # Total nodes we've allocated so far. + total_procs = 0 # Total processors we've allocated so far. + cmd = step_cmd # The step command we'll substitute into. for match in alloc_search: LOGGER.debug("Found a match: %s", match.group()) _nodes = None @@ -158,24 +176,28 @@ def _substitute_parallel_command(self, step_cmd, **kwargs): LOGGER.debug( "Legacy setup detected. (nodes=%s, procs=%s)", _nodes, - _procs + _procs, ) else: # We're dealing with the new style. # Make sure we only have at most one proc and node # allocation specified. if _alloc.count("p") > 1 or _alloc.count("n") > 1: - msg = "cmd: {}\n Invalid allocations specified ({})." \ - " Number of nodes and/or procs must only be " \ - "specified once." \ - .format(step_cmd, _alloc) + msg = ( + "cmd: {}\n Invalid allocations specified ({})." + " Number of nodes and/or procs must only be " + "specified once.".format(step_cmd, _alloc) + ) LOGGER.error(msg) raise ValueError(msg) if _alloc.count("p") < 1: - msg = "cmd: {}\n Invalid allocations specified ({})." \ - " Processors/tasks must be specified." \ - .format(step_cmd, _alloc) + msg = ( + "cmd: {}\n Invalid allocations specified ({})." + " Processors/tasks must be specified.".format( + step_cmd, _alloc + ) + ) LOGGER.error(msg) raise ValueError(msg) @@ -189,7 +211,7 @@ def _substitute_parallel_command(self, step_cmd, **kwargs): LOGGER.debug( "New setup detected. (nodes=%s, procs=%s)", _nodes, - _procs + _procs, ) msg = [] @@ -225,14 +247,18 @@ def _substitute_parallel_command(self, step_cmd, **kwargs): # Verify that the total nodes/procs used is within maximum. if total_procs > procs: - msg = "Total processors ({}) requested exceeds the " \ - "maximum requested ({})".format(total_procs, procs) + msg = ( + "Total processors ({}) requested exceeds the " + "maximum requested ({})".format(total_procs, procs) + ) LOGGER.error(msg) raise ValueError(msg) if total_nodes > nodes: - msg = "Total nodes ({}) requested exceeds the " \ - "maximum requested ({})".format(total_nodes, nodes) + msg = ( + "Total nodes ({}) requested exceeds the " + "maximum requested ({})".format(total_nodes, nodes) + ) LOGGER.error(msg) raise ValueError(msg) @@ -246,24 +272,27 @@ def _substitute_parallel_command(self, step_cmd, **kwargs): # Catch the case where the launcher token appears on its own if self.launcher_var in step_cmd: LOGGER.debug( - "'%s' found in cmd. Substituting", self.launcher_var) + "'%s' found in cmd. Substituting", self.launcher_var + ) return step_cmd.replace(self.launcher_var, pcmd) else: LOGGER.debug("The command did not specify an MPI command.") return step_cmd def get_scheduler_command(self, step): - """ - Generate the full parallelized command for use in a batch script. - - :param step: A StudyStep instance. - :returns: - 1. A Boolean value - True if command is to be scheduled, False - otherwise. - 2. A string representing the parallelized batch command for the - specified step command. - 3. A string representing the parallelized batch command for the - specified step restart command. + """Generate the full parallelized command for use in a batch script. + + Args: + step: A StudyStep instance. + + Returns: + 1. A Boolean value - True if command is to be scheduled, False + otherwise. + 2. A string representing the parallelized batch command for the + specified step command. + 3. A string representing the parallelized batch command for the + specified step restart command. + """ # We should never get a study step that doesn't have a run entry; but # better to be safe. @@ -279,8 +308,7 @@ def get_scheduler_command(self, step): if _nodes or _procs: to_be_scheduled = True cmd = self._substitute_parallel_command( - step.run["cmd"], - **step.run + step.run["cmd"], **step.run ) LOGGER.debug("Scheduling command: %s", cmd) @@ -288,8 +316,7 @@ def get_scheduler_command(self, step): restart = "" if step.run["restart"]: restart = self._substitute_parallel_command( - step.run["restart"], - **step.run + step.run["restart"], **step.run ) LOGGER.debug("Restart command: %s", cmd) LOGGER.info("Scheduling workflow step '%s'.", step.name) @@ -304,8 +331,7 @@ def get_scheduler_command(self, step): @abstractmethod def _write_script(self, ws_path, step): - """ - Write a script to the workspace of a workflow step. + """Write a script to the workspace of a workflow step. The job_map optional parameter is a map of workflow step names to job identifiers. This parameter so far is only planned to be used when a @@ -314,19 +340,26 @@ def _write_script(self, ws_path, step): the parameter may change depending on both future intended use and derived classes. - :param ws_path: Path to the workspace directory of the step. - :param step: An instance of a StudyStep. - :returns: Boolean value (True if the workflow step is to be scheduled, - False otherwise) and the path to the written script. + Args: + ws_path: Path to the workspace directory of the step. + step: An instance of a StudyStep. + + Returns: + Boolean value (True if the workflow step is to be scheduled, + False otherwise) and the path to the written script. + """ pass @abstractmethod def _state(self, job_state): - """ - Map a scheduler specific job state to a Study.State enum. + """Map a scheduler specific job state to a Study.State enum. + + Args: + job_state: String representation of scheduler job status. + + Returns: + A Study.State enum corresponding to parameter job_state. - :param job_state: String representation of scheduler job status. - :returns: A Study.State enum corresponding to parameter job_state. """ pass diff --git a/maestrowf/abstracts/interfaces/scriptadapter.py b/maestrowf/abstracts/interfaces/scriptadapter.py index dff79380c..a99c6a74b 100644 --- a/maestrowf/abstracts/interfaces/scriptadapter.py +++ b/maestrowf/abstracts/interfaces/scriptadapter.py @@ -39,8 +39,7 @@ @six.add_metaclass(ABCMeta) class ScriptAdapter(object): - """ - Abstract class representing the interface for constructing scripts. + """Abstract class representing the interface for constructing scripts. The ScriptAdapter abstract class is meant to provide a consistent high level interface to generate scripts automatically based on an ExecutionDAG. @@ -51,6 +50,11 @@ class ScriptAdapter(object): - Generating a script with the proper syntax to submit. - Submitting a script using the proper command. - Checking job status. + + Args: + + Returns: + """ def __init__(self, **kwargs): @@ -64,29 +68,34 @@ def __init__(self, **kwargs): @abstractmethod def check_jobs(self, joblist): - """ - For the given job list, query execution status. + """For the given job list, query execution status. + + Args: + joblist: A list of job identifiers to be queried. + + Returns: + The return code of the status query, and a dictionary of job + identifiers to their status. - :param joblist: A list of job identifiers to be queried. - :returns: The return code of the status query, and a dictionary of job - identifiers to their status. """ pass @abstractmethod def cancel_jobs(self, joblist): - """ - For the given job list, cancel each job. + """For the given job list, cancel each job. + + Args: + joblist: A list of job identifiers to be cancelled. + + Returns: + The return code to indicate if jobs were cancelled. - :param joblist: A list of job identifiers to be cancelled. - :returns: The return code to indicate if jobs were cancelled. """ pass @abstractmethod def _write_script(self, ws_path, step): - """ - Write a script to the workspace of a workflow step. + """Write a script to the workspace of a workflow step. The job_map optional parameter is a map of workflow step names to job identifiers. This parameter so far is only planned to be used when a @@ -95,25 +104,33 @@ def _write_script(self, ws_path, step): the parameter may change depending on both future intended use and derived classes. - :param ws_path: Path to the workspace directory of the step. - :param step: An instance of a StudyStep. - :returns: Boolean value (True if the workflow step is to be scheduled, - False otherwise) and the path to the written script. + Args: + ws_path: Path to the workspace directory of the step. + step: An instance of a StudyStep. + + Returns: + Boolean value (True if the workflow step is to be scheduled, + False otherwise) and the path to the written script. + """ pass def write_script(self, ws_path, step): - """ - Generate the script for the specified StudyStep. + """Generate the script for the specified StudyStep. + + Args: + ws_path: Workspace path for the step. + step: An instance of a StudyStep class. + + Returns: + A tuple containing a boolean set to True if step should be + scheduled (False otherwise), path to the generate script, and path + to the generated restart script (None if step cannot be restarted). - :param ws_path: Workspace path for the step. - :param step: An instance of a StudyStep class. - :returns: A tuple containing a boolean set to True if step should be - scheduled (False otherwise), path to the generate script, and path - to the generated restart script (None if step cannot be restarted). """ - to_be_scheduled, script_path, restart_path = \ - self._write_script(ws_path, step) + to_be_scheduled, script_path, restart_path = self._write_script( + ws_path, step + ) st = os.stat(script_path) os.chmod(script_path, st.st_mode | stat.S_IXUSR) @@ -127,14 +144,15 @@ def write_script(self, ws_path, step): "Restart path: %s\n" "Scheduled?: %s\n" "---------------------------------\n", - script_path, restart_path, to_be_scheduled + script_path, + restart_path, + to_be_scheduled, ) return to_be_scheduled, script_path, restart_path @abstractmethod def submit(self, step, path, cwd, job_map=None, env=None): - """ - Submit a script to the scheduler. + """Submit a script to the scheduler. If cwd is specified, the submit method will operate outside of the path specified by the 'cwd' parameter. @@ -142,30 +160,42 @@ def submit(self, step, path, cwd, job_map=None, env=None): variables for submission to the specified values. The 'env' parameter should be a dictionary of environment variables. - :param step: An instance of a StudyStep. - :param path: Path to the script to be executed. - :param cwd: Path to the current working directory. - :param job_map: A map of workflow step names to their job identifiers. - :param env: A dict containing a modified environment for execution. - :returns: The return code of the submission command and job identiifer. + Args: + step: An instance of a StudyStep. + path: Path to the script to be executed. + cwd: Path to the current working directory. + job_map: A map of workflow step names to their job identifiers. + (Default value = None) + env: A dict containing a modified environment for execution. + (Default value = None) + + Returns: + The return code of the submission command and job identiifer. + """ pass @abstractproperty def extension(self): - """ - Returns the extension that generated scripts will use. + """Returns the extension that generated scripts will use. + + Args: + + Returns: + A string of the extension - :returns: A string of the extension """ pass @abstractproperty def key(self): """ - Return the key name for a ScriptAdapter.. - This is used to register the adapter in the ScriptAdapterFactory - and when writing the workflow specification. + Args: + + Returns: + This is used to register the adapter in the ScriptAdapterFactory + and when writing the workflow specification. + """ pass diff --git a/maestrowf/abstracts/specification.py b/maestrowf/abstracts/specification.py index d613f464f..85a776483 100644 --- a/maestrowf/abstracts/specification.py +++ b/maestrowf/abstracts/specification.py @@ -35,96 +35,122 @@ @six.add_metaclass(ABCMeta) class Specification: - """ - Abstract class for loading and verifying a Study Specification - """ + """Abstract class for loading and verifying a Study Specification""" @abstractclassmethod def load_specification(cls, path): - """ - Method for loading a study specification from a file. + """Method for loading a study specification from a file. + + Args: + path: Path to a study specification. + + Returns: + A specification object containing the information loaded + from path. - :param path: Path to a study specification. - :returns: A specification object containing the information loaded - from path. """ @abstractclassmethod def load_specification_from_stream(cls, stream): - """ - Method for loading a study specification from a stream. + """Method for loading a study specification from a stream. + + Args: + stream: Raw text stream containing specification data. + + Returns: + A specification object containing the information in string. - :param stream: Raw text stream containing specification data. - :returns: A specification object containing the information in string. """ @abstractmethod def verify(self): - """ - Verify the whole specification. - """ + """Verify the whole specification.""" @abstractmethod def get_study_environment(self): - """ - Generate a StudyEnvironment object from the environment in the spec. + """Generate a StudyEnvironment object from the environment in the spec. + + Args: + + Returns: + A StudyEnvironment object with the data in the specification. - :returns: A StudyEnvironment object with the data in the specification. """ @abstractmethod def get_parameters(self): - """ - Generate a ParameterGenerator object from the global parameters. + """Generate a ParameterGenerator object from the global parameters. + + Args: + + Returns: + A ParameterGenerator with data from the specification. - :returns: A ParameterGenerator with data from the specification. """ @abstractmethod def get_study_steps(self): - """ - Generate a list of StudySteps from the study in the specification. + """Generate a list of StudySteps from the study in the specification. + + Args: + + Returns: + A list of StudyStep objects. - :returns: A list of StudyStep objects. """ @abstractproperty def output_path(self): - """ - Return the OUTPUT_PATH variable (if it exists). + """Return the OUTPUT_PATH variable (if it exists). + + Args: + + Returns: + Returns OUTPUT_PATH if it exists, empty string otherwise. - :returns: Returns OUTPUT_PATH if it exists, empty string otherwise. """ @abstractproperty def name(self): - """ - Getter for the name of a study specification. + """Getter for the name of a study specification. + + Args: + + Returns: + The name of the study described by the specification. - :returns: The name of the study described by the specification. """ @name.setter def name(self, value): - """ - Setter for the name of a study specification. + """Setter for the name of a study specification. + + Args: + value: String value representing the new name. + + Returns: - :param value: String value representing the new name. """ @abstractproperty def desc(self): - """ - Getter for the description of a study specification. + """Getter for the description of a study specification. + + Args: + + Returns: + A string containing the description of the study + specification. - :returns: A string containing the description of the study - specification. """ @desc.setter def desc(self, value): - """ - Setter for the description of a study specification. + """Setter for the description of a study specification. + + Args: + value: String value representing the new description. + + Returns: - :param value: String value representing the new description. """ diff --git a/maestrowf/conductor.py b/maestrowf/conductor.py index 10fc32a02..897e6874c 100644 --- a/maestrowf/conductor.py +++ b/maestrowf/conductor.py @@ -48,21 +48,37 @@ LOGGER = logging.getLogger(__name__) # Formatting of logger. -LFORMAT = "%(asctime)s - %(name)s:%(funcName)s:%(lineno)s - " \ - "%(levelname)s - %(message)s" +LFORMAT = ( + "%(asctime)s - %(name)s:%(funcName)s:%(lineno)s - " + "%(levelname)s - %(message)s" +) + + +def setup_logging( + name, + output_path, + log_lvl=2, + log_path=None, + log_stdout=False, + log_format=None, +): + """Set up logging in the Main class. + + Args: + args: A Namespace object created by a parsed ArgumentParser. + name: The name of the log file. + output_path: + log_lvl: (Default value = 2) + log_path: (Default value = None) + log_stdout: (Default value = False) + log_format: (Default value = None) + + Returns: - -def setup_logging(name, output_path, log_lvl=2, log_path=None, - log_stdout=False, log_format=None): - """ - Set up logging in the Main class. - :param args: A Namespace object created by a parsed ArgumentParser. - :param name: The name of the log file. """ # Check if the user has specified a custom log path. if log_path: - LOGGER.info( - "Log path overwritten by command line -- %s", log_path) + LOGGER.info("Log path overwritten by command line -- %s", log_path) else: log_path = os.path.join(output_path, "logs") @@ -96,41 +112,71 @@ def setup_logging(name, output_path, log_lvl=2, log_path=None, def setup_parser(): - """ - Set up the Conductors's argument parser. + """Set up the Conductors's argument parser. + + Args: + + Returns: + A ArgumentParser that's initialized with the conductor's CLI. - :returns: A ArgumentParser that's initialized with the conductor's CLI. """ # Set up the parser for our conductor here. - parser = ArgumentParser(prog="Conductor", - description="An application for checking and " - "managing an ExecutionDAG within an executing " - "study.", - formatter_class=RawTextHelpFormatter) - - parser.add_argument("directory", type=str, help="The directory where " - "a study has been set up and where a pickle file " - "of an ExecutionGraph is stored.") - parser.add_argument("-s", "--status", action="store_true", - help="Check the status of the ExecutionGraph " - "located as specified by the 'directory' " - "argument.") - parser.add_argument("-l", "--logpath", type=str, - help="Alternate path to store program logging.") - parser.add_argument("-d", "--debug_lvl", type=int, default=2, - help="Level of logging messages to be output:\n" - "5 - Critical\n" - "4 - Error\n" - "3 - Warning\n" - "2 - Info (Default)\n" - "1 - Debug") - parser.add_argument("-c", "--logstdout", action="store_true", - help="Output logging to stdout in addition to a " - "file.") - parser.add_argument("-t", "--sleeptime", type=int, default=60, - help="Amount of time (in seconds) for the manager" - " to wait between job status checks.") + parser = ArgumentParser( + prog="Conductor", + description="An application for checking and " + "managing an ExecutionDAG within an executing " + "study.", + formatter_class=RawTextHelpFormatter, + ) + + parser.add_argument( + "directory", + type=str, + help="The directory where " + "a study has been set up and where a pickle file " + "of an ExecutionGraph is stored.", + ) + parser.add_argument( + "-s", + "--status", + action="store_true", + help="Check the status of the ExecutionGraph " + "located as specified by the 'directory' " + "argument.", + ) + parser.add_argument( + "-l", + "--logpath", + type=str, + help="Alternate path to store program logging.", + ) + parser.add_argument( + "-d", + "--debug_lvl", + type=int, + default=2, + help="Level of logging messages to be output:\n" + "5 - Critical\n" + "4 - Error\n" + "3 - Warning\n" + "2 - Info (Default)\n" + "1 - Debug", + ) + parser.add_argument( + "-c", + "--logstdout", + action="store_true", + help="Output logging to stdout in addition to a " "file.", + ) + parser.add_argument( + "-t", + "--sleeptime", + type=int, + default=60, + help="Amount of time (in seconds) for the manager" + " to wait between job status checks.", + ) return parser @@ -153,26 +199,37 @@ def __init__(self, study): @property def output_path(self): - """ - Return the path representing the root of the study workspace. + """Return the path representing the root of the study workspace. + + Args: + + Returns: + A string containing the path to the study's root. - :returns: A string containing the path to the study's root. """ return self._study.output_path @property def study_name(self): - """ - Return the name of the study this Conductor instance is managing. + """Return the name of the study this Conductor instance is managing. + + Args: + + Returns: + A string containing the name of the study. - :returns: A string containing the name of the study. """ return self._study.name @classmethod def store_study(cls, study): - """ - Store a Maestro study instance in a way the Conductor can read it. + """Store a Maestro study instance in a way the Conductor can read it. + + Args: + study: + + Returns: + """ # Pickle up the Study pkl_name = "{}{}".format(study.name, cls._pkl_extension) @@ -181,11 +238,14 @@ def store_study(cls, study): @classmethod def load_batch(cls, out_path): - """ - Load the batch information for the study rooted in 'out_path'. + """Load the batch information for the study rooted in 'out_path'. + + Args: + out_path: A string containing the path to a study root. + + Returns: + A dict containing the batch information for the study. - :param out_path: A string containing the path to a study root. - :returns: A dict containing the batch information for the study. """ batch_path = os.path.join(out_path, cls._batch_info) @@ -194,24 +254,29 @@ def load_batch(cls, out_path): LOGGER.error(msg) raise Exception(msg) - with open(batch_path, 'r') as data: + with open(batch_path, "r") as data: try: batch_info = yaml.load(data, yaml.FullLoader) except AttributeError: LOGGER.warning( "*** PyYAML is using an unsafe version with a known " "load vulnerability. Please upgrade your installation " - "to a more recent version! ***") + "to a more recent version! ***" + ) batch_info = yaml.load(data) return batch_info @classmethod def store_batch(cls, out_path, batch): - """ - Store the specified batch information to the study in 'out_path'. + """Store the specified batch information to the study in 'out_path'. + + Args: + out_path: A string containing the patht to a study root. + batch: + + Returns: - :param out_path: A string containing the patht to a study root. """ path = os.path.join(out_path, cls._batch_info) with open(path, "wb") as batch_info: @@ -219,28 +284,34 @@ def store_batch(cls, out_path, batch): @classmethod def load_study(cls, out_path): - """ - Load the Study instance in the study root specified by 'out_path'. + """Load the Study instance in the study root specified by 'out_path'. + + Args: + out_path: A string containing the patht to a study root. + + Returns: + A string containing the path to the study's root. - :param out_path: A string containing the patht to a study root. - :returns: A string containing the path to the study's root. """ - study_glob = \ - glob.glob(os.path.join(out_path, "*{}".format(cls._pkl_extension))) + study_glob = glob.glob( + os.path.join(out_path, "*{}".format(cls._pkl_extension)) + ) if len(study_glob) == 1: # We only expect one result.If we only get one, let's assume and # check after. path = study_glob[0] - with open(path, 'rb') as pkl: + with open(path, "rb") as pkl: obj = dill.load(pkl) if not isinstance(obj, Study): - msg = \ - "Object loaded from {path} is of type {type}. Expected " \ - "an object of type '{cls}.'" \ - .format(path=path, type=type(obj), cls=type(Study)) + msg = ( + "Object loaded from {path} is of type {type}. Expected " + "an object of type '{cls}.'".format( + path=path, type=type(obj), cls=type(Study) + ) + ) LOGGER.error(msg) raise TypeError(msg) else: @@ -257,11 +328,15 @@ def load_study(cls, out_path): @classmethod def get_status(cls, output_path): - """ - Retrieve the status of the study rooted at 'out_path'. + """Retrieve the status of the study rooted at 'out_path'. + + Args: + out_path: A string containing the patht to a study root. + output_path: + + Returns: + A dictionary containing the status of the study. - :param out_path: A string containing the patht to a study root. - :returns: A dictionary containing the status of the study. """ stat_path = os.path.join(output_path, "status.csv") lock_path = os.path.join(output_path, ".status.lock") @@ -279,23 +354,30 @@ def get_status(cls, output_path): @classmethod def mark_cancelled(cls, output_path): - """ - Mark the study rooted at 'out_path'. + """Mark the study rooted at 'out_path'. + + Args: + out_path: A string containing the patht to a study root. + output_path: + + Returns: + A dictionary containing the status of the study. - :param out_path: A string containing the patht to a study root. - :returns: A dictionary containing the status of the study. """ lock_path = make_safe_path(output_path, cls._cancel_lock) - with open(lock_path, 'a'): + with open(lock_path, "a"): os.utime(lock_path, None) def initialize(self, batch_info, sleeptime=60): - """ - Initializes the Conductor instance based on the stored study. + """Initializes the Conductor instance based on the stored study. + + Args: + batch_info: A dict containing batch information. + sleeptime: The amount of sleep time between polling loops + [Default: 60s]. + + Returns: - :param batch_info: A dict containing batch information. - :param sleeptime: The amount of sleep time between polling loops - [Default: 60s]. """ # Set our conductor's sleep time. self.sleep_time = sleeptime @@ -309,18 +391,21 @@ def initialize(self, batch_info, sleeptime=60): def monitor_study(self): """Monitor a running study.""" if not self._setup: - msg = \ - "Study '{}' located in '{}' not initialized. Initialize " \ - "study before calling launching. Aborting." \ - .format(self.study_name, self.output_path) + msg = ( + "Study '{}' located in '{}' not initialized. Initialize " + "study before calling launching. Aborting.".format( + self.study_name, self.output_path + ) + ) LOGGER.error(msg) raise Exception(msg) # Set some fixed variables that monitor will use. cancel_lock_path = make_safe_path(self.output_path, self._cancel_lock) dag = self._exec_dag - pkl_path = \ - os.path.join(self._pkl_path, "{}.pkl".format(self._study.name)) + pkl_path = os.path.join( + self._pkl_path, "{}.pkl".format(self._study.name) + ) sleep_time = self.sleep_time LOGGER.debug( @@ -329,7 +414,10 @@ def monitor_study(self): "cancel path = %s\n" "sleep time = %s\n" "------------------------------------------\n", - pkl_path, cancel_lock_path, sleep_time) + pkl_path, + cancel_lock_path, + sleep_time, + ) completion_status = StudyStatus.RUNNING while completion_status == StudyStatus.RUNNING: @@ -361,6 +449,7 @@ def monitor_study(self): return completion_status def cleanup(self): + """ """ self._exec_dag.cleanup() @@ -373,8 +462,13 @@ def main(): parser = setup_parser() args = parser.parse_args() study = Conductor.load_study(args.directory) - setup_logging(study.name, args.directory, args.debug_lvl, - args.logpath, args.logstdout) + setup_logging( + study.name, + args.directory, + args.debug_lvl, + args.logpath, + args.logstdout, + ) batch_info = Conductor.load_batch(args.directory) conductor = Conductor(study) diff --git a/maestrowf/datastructures/core/__init__.py b/maestrowf/datastructures/core/__init__.py index c638d6b44..404e2c79e 100644 --- a/maestrowf/datastructures/core/__init__.py +++ b/maestrowf/datastructures/core/__init__.py @@ -46,10 +46,18 @@ """ from maestrowf.datastructures.core.executiongraph import ExecutionGraph -from maestrowf.datastructures.core.parameters import Combination, \ - ParameterGenerator +from maestrowf.datastructures.core.parameters import ( + Combination, + ParameterGenerator, +) from maestrowf.datastructures.core.study import Study, StudyStep from maestrowf.datastructures.core.studyenvironment import StudyEnvironment -__all__ = ("Combination", "ExecutionGraph", "ParameterGenerator", "Study", - "StudyEnvironment", "StudyStep") +__all__ = ( + "Combination", + "ExecutionGraph", + "ParameterGenerator", + "Study", + "StudyEnvironment", + "StudyStep", +) diff --git a/maestrowf/datastructures/core/executiongraph.py b/maestrowf/datastructures/core/executiongraph.py index 5fea2d68e..ab671fdb1 100644 --- a/maestrowf/datastructures/core/executiongraph.py +++ b/maestrowf/datastructures/core/executiongraph.py @@ -9,26 +9,38 @@ from filelock import FileLock, Timeout from maestrowf.abstracts import PickleInterface -from maestrowf.abstracts.enums import JobStatusCode, State, SubmissionCode, \ - CancelCode, StudyStatus +from maestrowf.abstracts.enums import ( + JobStatusCode, + State, + SubmissionCode, + CancelCode, + StudyStatus, +) from maestrowf.datastructures.dag import DAG from maestrowf.datastructures.environment import Variable from maestrowf.interfaces import ScriptAdapterFactory -from maestrowf.utils import create_parentdir, get_duration, \ - round_datetime_seconds +from maestrowf.utils import ( + create_parentdir, + get_duration, + round_datetime_seconds, +) LOGGER = logging.getLogger(__name__) SOURCE = "_source" class _StepRecord: - """ - A simple container object representing a workflow step record. + """A simple container object representing a workflow step record. The record contains all information used to generate associated scripts, and settings for execution of the record. The StepRecord is a utility class to the ExecutionGraph and maintains all information for any given step in the DAG. + + Args: + + Returns: + """ def __init__(self, workspace, step, **kwargs): @@ -70,12 +82,15 @@ def setup_workspace(self): create_parentdir(self.workspace.value) def generate_script(self, adapter, tmp_dir=""): - """ - Generate the script for executing the workflow step. + """Generate the script for executing the workflow step. + + Args: + adapter: Instance of adapter to be used for script generation. + tmp_dir: If specified, place generated script in the specified + temp directory. (Default value = "") + + Returns: - :param adapter: Instance of adapter to be used for script generation. - :param tmp_dir: If specified, place generated script in the specified - temp directory. """ if tmp_dir: scr_dir = tmp_dir @@ -85,12 +100,27 @@ def generate_script(self, adapter, tmp_dir=""): self.step.run["cmd"] = self.workspace.substitute(self.step.run["cmd"]) LOGGER.info("Generating script for %s into %s", self.name, scr_dir) - self.to_be_scheduled, self.script, self.restart_script = \ - adapter.write_script(scr_dir, self.step) - LOGGER.info("Script: %s\nRestart: %s\nScheduled?: %s", - self.script, self.restart_script, self.to_be_scheduled) + ( + self.to_be_scheduled, + self.script, + self.restart_script, + ) = adapter.write_script(scr_dir, self.step) + LOGGER.info( + "Script: %s\nRestart: %s\nScheduled?: %s", + self.script, + self.restart_script, + self.to_be_scheduled, + ) def execute(self, adapter): + """ + + Args: + adapter: + + Returns: + + """ self.mark_submitted() retcode, jobid = self._execute(adapter, self.script) @@ -100,6 +130,14 @@ def execute(self, adapter): return retcode def restart(self, adapter): + """ + + Args: + adapter: + + Returns: + + """ retcode, jobid = self._execute(adapter, self.restart_script) if retcode == SubmissionCode.OK: @@ -109,10 +147,13 @@ def restart(self, adapter): @property def can_restart(self): - """ - Get whether or not the record can be restarted. + """Get whether or not the record can be restarted. + + Args: + + Returns: + True if the record has a restart command assigned to it. - :returns: True if the record has a restart command assigned to it. """ if self.restart_script: return True @@ -120,14 +161,21 @@ def can_restart(self): return False def _execute(self, adapter, script): + """ + + Args: + adapter: + script: + + Returns: + + """ if self.to_be_scheduled: - srecord = adapter.submit( - self.step, script, self.workspace.value) + srecord = adapter.submit(self.step, script, self.workspace.value) else: self.mark_running() ladapter = ScriptAdapterFactory.get_adapter("local")() - srecord = ladapter.submit( - self.step, script, self.workspace.value) + srecord = ladapter.submit(self.step, script, self.workspace.value) retcode = srecord.submission_code jobid = srecord.job_identifier @@ -138,14 +186,16 @@ def mark_submitted(self): LOGGER.debug( "Marking %s as submitted (PENDING) -- previously %s", self.name, - self.status) + self.status, + ) self.status = State.PENDING if not self._submit_time: self._submit_time = round_datetime_seconds(datetime.now()) else: LOGGER.warning( "Cannot set the submission time of '%s' because it has " - "already been set.", self.name + "already been set.", + self.name, ) def mark_running(self): @@ -153,22 +203,27 @@ def mark_running(self): LOGGER.debug( "Marking %s as running (RUNNING) -- previously %s", self.name, - self.status) + self.status, + ) self.status = State.RUNNING if not self._start_time: self._start_time = round_datetime_seconds(datetime.now()) def mark_end(self, state): - """ - Mark the end time of the record with associated termination state. + """Mark the end time of the record with associated termination state. + + Args: + state: State enum corresponding to termination state. + + Returns: - :param state: State enum corresponding to termination state. """ LOGGER.debug( "Marking %s as finished (%s) -- previously %s", self.name, state, - self.status) + self.status, + ) self.status = state if not self._end_time: self._end_time = round_datetime_seconds(datetime.now()) @@ -178,12 +233,12 @@ def mark_restart(self): LOGGER.debug( "Marking %s as restarting (TIMEOUT) -- previously %s", self.name, - self.status) + self.status, + ) self.status = State.TIMEDOUT # Designating a restart limit of zero as an unlimited restart setting. # Otherwise, if we're less than restart limit, attempt another restart. - if self.restart_limit == 0 or \ - self._num_restarts < self.restart_limit: + if self.restart_limit == 0 or self._num_restarts < self.restart_limit: self._num_restarts += 1 return True else: @@ -191,7 +246,7 @@ def mark_restart(self): @property def is_local_step(self): - """Return whether or not this step executes locally.""" + """ """ return not self.to_be_scheduled @property @@ -208,10 +263,13 @@ def elapsed_time(self): @property def run_time(self): - """ - Compute the run time of a record (includes restart queue time). + """Compute the run time of a record (includes restart queue time). + + Args: + + Returns: + A string of the records's run time. - :returns: A string of the records's run time. """ if self._start_time and self._end_time: # If start and end time is set -- calculate run time. @@ -225,28 +283,37 @@ def run_time(self): @property def name(self): - """ - Get the name of the step represented by the record instance. + """Get the name of the step represented by the record instance. + + Args: + + Returns: + The name of the StudyStep contained within the record. - :returns: The name of the StudyStep contained within the record. """ return self.step.real_name @property def walltime(self): - """ - Get the requested wall time of the record instance. + """Get the requested wall time of the record instance. + + Args: + + Returns: + A string representing the requested computing time. - :returns: A string representing the requested computing time. """ return self.step.run["walltime"] @property def time_submitted(self): - """ - Get the time the step started. + """Get the time the step started. + + Args: + + Returns: + A formatted string of the date and time the step started. - :returns: A formatted string of the date and time the step started. """ if self._submit_time: return str(self._submit_time) @@ -255,10 +322,13 @@ def time_submitted(self): @property def time_start(self): - """ - Get the time the step started. + """Get the time the step started. + + Args: + + Returns: + A formatted string of the date and time the step started. - :returns: A formatted string of the date and time the step started. """ if self._start_time: return str(self._start_time) @@ -267,10 +337,13 @@ def time_start(self): @property def time_end(self): - """ - Get the time the step ended. + """Get the time the step ended. + + Args: + + Returns: + A formatted string of the date and time the step ended. - :returns: A formatted string of the date and time the step ended. """ if self._end_time: return str(self._end_time) @@ -279,17 +352,19 @@ def time_end(self): @property def restarts(self): - """ - Get the number of restarts the step has executed. + """Get the number of restarts the step has executed. + + Args: + + Returns: + An int representing the number of restarts. - :returns: An int representing the number of restarts. """ return self._num_restarts class ExecutionGraph(DAG, PickleInterface): - """ - Datastructure that tracks, executes, and reports on study execution. + """Datastructure that tracks, executes, and reports on study execution. The ExecutionGraph is used to manage, monitor, and interact with tasks and the scheduler. This class searches its graph for tasks that are ready to @@ -300,10 +375,20 @@ class ExecutionGraph(DAG, PickleInterface): should go. Essentially, if logic is needed to automatically manipulate the workflow in some fashion or additional monitoring is needed, this class is where that would go. + + Args: + + Returns: + """ - def __init__(self, submission_attempts=1, submission_throttle=0, - use_tmp=False, dry_run=False): + def __init__( + self, + submission_attempts=1, + submission_throttle=0, + use_tmp=False, + dry_run=False, + ): """ Initialize a new instance of an ExecutionGraph. @@ -352,20 +437,27 @@ def __init__(self, submission_attempts=1, submission_throttle=0, "Use temporary directory = %s\n" "Tmp Dir = %s\n" "------------------------------------------", - submission_attempts, submission_throttle, use_tmp, self._tmp_dir + submission_attempts, + submission_throttle, + use_tmp, + self._tmp_dir, ) # Error check that the submission values are valid. if self._submission_attempts < 1: - _msg = "Submission attempts should always be greater than 0. " \ - "Received a value of {}.".format(self._submission_attempts) + _msg = ( + "Submission attempts should always be greater than 0. " + "Received a value of {}.".format(self._submission_attempts) + ) LOGGER.error(_msg) raise ValueError(_msg) if self._submission_throttle < 0: - _msg = "Throttling should be 0 for unthrottled or a positive " \ - "integer for the number of allowed inflight jobs. " \ - "Received a value of {}.".format(self._submission_throttle) + _msg = ( + "Throttling should be 0 for unthrottled or a positive " + "integer for the number of allowed inflight jobs. " + "Received a value of {}.".format(self._submission_throttle) + ) LOGGER.error(_msg) raise ValueError(_msg) @@ -377,39 +469,48 @@ def _check_tmp_dir(self): self._tmp_dir = tempfile.mkdtemp() def add_step(self, name, step, workspace, restart_limit): - """ - Add a StepRecord to the ExecutionGraph. + """Add a StepRecord to the ExecutionGraph. + + Args: + name: Name of the step to be added. + step: StudyStep instance to be recorded. + workspace: Directory path for the step's working directory. + restart_limit: Upper limit on the number of restart attempts. + + Returns: - :param name: Name of the step to be added. - :param step: StudyStep instance to be recorded. - :param workspace: Directory path for the step's working directory. - :param restart_limit: Upper limit on the number of restart attempts. """ data = { - "step": step, - "state": State.INITIALIZED, - "workspace": workspace, - "restart_limit": restart_limit, - } + "step": step, + "state": State.INITIALIZED, + "workspace": workspace, + "restart_limit": restart_limit, + } record = _StepRecord(**data) self._dependencies[name] = set() super(ExecutionGraph, self).add_node(name, record) def add_connection(self, parent, step): - """ - Add a connection between two steps in the ExecutionGraph. + """Add a connection between two steps in the ExecutionGraph. + + Args: + parent: The parent step that is required to execute 'step' + step: The dependent step that relies on parent. + + Returns: - :param parent: The parent step that is required to execute 'step' - :param step: The dependent step that relies on parent. """ self.add_edge(parent, step) self._dependencies[step].add(parent) def set_adapter(self, adapter): - """ - Set the adapter used to interface for scheduling tasks. + """Set the adapter used to interface for scheduling tasks. + + Args: + adapter: Adapter name to be used when launching the graph. + + Returns: - :param adapter: Adapter name to be used when launching the graph. """ if not adapter: # If we have no adapter specified, assume sequential execution. @@ -423,19 +524,24 @@ def set_adapter(self, adapter): # Check to see that the adapter type is something the if adapter["type"] not in ScriptAdapterFactory.get_valid_adapters(): - msg = "'{}' adapter must be specfied in ScriptAdapterFactory." \ - .format(adapter) + msg = \ + "'{}' adapter must be specfied in ScriptAdapterFactory." \ + .format(adapter) LOGGER.error(msg) raise TypeError(msg) self._adapter = adapter def add_description(self, name, description, **kwargs): - """ - Add a study description to the ExecutionGraph instance. + """Add a study description to the ExecutionGraph instance. + + Args: + name: Name of the study. + description: Description of the study. + **kwargs: + + Returns: - :param name: Name of the study. - :param description: Description of the study. """ self._description["name"] = name self._description["description"] = description @@ -443,65 +549,86 @@ def add_description(self, name, description, **kwargs): @property def name(self): - """ - Return the name for the study in the ExecutionGraph instance. + """Return the name for the study in the ExecutionGraph instance. + + Args: + + Returns: + A string of the name of the study. - :returns: A string of the name of the study. """ return self._description["name"] @name.setter def name(self, value): - """ - Set the name for the study in the ExecutionGraph instance. + """Set the name for the study in the ExecutionGraph instance. + + Args: + name: A string of the name for the study. + value: + + Returns: - :param name: A string of the name for the study. """ self._description["name"] = value @property def description(self): - """ - Return the description for the study in the ExecutionGraph instance. + """Return the description for the study in the ExecutionGraph instance. + + Args: + + Returns: + A string of the description for the study. - :returns: A string of the description for the study. """ return self._description["description"] @description.setter def description(self, value): - """ - Set the description for the study in the ExecutionGraph instance. + """Set the description for the study in the ExecutionGraph instance. + + Args: + value: A string of the description for the study. + + Returns: - :param value: A string of the description for the study. """ self._description["description"] = value def log_description(self): """Log the description of the ExecutionGraph.""" - desc = ["{}: {}".format(key, value) - for key, value in self._description.items()] + desc = [ + "{}: {}".format(key, value) + for key, value in self._description.items() + ] desc = "\n".join(desc) LOGGER.info( "\n==================================================\n" "%s\n" "==================================================\n", - desc + desc, ) def generate_scripts(self): - """ - Generate the scripts for all steps in the ExecutionGraph. + """Generate the scripts for all steps in the ExecutionGraph. The generate_scripts method scans the ExecutionGraph instance and uses the stored adapter to write executable scripts for either local or scheduled execution. If a restart command is specified, a restart script will be generated for that record. + + Args: + + Returns: + """ # An adapter must be specified if not self._adapter: - msg = "Adapter not found. Specify a ScriptAdapter using " \ - "set_adapter." + msg = ( + "Adapter not found. Specify a ScriptAdapter using " + "set_adapter." + ) LOGGER.error(msg) raise ValueError(msg) @@ -520,19 +647,23 @@ def generate_scripts(self): record.generate_script(adapter, self._tmp_dir) def _execute_record(self, record, adapter, restart=False): - """ - Execute a StepRecord. + """Execute a StepRecord. - :param record: The StepRecord to be executed. - :param adapter: An instance of the adapter to be used for cluster + Args: + record: The StepRecord to be executed. + adapter: An instance of the adapter to be used for cluster submission. - :param restart: True if the record needs restarting, False otherwise. + restart: True if the record needs restarting, False otherwise. + (Default value = False) + + Returns: + """ # Logging for debugging. LOGGER.info("Calling execute for StepRecord '%s'", record.name) - num_restarts = 0 # Times this step has temporally restarted. - retcode = None # Execution return code. + num_restarts = 0 # Times this step has temporally restarted. + retcode = None # Execution return code. # While our submission needs to be submitted, keep trying: # 1. If the JobStatus is not OK. @@ -541,10 +672,13 @@ def _execute_record(self, record, adapter, restart=False): # Only set up the workspace the initial iteration. if not restart: - LOGGER.debug("Setting up workspace for '%s' at %s", - record.name, str(datetime.now())) + LOGGER.debug( + "Setting up workspace for '%s' at %s", + record.name, + str(datetime.now()), + ) # Generate the script for execution on the fly. - record.setup_workspace() # Generate the workspace. + record.setup_workspace() # Generate the workspace. record.generate_script(adapter, self._tmp_dir) if self.dry_run: @@ -552,22 +686,33 @@ def _execute_record(self, record, adapter, restart=False): self.completed_steps.add(record.name) return - while retcode != SubmissionCode.OK and \ - num_restarts < self._submission_attempts: - LOGGER.info("Attempting submission of '%s' (attempt %d of %d)...", - record.name, num_restarts + 1, - self._submission_attempts) + while ( + retcode != SubmissionCode.OK + and num_restarts < self._submission_attempts + ): + LOGGER.info( + "Attempting submission of '%s' (attempt %d of %d)...", + record.name, + num_restarts + 1, + self._submission_attempts, + ) # We're not restarting -- submit as usual. if not restart: - LOGGER.debug("Calling 'execute' on '%s' at %s", - record.name, str(datetime.now())) + LOGGER.debug( + "Calling 'execute' on '%s' at %s", + record.name, + str(datetime.now()), + ) retcode = record.execute(adapter) # Otherwise, it's a restart. else: # If the restart is specified, use the record restart script. - LOGGER.debug("Calling 'restart' on '%s' at %s", - record.name, str(datetime.now())) + LOGGER.debug( + "Calling 'restart' on '%s' at %s", + record.name, + str(datetime.now()), + ) # Generate the script for execution on the fly. record.generate_script(adapter, self._tmp_dir) retcode = record.restart(adapter) @@ -580,29 +725,44 @@ def _execute_record(self, record, adapter, restart=False): self.in_progress.add(record.name) if record.is_local_step: - LOGGER.info("Local step %s executed with status OK. Complete.", - record.name) + LOGGER.info( + "Local step %s executed with status OK. Complete.", + record.name, + ) record.mark_end(State.FINISHED) self.completed_steps.add(record.name) self.in_progress.remove(record.name) else: # Find the subtree, because anything dependent on this step now # failed. - LOGGER.warning("'%s' failed to submit properly. " - "Step failed.", record.name) + LOGGER.warning( + "'%s' failed to submit properly. " "Step failed.", record.name + ) path, parent = self.bfs_subtree(record.name) for node in path: self.failed_steps.add(node) self.values[node].mark_end(State.FAILED) # After execution state debug logging. - LOGGER.debug("After execution of '%s' -- New state is %s.", - record.name, record.status) + LOGGER.debug( + "After execution of '%s' -- New state is %s.", + record.name, + record.status, + ) def write_status(self, path): - """Write the status of the DAG to a CSV file.""" - header = "Step Name,Job ID,Workspace,State,Run Time,Elapsed Time," \ - "Start Time,Submit Time,End Time,Number Restarts" + """Write the status of the DAG to a CSV file. + + Args: + path: + + Returns: + + """ + header = ( + "Step Name,Job ID,Workspace,State,Run Time,Elapsed Time," + "Start Time,Submit Time,End Time,Number Restarts" + ) status = [header] keys = set(self.values.keys()) - set(["_source"]) for key in keys: @@ -613,12 +773,17 @@ def write_status(self, path): jobid_str = str(value.jobid[-1]) _ = [ - value.name, jobid_str, - os.path.split(value.workspace.value)[1], - str(value.status.name), value.run_time, value.elapsed_time, - value.time_start, value.time_submitted, value.time_end, - str(value.restarts) - ] + value.name, + jobid_str, + os.path.split(value.workspace.value)[1], + str(value.status.name), + value.run_time, + value.elapsed_time, + value.time_start, + value.time_submitted, + value.time_end, + str(value.restarts), + ] _ = ",".join(_) status.append(_) @@ -633,14 +798,16 @@ def write_status(self, path): pass def _check_study_completion(self): + """ """ # We cancelled, return True marking study as complete. if self.is_canceled: LOGGER.info("Cancelled -- completing study.") return StudyStatus.CANCELLED # check for completion of all steps - resolved_set = \ + resolved_set = ( self.completed_steps | self.failed_steps | self.cancelled_steps + ) if not set(self.values.keys()) - resolved_set: # some steps were cancelled and is_canceled wasn't set if len(self.cancelled_steps) > 0: @@ -649,8 +816,9 @@ def _check_study_completion(self): # some steps were failures indicating failure if len(self.failed_steps) > 0: - logging.info("'%s' is complete with failures. Returning.", - self.name) + logging.info( + "'%s' is complete with failures. Returning.", self.name + ) return StudyStatus.FAILURE # everything completed were are done @@ -660,8 +828,7 @@ def _check_study_completion(self): return StudyStatus.RUNNING def execute_ready_steps(self): - """ - Execute any steps whose dependencies are satisfied. + """Execute any steps whose dependencies are satisfied. The 'execute_ready_steps' method is the core of how the ExecutionGraph manages execution. This method does the following: @@ -672,7 +839,11 @@ def execute_ready_steps(self): based on satisfied dependencies and executes steps whose dependencies are met. - :returns: True if the study has completed, False otherwise. + Args: + + Returns: + True if the study has completed, False otherwise. + """ # TODO: We may want to move this to a singleton somewhere # so we can guarantee that all steps use the same adapter. @@ -707,8 +878,11 @@ def execute_ready_steps(self): if status == State.FINISHED: # Mark the step complete and notate its end time. record.mark_end(State.FINISHED) - LOGGER.info("Step '%s' marked as finished. Adding to " - "complete set.", name) + LOGGER.info( + "Step '%s' marked as finished. Adding to " + "complete set.", + name, + ) self.completed_steps.add(name) self.in_progress.remove(name) @@ -725,22 +899,29 @@ def execute_ready_steps(self): if record.mark_restart(): LOGGER.info( "Step '%s' timed out. Restarting (%s of %s).", - name, record.restarts, record.restart_limit + name, + record.restarts, + record.restart_limit, ) self._execute_record(record, adapter, restart=True) else: - LOGGER.info("'%s' has been restarted %s of %s " - "times. Marking step and all " - "descendents as failed.", - name, - record.restarts, - record.restart_limit) + LOGGER.info( + "'%s' has been restarted %s of %s " + "times. Marking step and all " + "descendents as failed.", + name, + record.restarts, + record.restart_limit, + ) self.in_progress.remove(name) cleanup_steps.update(self.bfs_subtree(name)[0]) # Otherwise, we can't restart so mark the step timed out. else: - LOGGER.info("'%s' timed out, but cannot be restarted." - " Marked as TIMEDOUT.", name) + LOGGER.info( + "'%s' timed out, but cannot be restarted." + " Marked as TIMEDOUT.", + name, + ) # Mark that the step ended due to TIMEOUT. record.mark_end(State.TIMEDOUT) # Remove from in progress since it no longer is. @@ -757,8 +938,11 @@ def execute_ready_steps(self): # TODO: Need to make sure that we do this a finite number # of times. # Resubmit the cmd. - LOGGER.warning("Hardware failure detected. Attempting to " - "resubmit step '%s'.", name) + LOGGER.warning( + "Hardware failure detected. Attempting to " + "resubmit step '%s'.", + name, + ) # We can just let the logic below handle submission with # everything else. self.ready_steps.append(name) @@ -767,7 +951,7 @@ def execute_ready_steps(self): LOGGER.warning( "Job failure reported. Aborting %s -- flagging all " "dependent jobs as failed.", - name + name, ) self.in_progress.remove(name) record.mark_end(State.FAILED) @@ -779,7 +963,9 @@ def execute_ready_steps(self): "Step '%s' found in UNKNOWN state. Step was found " "in '%s' state previously, marking as UNKNOWN. " "Adding to failed steps.", - name, record.status) + name, + record.status, + ) cleanup_steps.update(self.bfs_subtree(name)[0]) self.in_progress.remove(name) @@ -811,22 +997,28 @@ def execute_ready_steps(self): # If the record is only INITIALIZED, we have encountered a step # that needs consideration. if record.status == State.INITIALIZED: - LOGGER.debug("'%s' found to be initialized. Checking " - "dependencies. ", key) + LOGGER.debug( + "'%s' found to be initialized. Checking " "dependencies. ", + key, + ) LOGGER.debug( - "Unfulfilled dependencies: %s", - self._dependencies[key]) + "Unfulfilled dependencies: %s", self._dependencies[key] + ) s_completed = filter( lambda x: x in self.completed_steps, - self._dependencies[key]) - self._dependencies[key] = \ - self._dependencies[key] - set(s_completed) + self._dependencies[key], + ) + self._dependencies[key] = self._dependencies[key] - set( + s_completed + ) LOGGER.debug( "Completed dependencies: %s\n" "Remaining dependencies: %s", - s_completed, self._dependencies[key]) + s_completed, + self._dependencies[key], + ) # If the gating dependencies set is empty, we can execute. if not self._dependencies[key]: @@ -874,12 +1066,16 @@ def execute_ready_steps(self): return completion_status def check_study_status(self): - """ - Check the status of currently executing steps in the graph. + """Check the status of currently executing steps in the graph. This method is used to check the status of all currently in progress steps in the ExecutionGraph. Each ExecutionGraph stores the adapter used to generate and execute its scripts. + + Args: + + Returns: + """ # Set up the job list and the map to get back to step names. joblist = [] @@ -895,8 +1091,9 @@ def check_study_status(self): # Use the adapter to grab the job statuses. retcode, job_status = adapter.check_jobs(joblist) # Map the job identifiers back to step names. - step_status = {jobmap[jobid]: status - for jobid, status in job_status.items()} + step_status = { + jobmap[jobid]: status for jobid, status in job_status.items() + } # Based on return code, log something different. if retcode == JobStatusCode.OK: @@ -929,7 +1126,8 @@ def cancel_study(self): LOGGER.info("Successfully requested to cancel all jobs.") elif crecord.cancel_status == CancelCode.ERROR: LOGGER.error( - "Failed to cancel jobs. (Code = %s)", crecord.return_code) + "Failed to cancel jobs. (Code = %s)", crecord.return_code + ) else: LOGGER.error("Unknown Error (Code = %s)", crecord.return_code) diff --git a/maestrowf/datastructures/core/parameters.py b/maestrowf/datastructures/core/parameters.py index 76c6ec527..c6b73d7b1 100644 --- a/maestrowf/datastructures/core/parameters.py +++ b/maestrowf/datastructures/core/parameters.py @@ -43,13 +43,16 @@ class Combination(object): - """ - Class representing a combination of parameters. + """Class representing a combination of parameters. This class represents a combination of parameters generated by a class of type ParameterGenerator. The only time a user should ever get an instance of a Combination from the ParameterGenerator is when a combination of - parameters is VALID. + + Args: + + Returns: + """ def __init__(self, token="$"): @@ -82,29 +85,34 @@ def __init__(self, token="$"): self._token = token def add(self, key, name, value, label): - """ - Add a parameter to the Combination object. + """Add a parameter to the Combination object. + + Args: + key: Parameter key that identifies a replacement. + name: Custom name that identifies a parameter. + value: Value of the parameter in this combination. + label: Value of the parameter label for this combination. + + Returns: - :param key: Parameter key that identifies a replacement. - :param name: Custom name that identifies a parameter. - :param value: Value of the parameter in this combination. - :param label: Value of the parameter label for this combination. """ # For the combination being added, assign the expected parameterized # strings that the user would substitute in for. # Parameterized value: () - logger.debug("Adding parameter value to Combination with args: %s", - [key, name, value, label]) + logger.debug( + "Adding parameter value to Combination with args: %s", + [key, name, value, label], + ) var = "{}({})".format(self._token, key) - logger.debug('Parameter value: %s = %s', var, value) + logger.debug("Parameter value: %s = %s", var, value) self._params[var] = value # Parameterized label: (.label) var = "{}({}.label)".format(self._token, key) - logger.debug('Label value: %s = %s', var, label) + logger.debug("Label value: %s = %s", var, label) self._labels[var] = label # Parameterized name: (.name) var = "{}({}.name)".format(self._token, key) - logger.debug('Name value: %s = %s', var, name) + logger.debug("Name value: %s = %s", var, name) self._names[var] = name def __str__(self): @@ -116,11 +124,14 @@ def __str__(self): return ".".join(self._labels.values()) def get_param_string(self, params): - """ - Get the combination string for the specified parameters. + """Get the combination string for the specified parameters. + + Args: + params: A set of parameters to be used in the string. + + Returns: + A string containing the labels for the parameters in params. - :param params: A set of parameters to be used in the string. - :returns: A string containing the labels for the parameters in params. """ combo_str = [] for item in sorted(params): @@ -130,11 +141,14 @@ def get_param_string(self, params): return ".".join(combo_str) def apply(self, item): - """ - Apply the combination to an item. + """Apply the combination to an item. + + Args: + item: String that may contain parameters to be substituted. + + Returns: + String equal to item, except with parameters replaced. - :param item: String that may contain parameters to be substituted. - :returns: String equal to item, except with parameters replaced. """ # Apply the Combination's labels to the item. # These are substrings within item that are represented by the format @@ -161,8 +175,7 @@ def apply(self, item): class ParameterGenerator: - """ - Class for containing parameters and generating combinations. + """Class for containing parameters and generating combinations. The goal of this class is to provide one centralized location for managing and storing parameters. This implementation of the ParameterGenerator, @@ -200,6 +213,11 @@ class ParameterGenerator: the new ParameterGenerator and add parameters. 3. Setup, stage, and execute your study. 4. Profit. + + Args: + + Returns: + """ def __init__(self, token="$", ltoken="%%"): @@ -233,18 +251,22 @@ def __init__(self, token="$", ltoken="%%"): self.length = 0 def add_parameter(self, key, values, label=None, name=None): - """ - Add a parameter to the ParameterGenerator. + """Add a parameter to the ParameterGenerator. Currently, all parameters added to a ParameterGenerator instance must have a list of values that are the same length. Future improvements will add the ability to specify either types of parameters or provide different ParameterGenerators derivations that have unique behavior. - :param key: Parameter key to find for replacement. - :param values: List of values the parameter can take. - :param label: Label string for labeling the parameter. - :param name: Custom name for identifying parameter. + Args: + key: Parameter key to find for replacement. + values: List of values the parameter can take. + label: Label string for labeling the parameter. + (Default value = None) + name: Custom name for identifying parameter. (Default value = None) + + Returns: + """ if key in self.parameters: logger.warning("'%s' already in parameter set. Overriding.", key) @@ -254,10 +276,13 @@ def add_parameter(self, key, values, label=None, name=None): self.length = len(values) elif len(values) != self.length: - error = "Length of values list must be the same size as " \ - "the other parameters that exist in the " \ - "generators. Length of '{}' is {}. Aborting." \ - .format(name, len(values)) + error = ( + "Length of values list must be the same size as " + "the other parameters that exist in the " + "generators. Length of '{}' is {}. Aborting.".format( + name, len(values) + ) + ) logger.exception(error) raise ValueError(error) @@ -291,10 +316,13 @@ def __bool__(self): __nonzero__ = __bool__ def get_combinations(self): - """ - Generate all combinations of parameters. + """Generate all combinations of parameters. + + Args: + + Returns: + A generator with all combinations of parameters. - :returns: A generator with all combinations of parameters. """ for i in range(0, self.length): combo = Combination() @@ -303,18 +331,22 @@ def get_combinations(self): if isinstance(self.labels[key], list): tlabel = self.labels[key][i] else: - tlabel = self.labels[key].replace(self.label_token, - str(pvalue)) + tlabel = self.labels[key].replace( + self.label_token, str(pvalue) + ) name = self.names[key] combo.add(key, name, pvalue, tlabel) yield combo def _get_used_parameters(self, item, params): - """ - Find the parameters used by an item in a StudyStep. + """Find the parameters used by an item in a StudyStep. + + Args: + item: The item to search for parameters. + params: The current set of found parameters. + + Returns: - :param item: The item to search for parameters. - :param params: The current set of found parameters. """ if not item: return @@ -333,27 +365,35 @@ def _get_used_parameters(self, item, params): for each in item.values(): self._get_used_parameters(each, params) else: - msg = "Encountered an object of type '{}'. Expected a str, list," \ - " int, or dict.".format(type(item)) + msg = ( + "Encountered an object of type '{}'. Expected a str, list," + " int, or dict.".format(type(item)) + ) logger.error(msg) raise ValueError(msg) def get_used_parameters(self, step): - """ - Return the parameters used by a StudyStep. + """Return the parameters used by a StudyStep. + + Args: + step: A StudyStep instance to be checked. + + Returns: + A set of the parameter names used within the step parameter. - :param step: A StudyStep instance to be checked. - :returns: A set of the parameter names used within the step parameter. """ params = set() self._get_used_parameters(step.__dict__, params) return params def get_metadata(self): - """ - Produce metadata for the parameters in a generator instance. + """Produce metadata for the parameters in a generator instance. + + Args: + + Returns: + A dictionary containing metadata about the instance. - :returns: A dictionary containing metadata about the instance. """ meta = {} for combo in self.get_combinations(): diff --git a/maestrowf/datastructures/core/study.py b/maestrowf/datastructures/core/study.py index 51aa51d4b..f42245365 100644 --- a/maestrowf/datastructures/core/study.py +++ b/maestrowf/datastructures/core/study.py @@ -47,14 +47,11 @@ WSREGEX = re.compile( r"\$\(([-!\$%\^&\*\(\)_\+\|~=`{}\[\]:;<>\?,\.\/\w]+)\.workspace\)" ) -ALL_COMBOS = re.compile( - r"_\*|\*" -) +ALL_COMBOS = re.compile(r"_\*|\*") class StudyStep: - """ - Class that represents the data and API for a single study step. + """Class that represents the data and API for a single study step. This class is primarily a 1:1 mapping of a study step in the YAML spec in terms of data. The StudyStep's class API should capture all functions that @@ -63,6 +60,11 @@ class StudyStep: * Applying a combination of parameters to itself. * Tests for equality and non-equality to check for changes. * Other -- WIP + + Args: + + Returns: + """ def __init__(self): @@ -71,25 +73,28 @@ def __init__(self): self.description = "" self.nickname = "" self.run = { - "cmd": "", - "depends": "", - "pre": "", - "post": "", - "restart": "", - "nodes": "", - "procs": "", - "gpus": "", - "cores per task": "", - "walltime": "", - "reservation": "" - } + "cmd": "", + "depends": "", + "pre": "", + "post": "", + "restart": "", + "nodes": "", + "procs": "", + "gpus": "", + "cores per task": "", + "walltime": "", + "reservation": "", + } def apply_parameters(self, combo): - """ - Apply a parameter combination to the StudyStep. + """Apply a parameter combination to the StudyStep. + + Args: + combo: A Combination instance to be applied to a StudyStep. + + Returns: + A new StudyStep instance with combo applied to its members. - :param combo: A Combination instance to be applied to a StudyStep. - :returns: A new StudyStep instance with combo applied to its members. """ # Create a new StudyStep and populate it with substituted values. tmp = StudyStep() @@ -100,10 +105,13 @@ def apply_parameters(self, combo): @property def name(self): - """ - Get the name to assign to a task for this step. + """Get the name to assign to a task for this step. + + Args: + + Returns: + A utf-8 formatted string of the task name. - :returns: A utf-8 formatted string of the task name. """ if self.nickname: return self.nickname @@ -111,19 +119,25 @@ def name(self): @name.setter def name(self, value): - """ - Set the name of a StudyStep instance. + """Set the name of a StudyStep instance. + + Args: + value: A string value representing the name to give the step. + + Returns: - :param value: A string value representing the name to give the step. """ self._name = value @property def real_name(self): - """ - Get the real name of the step (ignore nickname). + """Get the real name of the step (ignore nickname). + + Args: + + Returns: + A string of the true name of a StudyStep instance. - :returns: A string of the true name of a StudyStep instance. """ return self._name @@ -154,8 +168,7 @@ def __ne__(self, other): class Study(DAG, PickleInterface): - """ - Collection of high level objects to perform study construction. + """Collection of high level objects to perform study construction. The Study class is part of the meat and potatoes of this whole package. A Study object is where the intersection of the major moving parts are @@ -191,10 +204,22 @@ class Study(DAG, PickleInterface): designed in whatever class ends up managing all of this to have machine learning applications pipe messages to spin up new studies using the same environment. + + Args: + + Returns: + """ - def __init__(self, name, description, - studyenv=None, parameters=None, steps=None, out_path="./"): + def __init__( + self, + name, + description, + studyenv=None, + parameters=None, + steps=None, + out_path="./", + ): """ Study object used to represent the full workflow of a study. @@ -257,10 +282,13 @@ def __init__(self, name, description, @property def output_path(self): - """ - Property method for the OUTPUT_PATH specified for the study. + """Property method for the OUTPUT_PATH specified for the study. + + Args: + + Returns: + The string path stored in the OUTPUT_PATH variable. - :returns: The string path stored in the OUTPUT_PATH variable. """ return self._out_path @@ -273,7 +301,7 @@ def store_metadata(self): path = os.path.join(self._meta_path, "study") create_parentdir(path) path = os.path.join(path, "env.pkl") - with open(path, 'wb') as pkl: + with open(path, "wb") as pkl: pickle.dump(self, pkl) # Construct other metadata related to study construction. @@ -284,8 +312,9 @@ def store_metadata(self): elif key in self.step_combos: _workspaces[key] = os.path.split(value)[-1] else: - _workspaces[key] = \ - os.path.sep.join(value.rsplit(os.path.sep)[-2:]) + _workspaces[key] = os.path.sep.join( + value.rsplit(os.path.sep)[-2:] + ) # Construct relative paths for the combinations and nest them in the # same way as the step combinations dictionary. @@ -300,8 +329,9 @@ def store_metadata(self): _step_combos[key] = {} for combo in value: _ws = self.workspaces[combo] - _step_combos[key][combo] = \ - os.path.sep.join(_ws.rsplit(os.path.sep)[-2:]) + _step_combos[key][combo] = os.path.sep.join( + _ws.rsplit(os.path.sep)[-2:] + ) metadata = { "dependencies": self.depends, @@ -332,13 +362,16 @@ def load_metadata(self): return path = os.path.join(self._meta_path, "study", "env.pkl") - with open(path, 'rb') as pkl: + with open(path, "rb") as pkl: env = pickle.load(pkl) if not isinstance(env, type(self)): - msg = "Object loaded from {path} is of type {type}. Expected an" \ - " object of type '{cls}.'".format(path=path, type=type(env), - cls=type(self)) + msg = ( + "Object loaded from {path} is of type {type}. Expected an" + " object of type '{cls}.'".format( + path=path, type=type(env), cls=type(self) + ) + ) LOGGER.error(msg) raise TypeError(msg) @@ -353,8 +386,7 @@ def load_metadata(self): self.step_combos = metadata["step_combinations"] def add_step(self, step): - """ - Add a step to a study. + """Add a step to a study. For this helper to be most effective, it recommended to apply steps in the order that they will be encountered. The method attempts to be @@ -362,38 +394,46 @@ def add_step(self, step): a step. When adding steps out of order it's recommended to just use the base class DAG functionality and manually make connections. - :param step: A StudyStep instance to be added to the Study instance. + Args: + step: A StudyStep instance to be added to the Study instance. + + Returns: + """ # Add the node to the DAG. self.add_node(step.real_name, step) - LOGGER.info( - "Adding step '%s' to study '%s'...", step.name, self.name) + LOGGER.info("Adding step '%s' to study '%s'...", step.name, self.name) # Apply the environment to the incoming step. - step.__dict__ = \ - apply_function(step.__dict__, self.environment.apply_environment) + step.__dict__ = apply_function( + step.__dict__, self.environment.apply_environment + ) # If the step depends on a prior step, create an edge. if "depends" in step.run and step.run["depends"]: for dependency in step.run["depends"]: - LOGGER.info("{0} is dependent on {1}. Creating edge (" - "{1}, {0})...".format(step.real_name, dependency)) + LOGGER.info( + "{0} is dependent on {1}. Creating edge (" + "{1}, {0})...".format(step.real_name, dependency) + ) if "*" not in dependency: self.add_edge(dependency, step.real_name) else: self.add_edge( - re.sub(ALL_COMBOS, "", dependency), - step.real_name + re.sub(ALL_COMBOS, "", dependency), step.real_name ) else: # Otherwise, if no other dependency, just execute the step. self.add_edge(SOURCE, step.real_name) def walk_study(self, src=SOURCE): - """ - Walk the study and create a spanning tree. + """Walk the study and create a spanning tree. + + Args: + src: Source node to start the walk. (Default value = SOURCE) + + Returns: + A generator of (parent, node name, node value) tuples. - :param src: Source node to start the walk. - :returns: A generator of (parent, node name, node value) tuples. """ # Get a DFS spanning tree of the study. This method should always # return a complete tree because _source is flagged as a dependency @@ -419,26 +459,39 @@ def setup_environment(self): LOGGER.debug("Environment is setting up.") self.environment.acquire_environment() - def configure_study(self, submission_attempts=1, restart_limit=1, - throttle=0, use_tmp=False, hash_ws=False, - dry_run=False): - """ - Perform initial configuration of a study. \ - - The method is used for going through and actually acquiring each \ - dependency, substituting variables, sources and labels. \ - - :param submission_attempts: Number of attempted submissions before \ - marking a step as failed. \ - :param restart_limit: Upper limit on the number of times a step with \ - a restart command can be resubmitted before it is considered failed. \ - :param throttle: The maximum number of in-progress jobs allowed. [0 \ - denotes no cap].\ - :param use_tmp: Boolean value specifying if the generated \ - ExecutionGraph dumps its information into a temporary directory. \ - :param dry_run: Boolean value that toggles dry run to just generate \ - study workspaces and scripts without execution or status checking. \ - :returns: True if the Study is successfully setup, False otherwise. \ + def configure_study( + self, + submission_attempts=1, + restart_limit=1, + throttle=0, + use_tmp=False, + hash_ws=False, + dry_run=False, + ): + """Perform initial configuration of a study. + + The method is used for going through and actually acquiring each + dependency, substituting variables, sources and labels. + + Args: + submission_attempts: Number of attempted submissions before marking + a step as failed. (Default value = 1) + restart_limit: Upper limit on the number of times a step with a + restart command can be resubmitted before it is considered failed. + (Default value = 1) + throttle: The maximum number of in-progress jobs allowed. [0 denotes + no cap]. (Default value = 0) + use_tmp: Boolean value specifying if the generated ExecutionGraph + dumps its information into a temporary directory. + (Default value = False) + dry_run: Boolean value that toggles dry run to just generate study + workspaces and scripts without execution or status checking. + (Default value = False) + hash_ws: (Default value = False) + + Returns: + True if the Study is successfully setup, False otherwise. + """ self._submission_attempts = submission_attempts @@ -458,27 +511,36 @@ def configure_study(self, submission_attempts=1, restart_limit=1, "Dry run enabled = %s\n" "Output path = %s\n" "------------------------------------------", - submission_attempts, restart_limit, throttle, - use_tmp, hash_ws, dry_run, self._out_path + submission_attempts, + restart_limit, + throttle, + use_tmp, + hash_ws, + dry_run, + self._out_path, ) self.is_configured = True def _stage(self, dag): - """ - Set up the ExecutionGraph of a parameterized study. + """Set up the ExecutionGraph of a parameterized study. + + Args: + throttle: Maximum number of in progress jobs allowed. + dag: + + Returns: + The path to the study's global workspace and an expanded + ExecutionGraph based on the parameters and parameterized workflow + steps. - :param throttle: Maximum number of in progress jobs allowed. - :returns: The path to the study's global workspace and an expanded - ExecutionGraph based on the parameters and parameterized workflow - steps. """ # Items to store that should be reset. LOGGER.info( "\n==================================================\n" "Constructing parameter study '%s'\n" "==================================================\n", - self.name + self.name, ) # Topological sorted list of steps. @@ -504,7 +566,7 @@ def _stage(self, dag): "\n==================================================\n" "Processing step '%s'\n" "==================================================\n", - step + step, ) # If we encounter SOURCE, just add it and continue. if step == SOURCE: @@ -520,7 +582,7 @@ def _stage(self, dag): self.step_combos[step] = set() s_params = self.parameters.get_used_parameters(node) - p_params = set() # Used parameters excluding the current step. + p_params = set() # Used parameters excluding the current step. # Iterate through dependencies to update the p_params LOGGER.debug("\n*** Processing dependencies ***") for parent in node.run["depends"]: @@ -539,11 +601,14 @@ def _stage(self, dag): # node because they may use parameters. These are likely to cause # a node to fall into the 'Parameter Dependent' case. used_spaces = re.findall( - WSREGEX, "{} {}".format(node.run["cmd"], node.run["restart"])) + WSREGEX, "{} {}".format(node.run["cmd"], node.run["restart"]) + ) for ws in used_spaces: if ws not in self.used_params: - msg = "Workspace for '{}' is being used before it would" \ - " be generated.".format(ws) + msg = ( + "Workspace for '{}' is being used before it would" + " be generated.".format(ws) + ) LOGGER.error(msg) raise Exception(msg) @@ -553,12 +618,16 @@ def _stage(self, dag): if ws in self.hub_depends[step]: LOGGER.info( "'%s' parameter independent association found. " - "Skipping.", ws) + "Skipping.", + ws, + ) continue LOGGER.debug( "Found workspace '%s' using parameters %s", - ws, self.used_params[ws]) + ws, + self.used_params[ws], + ) p_params |= self.used_params[ws] # Total parameters used for this step are the union of each parent @@ -577,7 +646,7 @@ def _stage(self, dag): "\n-------------------------------------------------\n" "Adding step '%s' (No parameters used)\n" "-------------------------------------------------\n", - step + step, ) # If we're not using any parameters at all, we do: # Copy the step and set to not modified. @@ -646,14 +715,17 @@ def _stage(self, dag): "-------- Used Parameters --------\n" "%s\n" "---------------------------------", - step, self.used_params[step] + step, + self.used_params[step], ) # Now we iterate over the combinations and expand the step. for combo in self.parameters: - LOGGER.info("\n**********************************\n" - "Combo [%s]\n" - "**********************************", - str(combo)) + LOGGER.info( + "\n**********************************\n" + "Combo [%s]\n" + "**********************************", + str(combo), + ) # Compute this step's combination name and workspace. nickname = None combo_str = combo.get_param_string(self.used_params[step]) @@ -662,11 +734,12 @@ def _stage(self, dag): if self._hash_ws: nickname = md5(combo_str.encode("utf-8")).hexdigest() workspace = make_safe_path( - self._out_path, - *[step, nickname]) + self._out_path, *[step, nickname] + ) else: - workspace = \ - make_safe_path(self._out_path, *[step, combo_str]) + workspace = make_safe_path( + self._out_path, *[step, combo_str] + ) LOGGER.debug("Workspace: %s", workspace) combo_str = "{}_{}".format(step, combo_str) self.workspaces[combo_str] = workspace @@ -701,15 +774,19 @@ def _stage(self, dag): # the unparameterized match. ws = self.workspaces[match] LOGGER.info( - "Found unparameterized workspace -- %s", match) + "Found unparameterized workspace -- %s", match + ) else: # Otherwise, we're dealing with a combination. ws = "{}_{}".format( match, - combo.get_param_string(self.used_params[match]) + combo.get_param_string( + self.used_params[match] + ), ) LOGGER.info( - "Found parameterized workspace -- %s", ws) + "Found parameterized workspace -- %s", ws + ) ws = self.workspaces[ws] # Replace in both the command and restart command. @@ -721,7 +798,8 @@ def _stage(self, dag): step_exp.run["restart"] = r_cmd # Add to the step to the DAG. dag.add_step( - step_exp.real_name, step_exp, workspace, rlimit) + step_exp.real_name, step_exp, workspace, rlimit + ) if self.depends[step] or self.hub_depends[step]: # So, because we don't have used parameters, we can @@ -731,7 +809,9 @@ def _stage(self, dag): if self.used_params[p]: p = "{}_{}".format( p, - combo.get_param_string(self.used_params[p]) + combo.get_param_string( + self.used_params[p] + ), ) LOGGER.info( "Adding edge (%s, %s)...", p, combo_str @@ -758,12 +838,16 @@ def _stage(self, dag): return dag def _stage_linear(self, dag): - """ - Execute a linear workflow without parameters. + """Execute a linear workflow without parameters. + + Args: + throttle: Maximum number of in progress jobs allowed. + dag: + + Returns: + The path to the study's global workspace and an + ExecutionGraph based on linear steps in the study. - :param throttle: Maximum number of in progress jobs allowed. - :returns: The path to the study's global workspace and an - ExecutionGraph based on linear steps in the study. """ # For each step in the Study # Walk the study and add the steps to the ExecutionGraph. @@ -824,27 +908,34 @@ def _stage_linear(self, dag): return dag def stage(self): - """ - Generate the execution graph for a Study. + """Generate the execution graph for a Study. Staging creates an ExecutionGraph based on the combinations generated by the ParameterGeneration object stored in an instance of a Study. The stage method also sets up individual working directories (or workspaces) for each node in the workflow that requires it. - :returns: An ExecutionGraph object with the expanded workflow. + Args: + + Returns: + An ExecutionGraph object with the expanded workflow. + """ # If the workspace doesn't exist, raise an exception. if not os.path.exists(self._out_path): - msg = "Study {} is not set up for staging. Workspace does not " \ - "exists (Output Dir = {}).".format(self.name, self._out_path) + msg = ( + "Study {} is not set up for staging. Workspace does not " + "exists (Output Dir = {}).".format(self.name, self._out_path) + ) LOGGER.error(msg) raise Exception(msg) # If the environment isn't set up, raise an exception. if not self.environment.is_set_up: - msg = "Study {} is not set up for staging. Environment is not " \ - "set up. Aborting.".format(self.name) + msg = ( + "Study {} is not set up for staging. Environment is not " + "set up. Aborting.".format(self.name) + ) LOGGER.error(msg) raise Exception(msg) @@ -871,7 +962,9 @@ def stage(self): dag = ExecutionGraph( submission_attempts=self._submission_attempts, submission_throttle=self._submission_throttle, - use_tmp=self._use_tmp, dry_run=self._dry_run) + use_tmp=self._use_tmp, + dry_run=self._dry_run, + ) dag.add_description(**self.description) dag.log_description() @@ -880,6 +973,7 @@ def stage(self): # the execution graph. Because the execution graph is constructed from # the study steps, it won't contain a cycle. def _pass_detect_cycle(self): + """ """ pass dag.detect_cycle = MethodType(_pass_detect_cycle, dag) diff --git a/maestrowf/datastructures/core/studyenvironment.py b/maestrowf/datastructures/core/studyenvironment.py index 06ef74ecf..47ba1c238 100644 --- a/maestrowf/datastructures/core/studyenvironment.py +++ b/maestrowf/datastructures/core/studyenvironment.py @@ -37,11 +37,15 @@ class StudyEnvironment: - """ - StudyEnvironment for managing a study environment. + """StudyEnvironment for managing a study environment. The StudyEnvironment provides the context where all study steps can find variables, sources, dependencies, etc. + + Args: + + Returns: + """ def __init__(self): @@ -71,18 +75,24 @@ def __bool__(self): @property def is_set_up(self): - """ - Check that the StudyEnvironment is set up. + """Check that the StudyEnvironment is set up. + + Args: + + Returns: + True is the instance is set up, False otherwise. - :returns: True is the instance is set up, False otherwise. """ return self._is_set_up def add(self, item): - """ - Add the item parameter to the StudyEnvironment. + """Add the item parameter to the StudyEnvironment. + + Args: + item: EnvObject to be added to the environment. + + Returns: - :param item: EnvObject to be added to the environment. """ # TODO: Need to revist this to make this better. A label can get lost # because the necessary variable could have not been added yet @@ -100,9 +110,9 @@ def add(self, item): LOGGER.debug("Tokens: %s", self._tokens) name = item.name LOGGER.debug("Adding %s of type %s.", item.name, type(item)) - if ( - isinstance(item.value, str) and - any(token in item.value for token in self._tokens)): + if isinstance(item.value, str) and any( + token in item.value for token in self._tokens + ): LOGGER.debug("Label detected. Adding %s to labels", item.name) self.labels[item.name] = item else: @@ -113,15 +123,18 @@ def add(self, item): LOGGER.debug("Item source: %s", item.source) self.sources.append(item) else: - error = "Received an item of type {}. Expected an item of base " \ - "type Substitution, Source, or Dependency." \ - .format(type(item)) + error = ( + "Received an item of type {}. Expected an item of base " + "type Substitution, Source, or Dependency.".format(type(item)) + ) LOGGER.exception(error) raise TypeError(error) if name and name in self._names: - error = "A duplicate name '{}' has been detected. All names " \ - "must be unique. Aborting.".format(name) + error = ( + "A duplicate name '{}' has been detected. All names " + "must be unique. Aborting.".format(name) + ) LOGGER.exception(error) raise ValueError(error) else: @@ -129,12 +142,15 @@ def add(self, item): self._names.add(name) def find(self, key): - """ - Find the environment object labeled by the specified key. + """Find the environment object labeled by the specified key. + + Args: + key: Name of the environment object to find. + + Returns: + The environment object labeled by key, None if key is not + found. - :param key: Name of the environment object to find. - :returns: The environment object labeled by key, None if key is not - found. """ LOGGER.debug("Looking for '%s'...", key) if key in self.dependencies: @@ -153,11 +169,14 @@ def find(self, key): return None def remove(self, key): - """ - Remove the environment object labeled by the specified key. + """Remove the environment object labeled by the specified key. + + Args: + key: Name of the environment object to remove. + + Returns: + The environment object labeled by key. - :param key: Name of the environment object to remove. - :returns: The environment object labeled by key. """ LOGGER.debug("Looking to remove '%s'...", key) @@ -196,11 +215,14 @@ def acquire_environment(self): self._is_set_up = True def apply_environment(self, item): - """ - Apply the environment to the specified item. + """Apply the environment to the specified item. + + Args: + item: String to apply environment to. + + Returns: + String with the environment applied. - :param item: String to apply environment to. - :returns: String with the environment applied. """ if not item: return item diff --git a/maestrowf/datastructures/dag.py b/maestrowf/datastructures/dag.py index 1b6bf5497..ec1a99a45 100644 --- a/maestrowf/datastructures/dag.py +++ b/maestrowf/datastructures/dag.py @@ -38,11 +38,15 @@ class DAG(Graph): - """ - A directed acyclic graph (DAG) data structure. + """A directed acyclic graph (DAG) data structure. The implementation of this DAG uses an adjacency map with a map to index the values (or objects) at each node. + + Args: + + Returns: + """ def __init__(self): @@ -51,16 +55,18 @@ def __init__(self): self.values = OrderedDict() def add_node(self, name, obj): - """ - Add node 'name' to the DAG. + """Add node 'name' to the DAG. + + Args: + name: String identifier of the node. + obj: An object representing the value of the node. + + Returns: - :param name: String identifier of the node. - :param obj: An object representing the value of the node. """ logging.debug("Adding %s...", name) if name in self.values: - logger.warning("Node %s already exists. Returning.", - name) + logger.warning("Node %s already exists. Returning.", name) return logger.debug("Node %s added. Value is of type %s.", name, type(obj)) @@ -68,22 +74,28 @@ def add_node(self, name, obj): self.adjacency_table[name] = [] def add_edge(self, src, dest): - """ - Add an edge to the DAG if edge (src, dest) is a valid edge. + """Add an edge to the DAG if edge (src, dest) is a valid edge. + + Args: + src: Source vertex name. + dest: Destination vertex name. + + Returns: - :param src: Source vertex name. - :param dest: Destination vertex name. """ # Disallow loops to the same node. if src == dest: - msg = "Cannot add self referring cycle edge ({}, {})" \ - .format(src, dest) + msg = "Cannot add self referring cycle edge ({}, {})".format( + src, dest + ) logger.error(msg) return # Disallow adding edges to the graph before nodes are added. - error = "Attempted to create edge ({src}, {dest}), but node {node}" \ - " does not exist." + error = ( + "Attempted to create edge ({src}, {dest}), but node {node}" + " does not exist." + ) if src not in self.adjacency_table: error = error.format(src=src, dest=dest, node=src) logger.error(error) @@ -108,33 +120,49 @@ def add_edge(self, src, dest): raise Exception(msg) def remove_edge(self, src, dest): - """ - Remove edge (src, dest) from the DAG. + """Remove edge (src, dest) from the DAG. + + Args: + src: Source vertex name. + dest: Destination vertex name. + + Returns: - :param src: Source vertex name. - :param dest: Destination vertex name. """ if src not in self.adjacency_table: - logger.warning("Attempted to remove an edge (%s, %s), but %s" - " does not exist.", src, dest, src) + logger.warning( + "Attempted to remove an edge (%s, %s), but %s" + " does not exist.", + src, + dest, + src, + ) return if dest not in self.adjacency_table: - logger.warning("Attempted to remove an edge from (%s, %s), but %s" - " does not exist.", src, dest, dest) + logger.warning( + "Attempted to remove an edge from (%s, %s), but %s" + " does not exist.", + src, + dest, + dest, + ) return logging.debug("Removing edge (%s, %s).", src, dest) self.adjacency_table[src].remove(dest) def dfs_subtree(self, src, par=None): - """ - Create a subtree of the DAG starting at src in DFS order. + """Create a subtree of the DAG starting at src in DFS order. + + Args: + src: Source node name to begin search. + par: Name of parent node to the specified source node. + (Default value = None) + + Returns: + A list representing the path taken by DFS. - :param src: Source node name to begin search. - :param par: Name of parent node to the specified source node. - :returns: A list representing the path taken by DFS. - :returns: A dictionary containing a mapping from node to parent node. """ path = [src] parent = {src: par} @@ -147,12 +175,14 @@ def dfs_subtree(self, src, par=None): return path, parent def bfs_subtree(self, src): - """ - Create a subtree of the DAG starting at src in BFS order. + """Create a subtree of the DAG starting at src in BFS order. + + Args: + src: Source node name to begin search. + + Returns: + A list representing the path taken by BFS. - :param src: Source node name to begin search. - :returns: A list representing the path taken by BFS. - :returns: A dictionary containing a mapping from node to parent node. """ queue = deque([src]) path = [src] @@ -171,13 +201,16 @@ def bfs_subtree(self, src): return path, parent def _topological_sort(self, v, visited, stack): - """ - Recur through the nodes to perform a toplogical sort. + """Recur through the nodes to perform a toplogical sort. + + Args: + v: The vertex previously visited. + visited: A dict of visited statuses. + stack: The current stack of vertices that have been sorted. + + Returns: + A list of the DAG's nodes in topologically sorted order. - :param v: The vertex previously visited. - :param visited: A dict of visited statuses. - :param stack: The current stack of vertices that have been sorted. - :returns: A list of the DAG's nodes in topologically sorted order. """ # Mark the node as visited. visited[v] = True @@ -192,10 +225,13 @@ def _topological_sort(self, v, visited, stack): stack.appendleft(v) def topological_sort(self): - """ - Perform a topological ordering of the vertices in the DAG. + """Perform a topological ordering of the vertices in the DAG. + + Args: + + Returns: + A list of the vertices sorted in topological order. - :returns: A list of the vertices sorted in topological order. """ v_stack = deque() v_visited = {key: False for key in self.values.keys()} @@ -220,12 +256,15 @@ def detect_cycle(self): return False def _detect_cycle(self, v, visited, rstack): - """ - Recurse through nodes testing for loops. + """Recurse through nodes testing for loops. + + Args: + v: Name of source vertex to search from. + visited: Set of the nodes we've visited so far. + rstack: Set of nodes currently on the path. + + Returns: - :param v: Name of source vertex to search from. - :param visited: Set of the nodes we've visited so far. - :param rstack: Set of nodes currently on the path. """ visited.add(v) rstack.add(v) @@ -234,16 +273,21 @@ def _detect_cycle(self, v, visited, rstack): if c not in visited: logging.debug("Visting node '%s' from '%s'.", c, v) if self._detect_cycle(c, visited, rstack): - logger.debug("Cycle detected --\n" - "rstack = %s\n" - "visited = %s", - rstack, visited) + logger.debug( + "Cycle detected --\n" "rstack = %s\n" "visited = %s", + rstack, + visited, + ) return True elif c in rstack: - logger.debug("Cycle detected ('%s' in rstack)--\n" - "rstack = %s\n" - "visited = %s", - c, rstack, visited) + logger.debug( + "Cycle detected ('%s' in rstack)--\n" + "rstack = %s\n" + "visited = %s", + c, + rstack, + visited, + ) return True rstack.remove(v) logger.debug("No cycle originating from '%s'", v) diff --git a/maestrowf/datastructures/environment/gitdependency.py b/maestrowf/datastructures/environment/gitdependency.py index 3967d6dd9..1b197dd5b 100644 --- a/maestrowf/datastructures/environment/gitdependency.py +++ b/maestrowf/datastructures/environment/gitdependency.py @@ -42,7 +42,7 @@ class GitDependency(Dependency): """Environment GitDependency class for substituting a git dependency.""" - def __init__(self, name, value, path, token='$', **kwargs): + def __init__(self, name, value, path, token="$", **kwargs): """ Initialize the GitDependency class. @@ -88,40 +88,50 @@ def __init__(self, name, value, path, token='$', **kwargs): self.tag = kwargs.pop("tag", "") self.branch = kwargs.pop("branch", "") - self._verification("PathDependency initialized without complete " - " settings. Set required [name, value] before " - "calling methods.") + self._verification( + "PathDependency initialized without complete " + " settings. Set required [name, value] before " + "calling methods." + ) self._is_acquired = False def get_var(self): - """ - Get the variable representation of the dependency's name. + """Get the variable representation of the dependency's name. + + Args: + + Returns: + String of the Dependencies's name in token form. - :returns: String of the Dependencies's name in token form. """ return "{}({})".format(self.token, self.name) def substitute(self, data): - """ - Substitute the dependency's value for its notation. + """Substitute the dependency's value for its notation. + + Args: + data: String to substitute dependency into. + + Returns: + String with the dependency's name replaced with its value. - :param data: String to substitute dependency into. - :returns: String with the dependency's name replaced with its value. """ if not self._verify(): - error = "Ensure that all required fields (name, value)," \ - "are populated and that value is a valid path." + error = ( + "Ensure that all required fields (name, value)," + "are populated and that value is a valid path." + ) logger.exception(error) raise ValueError(error) path = os.path.join(self.path, self.name) - logger.debug("%s: %s", self.get_var(), - data.replace(self.get_var(), path)) + logger.debug( + "%s: %s", self.get_var(), data.replace(self.get_var(), path) + ) return data.replace(self.get_var(), path) def acquire(self, substitutions=None): - """ - Acquire the dependency specified by the PathDependency. + """Acquire the dependency specified by the PathDependency. The GitDependency will clone the remote repository specified by the instance's value to the local repository specified by path. If a commit @@ -129,15 +139,22 @@ def acquire(self, substitutions=None): version described by the hash. Alternatively, if a tag is specfied acquire will attempt to checkout the version labeled by the tag. - :param substitutions: List of Substitution objects that can be applied. + Args: + substitutions: List of Substitution objects that can be applied. + (Default value = None) + + Returns: + """ if self._is_acquired: return if not self._verify(): - error = "Ensure that all required fields (name, value, " \ - "path), are populated and that value is a " \ - "valid path." + error = ( + "Ensure that all required fields (name, value, " + "path), are populated and that value is a " + "valid path." + ) logger.error(error) raise ValueError(error) @@ -151,8 +168,10 @@ def acquire(self, substitutions=None): # Moved the path existence here because git doesn't actually return a # specific enough error code. if os.path.exists(path): - msg = "Destination path '{}' already exists and is not an " \ - "empty directory.".format(path) + msg = ( + "Destination path '{}' already exists and is not an " + "empty directory.".format(path) + ) logger.error(msg) raise Exception(msg) @@ -160,10 +179,12 @@ def acquire(self, substitutions=None): p = start_process(["git", "ls-remote", self.url], shell=False) p.communicate() if p.returncode != 0: - msg = "Connectivity check failed. Check that you have " \ - "permissions to the specified repository, that the URL is " \ - "correct, and that you have network connectivity. (url = {})" \ + msg = ( + "Connectivity check failed. Check that you have " + "permissions to the specified repository, that the URL is " + "correct, and that you have network connectivity. (url = {})" .format(self.url) + ) logger.error(msg) raise RuntimeError(msg) logger.info("Connectivity achieved!") @@ -172,49 +193,59 @@ def acquire(self, substitutions=None): clone = start_process(["git", "clone", self.url, path], shell=False) clone.communicate() if clone.returncode != 0: - msg = "Failed to acquire GitDependency named '{}'. Check " \ - "that repository URL ({}) and repository local path ({}) " \ - "are valid.".format(self.name, self.url, path) + msg = ( + "Failed to acquire GitDependency named '{}'. Check " + "that repository URL ({}) and repository local path ({}) " + "are valid.".format(self.name, self.url, path) + ) logger.error(msg) raise Exception(msg) if self.hash: logger.info("Checking out SHA1 hash '%s'...", self.hash) - chkout = start_process(["git", "checkout", self.hash], - cwd=path, shell=False) + chkout = start_process( + ["git", "checkout", self.hash], cwd=path, shell=False + ) retcode = chkout.wait() if retcode != 0: - msg = "Unable to checkout SHA1 hash '{}' for the repository" \ - " located at {}." \ - .format(self.hash, self.url) + msg = ( + "Unable to checkout SHA1 hash '{}' for the repository" + " located at {}.".format(self.hash, self.url) + ) logger.error(msg) raise ValueError(msg) if self.tag: logger.info("Checking out git tag '%s'...", self.tag) tag = "tags/{}".format(self.tag) - chkout = start_process(["git", "checkout", tag], - cwd=path, shell=False) + chkout = start_process( + ["git", "checkout", tag], cwd=path, shell=False + ) retcode = chkout.wait() if retcode != 0: - msg = "Unable to checkout tag '{}' for the repository" \ - " located at {}".format(self.tag, self.url) + msg = ( + "Unable to checkout tag '{}' for the repository" + " located at {}".format(self.tag, self.url) + ) logger.error(msg) raise ValueError(msg) if self.branch: logger.info("Checking out git branch '%s'...", self.branch) - chkout = start_process(["git", "checkout", self.branch], - cwd=path, shell=False) + chkout = start_process( + ["git", "checkout", self.branch], cwd=path, shell=False + ) retcode = chkout.wait() if retcode != 0: - msg = "Unable to checkout branch '{}' for the repository" \ - " located at {}".format(self.tag, self.url) + msg = ( + "Unable to checkout branch '{}' for the repository" + " located at {}".format(self.tag, self.url) + ) logger.error(msg) raise ValueError(msg) @@ -226,22 +257,29 @@ def acquire(self, substitutions=None): self._is_acquired = True def _verify(self): - """ - Verify that the necessary Dependency fields are populated. + """Verify that the necessary Dependency fields are populated. + + Args: + + Returns: + True if Dependency is valid, False otherwise. - :returns: True if Dependency is valid, False otherwise. """ valid_param_pattern = re.compile(r"\w+") - required = bool(re.search(valid_param_pattern, self.name) and - re.search(valid_param_pattern, self.url) and - re.search(valid_param_pattern, self.path) and - self.token) + required = bool( + re.search(valid_param_pattern, self.name) + and re.search(valid_param_pattern, self.url) + and re.search(valid_param_pattern, self.path) + and self.token + ) opt_args = set([self.branch, self.hash, self.tag]) opt_args.discard("") if len(opt_args) > 1: - msg = "A GitDependency cannot specify both a commit hash and " \ - "release tag. Specify one or the other, but not both." + msg = ( + "A GitDependency cannot specify both a commit hash and " + "release tag. Specify one or the other, but not both." + ) logger.error(msg) raise ValueError(msg) elif self.hash: diff --git a/maestrowf/datastructures/environment/pathdependency.py b/maestrowf/datastructures/environment/pathdependency.py index a4c26b6de..2a1e4c16c 100644 --- a/maestrowf/datastructures/environment/pathdependency.py +++ b/maestrowf/datastructures/environment/pathdependency.py @@ -41,7 +41,7 @@ class PathDependency(Dependency): """Environment PathDependency class for substituting a path dependency.""" - def __init__(self, name, value, token='$'): + def __init__(self, name, value, token="$"): """ Initialize the PathDependency class. @@ -63,53 +63,70 @@ def __init__(self, name, value, token='$'): self.value = os.path.abspath(value) self.token = token - self._verification("PathDependency initialized without complete" - " settings. Set required [name, value] before " - "calling methods.") + self._verification( + "PathDependency initialized without complete" + " settings. Set required [name, value] before " + "calling methods." + ) self._is_acquired = False def get_var(self): - """ - Get the variable representation of the dependency's name. + """Get the variable representation of the dependency's name. + + Args: + + Returns: + String of the Dependencies's name in token form. - :returns: String of the Dependencies's name in token form. """ return "{}({})".format(self.token, self.name) def substitute(self, data): - """ - Substitute the dependency's value for its notation. + """Substitute the dependency's value for its notation. + + Args: + data: String to substitute dependency into. + + Returns: + String with the dependency's name replaced with its value. - :param data: String to substitute dependency into. - :returns: String with the dependency's name replaced with its value. """ if not self._verify(): - error = "Ensure that all required fields (name, value)," \ - "are populated and that value is a valid path." + error = ( + "Ensure that all required fields (name, value)," + "are populated and that value is a valid path." + ) logger.exception(error) raise ValueError(error) - logger.debug("%s: %s", self.get_var(), - data.replace(self.get_var(), self.value)) + logger.debug( + "%s: %s", self.get_var(), data.replace(self.get_var(), self.value) + ) return data.replace(self.get_var(), self.value) def acquire(self, substitutions=None): - """ - Acquire the dependency specified by the PathDependency. + """Acquire the dependency specified by the PathDependency. The PathDependency is simply a path that already exists, so the method doesn't actually acquire anything, but it does verify that the path exists. - :param substitutions: List of Substitution objects that can be applied. + Args: + substitutions: List of Substitution objects that can be applied. + (Default value = None) + + Returns: + """ if self._is_acquired: return if not self._verify(): - error = "Ensure that all required fields (name, " \ - "value), are populated and that value is a " \ - "valid path." + error = ( + "Ensure that all required fields (name, " + "value), are populated and that value is a " + "valid path." + ) logger.exception(error) raise ValueError(error) @@ -121,15 +138,20 @@ def acquire(self, substitutions=None): self._is_acquired = True def _verify(self): - """ - Verify that the necessary Dependency fields are populated. + """Verify that the necessary Dependency fields are populated. + + Args: + + Returns: + True if Dependency is valid, False otherwise. - :returns: True if Dependency is valid, False otherwise. """ valid_param_pattern = re.compile(r"\w+") - return bool(re.search(valid_param_pattern, self.name) and - re.search(valid_param_pattern, self.value) and - self.token) + return bool( + re.search(valid_param_pattern, self.name) + and re.search(valid_param_pattern, self.value) + and self.token + ) def __str__(self): """ diff --git a/maestrowf/datastructures/environment/script.py b/maestrowf/datastructures/environment/script.py index b518b3330..8f73433d1 100644 --- a/maestrowf/datastructures/environment/script.py +++ b/maestrowf/datastructures/environment/script.py @@ -48,23 +48,31 @@ def __init__(self, source): :params source: The command for changing the execution environment. """ self.source = source - self._verification("Script initialized without complete settings. Set" - " source before calling methods.") + self._verification( + "Script initialized without complete settings. Set" + " source before calling methods." + ) def apply(self, cmds): - """ - Apply the Script source to the specified list of commands. + """Apply the Script source to the specified list of commands. + + Args: + cmds: List of commands to add source to. + + Returns: + List of commands with the source prepended. - :param cmds: List of commands to add source to. - :returns: List of commands with the source prepended. """ return [self.source] + list(cmds) def _verify(self): - """ - Verify the Script object's contents. + """Verify the Script object's contents. + + Args: + + Returns: + True if the Script object is valid, False otherwise. - :returns: True if the Script object is valid, False otherwise. """ valid_param_pattern = re.compile(r"\w+") return bool(re.search(valid_param_pattern, self.source)) diff --git a/maestrowf/datastructures/environment/variable.py b/maestrowf/datastructures/environment/variable.py index d43267c83..ccf319046 100644 --- a/maestrowf/datastructures/environment/variable.py +++ b/maestrowf/datastructures/environment/variable.py @@ -37,14 +37,18 @@ class Variable(Substitution): - """ - Environment Variable class capable of substituting itself into strings. + """Environment Variable class capable of substituting itself into strings. Derived from the Substitution EnvObject class which requires that a substitution be able to inject itself into data. + + Args: + + Returns: + """ - def __init__(self, name, value, token='$'): + def __init__(self, name, value, token="$"): """ Initialize the Variable class. @@ -63,37 +67,52 @@ def __init__(self, name, value, token='$'): self.token = token if not self._verify(): - msg = "Variable initialized without complete settings. Set " \ - "required [name, value] before calling methods." + msg = ( + "Variable initialized without complete settings. Set " + "required [name, value] before calling methods." + ) logger.exception(msg) raise ValueError(msg) def get_var(self): - """ - Get the variable representation of the variable's name. + """Get the variable representation of the variable's name. + + Args: + + Returns: + String of the Variable's name in token form. - :returns: String of the Variable's name in token form. """ return "{}({})".format(self.token, self.name) def substitute(self, data): - """ - Substitute the variable's value for its notation. + """Substitute the variable's value for its notation. + + Args: + data: String to substitute variable into. + + Returns: + String with the variable's name replaced with its value. - :param data: String to substitute variable into. - :returns: String with the variable's name replaced with its value. """ - self._verification("Attempting to substitute a variable that is not" - " complete.") - logger.debug("%s: %s", self.get_var(), - data.replace(self.get_var(), str(self.value))) + self._verification( + "Attempting to substitute a variable that is not" " complete." + ) + logger.debug( + "%s: %s", + self.get_var(), + data.replace(self.get_var(), str(self.value)), + ) return data.replace(self.get_var(), str(self.value)) def _verify(self): - """ - Verify that the necessary Variable fields are populated. + """Verify that the necessary Variable fields are populated. + + Args: + + Returns: + True if Variable is valid, False otherwise. - :returns: True if Variable is valid, False otherwise. """ _valid = bool(self.name) and self.value is not None return _valid diff --git a/maestrowf/interfaces/__init__.py b/maestrowf/interfaces/__init__.py index 156760a6f..8f8192a16 100644 --- a/maestrowf/interfaces/__init__.py +++ b/maestrowf/interfaces/__init__.py @@ -39,43 +39,65 @@ def iter_adapters(): - """ - Based off of packaging.python.org loop over a namespace and find the + """Based off of packaging.python.org loop over a namespace and find the modules. This has been adapted for this particular use case of loading all classes implementing ScriptAdapter loaded from all modules in maestrowf.interfaces.script. :return: an iterable of the classes existing in the namespace + + Args: + + Returns: + """ # get loader for the script adapter package - loader = pkgutil.get_loader('maestrowf.interfaces.script') + loader = pkgutil.get_loader("maestrowf.interfaces.script") # get all of the modules in the package - mods = [(name, ispkg) for finder, name, ispkg in pkgutil.iter_modules( - loader.load_module('maestrowf.interfaces.script').__path__, - loader.load_module('maestrowf.interfaces.script').__name__ + ".")] + mods = [ + (name, ispkg) + for finder, name, ispkg in pkgutil.iter_modules( + loader.load_module("maestrowf.interfaces.script").__path__, + loader.load_module("maestrowf.interfaces.script").__name__ + ".", + ) + ] cs = [] for name, _ in mods: # get loader for every module m = pkgutil.get_loader(name).load_module(name) # get all classes that implement ScriptAdapter and are not abstract for n, cls in m.__dict__.items(): - if isinstance(cls, type) and issubclass(cls, ScriptAdapter) and \ - not inspect.isabstract(cls): + if ( + isinstance(cls, type) + and issubclass(cls, ScriptAdapter) + and not inspect.isabstract(cls) + ): cs.append(cls) return cs class ScriptAdapterFactory(object): - factories = { - adapter.key: adapter for adapter in iter_adapters() - } + """ """ + + factories = {adapter.key: adapter for adapter in iter_adapters()} @classmethod def get_adapter(cls, adapter_id): + """ + + Args: + adapter_id: + + Returns: + + """ if adapter_id.lower() not in cls.factories: - msg = "Adapter '{0}' not found. Specify an adapter that exists " \ - "or implement a new one mapping to the '{0}'" \ - .format(str(adapter_id)) + msg = ( + "Adapter '{0}' not found. Specify an adapter that exists " + "or implement a new one mapping to the '{0}'".format( + str(adapter_id) + ) + ) LOGGER.error(msg) raise Exception(msg) @@ -83,4 +105,5 @@ def get_adapter(cls, adapter_id): @classmethod def get_valid_adapters(cls): + """ """ return cls.factories.keys() diff --git a/maestrowf/interfaces/script/__init__.py b/maestrowf/interfaces/script/__init__.py index b141fa307..0e3ef8937 100644 --- a/maestrowf/interfaces/script/__init__.py +++ b/maestrowf/interfaces/script/__init__.py @@ -28,7 +28,7 @@ ############################################################################### """Module for interfaces that support various schedulers.""" -__path__ = __import__('pkgutil').extend_path(__path__, __name__) +__path__ = __import__("pkgutil").extend_path(__path__, __name__) import inspect import logging import pkgutil @@ -60,40 +60,52 @@ def __init__(self, subcode, retcode, jobid=-1): @property def job_identifier(self): - """ - Property for the job identifier for the record. + """Property for the job identifier for the record. + + Args: + + Returns: + A string representing the job identifier assigned by the + scheduler. - :returns: A string representing the job identifier assigned by the - scheduler. """ return self._info.get("jobid", None) @property def submission_code(self): - """ - Property for submission state for the record. + """Property for submission state for the record. + + Args: + + Returns: + A SubmissionCode enum representing the state of the + submission call. - :returns: A SubmissionCode enum representing the state of the - submission call. """ return self._subcode @property def return_code(self): - """ - Property for the raw return code returned from submission. + """Property for the raw return code returned from submission. + + Args: + + Returns: + An integer representing the state of the raw return code + from submission. - :returns: An integer representing the state of the raw return code - from submission. """ return self._info["retcode"] def add_info(self, key, value): - """ - Set additional informational key-value information. + """Set additional informational key-value information. + + Args: + key: Record key identifying data. + value: Data to be recorded. + + Returns: - :param key: Record key identifying data. - :param value: Data to be recorded. """ self._info[key] = value @@ -104,24 +116,29 @@ class CancellationRecord(Record): def __init__(self, cancel_status, retcode): """Initialize an empty CancellationRecord.""" self._status = { - CancelCode.OK: set(), - CancelCode.ERROR: set(), - } # Map of cancellation status to job set. + CancelCode.OK: set(), + CancelCode.ERROR: set(), + } # Map of cancellation status to job set. self._retcode = retcode self._cstatus = cancel_status def add_status(self, jobid, cancel_status): - """ - Add the cancellation status for a single job to a record. + """Add the cancellation status for a single job to a record. + + Args: + jobid: Unique job identifier for the job status to be added. + cancel_status: CancelCode designating how cancellation + terminated. + + Returns: - :param jobid: Unique job identifier for the job status to be added. - :param cancel_status: CancelCode designating how cancellation - terminated. """ if not isinstance(cancel_status, CancelCode): raise TypeError( "Parameter 'cancel_code' must be of type 'CancelCode'. " - "Received type '%s' instead.", type(cancel_status)) + "Received type '%s' instead.", + type(cancel_status), + ) self._status[cancel_status].add(jobid) @property @@ -135,11 +152,14 @@ def return_code(self): return self._retcode def lookup_status(self, cancel_status): - """ - Find the cancellation status of the job identified by jid. + """Find the cancellation status of the job identified by jid. + + Args: + cancel_status: The CancelCode to look up. + + Returns: + Set of job identifiers that match the requested status. - :param cancel_status: The CancelCode to look up. - :returns: Set of job identifiers that match the requested status. """ return self._status.get(cancel_status, set()) @@ -150,20 +170,30 @@ class FluxFactory(object): latest = "0.17.0" def _iter_flux(): - """ - Based off of packaging.python.org loop over a namespace and find the + """Based off of packaging.python.org loop over a namespace and find the modules. This has been adapted for this particular use case of loading all classes implementing FluxInterface loaded from all modules in maestrowf.interfaces.script._flux. :return: an iterable of the classes existing in the namespace + + Args: + + Returns: + """ # get loader for the script adapter package - loader = pkgutil.get_loader('maestrowf.interfaces.script._flux') + loader = pkgutil.get_loader("maestrowf.interfaces.script._flux") # get all of the modules in the package - mods = [(name, ispkg) for finder, name, ispkg in pkgutil.iter_modules( - loader.load_module('maestrowf.interfaces.script._flux').__path__, - loader.load_module( - 'maestrowf.interfaces.script._flux').__name__ + "." + mods = [ + (name, ispkg) + for finder, name, ispkg in pkgutil.iter_modules( + loader.load_module( + "maestrowf.interfaces.script._flux" + ).__path__, + loader.load_module( + "maestrowf.interfaces.script._flux" + ).__name__ + + ".", ) ] cs = [] @@ -172,22 +202,33 @@ def _iter_flux(): m = pkgutil.get_loader(name).load_module(name) # get all classes that implement ScriptAdapter and are not abstract for n, cls in m.__dict__.items(): - if isinstance(cls, type) and \ - issubclass(cls, FluxInterface) and \ - not inspect.isabstract(cls): + if ( + isinstance(cls, type) + and issubclass(cls, FluxInterface) + and not inspect.isabstract(cls) + ): cs.append(cls) return cs - factories = { - interface.key: interface for interface in _iter_flux() - } + factories = {interface.key: interface for interface in _iter_flux()} @classmethod def get_interface(cls, interface_id): + """ + + Args: + interface_id: + + Returns: + + """ if interface_id.lower() not in cls.factories: - msg = "Interface '{0}' not found. Specify a supported version " \ - "of Flux or implement a new one mapping to the '{0}'" \ - .format(str(interface_id)) + msg = ( + "Interface '{0}' not found. Specify a supported version " + "of Flux or implement a new one mapping to the '{0}'".format( + str(interface_id) + ) + ) LOGGER.error(msg) raise Exception(msg) @@ -195,8 +236,10 @@ def get_interface(cls, interface_id): @classmethod def get_valid_interfaces(cls): + """ """ return cls.factories.keys() @classmethod def get_latest_interface(cls): + """ """ return cls.factories[cls.latest] diff --git a/maestrowf/interfaces/script/_flux/flux0_17_0.py b/maestrowf/interfaces/script/_flux/flux0_17_0.py index 1b3fc4422..a1ba84f9d 100644 --- a/maestrowf/interfaces/script/_flux/flux0_17_0.py +++ b/maestrowf/interfaces/script/_flux/flux0_17_0.py @@ -2,8 +2,12 @@ import logging import os -from maestrowf.abstracts.enums import CancelCode, JobStatusCode, State, \ - SubmissionCode +from maestrowf.abstracts.enums import ( + CancelCode, + JobStatusCode, + State, + SubmissionCode, +) from maestrowf.abstracts.interfaces.flux import FluxInterface LOGGER = logging.getLogger(__name__) @@ -15,6 +19,8 @@ class FluxInterface_0170(FluxInterface): + """ """ + # This utility class is for Flux 0.17.0 key = "0.17.0" @@ -49,17 +55,39 @@ class FluxInterface_0170(FluxInterface): "status_abbrev": ("state", "result"), } - attrs = set( - _FIELDATTRS["userid"] + _FIELDATTRS["status"] - ) + attrs = set(_FIELDATTRS["userid"] + _FIELDATTRS["status"]) flux_handle = None @classmethod def submit( - cls, nodes, procs, cores_per_task, path, cwd, walltime, - ngpus=0, job_name=None, force_broker=False + cls, + nodes, + procs, + cores_per_task, + path, + cwd, + walltime, + ngpus=0, + job_name=None, + force_broker=False, ): + """ + + Args: + nodes: + procs: + cores_per_task: + path: + cwd: + walltime: + ngpus: (Default value = 0) + job_name: (Default value = None) + force_broker: (Default value = False) + + Returns: + + """ if not cls.flux_handle: cls.flux_handle = flux.Flux() LOGGER.debug("New Flux instance created.") @@ -73,20 +101,26 @@ def submit( if force_broker or nodes > 1: LOGGER.debug( "Launch under Flux sub-broker. [force_broker=%s, nodes=%d]", - force_broker, nodes + force_broker, + nodes, ) cmd_line = ["flux", "start", path] else: LOGGER.debug( "Launch under root Flux broker. [force_broker=%s, nodes=%d]", - force_broker, nodes + force_broker, + nodes, ) cmd_line = [path] LOGGER.debug("Handle address -- %s", hex(id(cls.flux_handle))) jobspec = flux.job.JobspecV1.from_command( - cmd_line, num_tasks=nodes, num_nodes=nodes, - cores_per_task=cores_per_task, gpus_per_task=ngpus) + cmd_line, + num_tasks=nodes, + num_nodes=nodes, + cores_per_task=cores_per_task, + gpus_per_task=ngpus, + ) jobspec.cwd = cwd jobspec.environment = dict(os.environ) @@ -95,16 +129,19 @@ def submit( try: # Submit our job spec. - jobid = \ - flux.job.submit(cls.flux_handle, jobspec, waitable=True) + jobid = flux.job.submit(cls.flux_handle, jobspec, waitable=True) submit_status = SubmissionCode.OK retcode = 0 - LOGGER.info("Submission returned status OK. -- " - "Assigned identifier (%s)", jobid) + LOGGER.info( + "Submission returned status OK. -- " + "Assigned identifier (%s)", + jobid, + ) except Exception as exception: LOGGER.error( - "Submission failed -- Message (%s).", exception.message) + "Submission failed -- Message (%s).", exception.message + ) jobid = -1 retcode = -1 submit_status = SubmissionCode.ERROR @@ -113,6 +150,16 @@ def submit( @classmethod def parallelize(cls, procs, nodes=None, **kwargs): + """ + + Args: + procs: + nodes: (Default value = None) + **kwargs: + + Returns: + + """ args = ["flux", "mini", "run", "-n", str(procs)] # if we've specified nodes, add that to wreckrun @@ -138,12 +185,21 @@ def parallelize(cls, procs, nodes=None, **kwargs): @staticmethod def status_callback(future, args): + """ + + Args: + future: + args: + + Returns: + + """ jobid, cb_args = args try: job = future.get_job() e_stat = "S" except EnvironmentError as err: - job = {'id': jobid} + job = {"id": jobid} if err.errno == errno.ENOENT: LOGGER.error("Flux Job identifier '%s' not found.", jobid) e_stat = "NF" @@ -158,26 +214,33 @@ def status_callback(future, args): @classmethod def get_statuses(cls, joblist): + """ + + Args: + joblist: + + Returns: + + """ # We need to import flux here, as it may not be installed on # all systems. if not cls.flux_handle: cls.flux_handle = flux.Flux() LOGGER.debug("New Flux instance created.") - LOGGER.debug( - "Handle address -- %s", hex(id(cls.flux_handle))) + LOGGER.debug("Handle address -- %s", hex(id(cls.flux_handle))) cb_args = { - "jobs": [], + "jobs": [], "handle": cls.flux_handle, - "count": 0, + "count": 0, "total": len(joblist), } for jobid in joblist: - rpc_handle = \ - flux.job.job_list_id( - cls.flux_handle, int(jobid), list(cls.attrs)) + rpc_handle = flux.job.job_list_id( + cls.flux_handle, int(jobid), list(cls.attrs) + ) rpc_handle.then(cls.status_callback, arg=(int(jobid), cb_args)) ret = cls.flux_handle.reactor_run(rpc_handle.get_reactor(), 0) @@ -195,13 +258,22 @@ def get_statuses(cls, joblist): statuses[job[1]["id"]] = State.UNKNOWN else: LOGGER.debug( - "Job checked with status '%s'\nEntry: %s", job[0], job[1]) - statuses[job[1]["id"]] = \ - cls.statustostr(job[1], True) + "Job checked with status '%s'\nEntry: %s", job[0], job[1] + ) + statuses[job[1]["id"]] = cls.statustostr(job[1], True) return chk_status, statuses @classmethod def resulttostr(cls, resultid, singlechar=False): + """ + + Args: + resultid: + singlechar: (Default value = False) + + Returns: + + """ # if result not returned, just return empty string back inner = __import__("flux.core.inner", fromlist=["raw"]) if resultid == "": @@ -209,17 +281,29 @@ def resulttostr(cls, resultid, singlechar=False): LOGGER.debug( "Calling 'inner.raw.flux_job_resulttostr' with (%s, %s)", - resultid, singlechar) + resultid, + singlechar, + ) ret = inner.raw.flux_job_resulttostr(resultid, singlechar) return ret.decode("utf-8") @classmethod def statustostr(cls, job_entry, abbrev=True): + """ + + Args: + job_entry: + abbrev: (Default value = True) + + Returns: + + """ flux = __import__("flux", fromlist=["constants"]) stateid = job_entry["state"] LOGGER.debug( - "JOBID [%d] -- Encountered (%s)", job_entry["id"], stateid) + "JOBID [%d] -- Encountered (%s)", job_entry["id"], stateid + ) if stateid & flux.constants.FLUX_JOB_PENDING: LOGGER.debug("Marking as PENDING.") @@ -230,18 +314,21 @@ def statustostr(cls, job_entry, abbrev=True): else: LOGGER.debug( "Found Flux INACTIVE state. Calling resulttostr (result=%s).", - job_entry["result"]) + job_entry["result"], + ) statusstr = cls.resulttostr(job_entry["result"], abbrev) return cls.state(statusstr) @classmethod def cancel(cls, joblist): - """ - Cancel a job using Flux 0.17.0 cancellation API. + """Cancel a job using Flux 0.17.0 cancellation API. + + Args: + joblist: A list of job identifiers to cancel. + + Returns: + CancelCode enumeration that reflects result of cancellation. - :param joblist: A list of job identifiers to cancel. - :return: CancelCode enumeration that reflects result of cancellation. - "return: A cancel return code indicating how cancellation call exited. """ # We need to import flux here, as it may not be installed on # all systems. @@ -249,11 +336,10 @@ def cancel(cls, joblist): cls.flux_handle = flux.Flux() LOGGER.debug("New Flux instance created.") - LOGGER.debug( - "Handle address -- %s", hex(id(cls.flux_handle))) + LOGGER.debug("Handle address -- %s", hex(id(cls.flux_handle))) LOGGER.debug( "Attempting to cancel jobs.\nJoblist:\n%s", - "\n".join(str(j) for j in joblist) + "\n".join(str(j) for j in joblist), ) cancel_code = CancelCode.OK @@ -271,6 +357,14 @@ def cancel(cls, joblist): @staticmethod def state(state): + """ + + Args: + state: + + Returns: + + """ if state == "CD": return State.FINISHED elif state == "F": diff --git a/maestrowf/interfaces/script/_flux/flux0_18_0.py b/maestrowf/interfaces/script/_flux/flux0_18_0.py index b9999397e..0baaa9143 100644 --- a/maestrowf/interfaces/script/_flux/flux0_18_0.py +++ b/maestrowf/interfaces/script/_flux/flux0_18_0.py @@ -4,8 +4,12 @@ from math import ceil import os -from maestrowf.abstracts.enums import CancelCode, JobStatusCode, State, \ - SubmissionCode +from maestrowf.abstracts.enums import ( + CancelCode, + JobStatusCode, + State, + SubmissionCode, +) from maestrowf.abstracts.interfaces.flux import FluxInterface LOGGER = logging.getLogger(__name__) @@ -19,6 +23,8 @@ class FluxInterface_0190(FluxInterface): + """ """ + # This utility class is for Flux 0.17.0 key = "0.19.0" @@ -53,17 +59,39 @@ class FluxInterface_0190(FluxInterface): "status_abbrev": ("state", "result"), } - attrs = set( - _FIELDATTRS["userid"] + _FIELDATTRS["status"] - ) + attrs = set(_FIELDATTRS["userid"] + _FIELDATTRS["status"]) flux_handle = None @classmethod def submit( - cls, nodes, procs, cores_per_task, path, cwd, walltime, - ngpus=0, job_name=None, force_broker=False + cls, + nodes, + procs, + cores_per_task, + path, + cwd, + walltime, + ngpus=0, + job_name=None, + force_broker=False, ): + """ + + Args: + nodes: + procs: + cores_per_task: + path: + cwd: + walltime: + ngpus: (Default value = 0) + job_name: (Default value = None) + force_broker: (Default value = False) + + Returns: + + """ if not cls.flux_handle: cls.flux_handle = Flux() LOGGER.debug("New Flux instance created.") @@ -78,20 +106,30 @@ def submit( if force_broker or nodes > 1: LOGGER.debug( "Launch under Flux sub-broker. [force_broker=%s, nodes=%d]", - force_broker, nodes + force_broker, + nodes, ) ngpus_per_slot = int(ceil(ngpus / nodes)) jobspec = flux_job.JobspecV1.from_nest_command( - [path], num_nodes=nodes, cores_per_slot=cores_per_task, - num_slots=nodes, gpus_per_slot=ngpus_per_slot) + [path], + num_nodes=nodes, + cores_per_slot=cores_per_task, + num_slots=nodes, + gpus_per_slot=ngpus_per_slot, + ) else: LOGGER.debug( "Launch under root Flux broker. [force_broker=%s, nodes=%d]", - force_broker, nodes + force_broker, + nodes, ) jobspec = flux_job.JobspecV1.from_command( - [path], num_tasks=procs, num_nodes=nodes, - cores_per_task=cores_per_task, gpus_per_task=ngpus) + [path], + num_tasks=procs, + num_nodes=nodes, + cores_per_task=cores_per_task, + gpus_per_task=ngpus, + ) LOGGER.debug("Handle address -- %s", hex(id(cls.flux_handle))) if job_name: @@ -109,16 +147,17 @@ def submit( try: # Submit our job spec. - jobid = \ - flux_job.submit(cls.flux_handle, jobspec, waitable=True) + jobid = flux_job.submit(cls.flux_handle, jobspec, waitable=True) submit_status = SubmissionCode.OK retcode = 0 - LOGGER.info("Submission returned status OK. -- " - "Assigned identifier (%s)", jobid) + LOGGER.info( + "Submission returned status OK. -- " + "Assigned identifier (%s)", + jobid, + ) except Exception as exception: - LOGGER.error( - "Submission failed -- Message (%s).", exception) + LOGGER.error("Submission failed -- Message (%s).", exception) jobid = -1 retcode = -1 submit_status = SubmissionCode.ERROR @@ -127,6 +166,16 @@ def submit( @classmethod def parallelize(cls, procs, nodes=None, **kwargs): + """ + + Args: + procs: + nodes: (Default value = None) + **kwargs: + + Returns: + + """ args = ["flux", "mini", "run", "-n", str(procs)] # if we've specified nodes, add that to wreckrun @@ -152,12 +201,21 @@ def parallelize(cls, procs, nodes=None, **kwargs): @staticmethod def status_callback(future, args): + """ + + Args: + future: + args: + + Returns: + + """ jobid, cb_args = args try: job = future.get_job() e_stat = "S" except EnvironmentError as err: - job = {'id': jobid} + job = {"id": jobid} if err.errno == errno.ENOENT: LOGGER.error("Flux Job identifier '%s' not found.", jobid) e_stat = "NF" @@ -172,25 +230,33 @@ def status_callback(future, args): @classmethod def get_statuses(cls, joblist): + """ + + Args: + joblist: + + Returns: + + """ # We need to import flux here, as it may not be installed on # all systems. if not cls.flux_handle: cls.flux_handle = Flux() LOGGER.debug("New Flux instance created.") - LOGGER.debug( - "Handle address -- %s", hex(id(cls.flux_handle))) + LOGGER.debug("Handle address -- %s", hex(id(cls.flux_handle))) cb_args = { - "jobs": [], + "jobs": [], "handle": cls.flux_handle, - "count": 0, + "count": 0, "total": len(joblist), } for jobid in joblist: rpc_handle = flux_job.job_list_id( - cls.flux_handle, int(jobid), list(cls.attrs)) + cls.flux_handle, int(jobid), list(cls.attrs) + ) rpc_handle.then(cls.status_callback, arg=(int(jobid), cb_args)) ret = cls.flux_handle.reactor_run(rpc_handle.get_reactor(), 0) @@ -209,13 +275,25 @@ def get_statuses(cls, joblist): else: LOGGER.debug( "Job checked with status '%s'\nEntry: %s", - job_entry[0], job_entry[1]) - statuses[job_entry[1]["id"]] = \ - cls.statustostr(job_entry[1], True) + job_entry[0], + job_entry[1], + ) + statuses[job_entry[1]["id"]] = cls.statustostr( + job_entry[1], True + ) return chk_status, statuses @classmethod def resulttostr(cls, resultid, singlechar=False): + """ + + Args: + resultid: + singlechar: (Default value = False) + + Returns: + + """ # if result not returned, just return empty string back inner = __import__("flux.core.inner", fromlist=["raw"]) if resultid == "": @@ -223,15 +301,27 @@ def resulttostr(cls, resultid, singlechar=False): LOGGER.debug( "Calling 'inner.raw.flux_job_resulttostr' with (%s, %s)", - resultid, singlechar) + resultid, + singlechar, + ) ret = inner.raw.flux_job_resulttostr(resultid, singlechar) return ret.decode("utf-8") @classmethod def statustostr(cls, job_entry, abbrev=True): + """ + + Args: + job_entry: + abbrev: (Default value = True) + + Returns: + + """ stateid = job_entry["state"] LOGGER.debug( - "JOBID [%d] -- Encountered (%s)", job_entry["id"], stateid) + "JOBID [%d] -- Encountered (%s)", job_entry["id"], stateid + ) if stateid & flux_constants.FLUX_JOB_PENDING: LOGGER.debug("Marking as PENDING.") @@ -242,18 +332,21 @@ def statustostr(cls, job_entry, abbrev=True): else: LOGGER.debug( "Found Flux INACTIVE state. Calling resulttostr (result=%s).", - job_entry["result"]) + job_entry["result"], + ) statusstr = cls.resulttostr(job_entry["result"], abbrev) return cls.state(statusstr) @classmethod def cancel(cls, joblist): - """ - Cancel a job using Flux 0.17.0 cancellation API. + """Cancel a job using Flux 0.17.0 cancellation API. + + Args: + joblist: A list of job identifiers to cancel. + + Returns: + CancelCode enumeration that reflects result of cancellation. - :param joblist: A list of job identifiers to cancel. - :return: CancelCode enumeration that reflects result of cancellation. - "return: A cancel return code indicating how cancellation call exited. """ # We need to import flux here, as it may not be installed on # all systems. @@ -261,11 +354,10 @@ def cancel(cls, joblist): cls.flux_handle = Flux() LOGGER.debug("New Flux instance created.") - LOGGER.debug( - "Handle address -- %s", hex(id(cls.flux_handle))) + LOGGER.debug("Handle address -- %s", hex(id(cls.flux_handle))) LOGGER.debug( "Attempting to cancel jobs.\nJoblist:\n%s", - "\n".join(str(j) for j in joblist) + "\n".join(str(j) for j in joblist), ) cancel_code = CancelCode.OK @@ -283,6 +375,14 @@ def cancel(cls, joblist): @staticmethod def state(state): + """ + + Args: + state: + + Returns: + + """ if state == "CD": return State.FINISHED elif state == "F": diff --git a/maestrowf/interfaces/script/fluxscriptadapter.py b/maestrowf/interfaces/script/fluxscriptadapter.py index 2cadc8478..d09affaf9 100644 --- a/maestrowf/interfaces/script/fluxscriptadapter.py +++ b/maestrowf/interfaces/script/fluxscriptadapter.py @@ -36,8 +36,11 @@ from maestrowf.abstracts.interfaces import SchedulerScriptAdapter from maestrowf.abstracts.enums import JobStatusCode, CancelCode -from maestrowf.interfaces.script import CancellationRecord, SubmissionRecord, \ - FluxFactory +from maestrowf.interfaces.script import ( + CancellationRecord, + SubmissionRecord, + FluxFactory, +) LOGGER = logging.getLogger(__name__) status_re = re.compile(r"Job \d+ status: (.*)$") @@ -73,7 +76,8 @@ def __init__(self, **kwargs): if not uri: raise ValueError( "Flux URI must be specified in batch or stored in the " - "environment under 'FLUX_URI'") + "environment under 'FLUX_URI'" + ) self.add_batch_parameter("flux_uri", uri) # NOTE: Host doesn"t seem to matter for FLUX. sbatch assumes that the @@ -97,37 +101,48 @@ def __init__(self, **kwargs): self.h = None # Store the interface we're using _version = kwargs.pop("version", FluxFactory.latest) - self.add_batch_parameter( - "version", _version) + self.add_batch_parameter("version", _version) self._interface = FluxFactory.get_interface(_version) @property def extension(self): + """ """ return self._extension def _convert_walltime_to_seconds(self, walltime): + """ + + Args: + walltime: + + Returns: + + """ if not walltime: LOGGER.debug("Encountered inf walltime!") return "inf" # Convert walltime to seconds. LOGGER.debug("Converting %s to seconds...", walltime) - wt = \ - (datetime.strptime(walltime, "%H:%M:%S") - datetime(1900, 1, 1)) + wt = datetime.strptime(walltime, "%H:%M:%S") - datetime(1900, 1, 1) return int(wt.total_seconds()) def get_header(self, step): - """ - Generate the header present at the top of Flux execution scripts. + """Generate the header present at the top of Flux execution scripts. + + Args: + step: A StudyStep instance. + + Returns: + A string of the header based on internal batch parameters and + the parameter step. - :param step: A StudyStep instance. - :returns: A string of the header based on internal batch parameters and - the parameter step. """ run = dict(step.run) batch_header = dict(self._batch) walltime = step.run.get("walltime", None) - batch_header["walltime"] = \ - str(self._convert_walltime_to_seconds(walltime)) + batch_header["walltime"] = str( + self._convert_walltime_to_seconds(walltime) + ) if run["nodes"]: batch_header["nodes"] = run.pop("nodes") @@ -143,31 +158,40 @@ def get_header(self, step): return "\n".join(modified_header) def get_parallelize_command(self, procs, nodes=None, **kwargs): - """ - Generate the FLUX parallelization segement of the command line. + """Generate the FLUX parallelization segement of the command line. + + Args: + procs: Number of processors to allocate to the parallel call. + nodes: Number of nodes to allocate to the parallel call + (default = 1). + **kwargs: + + Returns: + A string of the parallelize command configured using nodes + and procs. - :param procs: Number of processors to allocate to the parallel call. - :param nodes: Number of nodes to allocate to the parallel call - (default = 1). - :returns: A string of the parallelize command configured using nodes - and procs. """ ntasks = nodes if nodes else self._batch.get("nodes", 1) return self._interface.parallelize( - procs, nodes=ntasks, addtl_args=self._addl_args, **kwargs) + procs, nodes=ntasks, addtl_args=self._addl_args, **kwargs + ) def submit(self, step, path, cwd, job_map=None, env=None): - """ - Submit a script to the Flux scheduler. - - :param step: The StudyStep instance this submission is based on. - :param path: Local path to the script to be executed. - :param cwd: Path to the current working directory. - :param job_map: A dictionary mapping step names to their job - identifiers. - :param env: A dict containing a modified environment for execution. - :returns: The return status of the submission command and job - identiifer. + """Submit a script to the Flux scheduler. + + Args: + step: The StudyStep instance this submission is based on. + path: Local path to the script to be executed. + cwd: Path to the current working directory. + job_map: A dictionary mapping step names to their job + identifiers. (Default value = None) + env: A dict containing a modified environment for execution. + (Default value = None) + + Returns: + The return status of the submission command and job + identiifer. + """ # walltime = self._convert_walltime_to_seconds(step.run["walltime"]) nodes = step.run.get("nodes") @@ -181,7 +205,9 @@ def submit(self, step, path, cwd, job_map=None, env=None): cores_per_task = ceil(processors / nodes) LOGGER.warn( "'cores per task' set to a non-value. Populating with a " - "sensible default. (cores per task = %d", cores_per_task) + "sensible default. (cores per task = %d", + cores_per_task, + ) # Calculate ngpus ngpus = step.run.get("gpus", 0) @@ -192,36 +218,49 @@ def submit(self, step, path, cwd, job_map=None, env=None): # Check to make sure that cores_per_task matches if processors # is specified. if processors > 0 and processors > ncores: - msg = "Calculated ncores (nodes * cores per task) = {} " \ - "-- procs = {}".format(ncores, processors) + msg = ( + "Calculated ncores (nodes * cores per task) = {} " + "-- procs = {}".format(ncores, processors) + ) LOGGER.error(msg) raise ValueError(msg) # Raise an exception if ncores is 0 if ncores <= 0: - msg = "Invalid number of cores specified. " \ - "Aborting. (ncores = {})".format(ncores) + msg = ( + "Invalid number of cores specified. " + "Aborting. (ncores = {})".format(ncores) + ) LOGGER.error(msg) raise ValueError(msg) - jobid, retcode, submit_status = \ - self._interface.submit( - nodes, processors, cores_per_task, path, cwd, walltime, ngpus, - job_name=step.name, force_broker=force_broker - ) + jobid, retcode, submit_status = self._interface.submit( + nodes, + processors, + cores_per_task, + path, + cwd, + walltime, + ngpus, + job_name=step.name, + force_broker=force_broker, + ) return SubmissionRecord(submit_status, retcode, jobid) def check_jobs(self, joblist): - """ - For the given job list, query execution status. + """For the given job list, query execution status. This method uses the scontrol show job command and does a regex search for job information. - :param joblist: A list of job identifiers to be queried. - :returns: The return code of the status query, and a dictionary of job - identifiers to their status. + Args: + joblist: A list of job identifiers to be queried. + + Returns: + The return code of the status query, and a dictionary of job + identifiers to their status. + """ LOGGER.debug("Joblist type -- %s", type(joblist)) LOGGER.debug("Joblist contents -- %s", joblist) @@ -247,11 +286,14 @@ def check_jobs(self, joblist): return chk_status, status def cancel_jobs(self, joblist): - """ - For the given job list, cancel each job. + """For the given job list, cancel each job. + + Args: + joblist: A list of job identifiers to be cancelled. + + Returns: + The return code to indicate if jobs were cancelled. - :param joblist: A list of job identifiers to be cancelled. - :returns: The return code to indicate if jobs were cancelled. """ # If we don"t have any jobs to check, just return status OK. if not joblist: @@ -261,18 +303,21 @@ def cancel_jobs(self, joblist): return CancellationRecord(c_status, r_code) def _state(self, flux_state): - """ - Map a scheduler specific job state to a Study.State enum. + """Map a scheduler specific job state to a Study.State enum. + + Args: + flux_state: String representation of scheduler job status. + + Returns: + A Study.State enum corresponding to parameter job_state. - :param flux_state: String representation of scheduler job status. - :returns: A Study.State enum corresponding to parameter job_state. """ raise NotImplementedError( - "FluxScriptAdapter no longer uses the _state mapping.") + "FluxScriptAdapter no longer uses the _state mapping." + ) def _write_script(self, ws_path, step): - """ - Write a Flux script to the workspace of a workflow step. + """Write a Flux script to the workspace of a workflow step. The job_map optional parameter is a map of workflow step names to job identifiers. This parameter so far is only planned to be used when a @@ -280,11 +325,15 @@ def _write_script(self, ws_path, step): chain using a scheduler dependency setting). The functionality of the parameter may change depending on both future intended use. - :param ws_path: Path to the workspace directory of the step. - :param step: An instance of a StudyStep. - :returns: Boolean value (True if to be scheduled), the path to the - written script for run["cmd"], and the path to the script - written for run["restart"] (if it exists). + Args: + ws_path: Path to the workspace directory of the step. + step: An instance of a StudyStep. + + Returns: + Boolean value (True if to be scheduled), the path to the + written script for run["cmd"], and the path to the script + written for run["restart"] (if it exists). + """ to_be_scheduled, cmd, restart = self.get_scheduler_command(step) diff --git a/maestrowf/interfaces/script/localscriptadapter.py b/maestrowf/interfaces/script/localscriptadapter.py index b7c01a0be..d63d9b31d 100644 --- a/maestrowf/interfaces/script/localscriptadapter.py +++ b/maestrowf/interfaces/script/localscriptadapter.py @@ -31,8 +31,7 @@ import logging import os -from maestrowf.abstracts.enums import JobStatusCode, SubmissionCode, \ - CancelCode +from maestrowf.abstracts.enums import JobStatusCode, SubmissionCode, CancelCode from maestrowf.interfaces.script import CancellationRecord, SubmissionRecord from maestrowf.abstracts.interfaces import ScriptAdapter from maestrowf.utils import start_process @@ -60,8 +59,7 @@ def __init__(self, **kwargs): self._extension = ".sh" def _write_script(self, ws_path, step): - """ - Write a Slurm script to the workspace of a workflow step. + """Write a Slurm script to the workspace of a workflow step. The job_map optional parameter is a map of workflow step names to job identifiers. This parameter so far is only planned to be used when a @@ -69,11 +67,15 @@ def _write_script(self, ws_path, step): chain using a scheduler's dependency setting). The functionality of the parameter may change depending on both future intended use. - :param ws_path: Path to the workspace directory of the step. - :param step: An instance of a StudyStep. - :returns: False (will not be scheduled), the path to the - written script for run["cmd"], and the path to the script written - for run["restart"] (if it exists). + Args: + ws_path: Path to the workspace directory of the step. + step: An instance of a StudyStep. + + Returns: + False (will not be scheduled), the path to the + written script for run["cmd"], and the path to the script written + for run["restart"] (if it exists). + """ cmd = step.run["cmd"] restart = step.run["restart"] @@ -96,27 +98,32 @@ def _write_script(self, ws_path, step): return to_be_scheduled, script_path, restart_path def check_jobs(self, joblist): - """ - For the given job list, query execution status. + """For the given job list, query execution status. + + Args: + joblist: A list of job identifiers to be queried. + + Returns: + The return code of the status query, and a dictionary of job + identifiers to their status. - :param joblist: A list of job identifiers to be queried. - :returns: The return code of the status query, and a dictionary of job - identifiers to their status. """ return JobStatusCode.NOJOBS, {} def cancel_jobs(self, joblist): - """ - For the given job list, cancel each job. + """For the given job list, cancel each job. + + Args: + joblist: A list of job identifiers to be cancelled. + + Returns: + The return code to indicate if jobs were cancelled. - :param joblist: A list of job identifiers to be cancelled. - :returns: The return code to indicate if jobs were cancelled. """ return CancellationRecord(CancelCode.OK, 0) def submit(self, step, path, cwd, job_map=None, env=None): - """ - Execute the step locally. + """Execute the step locally. If cwd is specified, the submit method will operate outside of the path specified by the 'cwd' parameter. @@ -124,12 +131,18 @@ def submit(self, step, path, cwd, job_map=None, env=None): variables for submission to the specified values. The 'env' parameter should be a dictionary of environment variables. - :param step: An instance of a StudyStep. - :param path: Path to the script to be executed. - :param cwd: Path to the current working directory. - :param job_map: A map of workflow step names to their job identifiers. - :param env: A dict containing a modified environment for execution. - :returns: The return code of the submission command and job identiifer. + Args: + step: An instance of a StudyStep. + path: Path to the script to be executed. + cwd: Path to the current working directory. + job_map: A map of workflow step names to their job identifiers. + (Default value = None) + env: A dict containing a modified environment for execution. + (Default value = None) + + Returns: + The return code of the submission command and job identiifer. + """ LOGGER.debug("cwd = %s", cwd) LOGGER.debug("Script to execute: %s", path) @@ -158,4 +171,5 @@ def submit(self, step, path, cwd, job_map=None, env=None): @property def extension(self): + """ """ return self._extension diff --git a/maestrowf/interfaces/script/lsfscriptadapter.py b/maestrowf/interfaces/script/lsfscriptadapter.py index f924c8558..5d644d519 100644 --- a/maestrowf/interfaces/script/lsfscriptadapter.py +++ b/maestrowf/interfaces/script/lsfscriptadapter.py @@ -36,8 +36,12 @@ from subprocess import PIPE, Popen from maestrowf.abstracts.interfaces import SchedulerScriptAdapter -from maestrowf.abstracts.enums import CancelCode, JobStatusCode, State, \ - SubmissionCode +from maestrowf.abstracts.enums import ( + CancelCode, + JobStatusCode, + State, + SubmissionCode, +) from maestrowf.interfaces.script import CancellationRecord, SubmissionRecord @@ -88,22 +92,25 @@ def __init__(self, **kwargs): } self._cmd_flags = { - "cmd": "jsrun --bind rs", - "ntasks": "--tasks_per_rs {procs} --cpu_per_rs {procs}", - "nodes": "--nrs", - "gpus": "-g", - "reservation": "-J", + "cmd": "jsrun --bind rs", + "ntasks": "--tasks_per_rs {procs} --cpu_per_rs {procs}", + "nodes": "--nrs", + "gpus": "-g", + "reservation": "-J", } self._extension = ".lsf.sh" def get_header(self, step): - """ - Generate the header present at the top of LSF execution scripts. + """Generate the header present at the top of LSF execution scripts. + + Args: + step: A StudyStep instance. + + Returns: + A string of the header based on internal batch parameters and + the parameter step. - :param step: A StudyStep instance. - :returns: A string of the header based on internal batch parameters and - the parameter step. """ run = dict(step.run) batch_header = dict(self._batch) @@ -120,9 +127,9 @@ def get_header(self, step): # If wall time is specified in three parts, we'll just calculate # the minutes off of the seconds and then shift up to hours if # needed. - seconds_minutes = ceil(float(wt_split[2])/60) + seconds_minutes = ceil(float(wt_split[2]) / 60) total_minutes = int(wt_split[1]) + seconds_minutes - hours = int(wt_split[0]) + int(total_minutes/60) + hours = int(wt_split[0]) + int(total_minutes / 60) total_minutes %= 60 walltime = "{:02d}:{:02d}".format(hours, int(total_minutes)) @@ -135,64 +142,61 @@ def get_header(self, step): return "\n".join(modified_header) def get_parallelize_command(self, procs, nodes=None, **kwargs): - """ - Generate the LSF parallelization segement of the command line. + """Generate the LSF parallelization segement of the command line. + + Args: + procs: Number of processors to allocate to the parallel call. + nodes: Number of nodes to allocate to the parallel call + (default = 1). + **kwargs: + + Returns: + A string of the parallelize command configured using nodes + and procs. - :param procs: Number of processors to allocate to the parallel call. - :param nodes: Number of nodes to allocate to the parallel call - (default = 1). - :returns: A string of the parallelize command configured using nodes - and procs. """ args = [self._cmd_flags["cmd"]] if nodes: _nodes = nodes - args += [ - self._cmd_flags["nodes"], - str(nodes) - ] + args += [self._cmd_flags["nodes"], str(nodes)] else: _nodes = 1 # Compute the number of CPUs per node (rs) - _procs = int(procs)/int(_nodes) + _procs = int(procs) / int(_nodes) # Processors segment - args += [ - self._cmd_flags["ntasks"].format(procs=_procs) - ] + args += [self._cmd_flags["ntasks"].format(procs=_procs)] # If we have GPUs being requested, add them to the command. gpus = kwargs.get("gpus", 0) if gpus: - args += [ - self._cmd_flags["gpus"], - str(gpus) - ] + args += [self._cmd_flags["gpus"], str(gpus)] return " ".join(args) def submit(self, step, path, cwd, job_map=None, env=None): - """ - Submit a script to the LSF scheduler. - - :param step: The StudyStep instance this submission is based on. - :param path: Local path to the script to be executed. - :param cwd: Path to the current working directory. - :param job_map: A dictionary mapping step names to their job - identifiers. - :param env: A dict containing a modified environment for execution. - :returns: The return status of the submission command and job - identiifer. + """Submit a script to the LSF scheduler. + + Args: + step: The StudyStep instance this submission is based on. + path: Local path to the script to be executed. + cwd: Path to the current working directory. + job_map: A dictionary mapping step names to their job + identifiers. (Default value = None) + env: A dict containing a modified environment for execution. + (Default value = None) + + Returns: + The return status of the submission command and job + identiifer. + """ args = ["bsub"] if "reservation" in self._batch: - args += [ - "-U", - self._batch["reservation"] - ] + args += ["-U", self._batch["reservation"]] args += ["-cwd", cwd, "<", path] cmd = " ".join(args) @@ -210,22 +214,27 @@ def submit(self, step, path, cwd, job_map=None, env=None): if retcode == 0: LOGGER.info("Submission returned status OK.") return SubmissionRecord( - SubmissionCode.OK, retcode, - re.search('[0-9]+', output).group(0)) + SubmissionCode.OK, + retcode, + re.search("[0-9]+", output).group(0), + ) else: LOGGER.warning("Submission returned an error.") return SubmissionRecord(SubmissionCode.ERROR, retcode, -1) def check_jobs(self, joblist): - """ - For the given job list, query execution status. + """For the given job list, query execution status. This method uses the scontrol show job command and does a regex search for job information. - :param joblist: A list of job identifiers to be queried. - :returns: The return code of the status query, and a dictionary of job - identifiers to their status. + Args: + joblist: A list of job identifiers to be queried. + + Returns: + The return code of the status query, and a dictionary of job + identifiers to their status. + """ # TODO: This method needs to be updated to use sacct. # squeue options: @@ -233,9 +242,9 @@ def check_jobs(self, joblist): # -t = list of job states to search for. 'all' for all states. # -o = status output formatting o_format = "jobid:7 stat:5 exit_code:10 exit_reason:50 delimiter='|'" - stat_cmd = "bjobs -a -u $USER -o \"{}\"" + stat_cmd = 'bjobs -a -u $USER -o "{}"' cmd = stat_cmd.format(o_format) - LOGGER.debug("bjobs cmd = \"%s\"", cmd) + LOGGER.debug('bjobs cmd = "%s"', cmd) p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) output, err = p.communicate() retcode = p.wait() @@ -255,8 +264,10 @@ def check_jobs(self, joblist): # system is configured to return 0. no_jobs = re.search(self.NOJOB_REGEX, output) if no_jobs: - LOGGER.warning("User '%s' has no jobs executing. Returning.", - getpass.getuser()) + LOGGER.warning( + "User '%s' has no jobs executing. Returning.", + getpass.getuser(), + ) return JobStatusCode.NOJOBS, {} # Otherwise, we can just process as normal. @@ -272,8 +283,8 @@ def check_jobs(self, joblist): LOGGER.debug("Entry split: %s", job_split) if len(job_split) < 4: LOGGER.debug( - "Entry has less than 4 fields. Skipping.", - job_split) + "Entry has less than 4 fields. Skipping.", job_split + ) continue while job_split[0] == "": @@ -295,29 +306,36 @@ def check_jobs(self, joblist): else: _j_state = job_split[state_index] _state = self._state(_j_state) - LOGGER.debug("ID Found. %s -- %s", - job_split[state_index], - _state) + LOGGER.debug( + "ID Found. %s -- %s", job_split[state_index], _state + ) status[job_split[jobid_index]] = _state return JobStatusCode.OK, status # NOTE: We're keeping this here for now since we could see it in the # future... elif retcode == 255: - LOGGER.warning("User '%s' has no jobs executing. Returning.", - getpass.getuser()) + LOGGER.warning( + "User '%s' has no jobs executing. Returning.", + getpass.getuser(), + ) return JobStatusCode.NOJOBS, status else: - LOGGER.error("Error code '%s' seen. Unexpected behavior " - "encountered.", retcode) + LOGGER.error( + "Error code '%s' seen. Unexpected behavior " "encountered.", + retcode, + ) return JobStatusCode.ERROR, status def cancel_jobs(self, joblist): - """ - For the given job list, cancel each job. + """For the given job list, cancel each job. + + Args: + joblist: A list of job identifiers to be cancelled. + + Returns: + The return code to indicate if jobs were cancelled. - :param joblist: A list of job identifiers to be cancelled. - :returns: The return code to indicate if jobs were cancelled. """ # If we don't have any jobs to check, just return status OK. if not joblist: @@ -331,16 +349,22 @@ def cancel_jobs(self, joblist): if retcode == 0: return CancellationRecord(CancelCode.OK, retcode) else: - LOGGER.error("Error code '%s' seen. Unexpected behavior " - "encountered.", retcode) + LOGGER.error( + "Error code '%s' seen. Unexpected behavior " "encountered.", + retcode, + ) return CancellationRecord(CancelCode.ERROR, retcode) def _state(self, lsf_state): - """ - Map a scheduler specific job state to a Study.State enum. + """Map a scheduler specific job state to a Study.State enum. + + Args: + slurm_state: String representation of scheduler job status. + lsf_state: + + Returns: + A Study.State enum corresponding to parameter job_state. - :param slurm_state: String representation of scheduler job status. - :returns: A Study.State enum corresponding to parameter job_state. """ # NOTE: fdinatale -- If I'm understanding this correctly, there are # four naturally occurring states (excluding states of suspension.) @@ -368,8 +392,7 @@ def _state(self, lsf_state): return State.UNKNOWN def _write_script(self, ws_path, step): - """ - Write a LSF script to the workspace of a workflow step. + """Write a LSF script to the workspace of a workflow step. The job_map optional parameter is a map of workflow step names to job identifiers. This parameter so far is only planned to be used when a @@ -377,11 +400,15 @@ def _write_script(self, ws_path, step): chain using a scheduler's dependency setting). The functionality of the parameter may change depending on both future intended use. - :param ws_path: Path to the workspace directory of the step. - :param step: An instance of a StudyStep. - :returns: Boolean value (True if to be scheduled), the path to the - written script for run["cmd"], and the path to the script - written for run["restart"] (if it exists). + Args: + ws_path: Path to the workspace directory of the step. + step: An instance of a StudyStep. + + Returns: + Boolean value (True if to be scheduled), the path to the + written script for run["cmd"], and the path to the script + written for run["restart"] (if it exists). + """ to_be_scheduled, cmd, restart = self.get_scheduler_command(step) @@ -415,4 +442,5 @@ def _write_script(self, ws_path, step): @property def extension(self): + """ """ return self._extension diff --git a/maestrowf/interfaces/script/slurmscriptadapter.py b/maestrowf/interfaces/script/slurmscriptadapter.py index 21ebf0462..3fef0a28c 100644 --- a/maestrowf/interfaces/script/slurmscriptadapter.py +++ b/maestrowf/interfaces/script/slurmscriptadapter.py @@ -34,8 +34,12 @@ import re from maestrowf.abstracts.interfaces import SchedulerScriptAdapter -from maestrowf.abstracts.enums import JobStatusCode, State, SubmissionCode, \ - CancelCode +from maestrowf.abstracts.enums import ( + JobStatusCode, + State, + SubmissionCode, + CancelCode, +) from maestrowf.interfaces.script import CancellationRecord, SubmissionRecord from maestrowf.utils import start_process @@ -85,13 +89,12 @@ def __init__(self, **kwargs): "queue": "#SBATCH --partition={queue}", "bank": "#SBATCH --account={bank}", "walltime": "#SBATCH --time={walltime}", - "job-name": - "#SBATCH --job-name=\"{job-name}\"\n" - "#SBATCH --output=\"{job-name}.out\"\n" - "#SBATCH --error=\"{job-name}.err\"", - "comment": "#SBATCH --comment \"{comment}\"", - "reservation": "#SBATCH --reservation=\"{reservation}\"", - "gpus": "#SBATCH --gres=gpu:{gpus}" + "job-name": '#SBATCH --job-name="{job-name}"\n' + '#SBATCH --output="{job-name}.out"\n' + '#SBATCH --error="{job-name}.err"', + "comment": '#SBATCH --comment "{comment}"', + "reservation": '#SBATCH --reservation="{reservation}"', + "gpus": "#SBATCH --gres=gpu:{gpus}", } self._ntask_header = "#SBATCH --ntasks={procs}" @@ -109,12 +112,15 @@ def __init__(self, **kwargs): self._unsupported = set(["cmd", "depends", "ntasks", "nodes"]) def get_header(self, step): - """ - Generate the header present at the top of Slurm execution scripts. + """Generate the header present at the top of Slurm execution scripts. + + Args: + step: A StudyStep instance. + + Returns: + A string of the header based on internal batch parameters and + the parameter step. - :param step: A StudyStep instance. - :returns: A string of the header based on internal batch parameters and - the parameter step. """ resources = {} @@ -122,7 +128,8 @@ def get_header(self, step): procs_in_batch = bool("procs" in resources) resources.update( { - resource: value for (resource, value) in step.run.items() + resource: value + for (resource, value) in step.run.items() if value } ) @@ -131,10 +138,11 @@ def get_header(self, step): nodes = resources.get("nodes") if not procs and not nodes: - err_msg = \ - 'No explicit resources specified in {}. At least one' \ - ' of "procs" or "nodes" must be set to a non-zero' \ - ' value.'.format(step.name) + err_msg = ( + "No explicit resources specified in {}. At least one" + ' of "procs" or "nodes" must be set to a non-zero' + " value.".format(step.name) + ) LOGGER.error(err_msg) raise RuntimeError(err_msg) @@ -159,21 +167,25 @@ def get_header(self, step): return "\n".join(modified_header) def get_parallelize_command(self, procs, nodes=None, **kwargs): - """ - Generate the SLURM parallelization segement of the command line. + """Generate the SLURM parallelization segement of the command line. + + Args: + procs: Number of processors to allocate to the parallel call. + nodes: Number of nodes to allocate to the parallel call + (default = 1). + **kwargs: + + Returns: + A string of the parallelize command configured using nodes + and procs. - :param procs: Number of processors to allocate to the parallel call. - :param nodes: Number of nodes to allocate to the parallel call - (default = 1). - :returns: A string of the parallelize command configured using nodes - and procs. """ args = [ # SLURM srun command self._cmd_flags["cmd"], # Processors segment self._cmd_flags["ntasks"], - str(procs) + str(procs), ] if nodes: @@ -189,25 +201,26 @@ def get_parallelize_command(self, procs, nodes=None, **kwargs): LOGGER.warning("'%s' is not supported -- omitted.", key) continue if value: - args += [ - self._cmd_flags[key], - "{}".format(str(value)) - ] + args += [self._cmd_flags[key], "{}".format(str(value))] return " ".join(args) def submit(self, step, path, cwd, job_map=None, env=None): - """ - Submit a script to the Slurm scheduler. - - :param step: The StudyStep instance this submission is based on. - :param path: Local path to the script to be executed. - :param cwd: Path to the current working directory. - :param job_map: A dictionary mapping step names to their job - identifiers. - :param env: A dict containing a modified environment for execution. - :returns: The return status of the submission command and job - identiifer. + """Submit a script to the Slurm scheduler. + + Args: + step: The StudyStep instance this submission is based on. + path: Local path to the script to be executed. + cwd: Path to the current working directory. + job_map: A dictionary mapping step names to their job + identifiers. (Default value = None) + env: A dict containing a modified environment for execution. + (Default value = None) + + Returns: + The return status of the submission command and job + identiifer. + """ # Leading command is 'sbatch' cmd = ["sbatch"] @@ -232,23 +245,27 @@ def submit(self, step, path, cwd, job_map=None, env=None): if retcode == 0: LOGGER.info("Submission returned status OK.") - jid = re.search('[0-9]+', output).group(0) + jid = re.search("[0-9]+", output).group(0) return SubmissionRecord(SubmissionCode.OK, retcode, jid) else: LOGGER.warning( - "Submission returned an error (see next line).\n%s", err) + "Submission returned an error (see next line).\n%s", err + ) return SubmissionRecord(SubmissionCode.ERROR, retcode) def check_jobs(self, joblist): - """ - For the given job list, query execution status. + """For the given job list, query execution status. This method uses the scontrol show job command and does a regex search for job information. - :param joblist: A list of job identifiers to be queried. - :returns: The return code of the status query, and a dictionary of job - identifiers to their status. + Args: + joblist: A list of job identifiers to be queried. + + Returns: + The return code of the status query, and a dictionary of job + identifiers to their status. + """ # TODO: This method needs to be updated to use sacct. # squeue options: @@ -290,27 +307,37 @@ def check_jobs(self, joblist): continue if job_split[jobid_index] in status: - LOGGER.debug("ID Found. %s -- %s", job_split[state_index], - self._state(job_split[state_index])) - status[job_split[jobid_index]] = \ - self._state(job_split[state_index]) + LOGGER.debug( + "ID Found. %s -- %s", + job_split[state_index], + self._state(job_split[state_index]), + ) + status[job_split[jobid_index]] = self._state( + job_split[state_index] + ) return JobStatusCode.OK, status elif retcode == 1: - LOGGER.warning("User '%s' has no jobs executing. Returning.", - getpass.getuser()) + LOGGER.warning( + "User '%s' has no jobs executing. Returning.", + getpass.getuser(), + ) return JobStatusCode.NOJOBS, status else: - LOGGER.error("Error code '%s' seen. Unexpected behavior " - "encountered.") + LOGGER.error( + "Error code '%s' seen. Unexpected behavior " "encountered." + ) return JobStatusCode.ERROR, status def cancel_jobs(self, joblist): - """ - For the given job list, cancel each job. + """For the given job list, cancel each job. + + Args: + joblist: A list of job identifiers to be cancelled. + + Returns: + The return code to indicate if jobs were cancelled. - :param joblist: A list of job identifiers to be cancelled. - :returns: The return code to indicate if jobs were cancelled. """ # If we don't have any jobs to check, just return status OK. if not joblist: @@ -324,18 +351,22 @@ def cancel_jobs(self, joblist): if retcode == 0: _record = CancellationRecord(CancelCode.OK, retcode) else: - LOGGER.error("Error code '%s' seen. Unexpected behavior " - "encountered.") + LOGGER.error( + "Error code '%s' seen. Unexpected behavior " "encountered." + ) _record = CancellationRecord(CancelCode.ERROR, retcode) return _record def _state(self, slurm_state): - """ - Map a scheduler specific job state to a Study.State enum. + """Map a scheduler specific job state to a Study.State enum. + + Args: + slurm_state: String representation of scheduler job status. + + Returns: + A Study.State enum corresponding to parameter job_state. - :param slurm_state: String representation of scheduler job status. - :returns: A Study.State enum corresponding to parameter job_state. """ LOGGER.debug("Received SLURM State -- %s", slurm_state) if slurm_state == "R": @@ -358,8 +389,7 @@ def _state(self, slurm_state): return State.UNKNOWN def _write_script(self, ws_path, step): - """ - Write a Slurm script to the workspace of a workflow step. + """Write a Slurm script to the workspace of a workflow step. The job_map optional parameter is a map of workflow step names to job identifiers. This parameter so far is only planned to be used when a @@ -367,11 +397,15 @@ def _write_script(self, ws_path, step): chain using a scheduler's dependency setting). The functionality of the parameter may change depending on both future intended use. - :param ws_path: Path to the workspace directory of the step. - :param step: An instance of a StudyStep. - :returns: Boolean value (True if to be scheduled), the path to the - written script for run["cmd"], and the path to the script written - for run["restart"] (if it exists). + Args: + ws_path: Path to the workspace directory of the step. + step: An instance of a StudyStep. + + Returns: + Boolean value (True if to be scheduled), the path to the + written script for run["cmd"], and the path to the script written + for run["restart"] (if it exists). + """ to_be_scheduled, cmd, restart = self.get_scheduler_command(step) @@ -400,4 +434,5 @@ def _write_script(self, ws_path, step): @property def extension(self): + """ """ return self._extension diff --git a/maestrowf/maestro.py b/maestrowf/maestro.py index 6ab00cf27..02fdd19c0 100644 --- a/maestrowf/maestro.py +++ b/maestrowf/maestro.py @@ -43,9 +43,13 @@ from maestrowf.specification import YAMLSpecification from maestrowf.datastructures.core import Study from maestrowf.datastructures.environment import Variable -from maestrowf.utils import \ - create_parentdir, create_dictionary, LoggerUtility, make_safe_path, \ - start_process +from maestrowf.utils import ( + create_parentdir, + create_dictionary, + LoggerUtility, + make_safe_path, + start_process, +) # Program Globals @@ -53,14 +57,22 @@ LOG_UTIL = LoggerUtility(LOGGER) # Configuration globals -DEBUG_FORMAT = "[%(asctime)s: %(levelname)s] " \ - "[%(module)s: %(lineno)d] %(message)s" +DEBUG_FORMAT = ( + "[%(asctime)s: %(levelname)s] " "[%(module)s: %(lineno)d] %(message)s" +) LFORMAT = "[%(asctime)s: %(levelname)s] %(message)s" ACCEPTED_INPUT = set(["yes", "y"]) def status_study(args): - """Check and print the status of an executing study.""" + """Check and print the status of an executing study. + + Args: + args: + + Returns: + + """ # Force logging to Warning and above LOG_UTIL.configure(LFORMAT, log_lvl=3) @@ -83,20 +95,29 @@ def status_study(args): print( "\nNo status to report -- the Maestro study in this path " "either unexpectedly crashed or the path does not contain " - "a Maestro study.") + "a Maestro study." + ) print("") print(header_format) else: print( "Path(s) or glob(s) did not resolve to a directory(ies) that " - "exists.") + "exists." + ) return 1 return 0 def cancel_study(args): - """Flag a study to be cancelled.""" + """Flag a study to be cancelled. + + Args: + args: + + Returns: + + """ # Force logging to Warning and above LOG_UTIL.configure(LFORMAT, log_lvl=3) @@ -110,7 +131,8 @@ def cancel_study(args): if not os.path.isdir(abs_path): print( f"Attempted to cancel '{abs_path}' " - "-- study directory not found.") + "-- study directory not found." + ) ret_code = 1 else: print(f"Study in '{abs_path}' to be cancelled.") @@ -136,21 +158,25 @@ def cancel_study(args): def load_parameter_generator(path, env, kwargs): - """ - Import and load custom parameter Python files. + """Import and load custom parameter Python files. - :param path: Path to a Python file containing the function \ + Args: + path: Path to a Python file containing the function \ 'get_custom_generator'. - :param env: A StudyEnvironment object containing custom information. - :param kwargs: Dictionary containing keyword arguments for the function \ + env: A StudyEnvironment object containing custom information. + kwargs: Dictionary containing keyword arguments for the function \ 'get_custom_generator'. - :returns: A populated ParameterGenerator instance. + + Returns: + A populated ParameterGenerator instance. + """ path = os.path.abspath(path) LOGGER.info("Loading custom parameter generator from '%s'", path) try: # Python 3.5 import importlib.util + LOGGER.debug("Using Python 3.5 importlib...") spec = importlib.util.spec_from_file_location("custom_gen", path) f = importlib.util.module_from_spec(spec) @@ -160,12 +186,14 @@ def load_parameter_generator(path, env, kwargs): try: # Python 3.3 from importlib.machinery import SourceFileLoader + LOGGER.debug("Using Python 3.4 SourceFileLoader...") f = SourceFileLoader("custom_gen", path).load_module() return f.get_custom_generator(env, **kwargs) except ImportError: # Python 2 import imp + LOGGER.debug("Using Python 2 imp library...") f = imp.load_source("custom_gen", path) return f.get_custom_generator(env, **kwargs) @@ -175,7 +203,14 @@ def load_parameter_generator(path, env, kwargs): def run_study(args): - """Run a Maestro study.""" + """Run a Maestro study. + + Args: + args: + + Returns: + + """ # Report log lvl LOGGER.info("INFO Logging Level -- Enabled") LOGGER.warning("WARNING Logging Level -- Enabled") @@ -206,7 +241,8 @@ def run_study(args): else: uinput = six.moves.input( "Output path already exists. Would you like to overwrite " - "it? [yn] ") + "it? [yn] " + ) if uinput.lower() in ACCEPTED_INPUT: print("Cleaning up existing out path...") @@ -224,8 +260,7 @@ def run_study(args): out_dir = os.path.abspath(out_dir.value) out_name = "{}_{}".format( - spec.name.replace(" ", "_"), - time.strftime("%Y%m%d-%H%M%S") + spec.name.replace(" ", "_"), time.strftime("%Y%m%d-%H%M%S") ) output_path = make_safe_path(out_dir, *[out_name]) environment.add(Variable("OUTPUT_PATH", output_path)) @@ -264,36 +299,52 @@ def run_study(args): parameters = spec.get_parameters() # Setup the study. - study = Study(spec.name, spec.description, studyenv=environment, - parameters=parameters, steps=steps, out_path=output_path) + study = Study( + spec.name, + spec.description, + studyenv=environment, + parameters=parameters, + steps=steps, + out_path=output_path, + ) # Check if the submission attempts is greater than 0: if args.attempts < 1: - _msg = "Submission attempts must be greater than 0. " \ - "'{}' provided.".format(args.attempts) + _msg = ( + "Submission attempts must be greater than 0. " + "'{}' provided.".format(args.attempts) + ) LOGGER.error(_msg) raise ArgumentError(_msg) # Check if the throttle is zero or greater: if args.throttle < 0: - _msg = "Submission throttle must be a value of zero or greater. " \ - "'{}' provided.".format(args.throttle) + _msg = ( + "Submission throttle must be a value of zero or greater. " + "'{}' provided.".format(args.throttle) + ) LOGGER.error(_msg) raise ArgumentError(_msg) # Check if the restart limit is zero or greater: if args.rlimit < 0: - _msg = "Restart limit must be a value of zero or greater. " \ - "'{}' provided.".format(args.rlimit) + _msg = ( + "Restart limit must be a value of zero or greater. " + "'{}' provided.".format(args.rlimit) + ) LOGGER.error(_msg) raise ArgumentError(_msg) # Set up the study workspace and configure it for execution. study.setup_workspace() study.configure_study( - throttle=args.throttle, submission_attempts=args.attempts, - restart_limit=args.rlimit, use_tmp=args.usetmp, hash_ws=args.hashws, - dry_run=args.dry) + throttle=args.throttle, + submission_attempts=args.attempts, + restart_limit=args.rlimit, + use_tmp=args.usetmp, + hash_ws=args.hashws, + dry_run=args.dry, + ) study.setup_environment() if args.dry: @@ -335,14 +386,21 @@ def run_study(args): else: # Launch manager with nohup log_path = make_safe_path( + study.output_path, *["{}.txt".format(study.name)] + ) + + cmd = [ + "nohup", + "conductor", + "-t", + str(sleeptime), + "-d", + str(args.debug_lvl), study.output_path, - *["{}.txt".format(study.name)]) - - cmd = ["nohup", "conductor", - "-t", str(sleeptime), - "-d", str(args.debug_lvl), - study.output_path, - ">", log_path, "2>&1"] + ">", + log_path, + "2>&1", + ] LOGGER.debug(" ".join(cmd)) start_process(" ".join(cmd)) @@ -359,113 +417,193 @@ def setup_argparser(): prog="maestro", description="The Maestro Workflow Conductor for specifying, launching" ", and managing general workflows.", - formatter_class=RawTextHelpFormatter) - subparsers = parser.add_subparsers(dest='subparser') + formatter_class=RawTextHelpFormatter, + ) + subparsers = parser.add_subparsers(dest="subparser") # subparser for a cancel subcommand - cancel = subparsers.add_parser( - 'cancel', - help="Cancel all running jobs.") + cancel = subparsers.add_parser("cancel", help="Cancel all running jobs.") cancel.add_argument( - "directory", type=str, nargs="+", - help="Directory containing a launched study.") + "directory", + type=str, + nargs="+", + help="Directory containing a launched study.", + ) cancel.set_defaults(func=cancel_study) # subparser for a run subcommand - run = subparsers.add_parser('run', - help="Launch a study based on a specification") - run.add_argument("-a", "--attempts", type=int, default=1, - help="Maximum number of submission attempts before a " - "step is marked as failed. [Default: %(default)d]") - run.add_argument("-r", "--rlimit", type=int, default=1, - help="Maximum number of restarts allowed when steps. " - "specify a restart command (0 denotes no limit). " - "[Default: %(default)d]") - run.add_argument("-t", "--throttle", type=int, default=0, - help="Maximum number of inflight jobs allowed to execute " - "simultaneously (0 denotes not throttling). " - "[Default: %(default)d]") - run.add_argument("-s", "--sleeptime", type=int, default=60, - help="Amount of time (in seconds) for the manager to " - "wait between job status checks. [Default: %(default)d]") - run.add_argument("--dry", action="store_true", default=False, - help="Generate the directory structure and scripts for a " - "study but do not launch it. [Default: %(default)s]") - run.add_argument("-p", "--pgen", type=str, - help="Path to a Python code file containing a function " - "that returns a custom filled ParameterGenerator " - "instance.") - run.add_argument("--pargs", type=str, action="append", default=[], - help="A string that represents a single argument to pass " - "a custom parameter generation function. Reuse '--parg' " - "to pass multiple arguments. [Use with '--pgen']") - run.add_argument("-o", "--out", type=str, - help="Output path to place study in. [NOTE: overrides " - "OUTPUT_PATH in the specified specification]") - run.add_argument("-fg", action="store_true", default=False, - help="Runs the backend conductor in the foreground " - "instead of using nohup. [Default: %(default)s]") - run.add_argument("--hashws", action="store_true", default=False, - help="Enable hashing of subdirectories in parameterized " - "studies (NOTE: breaks commands that use parameter labels" - " to search directories). [Default: %(default)s]") + run = subparsers.add_parser( + "run", help="Launch a study based on a specification" + ) + run.add_argument( + "-a", + "--attempts", + type=int, + default=1, + help="Maximum number of submission attempts before a " + "step is marked as failed. [Default: %(default)d]", + ) + run.add_argument( + "-r", + "--rlimit", + type=int, + default=1, + help="Maximum number of restarts allowed when steps. " + "specify a restart command (0 denotes no limit). " + "[Default: %(default)d]", + ) + run.add_argument( + "-t", + "--throttle", + type=int, + default=0, + help="Maximum number of inflight jobs allowed to execute " + "simultaneously (0 denotes not throttling). " + "[Default: %(default)d]", + ) + run.add_argument( + "-s", + "--sleeptime", + type=int, + default=60, + help="Amount of time (in seconds) for the manager to " + "wait between job status checks. [Default: %(default)d]", + ) + run.add_argument( + "--dry", + action="store_true", + default=False, + help="Generate the directory structure and scripts for a " + "study but do not launch it. [Default: %(default)s]", + ) + run.add_argument( + "-p", + "--pgen", + type=str, + help="Path to a Python code file containing a function " + "that returns a custom filled ParameterGenerator " + "instance.", + ) + run.add_argument( + "--pargs", + type=str, + action="append", + default=[], + help="A string that represents a single argument to pass " + "a custom parameter generation function. Reuse '--parg' " + "to pass multiple arguments. [Use with '--pgen']", + ) + run.add_argument( + "-o", + "--out", + type=str, + help="Output path to place study in. [NOTE: overrides " + "OUTPUT_PATH in the specified specification]", + ) + run.add_argument( + "-fg", + action="store_true", + default=False, + help="Runs the backend conductor in the foreground " + "instead of using nohup. [Default: %(default)s]", + ) + run.add_argument( + "--hashws", + action="store_true", + default=False, + help="Enable hashing of subdirectories in parameterized " + "studies (NOTE: breaks commands that use parameter labels" + " to search directories). [Default: %(default)s]", + ) prompt_opts = run.add_mutually_exclusive_group() prompt_opts.add_argument( - "-n", "--autono", action="store_true", default=False, - help="Automatically answer no to input prompts.") + "-n", + "--autono", + action="store_true", + default=False, + help="Automatically answer no to input prompts.", + ) prompt_opts.add_argument( - "-y", "--autoyes", action="store_true", default=False, - help="Automatically answer yes to input prompts.") + "-y", + "--autoyes", + action="store_true", + default=False, + help="Automatically answer yes to input prompts.", + ) # The only required positional argument for 'run' is a specification path. run.add_argument( - "specification", type=str, + "specification", + type=str, help="The path to a Study YAML specification that will be loaded and " - "executed.") + "executed.", + ) run.add_argument( - "--usetmp", action="store_true", default=False, + "--usetmp", + action="store_true", + default=False, help="Make use of a temporary directory for dumping scripts and other " - "Maestro related files.") + "Maestro related files.", + ) run.set_defaults(func=run_study) # subparser for a status subcommand status = subparsers.add_parser( - 'status', - help="Check the status of a running study.") + "status", help="Check the status of a running study." + ) status.add_argument( - "directory", type=str, nargs="+", - help="Directory containing a launched study.") + "directory", + type=str, + nargs="+", + help="Directory containing a launched study.", + ) status.set_defaults(func=status_study) # global options parser.add_argument( - "-l", "--logpath", type=str, - help="Alternate path to store program logging.") + "-l", + "--logpath", + type=str, + help="Alternate path to store program logging.", + ) parser.add_argument( - "-d", "--debug_lvl", type=int, default=2, + "-d", + "--debug_lvl", + type=int, + default=2, help="Level of logging messages to be output:\n" "5 - Critical\n" "4 - Error\n" "3 - Warning\n" "2 - Info (Default)\n" - "1 - Debug") + "1 - Debug", + ) parser.add_argument( - "-c", "--logstdout", action="store_true", default=True, - help="Log to stdout in addition to a file. [Default: %(default)s]") + "-c", + "--logstdout", + action="store_true", + default=True, + help="Log to stdout in addition to a file. [Default: %(default)s]", + ) parser.add_argument( - "-v", "--version", action="version", version='%(prog)s ' + __version__) + "-v", "--version", action="version", version="%(prog)s " + __version__ + ) return parser def main(): - """ - Execute the main program's functionality. + """Execute the main program's functionality. This function uses command line arguments to locate the study description. It makes use of the maestrowf core data structures as a high level class interface. + + Args: + + Returns: + """ # Set up the necessary base data structures to begin study set up. parser = setup_argparser() diff --git a/maestrowf/specification/yamlspecification.py b/maestrowf/specification/yamlspecification.py index d7702bc42..a11aea393 100644 --- a/maestrowf/specification/yamlspecification.py +++ b/maestrowf/specification/yamlspecification.py @@ -50,8 +50,7 @@ class YAMLSpecification(Specification): - """ - Class for loading and verifying a Study Specification. + """Class for loading and verifying a Study Specification. The Specification class provides an abstracted interface for constructing and managing studies. The Specification class makes use of a YAML file @@ -75,6 +74,11 @@ class YAMLSpecification(Specification): is an example of how you would use an interface (of whatever type) to construct the core structures and make use of them to run a study. + + Args: + + Returns: + """ def __init__(self): @@ -92,11 +96,14 @@ def __init__(self): @classmethod def load_specification(cls, path): - """ - Load a study specification. + """Load a study specification. + + Args: + path: Path to a study specification. + + Returns: + A specification object containing the information from path. - :param path: Path to a study specification. - :returns: A specification object containing the information from path. """ logger.info("Loading specification -- path = %s", path) try: @@ -114,12 +121,15 @@ def load_specification(cls, path): @classmethod def load_specification_from_stream(cls, stream): - """ - Load a study specification. + """Load a study specification. + + Args: + stream: Raw text stream to study YAML specification data. + + Returns: + A specification object containing the information from the + passed stream. - :param stream: Raw text stream to study YAML specification data. - :returns: A specification object containing the information from the - passed stream. """ try: @@ -169,11 +179,16 @@ def verify(self): ) def verify_description(self, schema): - """ - Verify the description in the specification. + """Verify the description in the specification. The description is required to have both a name and a description. If either is missing, the specification is considered invalid. + + Args: + schema: + + Returns: + """ # Verify that the top level structure contains a name, description # and study. @@ -188,14 +203,17 @@ def verify_description(self, schema): logger.debug("Study description verified -- \n%s", self.description) def _verify_variables(self): - """ - Verify the variables section of env in a specification. + """Verify the variables section of env in a specification. The criteria for each variable is as follows: 1. Each variable must have a name and value (non-empty strings) 2. A variable name cannot be repeated. - :returns: A set of keys encountered in the variables section. + Args: + + Returns: + A set of keys encountered in the variables section. + """ keys_seen = set() if "variables" not in self.environment: @@ -236,15 +254,18 @@ def _verify_sources(self): pass def _verify_dependencies(self, keys_seen): - """ - Verify the dependencies section of env in a specification. + """Verify the dependencies section of env in a specification. A dependency is required to have at least a name in all cases. Other required keys are entirely dependent on the type of dependency. - :param keys_seen: A set of the keys seen in other parts of the - specification. - :returns: A set of variable names seen. + Args: + keys_seen: A set of the keys seen in other parts of the + specification. + + Returns: + A set of variable names seen. + """ dep_types = ["path", "git", "spack"] @@ -274,11 +295,16 @@ def _verify_dependencies(self, keys_seen): return keys_seen def verify_environment(self, schema): - """Verify that the environment in a specification is valid.""" + """Verify that the environment in a specification is valid. + + Args: + schema: + + Returns: + + """ # validate environment against json schema - YAMLSpecification.validate_schema( - "env", self.environment, schema - ) + YAMLSpecification.validate_schema("env", self.environment, schema) # Verify the variables section of the specification. keys_seen = self._verify_variables() # Verify the sources section of the specification. @@ -287,7 +313,14 @@ def verify_environment(self, schema): self._verify_dependencies(keys_seen) def verify_study(self, schema): - """Verify the each step of the study in the specification.""" + """Verify the each step of the study in the specification. + + Args: + schema: + + Returns: + + """ # The workflow must have at least one step in it, otherwise, it's # not a workflow... try: @@ -307,19 +340,22 @@ def verify_study(self, schema): raise def _verify_steps(self, schema): - """ - Verify each study step in the specification. + """Verify each study step in the specification. A study step is required to have a name, description, and a command. If any are missing, the specification is considered invalid. + + Args: + schema: + + Returns: + """ try: for step in self.study: # validate step against json schema YAMLSpecification.validate_schema( - "study step '{}'".format(step["name"]), - step, - schema, + "study step '{}'".format(step["name"]), step, schema, ) except Exception as e: @@ -329,8 +365,7 @@ def _verify_steps(self, schema): logger.debug("Verified steps") def verify_parameters(self, schema): - """ - Verify the parameters section of the specification. + """Verify the parameters section of the specification. Verify that (if globals exist) they conform to the following: Each parameter must have: @@ -343,6 +378,12 @@ def verify_parameters(self, schema): 1. All global names must be unique. 2. Each list of values must be the same length. 3. If the label is a list, its length must match the value length. + + Args: + schema: + + Returns: + """ try: if self.globals: @@ -358,9 +399,7 @@ def verify_parameters(self, schema): # validate parameters against json schema YAMLSpecification.validate_schema( - "global.params.{}".format(name), - value, - schema, + "global.params.{}".format(name), value, schema, ) # If label is a list, check its length against values. @@ -400,9 +439,16 @@ def verify_parameters(self, schema): @staticmethod def validate_schema(parent_key, instance, schema): - """ - Given a parent key, an instance of a spec section, and a json schema + """Given a parent key, an instance of a spec section, and a json schema for that section, validate the instance against the schema. + + Args: + parent_key: + instance: + schema: + + Returns: + """ validator = jsonschema.Draft7Validator(schema) errors = validator.iter_errors(instance) @@ -413,8 +459,9 @@ def validate_schema(parent_key, instance, schema): re.search(r"'.+'", error.message).group(0).strip("'") ) raise jsonschema.ValidationError( - "Unrecognized key '{0}' found in {1}." - .format(unrecognized, parent_key) + "Unrecognized key '{0}' found in {1}.".format( + unrecognized, parent_key + ) ) elif error.validator == "type": @@ -425,8 +472,9 @@ def validate_schema(parent_key, instance, schema): .strip("'") ) raise jsonschema.ValidationError( - "In {0}, {1} must be of type '{2}'." - .format(parent_key, path, expected_type) + "In {0}, {1} must be of type '{2}'.".format( + parent_key, path, expected_type + ) ) elif error.validator == "required": @@ -441,27 +489,34 @@ def validate_schema(parent_key, instance, schema): elif error.validator == "uniqueItems": raise jsonschema.ValidationError( - "Non-unique step names in {0}.run.depends." - .format(parent_key) + "Non-unique step names in {0}.run.depends.".format( + parent_key + ) ) elif error.validator == "minLength": raise jsonschema.ValidationError( - "In {0}, empty string found as value for {1}." - .format(parent_key, path) + "In {0}, empty string found as value for {1}.".format( + parent_key, path + ) ) elif error.validator == "anyOf": path = ".".join(list(error.path)) context_message = error.context[0].message - context_message = re.sub(r"'.+' ", "'{0}' ".format( - path - ), context_message) + context_message = re.sub( + r"'.+' ", "'{0}' ".format(path), context_message + ) raise jsonschema.ValidationError( - ("The value '{0}' in field {1} of {2} is not of type " - "'{3}' or does not conform to the format '$(VARNAME)'.") - .format(error.instance, path, parent_key, - error.validator_value[0]["type"]) + ( + "The value '{0}' in field {1} of {2} is not of type " + "'{3}' or does not conform to the format '$(VARNAME)'." + ).format( + error.instance, + path, + parent_key, + error.validator_value[0]["type"], + ) ) else: @@ -469,10 +524,13 @@ def validate_schema(parent_key, instance, schema): @property def output_path(self): - """ - Return the OUTPUT_PATH variable (if it exists). + """Return the OUTPUT_PATH variable (if it exists). + + Args: + + Returns: + Returns OUTPUT_PATH if it exists, empty string otherwise. - :returns: Returns OUTPUT_PATH if it exists, empty string otherwise. """ if "variables" in self.environment: if "OUTPUT_PATH" in self.environment["variables"]: @@ -485,46 +543,61 @@ def output_path(self): @property def name(self): - """ - Getter for the name of a study specification. + """Getter for the name of a study specification. + + Args: + + Returns: + The name of the study described by the specification. - :returns: The name of the study described by the specification. """ return self.description["name"] @name.setter def name(self, value): - """ - Setter for the name of a study specification. + """Setter for the name of a study specification. + + Args: + value: String value representing the new name. + + Returns: - :param value: String value representing the new name. """ self.description["name"] = value @property def desc(self): - """ - Getter for the description of a study specification. + """Getter for the description of a study specification. + + Args: + + Returns: + A string containing the description of the study + specification. - :returns: A string containing the description of the study - specification. """ return self.description["description"] @desc.setter def desc(self, value): - """ - Setter for the description of a study specification. + """Setter for the description of a study specification. + + Args: + value: String value representing the new description. + + Returns: - :param value: String value representing the new description. """ self.description["description"] = value def get_study_environment(self): - """ - Generate a StudyEnvironment object from the environment in the spec. + """Generate a StudyEnvironment object from the environment in the spec. + + Args: + + Returns: + A StudyEnvironment object with the data in the specification. - :returns: A StudyEnvironment object with the data in the specification. """ env = StudyEnvironment() if "variables" in self.environment: @@ -564,10 +637,13 @@ def get_study_environment(self): return env def get_parameters(self): - """ - Generate a ParameterGenerator object from the global parameters. + """Generate a ParameterGenerator object from the global parameters. + + Args: + + Returns: + A ParameterGenerator with data from the specification. - :returns: A ParameterGenerator with data from the specification. """ params = ParameterGenerator() for key, value in self.globals.items(): @@ -581,10 +657,13 @@ def get_parameters(self): return params def get_study_steps(self): - """ - Generate a list of StudySteps from the study in the specification. + """Generate a list of StudySteps from the study in the specification. + + Args: + + Returns: + A list of StudyStep objects. - :returns: A list of StudyStep objects. """ steps = [] for step in self.study: diff --git a/maestrowf/utils.py b/maestrowf/utils.py index cd7322282..3c3d415d0 100644 --- a/maestrowf/utils.py +++ b/maestrowf/utils.py @@ -44,11 +44,15 @@ def get_duration(time_delta): - """ - Convert durations to HH:MM:SS format. + """Convert durations to HH:MM:SS format. + + Args: + s: time_delta: A time difference in datatime format. + time_delta: + + Returns: + A formatted string in HH:MM:SS - :params time_delta: A time difference in datatime format. - :returns: A formatted string in HH:MM:SS """ duration = time_delta.total_seconds() days = int(duration / 86400) @@ -56,19 +60,24 @@ def get_duration(time_delta): minutes = int((duration % 86400 % 3600) / 60) seconds = int((duration % 86400 % 3600) % 60) - return "{:d}d:{:02d}h:{:02d}m:{:02d}s" \ - .format(days, hours, minutes, seconds) + return "{:d}d:{:02d}h:{:02d}m:{:02d}s".format( + days, hours, minutes, seconds + ) def round_datetime_seconds(input_datetime): - """ - Round datetime to the nearest whole second. + """Round datetime to the nearest whole second. Solution referenced from: https://stackoverflow.com/questions/47792242/ rounding-time-off-to-the-nearest-second-python. - :params input_datetime: A datetime in datatime format. - :returns: ``input_datetime`` rounded to the nearest whole second + Args: + s: input_datetime: A datetime in datatime format. + input_datetime: + + Returns: + input_datetime`` rounded to the nearest whole second + """ new_datetime = input_datetime @@ -79,11 +88,14 @@ def round_datetime_seconds(input_datetime): def generate_filename(path, append_time=True): - """ - Generate a non-conflicting file name. + """Generate a non-conflicting file name. + + Args: + path: Path to file. + append_time: Setting to append a timestamp. (Default value = True) + + Returns: - :param path: Path to file. - :param append_time: Setting to append a timestamp. """ LOGGER.debug("Parameter path = %s", path) path = os.path.expanduser(path) @@ -97,9 +109,9 @@ def generate_filename(path, append_time=True): LOGGER.debug("Filename = %s", fname) index = 0 - timestamp = '' + timestamp = "" if append_time: - timestamp = '_{0}'.format(time.strftime("%Y%m%d-%H%M%S")) + timestamp = "_{0}".format(time.strftime("%Y%m%d-%H%M%S")) candidate = "{0}{1}{2}".format(fname, timestamp, ext) ls_files = set(os.listdir(parent)) @@ -112,25 +124,32 @@ def generate_filename(path, append_time=True): def create_parentdir(path): - """ - Recursively create parent directories. + """Recursively create parent directories. + + Args: + path: Path to a directory to be created. + + Returns: - :param path: Path to a directory to be created. """ if not os.path.exists(path): - LOGGER.info("Directory does not exist. Creating directories to %s", - path) + LOGGER.info( + "Directory does not exist. Creating directories to %s", path + ) path = os.path.expanduser(path) os.makedirs(path) def apply_function(item, func): - """ - Apply a function to items depending on type. + """Apply a function to items depending on type. - :param item: A Python primitive to apply a function to. - :param func: Function that returns takes item as a parameter and returns + Args: + item: A Python primitive to apply a function to. + func: Function that returns takes item as a parameter and returns item modified in some way. + + Returns: + """ if not item: return item @@ -140,23 +159,29 @@ def apply_function(item, func): return [apply_function(x, func) for x in item] elif isinstance(item, dict): return { - key: apply_function(value, func) for key, value in item.items()} + key: apply_function(value, func) for key, value in item.items() + } elif isinstance(item, int): return item else: - msg = "Encountered an object of type '{}'. Expected a str, list, int" \ - ", or dict.".format(type(item)) + msg = ( + "Encountered an object of type '{}'. Expected a str, list, int" + ", or dict.".format(type(item)) + ) LOGGER.error(msg) raise ValueError(msg) def csvtable_to_dict(fstream): - """ - Convert a csv file stream into an in memory dictionary. + """Convert a csv file stream into an in memory dictionary. + + Args: + fstream: An open file stream to a csv table (with header) + + Returns: + A dictionary with a key for each column header and a list of + column values for each key. - :param fstream: An open file stream to a csv table (with header) - :returns: A dictionary with a key for each column header and a list of - column values for each key. """ # Read in the lines from the file stream. lines = fstream.readlines() @@ -188,12 +213,17 @@ def csvtable_to_dict(fstream): def make_safe_path(base_path, *args): - """ - Construct a subpath that is path safe. + """Construct a subpath that is path safe. + + Args: + s: base_path: The base path to append args to. + s: args: Path components to join into a path. + base_path: + *args: + + Returns: + A joined subpath with invalid characters stripped. - :params base_path: The base path to append args to. - :params args: Path components to join into a path. - :returns: A joined subpath with invalid characters stripped. """ valid = "-_.() {}{}".format(string.ascii_letters, string.digits) path = [base_path] @@ -205,23 +235,29 @@ def make_safe_path(base_path, *args): def start_process(cmd, cwd=None, env=None, shell=True): - """ - Start a new process using a specified command. + """Start a new process using a specified command. + + Args: + cmd: A string or a list representing the command to be run. + cwd: Current working path that the process will be started in. + (Default value = None) + env: A dictionary containing the environment the process will use. + (Default value = None) + shell: Boolean that determines if the process will run a shell. + (Default value = True) + + Returns: - :param cmd: A string or a list representing the command to be run. - :param cwd: Current working path that the process will be started in. - :param env: A dictionary containing the environment the process will use. - :param shell: Boolean that determines if the process will run a shell. """ if isinstance(cmd, list): shell = False # Define kwargs for the upcoming Popen call. kwargs = { - "shell": shell, - "universal_newlines": True, - "stdout": PIPE, - "stderr": PIPE, + "shell": shell, + "universal_newlines": True, + "stdout": PIPE, + "stderr": PIPE, } # Individually check if cwd and env are set -- this prevents us from @@ -237,10 +273,13 @@ def start_process(cmd, cwd=None, env=None, shell=True): def ping_url(url): - """ - Load a webpage to test that it is accessible. + """Load a webpage to test that it is accessible. + + Args: + url: URL string to be loaded. + + Returns: - :param url: URL string to be loaded. """ try: response = urlopen(url) @@ -250,7 +289,10 @@ def ping_url(url): except URLError as e: LOGGER.error( "Check specified URL (%s) and that you are connected to the " - "internet. (%s)", url, e.code) + "internet. (%s)", + url, + e.code, + ) raise e else: response.read() @@ -258,12 +300,15 @@ def ping_url(url): def create_dictionary(list_keyvalues, token=":"): - """ - Create a dictionary from a list of key-value pairs. + """Create a dictionary from a list of key-value pairs. + + Args: + list_keyvalues: List of token separates key-values. + token: The token to split each key-value by. (Default value = ":") + + Returns: + A dictionary containing the key-value pairings in list_keyvalues. - :param list_keyvalues: List of token separates key-values. - :param token: The token to split each key-value by. - :returns: A dictionary containing the key-value pairings in list_keyvalues. """ _dict = {} for item in list_keyvalues: @@ -271,9 +316,11 @@ def create_dictionary(list_keyvalues, token=":"): key, value = [i.strip() for i in item.split(token, 1)] _dict[key] = value except ValueError: - msg = "'{}' is not capable of being split by the token '{}'. " \ - "Verify that all other parameters are formatted properly." \ - .format(item, token) + msg = ( + "'{}' is not capable of being split by the token '{}'. " + "Verify that all other parameters are formatted properly." + .format(item, token) + ) LOGGER.exception(msg) raise ValueError(msg) @@ -292,23 +339,35 @@ def __init__(self, logger): self._logger = logger def configure(self, log_format, log_lvl=2, colors=True): - """ - Configures the general logging facility. + """Configures the general logging facility. + + Args: + log_format: String containing the desired logging format. + log_lvl: Integer level (1-5) to set the logger to. + (Default value = 2) + colors: (Default value = True) + + Returns: - :param log_format: String containing the desired logging format. - :param log_lvl: Integer level (1-5) to set the logger to. """ logging.basicConfig(level=self.map_level(log_lvl), format=log_format) if colors: - coloredlogs.install(level=self.map_level(log_lvl), - logger=self._logger, fmt=log_format) + coloredlogs.install( + level=self.map_level(log_lvl), + logger=self._logger, + fmt=log_format, + ) def add_stream_handler(self, log_format, log_lvl=2): - """ - Add a stream handler to logging. + """Add a stream handler to logging. + + Args: + log_format: String containing the desired logging format. + log_lvl: Integer level (1-5) to set the logger to. + (Default value = 2) + + Returns: - :param log_format: String containing the desired logging format. - :param log_lvl: Integer level (1-5) to set the logger to. """ # Create the FileHandler and add it to the logger. sh = logging.StreamHandler() @@ -317,12 +376,16 @@ def add_stream_handler(self, log_format, log_lvl=2): self._logger.addHandler(sh) def add_file_handler(self, log_path, log_format, log_lvl=2): - """ - Add a file handler to logging. + """Add a file handler to logging. + + Args: + log_path: String containing the file path to store logging. + log_format: String containing the desired logging format. + log_lvl: Integer level (1-5) to set the logger to. + (Default value = 2) + + Returns: - :param log_path: String containing the file path to store logging. - :param log_format: String containing the desired logging format. - :param log_lvl: Integer level (1-5) to set the logger to. """ # Create the FileHandler and add it to the logger. formatter = logging.Formatter(log_format) @@ -334,10 +397,13 @@ def add_file_handler(self, log_path, log_format, log_lvl=2): @staticmethod def map_level(log_lvl): - """ - Map level 1-5 to their respective logging enumerations. + """Map level 1-5 to their respective logging enumerations. + + Args: + log_lvl: Integer level (1-5) representing logging verbosity. + + Returns: - :param log_lvl: Integer level (1-5) representing logging verbosity. """ if log_lvl == 1: return logging.DEBUG