Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions maestrowf/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,24 @@
try: # Python 2.7+
from logging import NullHandler
except ImportError:

class NullHandler(logging.Handler):
"""Null logging handler for Python 3+."""

def emit(self, record):
"""Override so that logging outputs nothing."""
"""Override so that logging outputs nothing.

Args:
record:

Returns:

"""
pass


LOGGER = logging.getLogger(__name__)
LOGGER.addHandler(NullHandler())

__version_info__ = ("1", "1", "9dev0")
__version__ = '.'.join(__version_info__)
__version__ = ".".join(__version_info__)
52 changes: 36 additions & 16 deletions maestrowf/abstracts/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,16 @@
from maestrowf.abstracts.specification import Specification


__all__ = ("abstractclassmethod", "Dependency", "Graph", "PickleInterface",
"Singleton", "Source", "Specification", "Substitution")
__all__ = (
"abstractclassmethod",
"Dependency",
"Graph",
"PickleInterface",
"Singleton",
"Source",
"Specification",
"Substitution",
)

LOGGER = logging.getLogger(__name__)

Expand All @@ -57,44 +65,56 @@ class PickleInterface:

@classmethod
def unpickle(cls, path):
"""
Load a pickled instance from a pickle file.
"""Load a pickled instance from a pickle file.

Args:
path: Path to a pickle file containing a class instance.

Returns:

:param path: Path to a pickle file containing a class instance.
"""
with open(path, 'rb') as pkl:
with open(path, "rb") as pkl:
obj = dill.load(pkl)

if not isinstance(obj, cls):
msg = "Object loaded from {path} is of type {type}. Expected an" \
" object of type '{cls}.'".format(path=path, type=type(obj),
cls=type(cls))
msg = (
"Object loaded from {path} is of type {type}. Expected an"
" object of type '{cls}.'".format(
path=path, type=type(obj), cls=type(cls)
)
)
LOGGER.error(msg)
raise TypeError(msg)

return obj

def pickle(self, path):
"""
Generate a pickle file of of a class instance.
"""Generate a pickle file of of a class instance.

Args:
path: The path to write the pickle to.

Returns:

:param path: The path to write the pickle to.
"""
with open(path, 'wb') as pkl:
with open(path, "wb") as pkl:
dill.dump(self, pkl)


class _Singleton(type):
""" """

_instances = {}

def __call__(cls, *args, **kwargs):
if cls not in cls._instances:
cls._instances[cls] = super(_Singleton, cls).__call__(*args,
**kwargs)
cls._instances[cls] = super(_Singleton, cls).__call__(
*args, **kwargs
)
return cls._instances[cls]


class Singleton(_Singleton('SingletonMeta', (object,), {})):
class Singleton(_Singleton("SingletonMeta", (object,), {})):
"""Single type to allow for classes to be typed as a singleton."""

pass
8 changes: 7 additions & 1 deletion maestrowf/abstracts/abstractclassmethod.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,13 @@ class abstractclassmethod(classmethod):
"""Python 2.7 does not include built in @abstractclassmethod so we create
our own class that extends @classmethod and then attaches a
__isabstractmethod variable to the callable function. See ref:
https://stackoverflow.com/a/11218474"""
https://stackoverflow.com/a/11218474

Args:

Returns:

"""

__isabstractmethod__ = True

Expand Down
16 changes: 9 additions & 7 deletions maestrowf/abstracts/containers/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,14 +38,16 @@ def __init__(self):
self._info = {}

def get(self, key, default=None):
"""
Get information by key in a record.
"""Get information by key in a record.

Args:
key: The key to look up in a Record's stored information.
default: The default value to return if the key is not found
(Default: None).

:param key: The key to look up in a Record's stored information.
:param default: The default value to return if the key is not found
(Default: None).
:returns: The information labeled by parameter key. Default if key does
not exist.
Returns:
The information labeled by parameter key. Default if key does
not exist.

"""
return self._info.get(key, default)
13 changes: 10 additions & 3 deletions maestrowf/abstracts/enums/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,16 +34,22 @@


class SubmissionCode(Enum):
""" """

OK = 0
ERROR = 1


class CancelCode(Enum):
""" """

OK = 0
ERROR = 1


class JobStatusCode(Enum):
""" """

OK = 0
NOJOBS = 1
ERROR = 2
Expand Down Expand Up @@ -71,7 +77,8 @@ class State(Enum):

class StudyStatus(Enum):
"""Workflow status enumeration"""
FINISHED = 0 # The Study has finished successfully, all steps ran
RUNNING = 1 # The Study is currently running
FAILURE = 2 # The Study has finished, but 1 or more steps failed

FINISHED = 0 # The Study has finished successfully, all steps ran
RUNNING = 1 # The Study is currently running
FAILURE = 2 # The Study has finished, but 1 or more steps failed
CANCELLED = 3 # The Study has finished, but was cancelled
76 changes: 52 additions & 24 deletions maestrowf/abstracts/envobject.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,36 +38,46 @@

@six.add_metaclass(ABCMeta)
class EnvObject:
"""
An abstract class representing objects that exist in a study's environment.
"""An abstract class representing objects that exist in a study's environment.

The EnvObject is meant to be used to represent entities in the larger
environment that affect the execution of a study (and therefore jobs).
This abstract base class should be used to represent things such as data
dependencies, code dependencies, variables, aliases, etc. The only method
we require is _verify, to allow users to verify that they've provided the
minimal information for the object to be valid.

Args:

Returns:

"""

@abstractmethod
def _verify(self):
"""
Verify that the object is valid.
"""Verify that the object is valid.

Subclasses that inherit from the EnvObject abstract class are expected
to provide a method for asserting that the contents contained within
the object are valid. Valid can range anywhere from asserting that all
expected member variables are populated to asserting specific values of
members, etc.

:returns: True if the EnvObject is verified, False otherwise.
Args:

Returns:
True if the EnvObject is verified, False otherwise.

"""

def _verification(self, error):
"""
A wrapper method for verifying for using a custom error message.
"""A wrapper method for verifying for using a custom error message.

Args:
error: String containing a custom error message.

Returns:

:param error: String containing a custom error message.
"""
if not self._verify():
LOGGER.exception(error)
Expand All @@ -80,22 +90,24 @@ class Substitution(EnvObject):

@abstractmethod
def substitute(self, data):
"""
Perform a replacement of some substring into data.
"""Perform a replacement of some substring into data.

The method takes the input string data and performs a replacement. This
API is used to represent concepts such as variables or parameters that
would want to be replaced within the string data.

:param data: A string to perform a replacement on.
:returns: A string equal to the original string data with substitutions
made (if any were performed).
Args:
data: A string to perform a replacement on.

Returns:
A string equal to the original string data with substitutions
made (if any were performed).

"""


class Source(EnvObject):
"""
Abstract class representing classes that alter environment sourcing.
"""Abstract class representing classes that alter environment sourcing.

WARNING: The API for this class is still in development.
The Source environment class is meant to provide a way to programmatically
Expand All @@ -104,20 +116,28 @@ class Source(EnvObject):

* Exporting of shell/environment variables (using 'export')
* Setting of an environment package with the 'use' command

Args:

Returns:

"""

@abstractmethod
def apply(self, data):
"""
Apply the Source to some string data.
"""Apply the Source to some string data.

Subclasses of Source should use this method in order to apply an
environment altering change. The 'data' parameter should be a string
representing a command to apply Source to or a list of other commands
that Source should be included with.

:param data: A string representing a command or set of other sources.
:returns: A string with the Source applied.
Args:
data: A string representing a command or set of other sources.

Returns:
A string with the Source applied.

"""
# NOTE: This functionality has not been settled yet. The use of this
# class or this design may not be the best for applying script sources
Expand All @@ -126,8 +146,7 @@ def apply(self, data):

@six.add_metaclass(ABCMeta)
class Dependency(Substitution):
"""
Abstract object representing a dependency.
"""Abstract object representing a dependency.

The Dependency base class is intended to be used to capture external items
the workflow is dependent on. These items include (but are not limited to):
Expand All @@ -140,17 +159,26 @@ class Dependency(Substitution):

The goal of this base class is to make it so that this package is able to
pull external dependencies in a consistent manner.

Args:

Returns:

"""

@abstractmethod
def acquire(self, substitutions=None):
"""
Acquire the dependency as specfied by the class instance.
"""Acquire the dependency as specfied by the class instance.

Subclasses that implement this interface should raise exceptions during
acquisition should they be unable to retrieve their specified
dependency. It is assumed that if acquiring throws an exception that
the study cannot proceed forward.

:param substitutions: List of Substitution objects that can be applied.
Args:
substitutions: List of Substitution objects that can be applied.
(Default value = None)

Returns:

"""
33 changes: 21 additions & 12 deletions maestrowf/abstracts/graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,27 +44,36 @@ class Graph:

@abstractmethod
def add_node(self, name, obj):
"""
Method to add a node to the graph.
"""Method to add a node to the graph.

Args:
name: String identifier of the node.
obj: An object representing the value of the node.

Returns:

:param name: String identifier of the node.
:param obj: An object representing the value of the node.
"""

@abstractmethod
def add_edge(self, src, dest):
"""
Add the edge (src, dest) to the graph.
"""Add the edge (src, dest) to the graph.

Args:
src: Source vertex name.
dest: Destination vertex name.

Returns:

:param src: Source vertex name.
:param dest: Destination vertex name.
"""

@abstractmethod
def remove_edge(self, src, dest):
"""
Remove edge (src, dest) from the graph.
"""Remove edge (src, dest) from the graph.

Args:
src: Source vertex name.
dest: Destination vertex name.

Returns:

:param src: Source vertex name.
:param dest: Destination vertex name.
"""
Loading