diff --git a/AGENTS.md b/AGENTS.md index 4197daadea..06af64164d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -57,7 +57,7 @@ A few distinctions are easy to confuse. Keep them straight before changing engin * **Graph = DAG of attributes.** A **`Graph`** (`meshroom/core/graph.py`) holds nodes connected by **`Edge`**s that link one node's *output* attribute to another node's *input* attribute. A connected input reads its value from upstream; the graph is a DAG and defines evaluation order. Each node computes a content-based **UID** (hash of its inputs) so unchanged nodes can be cached and skipped on recompute. * **Execution = chunks.** Work is split into **`NodeChunk`**s for parallelism (driven by the descriptor's `size`/`parallelization`). `desc.Node` subclasses implement **`processChunk(chunk)`** in Python; `desc.CommandLineNode` builds a command line from its `commandLine` template plus the chunk's range and runs the external binary. **`TaskManager`** (`meshroom/core/taskManager.py`) orchestrates execution — either **locally** (`compute`) or by **submitting** to a render farm (`submit`) via `meshroom/submitters/`. * **Persistence.** Graphs are saved as **`.mg`** JSON files (`meshroom/core/graphIO.py`): a versioned `header` plus a `graph` payload. On load, `nodeFactory` reconciles descriptor changes so old projects still open — a node that no longer matches its current descriptor becomes a **`CompatibilityNode`** instead of failing. Pipeline **templates** are just `.mg` files registered at startup. -* **Discovery.** Node types and pipeline templates are loaded at import time (`meshroom/core/__init__.py`, `meshroom/core/plugins.py`) from the built-in `meshroom/nodes/` plus any plugin/template paths (see [INSTALL_PLUGINS.md](INSTALL_PLUGINS.md)). +* **Discovery.** Node types and pipeline templates are loaded at import time (`meshroom/core/__init__.py`, `meshroom/core/plugins/`) from the built-in `meshroom/nodes/` plus any plugin/template paths (see [INSTALL_PLUGINS.md](INSTALL_PLUGINS.md)). * **UI bridge.** The engine is Qt-agnostic: `meshroom/common/` selects either a Qt or a headless backend for `BaseObject`, so the core runs without a UI. The `meshroom/ui/*.py` layer wraps it for Qt — `app.py` (application/entry), `graph.py` (a `UIGraph` exposing the core `Graph` as Qt models + async compute), `commands.py` (undo/redo command stack), `scene.py`. QML in `meshroom/ui/qml/` binds to these. ## Development & Verification diff --git a/INSTALL_PLUGINS.md b/INSTALL_PLUGINS.md index 59defad7cb..948e625e41 100644 --- a/INSTALL_PLUGINS.md +++ b/INSTALL_PLUGINS.md @@ -5,7 +5,7 @@ Plugins are collections of nodes and templates with their own dependencies. Plug ## Required Structure - **Meshroom folder**: All plugin nodes and templates must be placed within a `./meshroom/` directory -- **Configuration file (optional)**: `./meshroom/config.json` file allows to define custom environment variables for the plugin +- **Configuration file (optional)**: `./meshroom/config.json` file allows to define the plugin's name, version and custom environment variables - **Virtual environment (optional)**: If you have specific dependencies, you can create a virtual environment named "venv" in a folder and this Python will be used when computing the node. ## Example Structure @@ -18,8 +18,7 @@ For a plugin named "customPlugin", Meshroom expects this layout: │ │ │ ├── __init__.py # Required to be a python module │ │ │ ├── NodeA.py │ │ │ ├── NodeB.py -│ │ ├── customNodes2/ # Another set of nodes if needed -│ │ │ ├── __init__.py +│ │ ├── customNodes2/ # Another set of standalone nodes │ │ │ ├── NodeC.py │ │ │ ├── NodeD.py │ │ ├── customTemplate1.mg # Ready-to-use pipeline templates @@ -29,6 +28,28 @@ For a plugin named "customPlugin", Meshroom expects this layout: │ └── ... # Custom code (any structure) ``` +## Configuration File + +`config.json` can be written in two formats: + +- A plain list of environment variable entries: + ```json + [ + { "key": "MY_VAR", "type": "string", "value": "myValue" }, + { "key": "MY_PATH", "type": "path", "value": "relativeOrAbsolutePath" } + ] + ``` +- An object with optional `name`, `version`, and `env` keys, `env` using the same entry format as above: + ```json + { + "name": "customPlugin", + "version": "1.0.0", + "env": [ + { "key": "MY_VAR", "type": "string", "value": "myValue" } + ] + } + ``` + ## Loading the Plugin The "customPlugin" will be loaded automatically when Meshroom starts by setting the `MESHROOM_PLUGINS_PATH` environment variable: diff --git a/meshroom/core/__init__.py b/meshroom/core/__init__.py index ae9d23f76d..36addb0bd8 100644 --- a/meshroom/core/__init__.py +++ b/meshroom/core/__init__.py @@ -1,13 +1,8 @@ -from contextlib import contextmanager import hashlib -import importlib -import inspect import logging import os from pathlib import Path -import pkgutil import sys -import traceback import uuid try: @@ -18,7 +13,7 @@ except Exception: pass -from meshroom.core.plugins import NodePlugin, NodePluginManager, Plugin, processEnvFactory, formatNodeDescriptionErrorMessage +from meshroom.core.plugins.manager import PluginManager from meshroom.core.submitter import BaseSubmitter from meshroom.env import EnvVar, meshroomFolder from . import desc @@ -31,7 +26,7 @@ sessionUid = str(uuid.uuid1()) cacheFolderName = 'MeshroomCache' -pluginManager: NodePluginManager = NodePluginManager() +pluginManager: PluginManager = PluginManager() submitters: dict[str, BaseSubmitter] = {} pipelineTemplates: dict[str, str] = {} @@ -42,161 +37,6 @@ def hashValue(value) -> str: return hashObject.hexdigest() -@contextmanager -def add_to_path(p, packageName=None, pluginUid: str = None): - import sys - old_path = sys.path - sys.path = sys.path[:] - sys.path.insert(0, p) - try: - yield - finally: - sys.path = old_path - # Rename all meshroom plugins modules so that they all - # have a unique module name. - if packageName is not None: - for modName in list(sys.modules): - if modName == packageName or modName.startswith(packageName + "."): - mod = sys.modules.pop(modName) - uniqueModName = f"{pluginUid}_{modName}" - mod.__name__ = uniqueModName - sys.modules[uniqueModName] = mod - # Update the spec name, required for module reloading - mod.__spec__.name = uniqueModName - # Update classes __module__ so that all functions using - # __module__ string lookup resolve correctly. - for attrName in dir(mod): - attr = getattr(mod, attrName) - if isinstance(attr, type) and attr.__module__ == modName: - attr.__module__ = uniqueModName - - -def loadClasses(folder: str, packageName: str, classType: type, pluginUid: str = None) -> list[type]: - """ - Go over the Python module named "packageName" located in "folder" to find files - that contain classes of type "classType" and return these classes in a list. - - Args: - folder: the folder to load the module from. - packageName: the name of the module to look for nodes in. - classType: the class to look for in the files that are inspected. - pluginUid: (optional) A unique node for the plugin where will be the nodes. - """ - classes = [] - errors = [] - - resolvedFolder = str(Path(folder).resolve()) - # temporarily add folder to python path - with add_to_path(resolvedFolder, packageName, pluginUid): - # import node package - - try: - package = importlib.import_module(packageName) - packageName = package.packageName if hasattr(package, "packageName") \ - else package.__name__ - packagePath = os.path.dirname(package.__file__) - except Exception as exc: - tb = traceback.extract_tb(exc.__traceback__) - last_call = tb[-1] - logging.warning(f' * Failed to load package "{packageName}" from folder "{resolvedFolder}" ({type(exc).__name__}): {str(exc)}\n' - # filename:lineNumber functionName - f'{last_call.filename}:{last_call.lineno} {last_call.name}\n' - # line of code with the error - f'{last_call.line}' - # Full traceback - f'\n{traceback.format_exc()}\n\n' - ) - return [] - - for _, pluginName, _ in pkgutil.iter_modules(package.__path__): - pluginModuleName = "." + pluginName - - try: - pluginMod = importlib.import_module(pluginModuleName, package=package.__name__) - plugins = [plugin for _, plugin in inspect.getmembers(pluginMod, inspect.isclass) - if plugin.__module__ == f"{package.__name__}.{pluginName}" - and issubclass(plugin, classType)] - - if not plugins: - # Only packages/folders have __path__, single module/file do not have it. - isPackage = hasattr(pluginMod, "__path__") - # Sub-folders/Packages should not raise a warning - if not isPackage: - logging.debug(f"No class defined in plugin: {package.__name__}.{pluginName} ('{pluginMod.__file__}')") - - for p in plugins: - p.packageName = f"{pluginUid}_{packageName}" - p.packagePath = packagePath - if classType == desc.BaseNode: - nodePlugin = NodePlugin(p) - if nodePlugin.errors: - explicitErrors = [] - for err in nodePlugin.errors: - explicitErrors.append(f"\n\t - {formatNodeDescriptionErrorMessage(err)}") - errors.append(f" * {pluginName}: The following parameters have issues: {''.join(explicitErrors)}") - classes.append(nodePlugin) - else: - classes.append(p) - except Exception as exc: - if classType == BaseSubmitter: - logging.warning(f" Could not load submitter {pluginName} from package '{package.__name__}'\n{exc}") - else: - tb = traceback.extract_tb(exc.__traceback__) - last_call = tb[-1] - errors.append(f' * {pluginName} ({type(exc).__name__}): {exc}\n' - # filename:lineNumber functionName - f'{last_call.filename}:{last_call.lineno} {last_call.name}\n' - # line of code with the error - f'{last_call.line}' - # Full traceback - f'\n{traceback.format_exc()}\n\n' - ) - - if errors: - logging.warning(' The following "{package}" plugins could not be loaded:\n' - '{errorMsg}\n' - .format(package=packageName, errorMsg='\n'.join(errors))) - - return classes - - -def loadClassesNodes(folder: str, packageName: str, pluginUid: str) -> list[NodePlugin]: - """ - Return the list of all the NodePlugins that were created following the search of the - Python module named "packageName" located in the folder "folder". - A NodePlugin is created when a file within "packageName" that contains a class inheriting - desc.BaseNode is found. - - Args: - folder: the folder to load the module from. - packageName: the name of the module to look for nodes in. - pluginUid: A unique node for the plugin where will be the nodes. - - Returns: - list[NodePlugin]: a list of all the NodePlugins that were created based on the - module's search. If none has been created, an empty list is returned. - """ - return loadClasses(folder, packageName, desc.BaseNode, pluginUid=pluginUid) - - -def loadClassesSubmitters(folder: str, packageName: str) -> list[BaseSubmitter]: - """ - Return the list of all the submitters that were found during the search of the - Python module named "packageName" that located in the folder "folder". - A submitter is found if a file within "packageName" contains a class inheriting - from BaseSubmitter. - - Args: - folder: the folder to load the module from. - packageName: the name of the module to look for nodes in. - - Returns: - list[BaseSubmitter]: a list of all the submitters that were found during the - module's search - """ - return loadClasses(folder, packageName, BaseSubmitter) - - class Version: """ Version provides convenient properties and methods to manipulate and compare versions. @@ -343,74 +183,6 @@ def nodeVersion(nodeDesc: desc.Node, default=None): return moduleVersion(nodeDesc.__module__, default) -def loadNodes(folder, packageName, pluginUid) -> list[NodePlugin]: - if not os.path.isdir(folder): - logging.error(f"Node folder '{folder}' does not exist.") - return [] - - nodes = loadClassesNodes(folder, packageName, pluginUid) - return nodes - - -def loadAllNodes(folder) -> list[Plugin]: - plugins = [] - for _, package, ispkg in pkgutil.iter_modules([folder]): - if ispkg: - plugin = Plugin(package, folder) - nodePlugins = loadNodes(folder, package, plugin.uid) - if nodePlugins: - for node in nodePlugins: - plugin.addNodePlugin(node) - nodesStr = ', '.join([node.nodeDescriptor.__name__ for node in nodePlugins]) - logging.debug(f'Nodes loaded [{package}]: {nodesStr}') - plugins.append(plugin) - return plugins - - -def loadPluginFolder(folder, userPlugin: bool = False) -> list[Plugin]: - if not os.path.isdir(folder): - logging.info(f"Plugin folder '{folder}' does not exist.") - return [] - - mrFolder = Path(folder, 'meshroom') - if not mrFolder.exists(): - logging.info(f"Plugin folder '{folder}' does not contain a 'meshroom' folder.") - return [] - - plugins = loadAllNodes(folder=mrFolder) - if plugins: - for plugin in plugins: - plugin.isUserPlugin = userPlugin - pluginManager.addPlugin(plugin) - pipelineTemplates.update(plugin.templates) - - return plugins - - -def registerSubmitter(s: BaseSubmitter): - if s.name in submitters: - logging.error(f"Submitter {s.name} is already registered.") - submitters[s.name] = s - - -def loadSubmitters(folder, packageName) -> list[BaseSubmitter]: - if not os.path.isdir(folder): - logging.error(f"Submitters folder '{folder}' does not exist.") - return - - return loadClassesSubmitters(folder, packageName) - - -def loadAllSubmitters(folder) -> list[BaseSubmitter]: - submitters = [] - for _, package, ispkg in pkgutil.iter_modules([folder]): - if ispkg: - subs = loadSubmitters(folder, package) - if subs: - submitters.extend(subs) - return submitters - - def loadPipelineTemplates(folder: str): if not os.path.isdir(folder): logging.error(f"Pipeline templates folder '{folder}' does not exist.") @@ -421,27 +193,30 @@ def loadPipelineTemplates(folder: str): def initNodes(): - additionalNodesPath = EnvVar.getList(EnvVar.MESHROOM_NODES_PATH) - nodesFolders = [os.path.join(meshroomFolder, "nodes")] + additionalNodesPath - for f in nodesFolders: - plugins = loadAllNodes(folder=f) - if plugins: - for plugin in plugins: - pluginManager.addPlugin(plugin) + nodesFolder = os.path.join(meshroomFolder, "nodes") # Built-in nodes + additionalNodesFolders = EnvVar.getList(EnvVar.MESHROOM_NODES_PATH) + for folder in [nodesFolder] + additionalNodesFolders: + if not os.path.isdir(folder): + continue + # Load each subfolder as a built-in plugin + for subfolderPath in sorted(p for p in Path(folder).iterdir() + if p.is_dir() and not p.name.startswith("__")): + pluginManager.addPluginFromBuiltInFolder(subfolderPath.name, str(subfolderPath)) def initSubmitters(): - """ Detect and register submitter plugins - Note: Make sure the package name (folder inside the additionalPaths folders) - are unique : so we cannot name them "submitters" because it is already taken - by the submitters package inside meshroom - """ - # Load submitters - submitterPaths = EnvVar.getList(EnvVar.MESHROOM_SUBMITTERS_PATH) - for folder in submitterPaths: - subs = loadAllSubmitters(folder) - for sub in subs: - registerSubmitter(sub()) + # For now we do not want meshroom/submitters always loaded + # submittersFolder = os.path.join(meshroomFolder, "submitters") # Built-in submitters + additionalSubmittersFolders = EnvVar.getList(EnvVar.MESHROOM_SUBMITTERS_PATH) + for folder in additionalSubmittersFolders: + if not os.path.isdir(folder): + continue + # Load each subfolder as a built-in plugin + for subfolderPath in sorted(p for p in Path(folder).iterdir() + if p.is_dir() and not p.name.startswith("__")): + pluginManager.addPluginFromBuiltInFolder(subfolderPath.name, str(subfolderPath)) + + submitters.update({provider.name: provider.instance for provider in pluginManager.getSubmitterProviders().values()}) def initPipelines(): @@ -450,55 +225,42 @@ def initPipelines(): pipelineTemplatesFolders = EnvVar.getList(EnvVar.MESHROOM_PIPELINE_TEMPLATES_PATH) for f in pipelineTemplatesFolders: loadPipelineTemplates(f) - for plugin in pluginManager.getPlugins().values(): - pipelineTemplates.update(plugin.templates) + pipelineTemplates.update(pluginManager.getPipelineTemplates()) def initPlugins(): - # Classic plugins (with a DirTreeProcessEnv) + # Plugin paths + # Using DirTreeProcessEnv additionalPluginsPath = EnvVar.getList(EnvVar.MESHROOM_PLUGINS_PATH) pluginsFolders = [os.path.join(meshroomFolder, "plugins")] + additionalPluginsPath for folder in pluginsFolders: - plugins = loadPluginFolder(folder) - # Set the ProcessEnv for each plugin - if plugins: - for plugin in plugins: - plugin.processEnv = processEnvFactory(folder, plugin.configEnv, plugin.name) + # Use folder name as default plugin name + pluginManager.addPluginFromPath(Path(folder).name, folder, isUserPlugin=False) - # User plugins (with a DirTreeProcessEnv) + # User plugin paths + # Using DirTreeProcessEnv userPluginsFolders = EnvVar.getList(EnvVar.MESHROOM_USER_PLUGINS_PATH) for folder in userPluginsFolders: - plugins = loadPluginFolder(folder, userPlugin=True) - # Set the ProcessEnv for each user plugin - if plugins: - for plugin in plugins: - plugin.processEnv = processEnvFactory(folder, plugin.configEnv, plugin.name) - - # Rez plugins (with a RezProcessEnv) - rezPlugins = initRezPlugins() - - -def initRezPlugins(): - rezPlugins = {} - rezList = EnvVar.getList(EnvVar.MESHROOM_REZ_PLUGINS) - - for p in rezList: - name, folder = p.split("=") - rezPlugins[name] = folder # "name" is the name of the Rez package - plugins = loadPluginFolder(folder) - # Set the ProcessEnv for Rez plugins - if plugins: - for plugin in plugins: - plugin.processEnv = processEnvFactory(folder, plugin.configEnv, plugin.name, envType="rez", uri=name) - - userRezList = EnvVar.getList(EnvVar.MESHROOM_USER_REZ_PLUGINS) - for p in userRezList: - name, folder = p.split("=") - rezPlugins[name] = folder # "name" is the name of the Rez package - plugins = loadPluginFolder(folder, userPlugin=True) - # Set the ProcessEnv for user Rez plugins - if plugins: - for plugin in plugins: - plugin.processEnv = processEnvFactory(folder, plugin.configEnv, plugin.name, envType="rez", uri=name) - - return rezPlugins + # Use folder name as default plugin name + pluginManager.addPluginFromPath(Path(folder).name, folder, isUserPlugin=True) + + # Rez plugins + # Using RezProcessEnv + rezPluginList = EnvVar.getList(EnvVar.MESHROOM_REZ_PLUGINS) + for entry in rezPluginList: + # Use the REZ package name as plugin name + rezPackageNameVersion, rezPackageFolder = entry.split("=") + rezPackageName, _, rezPackageVersion = rezPackageNameVersion.partition("-") + pluginManager.addPluginFromRez(rezPackageName, rezPackageVersion, rezPackageFolder, isUserPlugin=False) + + # Rez user plugins + # Using RezProcessEnv + rezUserPluginList = EnvVar.getList(EnvVar.MESHROOM_USER_REZ_PLUGINS) + for entry in rezUserPluginList: + # Use the REZ package name as plugin name + rezPackageNameVersion, rezPackageFolder = entry.split("=") + rezPackageName, _, rezPackageVersion = rezPackageNameVersion.partition("-") + pluginManager.addPluginFromRez(rezPackageName, rezPackageVersion, rezPackageFolder, isUserPlugin=True) + + # Update pipeline templates + pipelineTemplates.update(pluginManager.getPipelineTemplates()) diff --git a/meshroom/core/attribute.py b/meshroom/core/attribute.py index 2836554749..d46cfaa7f5 100644 --- a/meshroom/core/attribute.py +++ b/meshroom/core/attribute.py @@ -218,7 +218,7 @@ def _getEvalValue(self): For string, expressions will be evaluated. """ if isinstance(self.value, str): - env = self.node.nodePlugin.configFullEnv if self.node.nodePlugin else os.environ + env = self.node.nodeDescProvider.configFullEnv if self.node.nodeDescProvider else os.environ substituted = Template(self.value).safe_substitute(env) try: varResolved = substituted.format(**self.node._expVars, **self.node._staticExpVars) diff --git a/meshroom/core/desc/node.py b/meshroom/core/desc/node.py index d3710551f5..b27463aa85 100644 --- a/meshroom/core/desc/node.py +++ b/meshroom/core/desc/node.py @@ -251,8 +251,11 @@ class BaseNode(object): parallelization = None documentation = "" category = "Other" - plugin = None nodeVersionType: NodeVersionType = NodeVersionType.UNKNOWN + # The plugin NodeDescProvider that supplied this node + provider = None + # The plugin that contains this node + plugin = None # Licenses required to run the plugin # Only used to select machines on the farm when the node is submitted _licenses = [] @@ -286,7 +289,7 @@ def getMrNodeType(self): def getNodeInfo(cls): info = OrderedDict([ ("module", cls.__module__), - ("modulePath", cls.plugin.path if cls.plugin else ""), + ("modulePath", cls.provider.path if cls.provider else ""), ]) # > Info from the plugin module plugin_module = sys.modules.get(cls.__module__) @@ -579,9 +582,9 @@ def processChunkInEnvironment(self, chunk): elif len(chunk.node.getChunks()) >= 1: meshroomComputeCmd += f" --iteration {chunk.range.iteration}" - runtimeEnv = chunk.node.nodeDesc.plugin.runtimeEnv - cmdPrefix = chunk.node.nodeDesc.plugin.commandPrefix - cmdSuffix = chunk.node.nodeDesc.plugin.commandSuffix + runtimeEnv = chunk.node.nodeDesc.provider.runtimeEnv + cmdPrefix = chunk.node.nodeDesc.provider.commandPrefix + cmdSuffix = chunk.node.nodeDesc.provider.commandSuffix self.executeChunkCommandLine(chunk, cmdPrefix + meshroomComputeCmd + cmdSuffix, env=runtimeEnv) @@ -604,9 +607,9 @@ def buildCommandLine(self, chunk) -> str: cmdLineVars = chunk.node.createCmdLineVars() cmdPrefix = "" cmdSuffix = "" - if chunk.node.nodeDesc.plugin: - cmdPrefix = chunk.node.nodeDesc.plugin.commandPrefix - cmdSuffix = chunk.node.nodeDesc.plugin.commandSuffix + if chunk.node.nodeDesc.provider: + cmdPrefix = chunk.node.nodeDesc.provider.commandPrefix + cmdSuffix = chunk.node.nodeDesc.provider.commandSuffix if chunk.node.isParallelized and chunk.node.size > 1: cmdSuffix = " " + self.commandLineRange.format(**chunk.range.toDict()) + " " + cmdSuffix @@ -622,7 +625,7 @@ def buildCommandLine(self, chunk) -> str: def processChunk(self, chunk): cmd = self.buildCommandLine(chunk) - runtimeEnv = chunk.node.nodeDesc.plugin.runtimeEnv + runtimeEnv = chunk.node.nodeDesc.provider.runtimeEnv self.executeChunkCommandLine(chunk, cmd, env=runtimeEnv) diff --git a/meshroom/core/graph.py b/meshroom/core/graph.py index 6f63dc9a80..fa9becad40 100644 --- a/meshroom/core/graph.py +++ b/meshroom/core/graph.py @@ -847,7 +847,7 @@ def upgradeAllNodes(self): for nodeName in nodeNames: self.upgradeNode(nodeName) - def reloadNodePlugins(self, nodeTypes: list[str]): + def reloadNodeDescProviders(self, nodeTypes: list[str]): """ Replace all the node instances of "nodeTypes" in the current graph with new node instances of the same type. If the description of the nodes has changed, the reloaded nodes will reflect theses diff --git a/meshroom/core/node.py b/meshroom/core/node.py index d3c99c1185..4f563abf3b 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -859,12 +859,12 @@ def __init__(self, nodeType: str, position: Position = None, parent: BaseObject super().__init__(parent) self._nodeType: str = nodeType self.nodeDesc: desc.BaseNode = None - self.nodePlugin: plugins.Plugin = None + self.nodeDescProvider: plugins.base.NodeDescProvider = None # instantiate node description if nodeType is valid - if meshroom.core.pluginManager.getRegisteredNodePlugin(nodeType): - self.nodeDesc = meshroom.core.pluginManager.getRegisteredNodePlugin(nodeType).nodeDescriptor() - self.nodePlugin = meshroom.core.pluginManager.getRegisteredNodePlugin(nodeType) + if meshroom.core.pluginManager.getNodeDescProvider(nodeType): + self.nodeDesc = meshroom.core.pluginManager.getNodeDescProvider(nodeType).nodeDescClass() + self.nodeDescProvider = meshroom.core.pluginManager.getNodeDescProvider(nodeType) self.packageName: str = "" self._internalFolder: str = "" diff --git a/meshroom/core/nodeFactory.py b/meshroom/core/nodeFactory.py index 363a9d1292..0b8fda2335 100644 --- a/meshroom/core/nodeFactory.py +++ b/meshroom/core/nodeFactory.py @@ -72,8 +72,8 @@ def __init__( self.position = Position(*self.nodeData.get("position", [])) self.uid = self.nodeData.get("uid", None) self.nodeDesc = None - if meshroom.core.pluginManager.isRegistered(self.nodeType): - self.nodeDesc = meshroom.core.pluginManager.getRegisteredNodePlugin(self.nodeType).nodeDescriptor + if meshroom.core.pluginManager.isNodeDescRegistered(self.nodeType): + self.nodeDesc = meshroom.core.pluginManager.getNodeDescProvider(self.nodeType).nodeDescClass def create(self) -> Union[Node, BackdropNode, CompatibilityNode]: compatibilityIssue = self._checkCompatibilityIssues() @@ -93,7 +93,7 @@ def _normalizeNodeData(self): def _checkCompatibilityIssues(self) -> Optional[CompatibilityIssue]: if self.nodeDesc is None: - if meshroom.core.pluginManager.belongsToPlugin(self.nodeType) is not None: + if meshroom.core.pluginManager.getPluginFromNodeDesc(self.nodeType) is not None: return CompatibilityIssue.PluginIssue return CompatibilityIssue.UnknownNodeType diff --git a/meshroom/core/plugins.py b/meshroom/core/plugins.py deleted file mode 100644 index 907d28c991..0000000000 --- a/meshroom/core/plugins.py +++ /dev/null @@ -1,799 +0,0 @@ -from __future__ import annotations - -import glob -import importlib -import json -import logging -import os -import re -import sys - -from enum import Enum -from inspect import getfile -from pathlib import Path - -from meshroom.common import BaseObject -from meshroom.core import desc -from meshroom.core.desc.attribute import ValueTypeErrors -from meshroom import _MESHROOM_ROOT -from meshroom.core.desc.node import _MESHROOM_COMPUTE_DEPS - - -def validateNodeDesc(nodeDesc: desc.BaseNode) -> list[tuple[str, ValueTypeErrors]]: - """ - Check that the node has a valid description before being loaded. For the description - to be valid, the default value of every parameter needs to correspond to the type - of the parameter. - An empty returned list means that every parameter is valid, and so is the node's description. - If it is not valid, the returned list contains the names of the invalid parameters. In case - of nested parameters (parameters in groups or lists, for example), the name of the parameter - follows the name of the parent attributes. For example, if the attribute "x", contained in group - "group", is invalid, then it will be added to the list as "group:x". - - Args: - nodeDesc: Description of the node. - - Returns: - errors: The list of invalid parameters if there are any, empty list otherwise. - """ - errors = [] - - for param in nodeDesc.inputs: - errMsg, errType = param.checkValueTypes() - if errMsg: - errors.append((errMsg, errType)) - - for param in nodeDesc.outputs: - if param.value is None: - if issubclass(nodeDesc, desc.InitNode): - errors.append((f"{param.name}", ValueTypeErrors.DYNAMIC_OUTPUT)) - continue - errMsg, errType = param.checkValueTypes() - if errMsg: - errors.append((errMsg, errType)) - - return errors - -def formatNodeDescriptionErrorMessage(error: tuple[str, ValueTypeErrors]) -> str: - """ - Format a node description error message from a tuple containing the error message (name of the attribute) and type. - - Args: - error: Tuple containing the name of the parameter that was rejected, and the type of the error. - - Returns: - str: Formatted error message. - """ - errMsg, errType = error - - if errType == ValueTypeErrors.TYPE: - return f"'value': Invalid type for parameter '{errMsg}'." - if errType == ValueTypeErrors.RANGE: - return f"'range': Invalid range value for parameter '{errMsg}'." - if errType == ValueTypeErrors.DYNAMIC_OUTPUT: - return f"'value': Unsupported dynamic output for parameter '{errMsg}'." - return f"Unknown error for parameter '{errMsg}'." - - -class ProcessEnvType(Enum): - """ Supported process environments. """ - DIRTREE = "dirtree", - REZ = "rez" - - -class ProcessEnv(BaseObject): - """ - Describes the environment required by a node's process. - - Args: - folder: the source folder for the process. - configEnv: the dictionary containing the environment variables defined in a configuration file - for the process to run. - pluginName: the name of the plugin object. - envType: (optional) the type of process environment. - uri: (optional) the Unique Resource Identifier to activate the environment. - """ - - def __init__(self, folder: str, configEnv: dict[str, str], pluginName: str, - envType: ProcessEnvType = ProcessEnvType.DIRTREE, uri: str = ""): - super().__init__() - self._folder: str = folder - self._configEnv: dict[str: str] = configEnv - self.pluginName: str = pluginName - self._processEnvType: ProcessEnvType = envType - self.uri: str = uri - self._env: dict = None - - def getEnvDict(self) -> dict: - """ Return the environment dictionary if it has been modified, None otherwise. """ - return self._env - - def getCommandPrefix(self) -> str: - """ Return the prefix to the command line that will be executed by the process. """ - return "" - - def getCommandSuffix(self) -> str: - """ Return the suffix to the command line that will be executed by the process. """ - return "" - - -class DirTreeProcessEnv(ProcessEnv): - """ - """ - def __init__(self, folder: str, configEnv: dict[str: str], pluginName: str): - super().__init__(folder, configEnv, pluginName, envType=ProcessEnvType.DIRTREE) - - # If there is a virtual environment, it is expected to be named "venv". - # Beside the virtual environment, a standard "bin"/"lib"/"lib64" hierarchy at - # the top level of the plugin folder is expected. - venvFolder = Path(folder, "venv") - - # Find all the libs that are not directly at the "lib*"-level - envLibPaths = glob.glob(f'{folder}/lib*/python[0-9].[0-9]*/site-packages', - recursive=False) - venvLibPaths = glob.glob(f'{venvFolder}/lib*/python[0-9].[0-9]*/site-packages', - recursive=False) - - self.binPaths: list = [str(Path(folder, "bin")), str(Path(venvFolder, "bin"))] - self.libPaths: list = [str(Path(folder, "lib")), str(Path(folder, "lib64")), - str(Path(venvFolder, "lib")), str(Path(venvFolder, "lib64"))] - self.pythonPaths: list = [str(Path(folder)), str(Path(venvFolder))] + \ - self.binPaths + envLibPaths + venvLibPaths - - if sys.platform == "win32": - # For Windows platforms, try and include the content of the virtual env if it exists - # The virtual env is expected to be named "venv" - venvLibPath = Path(venvFolder, "Lib", "site-packages") - if venvLibPath.exists(): - self.pythonPaths.append(venvLibPath.as_posix()) - else: - # For Linux platforms, lib paths may need to be discovered recursively to be properly - # added to LD_LIBRARY_PATH - extraLibPaths = [] - regex = re.compile(r"^lib(\d{2})?$") - for envPath in envLibPaths + venvLibPaths: - for path, directories, _ in os.walk(envPath): - for directory in directories: - if re.match(regex, directory): - extraLibPaths.append(os.path.join(path, directory)) - self.libPaths = self.libPaths + extraLibPaths - - # Setup the environment dictionary - self._env = os.environ.copy() - self._env["PYTHONPATH"] = os.pathsep.join( - [f"{_MESHROOM_ROOT}"] + self.pythonPaths + [os.getenv('PYTHONPATH', '')]) - self._env["LD_LIBRARY_PATH"] = f"{os.pathsep.join(self.libPaths)}{os.pathsep}{os.getenv('LD_LIBRARY_PATH', '')}" - self._env["PATH"] = f"{os.pathsep.join(self.binPaths)}{os.pathsep}{os.getenv('PATH', '')}" - - for k, val in self._configEnv.items(): - # Preserve user-defined environment variables: - # manually set environment variable values take precedence over config file defaults. - if k in self._env: - continue - - self._env[k] = val - - -class RezProcessEnv(ProcessEnv): - """ - """ - - REZ_DELIMITER_PATTERN = re.compile(r"-|==|>=|>|<=|<") - - def __init__(self, folder: str, configEnv: dict[str: str], pluginName: str, uri: str = ""): - if not uri: - raise RuntimeError("Missing name of the Rez environment needs to be provided.") - super().__init__(folder, configEnv, pluginName, envType=ProcessEnvType.REZ, uri=uri) - - def resolveRezSubrequires(self) -> list[str]: - """ - Return the list of packages defined for the node execution. These execution packages are - named subrequires. - Note: If a package does not have a version number, the version is aligned with the main - Meshroom environment (if this package is defined). - """ - if os.getenv(f"{self.uri.upper()}_{self.pluginName.upper()}_SUBREQUIRES"): - subrequires = os.environ.get(f"{self.uri.upper()}_{self.pluginName.upper()}_SUBREQUIRES", "").split(os.pathsep) - else: - subrequires = os.environ.get(f"{self.uri.upper()}_SUBREQUIRES", "").split(os.pathsep) - if not subrequires: - return [] - - packages = [] - # Packages that are resolved in the current environment - currentEnvPackages = [] - resolvedVersions = {} - if "REZ_USED_RESOLVE" in os.environ: - resolvedPackages = os.getenv("REZ_USED_RESOLVE", "").split() - for package in resolvedPackages: - if package.startswith("~"): - continue - currentEnvPackages.append(package) - name, version = self.REZ_DELIMITER_PATTERN.split(package, maxsplit=1) - resolvedVersions[name] = version - logging.debug("Packages in the current environment: " + ", ".join(currentEnvPackages)) - - # Take packages with the set versions for those which have one, and try to take packages - # in the current environment (if they are resolved in it) - for package in subrequires: - packageTuple = self.REZ_DELIMITER_PATTERN.split(package, maxsplit=1) - if len(packageTuple) == 1: - # Only the package name in the subrequires. - # Search for a corresponding version in the parent environment. - packageName = packageTuple[0] - parentResolvedVersion = resolvedVersions.get(packageName) - if parentResolvedVersion: - packages.append(f"{packageName}=={parentResolvedVersion}") - else: - packages.append(package) - elif len(packageTuple) == 2: - # The subrequires ask for a specific version - packages.append(package) - - def extractPackageName(packageString: str) -> str: - return self.REZ_DELIMITER_PATTERN.split(packageString, maxsplit=1)[0] - packageNames = [extractPackageName(package) for package in packages] - - for package in _MESHROOM_COMPUTE_DEPS: - # For packages that are required by meshroom_compute, do not specify any version - # or align it with Meshroom's: the version will be found during the resolution of - # the environment based on the other packages. - # If any of these packages is already part of the environment a plugin's dependency, - # do not add it - if package not in packageNames: - packages.append(package) - - logging.debug("Packages for the execution environment: " + ", ".join(packages)) - return packages - - def getCommandPrefix(self): - # TODO: make Windows-compatible - - # Use the PYTHONPATH from the subrequires' environment (which will only be resolved once - # inside the execution environment) and add MESHROOM_ROOT and the plugin's folder itself - # to it - pythonPaths = f"{os.pathsep.join(['$PYTHONPATH', f'{_MESHROOM_ROOT}', f'{self._folder}'])}" - - return f"rez env {' '.join(self.resolveRezSubrequires())} -c 'PYTHONPATH={pythonPaths} " - - def getCommandSuffix(self): - return "'" - - -def processEnvFactory(folder: str, configEnv: dict[str: str], pluginName: str, envType: str = "dirtree", uri: str = "") -> ProcessEnv: - if envType == "dirtree": - return DirTreeProcessEnv(folder, configEnv, pluginName) - return RezProcessEnv(folder, configEnv, pluginName, uri=uri) - - -class NodePluginStatus(Enum): - """ - Loading status for NodePlugin objects. - """ - NOT_LOADED = 0 # The node plugin exists but is not loaded and cannot be used (not registered) - LOADED = 1 # The node plugin is currently loaded and functional (it has been registered) - DESC_ERROR = 2 # The node plugin exists but has an invalid description - LOADING_ERROR = 3 # The node plugin exists and is valid but could not be successfully registered - ERROR = 4 # Error when importing the node plugin from its module - - -class Plugin(BaseObject): - """ - A collection of node plugins. - - Members: - name: the name of the plugin (e.g. name of the Python module containing the node plugins) - path: the absolute path of the plugin - user: whether the plugin is a user plugin (not maintained by the core Meshroom team) - nodePlugins: dictionary mapping the name of a node plugin contained in the plugin - to its corresponding NodePlugin object - templates: dictionary mapping the name of templates (.mg files) associated to the plugin - with their absolute paths - configEnv: the environment variables and their values, as described in the plugin's - configuration file - configFullEnv: the static merge of os.environ and configEnv, with os.environ taking precedence - processEnv: the environment required for the nodes' processes to be correctly executed - """ - - _instancesCount = 0 - - def __init__(self, name: str, path: str): - super().__init__() - - Plugin._instancesCount += 1 - self._uid: str = f"{Plugin._instancesCount:04d}" - self._name: str = name - self._path: str = path - self._user: bool = False - - self._nodePlugins: dict[str: NodePlugin] = {} - self._templates: dict[str: str] = {} - self._configEnv: dict[str: str] = {} - self._configFullEnv: dict[str: str] = {} - self._processEnv: ProcessEnv = ProcessEnv(path, self._configEnv, self._name) - - self.loadTemplates() - self.loadConfig() - - def __repr__(self): - return f"" - - @property - def uid(self): - return self._uid - - @property - def name(self): - """ Return the name of the plugin. """ - return self._name - - @property - def uname(self): - """ Return the unique name of the plugin. """ - return f"{self._uid}_{self._name}" - - @property - def path(self): - """ Return the absolute path of the plugin. """ - return self._path - - @property - def isUserPlugin(self): - """ Return whether the plugin is a user plugin (not maintained by the core Meshroom team). """ - return self._user - - @isUserPlugin.setter - def isUserPlugin(self, user: bool): - """ Set whether the plugin is a user plugin. """ - self._user = user - - @property - def nodes(self): - """ - Return the dictionary containing the NodePlugin objects associated to - the plugin. - """ - return self._nodePlugins - - @property - def templates(self): - """ Return the list of templates associated to the plugin. """ - return self._templates - - @property - def processEnv(self): - """ Return the environment required to successfully execute processes. """ - return self._processEnv - - @processEnv.setter - def processEnv(self, processEnv: ProcessEnv): - """ Set the environment required to successfully execute processes. """ - self._processEnv = processEnv - - @property - def configEnv(self): - """ - Return the dictionary containing the environment variables and their values - provided in the plugin's configuration file. - """ - return self._configEnv - - @property - def configFullEnv(self): - """ Return the fusion of the os.environ dictionary with the configEnv dictionary. """ - return self._configFullEnv - - def addNodePlugin(self, nodePlugin: NodePlugin): - """ - Add a node plugin to the current plugin object and assign it as its containing plugin. - The node plugin is added to the dictionary of node plugins with the name of the node - descriptor as its key. - - Args: - nodePlugin: the NodePlugin object to add to the Plugin. - """ - self._nodePlugins[nodePlugin.nodeDescriptor.__name__] = nodePlugin - nodePlugin.plugin = self - - def removeNodePlugin(self, name: str): - """ - Remove a node plugin from the current plugin object and delete any container relationship. - - Args: - name: the name of the NodePlugin to remove. - """ - if name in self._nodePlugins: - self._nodePlugins[name].plugin = None - del self._nodePlugins[name] - else: - logging.warning(f"Node plugin {name} is not part of the plugin {self.name}.") - - def loadTemplates(self): - """ - Load all the pipeline templates that are available within the plugin folder. - Whenever this method is called, the list of templates for the plugin is cleared, - before being filled again. - """ - self._templates.clear() - for file in os.listdir(self.path): - if file.endswith(".mg"): - self._templates[os.path.splitext(file)[0]] = os.path.join(self.path, file) - - def loadConfig(self): - """ - Load the plugin's configuration file if it exists and saves all its environment variables - and their values, if they are valid. - The configuration file is expected to be named "config.json", located at the top-level of - the plugin. - """ - try: - with open(os.path.join(self.path, "config.json")) as config: - content = json.load(config) - for entry in content: - # An entry is expected to be formatted as follows: - # { "key": "key_of_var", "type": "type_of_value", "value": "var_value" } - # If "type" is not provided, it is assumed to be "string" - k = entry.get("key", None) - t = entry.get("type", None) - val = entry.get("value", None) - - if not k or not val: - logging.warning(f"Invalid entry in configuration file for {self.name}: {entry}.") - continue - - if t == "path": - if os.path.isabs(val): - resolvedPath = Path(val).resolve() - else: - resolvedPath = Path(os.path.join(self.path, val)).resolve() - - if resolvedPath.exists(): - val = resolvedPath.as_posix() - else: - logging.debug(f"{k}: {resolvedPath.as_posix()} does not exist " - f"(path before resolution: {val}).") - - self._configEnv[k] = str(val) - - except FileNotFoundError: - logging.debug(f"No configuration file 'config.json' was found for {self.name}.") - except json.JSONDecodeError as err: - logging.error(f"Malformed JSON in the configuration file for {self.name}: {err}") - except IOError as err: - logging.error(f"Error while accessing the configuration file for {self.name}: {err}") - - # If both dictionaries have identical keys, os.environ overwrites existing values from _configEnv - self._configFullEnv = self._configEnv | os.environ - - def containsNodePlugin(self, name: str) -> bool: - """ - Return whether the node plugin "name" is part of the plugin, independently from its - status. - - Args: - name: the name of the node plugin to be checked. - """ - return name in self._nodePlugins - - -class NodePlugin(BaseObject): - """ - Based on a node description, a NodePlugin represents a loadable node. - - Members: - plugin: the Plugin object that contains this node plugin - path: absolute path to the file containing the node's description - nodeDescriptor: the description of the node - status: the loading status on the node plugin - errors: the list of errors (if there are any) when validating the description - of the node or attempting to load it - processEnv: the environment required for the node plugin's process. It can either - be specific to this node plugin, or be common for all the node plugins within - the plugin - timestamp: the timestamp corresponding to the last time the node description's file has been - modified - """ - - def __init__(self, nodeDesc: desc.BaseNode, plugin: Plugin = None): - super().__init__() - self.plugin: Plugin = plugin - self.path: str = Path(getfile(nodeDesc)).resolve().as_posix() - self.nodeDescriptor: desc.BaseNode = nodeDesc - self.nodeDescriptor.plugin = self - - self.status: NodePluginStatus = NodePluginStatus.NOT_LOADED - self.errors: list[tuple[str, ValueTypeErrors]] = validateNodeDesc(nodeDesc) - - if self.errors: - self.status = NodePluginStatus.DESC_ERROR - - self._processEnv = None - self._timestamp = os.path.getmtime(self.path) - - def reload(self) -> bool: - """ - Reload the node plugin and update its status accordingly. If the timestamp of the node plugin's - path has not changed since the last time the plugin has been loaded, then nothing will happen. - - Returns: - bool: True if the node plugin has successfully been reloaded (i.e. there was no error, and - some changes were made since its last loading), False otherwise. - """ - timestamp = 0.0 - try: - timestamp = os.path.getmtime(self.path) - except FileNotFoundError: - self.status = NodePluginStatus.ERROR - logging.error(f"[Reload] {self.nodeDescriptor.__name__}: The path at {self.path} was not " - f"not found.") - return False - - if self._timestamp == timestamp: - logging.info(f"[Reload] {self.nodeDescriptor.__name__}: Not reloading. The node description " - f"at {self.path} has not been modified since the last load.") - return False - - try: - updated = importlib.reload(sys.modules.get(self.nodeDescriptor.__module__)) - except Exception as exc: - logging.error(f"[Reload] {self.nodeDescriptor.__name__}: {exc} ({type(exc).__name__})") - self.status = NodePluginStatus.DESC_ERROR - return False - descriptor = getattr(updated, self.nodeDescriptor.__name__) - - if not descriptor: - self.status = NodePluginStatus.ERROR - logging.error(f"[Reload] {self.nodeDescriptor.__name__}: The node description at {self.path} " - f"was not found.") - return False - - self.errors = validateNodeDesc(descriptor) - if self.errors: - self.status = NodePluginStatus.DESC_ERROR - logging.error(f"[Reload] {self.nodeDescriptor.__name__}: The node description at {self.path} " - f"has description errors.") - return False - - self.nodeDescriptor = descriptor - self.nodeDescriptor.plugin = self - self._timestamp = timestamp - self.status = NodePluginStatus.NOT_LOADED - logging.info(f"[Reload] {self.nodeDescriptor.__name__}: Successful reloading.") - return True - - @property - def plugin(self): - """ - Return the Plugin object that contains this node plugin. - If the node plugin has not been assigned to a plugin yet, this value will - be set to None. - """ - return self._plugin - - @plugin.setter - def plugin(self, plugin: Plugin): - """ Assign this node plugin to a containing Plugin object. """ - self._plugin = plugin - - @property - def isUserPlugin(self): - """ Return whether the node plugin belongs to a user plugin. """ - if self.plugin: - return self.plugin.isUserPlugin - return False - - @property - def processEnv(self): - """" - Return the process environment that is specific to the node plugin if it has any. - Otherwise, the Plugin's is returned. - """ - if self._processEnv: - return self._processEnv - if self.plugin: - return self.plugin.processEnv - return None - - @property - def runtimeEnv(self) -> dict: - """ Return the environment dictionary for the runtime. """ - return self.processEnv.getEnvDict() - - @property - def commandPrefix(self) -> str: - """ Return the command prefix for the NodePlugin's execution. """ - if not self.processEnv: - return "" - return self.processEnv.getCommandPrefix() - - @property - def commandSuffix(self) -> str: - """ Return the command suffix for the NodePlugin's execution. """ - if not self.processEnv: - return "" - return self.processEnv.getCommandSuffix() - - @property - def configFullEnv(self) -> dict[str: str]: - """ Return the plugin's full environment dictionary. """ - if not self.plugin: - return {} - return self.plugin.configFullEnv - -class NodePluginManager(BaseObject): - """ - Manager for all the loaded Plugin objects as well as the registered NodePlugin objects. - - Members: - plugins: dictionary containing all the loaded Plugins, with their name as the key - nodePlugins: dictionary containing all the NodePlugins that have been registered - (a NodePlugin may exist without having been registered) with their name as - the key - """ - - def __init__(self): - super().__init__() - - self._plugins: dict[str: Plugin] = {} # loaded plugins - self._nodePlugins: dict[str: NodePlugin] = {} # registered node plugins - - def isRegistered(self, name: str) -> bool: - """ - Return whether the node plugin has been registered already. - - Args: - name: the name of the node plugin whose registration needs to be checked. - """ - return name in self._nodePlugins - - def belongsToPlugin(self, name: str) -> Plugin: - """ - Check whether the node plugin belongs to a loaded plugin, independently from - whether it has been registered or not. - - Args: - name: the name of the node plugin that needs to be searched for across plugins. - - Returns: - Plugin | None: the Plugin the node belongs to if it exists, None otherwise. - """ - for plugin in self._plugins.values(): - if plugin.containsNodePlugin(name): - return plugin - return None - - def getPlugins(self) -> dict[str: Plugin]: - """ - Return a dictionary containing all the loaded Plugins, with {key, value} = - {name, Plugin}. - """ - return self._plugins - - def getPlugin(self, name: str, uname: bool = True) -> Plugin: - """ - Return the loaded Plugin object with "name". - - Args: - name: the unique name of the Plugin, used upon its loading. - uname: the name passed as argument is the unique name of the plugin. - if set to False, we will search for any plugin with this name - but this means there can be a collision. To avoid any confusion - use this function with the unique name as much as possible. - - Returns: - Plugin | None: the loaded Plugin object if it exists, None otherwise. - """ - if uname: - # Find plugin with unique name - if name in self._plugins: - return self._plugins[name] - else: - for plugin in self._plugins.values(): - if plugin.name == name: - return plugin - return None - - def addPlugin(self, plugin: Plugin, registerNodePlugins: bool = True): - """ - Load a Plugin object. - - Args: - plugin: the Plugin to load and add to the list of loaded plugins. - registerNodePlugins: True if all the NodePlugins from the plugin should be registered - at the same time the plugin is being loaded. Otherwise, the - NodePlugins will have to be registered at a later occasion. - """ - pluginUName = plugin.uname - if self.getPlugin(pluginUName): - logging.warning(f"Plugin {pluginUName} is already registered.") - return - self._plugins[pluginUName] = plugin - if registerNodePlugins: - for node in plugin.nodes: - self.registerNode(plugin.nodes[node]) - - def removePlugin(self, plugin: Plugin, unregisterNodePlugins: bool = True): - """ - Remove a loaded Plugin object. - - Args: - plugin: the Plugin to remove from the list of loaded plugins. - unregisterNodePlugins: True if all the nodes from the plugin should be unregistered (if they - are registered) at the same time as the plugin is unloaded. Otherwise, - the registered NodePlugins will remain while the Plugin itself will - be unloaded. - """ - if self.getPlugin(plugin.uname): - if unregisterNodePlugins: - for node in plugin.nodes.values(): - self.unregisterNode(node) - del self._plugins[plugin.uname] - - def getRegisteredNodePlugins(self) -> dict[str: NodePlugin]: - """ - Return a dictionary containing all the registered NodePlugins, with - {key, value} = {name, NodePlugin}. - """ - return self._nodePlugins - - def getRegisteredNodePlugin(self, name: str) -> NodePlugin: - """ - Return the NodePlugin object that has been registered under the name "name" if it exists. - - Args: - name: the name of the NodePlugin used for its registration. - - Returns: - NodePlugin | None: the loaded NodePlugin object if it exists, None otherwise. - """ - if self.isRegistered(name): - return self._nodePlugins[name] - return None - - def registerNode(self, nodePlugin: NodePlugin): - """ - Register a node plugin. A registered node plugin will become instantiable. - If it is already registered, or if there is an issue with the node description, - the node plugin will not be registered and its status will be updated. - - Args: - nodePlugin: the node plugin to register. - """ - name = nodePlugin.nodeDescriptor.__name__ - if self.isRegistered(name): - existingPlugin: NodePlugin = self._nodePlugins[name] - logging.warning( - f"Could not register node {name} ({nodePlugin.path}) " - f"because another node is already registered with this name ({existingPlugin.path})" - ) - return - if nodePlugin.status in (NodePluginStatus.DESC_ERROR, - NodePluginStatus.ERROR): - logging.warning( - f"Could not register node {name} ({nodePlugin.path}) " - f"because the node is in error ({nodePlugin.status})." - ) - return - - try: - self._nodePlugins[name] = nodePlugin - nodePlugin.status = NodePluginStatus.LOADED - except Exception as exc: - logging.error(f"NodePlugin {name} could not be loaded: {exc}") - nodePlugin.status = NodePluginStatus.LOADING_ERROR - - def unregisterNode(self, nodePlugin: NodePlugin): - """ - Unregister a node plugin. When unregistered, a node plugin cannot be instantiated anymore. - If it is not registered already, nothing happens. - - Args: - nodePlugin: the node plugin to unregister. - """ - name = nodePlugin.nodeDescriptor.__name__ - if self.isRegistered(name): - if nodePlugin.status != NodePluginStatus.LOADED: - logging.warning(f"NodePlugin {name} is registered but is not correctly loaded.") - else: - nodePlugin.status = NodePluginStatus.NOT_LOADED - del self._nodePlugins[name] diff --git a/tests/plugins/meshroom/pluginA/__init__.py b/meshroom/core/plugins/__init__.py similarity index 100% rename from tests/plugins/meshroom/pluginA/__init__.py rename to meshroom/core/plugins/__init__.py diff --git a/meshroom/core/plugins/base.py b/meshroom/core/plugins/base.py new file mode 100644 index 0000000000..70231f8bc6 --- /dev/null +++ b/meshroom/core/plugins/base.py @@ -0,0 +1,551 @@ +from __future__ import annotations + +import importlib +import logging +import os +import sys + +from enum import Enum +from inspect import getfile +from pathlib import Path +from typing import Optional + +from meshroom.common import BaseObject +from meshroom.core import desc +from meshroom.core.desc.attribute import ValueTypeErrors +from meshroom.core.submitter import BaseSubmitter +from meshroom.core.plugins.config import PluginConfig +from meshroom.core.plugins.env import ProcessEnv, processEnvFactory + + +class PluginType(Enum): + """ + Determines how a plugin is discovered and how its process environment is configured. + """ + BUILTIN = 1 # Plugin folder using meshroom environment + PATH = 2 # Plugin provided by a path + REZ = 3 # Plugin provided by a rez package + + +class Plugin(BaseObject): + """ + A centralized container that manages the plugin collection of NodeDescProvider objects and + SubmitterProvider objects. Alongside plugin name, version, type, templates and configuration. + + Members: + name: the name of the plugin (e.g. name of the Python module containing the node plugins) + rootPath: the absolute path of the plugin's root folder + path: the absolute path of the plugin's modules (its "meshroom" folder) + version: the version of the plugin, or "unknown" if none was provided + isUserPlugin: whether the plugin is a user plugin (not maintained by the core Meshroom team) + type: the PluginType describing how the plugin was discovered and how its process + environment is configured + nodeDescProviders: dictionary mapping the name of a node descriptor provider contained in the + plugin to its corresponding NodeDescProvider object + submitterProviders: dictionary mapping the name of a submitter provider contained in the + plugin to its corresponding SubmitterProvider object + templates: dictionary mapping the name of templates (.mg files) associated to the plugin + with their absolute paths + configEnv: the environment variables and their values, as described in the plugin's + configuration file + configFullEnv: the static merge of os.environ and configEnv, with os.environ taking precedence + processEnv: the environment required for the nodes' processes to be correctly executed + """ + + def __init__(self, name: str, rootPath: str, path: str, type: PluginType, + version: Optional[str] = None, isUserPlugin: bool = False, + config: Optional[PluginConfig] = None): + super().__init__() + + self._name: str = name + self._rootPath: str = rootPath + self._path: str = path + self._type: PluginType = type + self._version: str = version + self._isUserPlugin: bool = isUserPlugin + self._nodeDescProviders: dict[str, NodeDescProvider] = {} + self._submitterProviders: dict[str, SubmitterProvider] = {} + self._templates: dict[str, str] = {} + self._configEnv: dict[str, str] = {} + + # Get environment variables from config + if config: + self._configEnv = config.resolveEnv(self._path, self._name) + # If both dictionaries have identical keys, os.environ overwrites existing values from _configEnv + self._configFullEnv: dict[str, str] = self._configEnv | os.environ + + self.loadTemplates() + + envType = "rez" if type is PluginType.REZ else "dirtree" + self._processEnv: ProcessEnv = processEnvFactory(self._rootPath, self._configEnv, self._name, + envType=envType) + + def __repr__(self): + return f"" + + @property + def name(self): + """ Return the name of the plugin. """ + return self._name + + @property + def path(self): + """ Return the absolute path of the plugin's modules (its "meshroom" folder). """ + return self._path + + @property + def rootPath(self): + """ + Return the absolute path of the plugin's root folder, containing python modules + as well as any "bin"/"lib"/"lib64"/"venv" dependency folders. + """ + return self._rootPath + + @property + def type(self): + """ Return the PluginType describing how the plugin was discovered. """ + return self._type + + @property + def version(self): + """ Return the version of the plugin, or "unknown" if none was provided. """ + if self._version and len(self._version) > 0: + return self._version + return "unknown" + + @property + def isUserPlugin(self): + """ Return whether the plugin is a user plugin (not maintained by the core Meshroom team). """ + return self._isUserPlugin + + @property + def nodeDescProviders(self): + """ + Return the dictionary containing the NodeDescProvider objects associated to + the plugin. + """ + return self._nodeDescProviders + + @property + def submitterProviders(self): + """ + Return the dictionary containing the SubmitterProvider objects associated to + the plugin. + """ + return self._submitterProviders + + @property + def templates(self): + """ Return the list of templates associated to the plugin. """ + return self._templates + + @property + def processEnv(self): + """ Return the environment required to successfully execute processes. """ + return self._processEnv + + @property + def configEnv(self): + """ + Return the dictionary containing the environment variables and their values + provided in the plugin's configuration file. + """ + return self._configEnv + + @property + def configFullEnv(self): + """ Return the fusion of the os.environ dictionary with the configEnv dictionary. """ + return self._configFullEnv + + def addNodeDescProvider(self, nodeDescClass: type[desc.BaseNode]) -> NodeDescProvider: + """ + Create a NodeDescProvider for "nodeDescClass" and add it to the current plugin object, + assigning the plugin as its container. The node descriptor provider is added to the dictionary + of node descriptor providers with the name of the node descriptor as its key. + + Args: + nodeDescClass: the desc.BaseNode subclass to create a NodeDescProvider for. + + Returns: + NodeDescProvider: the created node descriptor provider. + """ + nodeDescProvider = NodeDescProvider(nodeDescClass, self) + self._nodeDescProviders[nodeDescProvider.name] = nodeDescProvider + return nodeDescProvider + + def removeNodeDescProvider(self, name: str): + """ + Remove a node descriptor provider from the current plugin object and delete any container + relationship. + + Args: + name: the name of the NodeDescProvider to remove. + """ + if name in self._nodeDescProviders: + self._nodeDescProviders[name].plugin = None + del self._nodeDescProviders[name] + else: + logging.warning(f"Node descriptor provider {name} is not part of the plugin {self.name}.") + + def containsNodeDescProvider(self, name: str) -> bool: + """ + Return whether the node descriptor provider "name" is part of the plugin, independently + from its status. + + Args: + name: the name of the node descriptor provider to be checked. + """ + return name in self._nodeDescProviders + + def addSubmitterProvider(self, submitterClass: type[BaseSubmitter]) -> SubmitterProvider: + """ + Create a SubmitterProvider for "submitterClass" and add it to the current plugin object, + assigning the plugin as its container. The submitter provider is added to the dictionary + of submitter providers with the name of the submitter class as its key. + + Args: + submitterClass: the BaseSubmitter subclass to create a SubmitterProvider for. + + Returns: + SubmitterProvider: the created submitter provider. + """ + submitterProvider = SubmitterProvider(submitterClass, self) + self._submitterProviders[submitterProvider.name] = submitterProvider + return submitterProvider + + def removeSubmitterProvider(self, name: str): + """ + Remove a submitter provider from the current plugin object. + + Args: + name: the name of the SubmitterProvider to remove. + """ + if name in self._submitterProviders: + del self._submitterProviders[name] + else: + logging.warning(f"submitter provider {name} is not part of the plugin {self.name}.") + + def containsSubmitterProvider(self, name: str) -> bool: + """ + Return whether the submitter provider "name" is part of the plugin, independently from + its status. + + Args: + name: the name of the submitter provider to be checked. + """ + return name in self._submitterProviders + + def loadTemplates(self): + """ + Load all the pipeline templates that are available within the plugin folder. + Whenever this method is called, the list of templates for the plugin is cleared, + before being filled again. + """ + self._templates.clear() + for file in os.listdir(self.path): + if file.endswith(".mg"): + self._templates[os.path.splitext(file)[0]] = os.path.join(self.path, file) + + +class NodeDescProviderStatus(Enum): + """ + Validity status for NodeDescProvider objects. + """ + VALID = 0 # The node description is valid and can be instantiated + DESC_ERROR = 1 # The node provider exists but has an invalid description + ERROR = 2 # Error when importing the node provider from its module + + +class NodeDescProvider(BaseObject): + """ + Based on a node description, a NodeDescProvider represents a loadable node. + + Members: + plugin: the Plugin object that contains this node descriptor provider + name: the name of the node descriptor, as declared by its class + path: absolute path to the file containing the node's description + nodeDescClass: the description of the node + status: the loading status on the node descriptor provider + error: a single formatted message combining every description "errors", or None if valid + processEnv: the environment required for the node descriptor provider's process. It can either + be specific to this node descriptor provider, or be common for all the node + descriptor providers within the plugin + runtimeEnv: the environment dictionary for the runtime, derived from processEnv + commandPrefix: the command prefix for the node provider's execution, derived from processEnv + commandSuffix: the command suffix for the node provider's execution, derived from processEnv + configFullEnv: the plugin's full environment dictionary + timestamp: the timestamp corresponding to the last time the node description's file has been + modified + """ + + @staticmethod + def __validateNodeDescClass(nodeDescClass: type[desc.BaseNode]) -> Optional[str]: + """ + Check that the node description class is a valid description. + To be valid, the default value of every parameter needs to correspond to the type + of the parameter. In case of nested parameters (parameters in groups or lists, for example), + the name of the parameter follows the name of the parent attributes. For example, if the attribute + "x", contained in group "group", is invalid, then it will be added to the list as "group:x". + + Args: + nodeDescClass: Description class of a node. + + Returns: + error: The list of invalid parameters in a formatted error message. + """ + errors: list[tuple[str, ValueTypeErrors]] = [] + for param in nodeDescClass.inputs: + errMsg, errType = param.checkValueTypes() + if errMsg: + errors.append((errMsg, errType)) + for param in nodeDescClass.outputs: + if param.value is None: + if issubclass(nodeDescClass, desc.InitNode): + errors.append((f"{param.name}", ValueTypeErrors.DYNAMIC_OUTPUT)) + continue + errMsg, errType = param.checkValueTypes() + if errMsg: + errors.append((errMsg, errType)) + errorMessages: list[str] = [] + for error in errors: + errMsg, errType = error + if errType == ValueTypeErrors.TYPE: + errorMessages.append(f" - 'value': Invalid type for parameter '{errMsg}'.") + elif errType == ValueTypeErrors.RANGE: + errorMessages.append(f" - 'range': Invalid range value for parameter '{errMsg}'.") + elif errType == ValueTypeErrors.DYNAMIC_OUTPUT: + errorMessages.append(f" - 'value': Unsupported dynamic output for parameter '{errMsg}'.") + else: + errorMessages.append(f" - Unknown error for parameter '{errMsg}'.") + if errorMessages: + return f"NodeDescProvider of '{nodeDescClass.__name__}' could not be validated:\n" + "\n".join(errorMessages) + return None + + def __init__(self, nodeDescClass: type[desc.BaseNode], plugin: Plugin = None): + super().__init__() + self._plugin: Plugin = plugin + self.path: str = Path(getfile(nodeDescClass)).resolve().as_posix() + self.nodeDescClass: desc.BaseNode = nodeDescClass + self.nodeDescClass.provider = self + self.nodeDescClass.plugin = plugin + self.nodeDescClass.packageName = plugin.name if plugin else "" + + self.status: NodeDescProviderStatus = NodeDescProviderStatus.VALID + self.error: Optional[str] = self.__validateNodeDescClass(nodeDescClass) + + if self.error: + self.status = NodeDescProviderStatus.DESC_ERROR + + # A "dirtree" env only depends on the plugin's folder/configEnv, not on the node's subpackage, + # so it is identical for every node of the plugin: reuse plugin.processEnv (via the property's + # fallback below) instead of rebuilding it for each node. Only a "rez" env genuinely needs its + # own instance, since its subrequires resolution is subpackage-specific. + self._processEnv = None + if plugin and plugin.type is PluginType.REZ: + self._processEnv: ProcessEnv = processEnvFactory(plugin.rootPath, plugin.configEnv, plugin.name, + pluginSubPackage=self.relativePackage, envType="rez") + self._timestamp = os.path.getmtime(self.path) + + def reload(self) -> bool: + """ + Reload the node descriptor provider and update its status accordingly. If the timestamp of the + node descriptor provider's path has not changed since the last time the plugin has been loaded, + then nothing will happen. + + Returns: + bool: True if the node descriptor provider has successfully been reloaded (i.e. there was + no error, and some changes were made since its last loading), False otherwise. + """ + timestamp = 0.0 + try: + timestamp = os.path.getmtime(self.path) + except FileNotFoundError: + self.status = NodeDescProviderStatus.ERROR + logging.error(f"[Reload] {self.name}: The path at {self.path} was not " + f"not found.") + return False + + if self._timestamp == timestamp: + logging.info(f"[Reload] {self.name}: Not reloading. The node description " + f"at {self.path} has not been modified since the last load.") + return False + + try: + updated = importlib.reload(sys.modules.get(self.nodeDescClass.__module__)) + except Exception as exc: + logging.error(f"[Reload] {self.name}: {exc} ({type(exc).__name__})") + self.status = NodeDescProviderStatus.DESC_ERROR + return False + descriptor = getattr(updated, self.name) + + if not descriptor: + self.status = NodeDescProviderStatus.ERROR + logging.error(f"[Reload] {self.name}: The node description at {self.path} " + f"was not found.") + return False + + self.error = self.__validateNodeDescClass(descriptor) + if self.error: + self.status = NodeDescProviderStatus.DESC_ERROR + logging.error(f"[Reload] {self.name}: The node description at {self.path} " + f"has description errors.") + return False + + self.nodeDescClass = descriptor + self.nodeDescClass.provider = self + self.nodeDescClass.plugin = self.plugin + self.nodeDescClass.packageName = self._plugin.name if self._plugin else "" + self._timestamp = timestamp + self.status = NodeDescProviderStatus.VALID + logging.info(f"[Reload] {self.name}: Successful reloading.") + return True + + @property + def plugin(self): + """ + Return the Plugin object that contains this node descriptor provider. + If the node descriptor provider has not been assigned to a plugin yet, this value will + be set to None. + """ + return self._plugin + + @property + def name(self) -> str: + """ Return the name of the node descriptor, as declared by its class. """ + return self.nodeDescClass.__name__ + + @property + def absolutePackage(self) -> str: + """ + Return the full dotted path of the package containing the node's description class. + + Only strip the last dotted component of the class' module name if that module is a leaf + file within a package: if the class is declared directly in a package's "__init__.py", + its module name already is that package's dotted path and must be kept as-is. + """ + moduleName = self.nodeDescClass.__module__ + module = sys.modules.get(moduleName) + if module is not None and hasattr(module, "__path__"): + return moduleName + return moduleName.rsplit(".", 1)[0] + + @property + def relativePackage(self) -> str: + """ + Return the dotted path of the package containing the node's description class, + relative to the plugin's root (i.e. without the "_meshroomPlugins." prefix). + """ + return ".".join(self.absolutePackage.split(".")[2:]) + + @property + def processEnv(self): + """" + Return the process environment that is specific to the node descriptor provider if it has any. + Otherwise, the Plugin's is returned. + """ + if self._processEnv: + return self._processEnv + if self.plugin: + return self.plugin.processEnv + return None + + @property + def runtimeEnv(self) -> dict: + """ Return the environment dictionary for the runtime. """ + return self.processEnv.getEnvDict() + + @property + def commandPrefix(self) -> str: + """ Return the command prefix for the NodeDescProvider's execution. """ + if not self.processEnv: + return "" + return self.processEnv.getCommandPrefix() + + @property + def commandSuffix(self) -> str: + """ Return the command suffix for the NodeDescProvider's execution. """ + if not self.processEnv: + return "" + return self.processEnv.getCommandSuffix() + + @property + def configFullEnv(self) -> dict[str, str]: + """ Return the plugin's full environment dictionary. """ + if not self.plugin: + return {} + return self.plugin.configFullEnv + + +class SubmitterProviderStatus(Enum): + """ + Validity status for SubmitterProvider objects. + """ + VALID = 0 # The submitter was successfully instantiated + ERROR = 1 # Error when instantiating the submitter + + +class SubmitterProvider(BaseObject): + """ + Based on a BaseSubmitter subclass, a SubmitterProvider represents a loadable submitter. + + Members: + plugin: the Plugin object that contains this submitter provider. Set once at + construction and constant for the lifetime of the submitter provider. + path: absolute path to the file containing the submitter's class + submitterClass: the BaseSubmitter subclass + name: the name of the submitter, as declared by its class + status: the validity status of the submitter provider + error: a single formatted message combining every submitter "errors", or None if valid + instance: the instantiated submitter, or None if instantiation failed + """ + + def __init__(self, submitterClass: type[BaseSubmitter], plugin: Plugin): + super().__init__() + self._plugin: Plugin = plugin + self.path: str = Path(getfile(submitterClass)).resolve().as_posix() + self.submitterClass: type[BaseSubmitter] = submitterClass + self.status: SubmitterProviderStatus = SubmitterProviderStatus.VALID + self.error: Optional[str] = None + + try: + self.instance: Optional[BaseSubmitter] = submitterClass() + except Exception as exc: + self.error = f"SubmitterProvider of '{submitterClass.__name__}' could not be instantiated:\n{exc}" + self.instance = None + self.status = SubmitterProviderStatus.ERROR + + @property + def plugin(self): + """ + Return the Plugin object that contains this submitter provider. Set once at + construction and constant for the lifetime of the submitter provider. + """ + return self._plugin + + @property + def name(self) -> str: + """ Return the name of the submitter, as declared by its class. """ + return self.submitterClass._name + + @property + def absolutePackage(self) -> str: + """ + Return the full dotted path of the package containing the submitter class. + + Only strip the last dotted component of the class' module name if that module is a leaf + file within a package: if the class is declared directly in a package's "__init__.py", + its module name already is that package's dotted path and must be kept as-is. + """ + moduleName = self.submitterClass.__module__ + module = sys.modules.get(moduleName) + if module is not None and hasattr(module, "__path__"): + return moduleName + return moduleName.rsplit(".", 1)[0] + + @property + def relativePackage(self) -> str: + """ + Return the dotted path of the package containing the submitter class, + relative to the plugin's root (i.e. without the "_meshroomPlugins." prefix). + """ + return ".".join(self.absolutePackage.split(".")[2:]) \ No newline at end of file diff --git a/meshroom/core/plugins/config.py b/meshroom/core/plugins/config.py new file mode 100644 index 0000000000..cfe180f6e8 --- /dev/null +++ b/meshroom/core/plugins/config.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +import json +import logging +import os +import re + +from pathlib import Path +from typing import NamedTuple, Optional + +# Plugin name pattern for config.json. +# Only letters and digits are allowed. +_NAME_PATTERN = re.compile(r"^[A-Za-z0-9]+$") + +# Plugin version pattern for config.json. +# Only letters and digits are allowed or "major.minor.patch". +_VERSION_PATTERN = re.compile(r"^([A-Za-z0-9]+|\d+\.\d+\.\d+)$") + + +class PluginConfig(NamedTuple): + """ + The parsed content of a plugin's "config.json" file. + + Members: + name: the plugin's name, if provided and valid. None if absent, invalid, or not + applicable (e.g. only an env list). + version: the plugin's version, if provided and valid. None if absent, invalid, or not + applicable (e.g. only an env list). + env: the list of environment variable entries declared in the file (at the top level + of the file, or under the "env" key). + """ + name: Optional[str] + version: Optional[str] + env: list[dict] + + def resolveEnv(self, basePath: Path, pluginName: str) -> dict[str, str]: + """ + Resolve "env" into a dictionary of environment variable names to values. + + Args: + basePath: the folder to resolve against when entry value is not absolute. + pluginName: the name of the plugin "env" belongs to, used in log messages. + + Returns: + dict[str, str]: the resolved environment variables. + """ + configEnv: dict[str, str] = {} + for entry in self.env: + # An entry is expected to be formatted as follows: + # { "key": "key_of_var", "type": "type_of_value", "value": "var_value" } + # If "type" is not provided, it is assumed to be "string" + k = entry.get("key", None) + t = entry.get("type", None) + val = entry.get("value", None) + + if not k or not val: + logging.warning(f"Invalid entry in configuration file for {pluginName}: {entry}.") + continue + + if t == "path": + if os.path.isabs(val): + resolvedPath = Path(val).resolve() + else: + resolvedPath = Path(os.path.join(basePath, val)).resolve() + + if resolvedPath.exists(): + val = resolvedPath.as_posix() + else: + logging.debug(f"{k}: {resolvedPath.as_posix()} does not exist " + f"(path before resolution: {val}).") + + configEnv[k] = str(val) + + return configEnv + + @staticmethod + def load(configPath: Path) -> PluginConfig: + """ + Parse the plugin configuration file at "configPath" into a PluginConfig. + + The file can either be: + - a plain list of environment variable entries (array), in which case "name" + and "version" are None. + - an object with optional "name" (str), "version" (str), and "env" + (list of environment variable) keys. + + Args: + configPath: the absolute path of the "config.json" file to parse. + + Returns: + PluginConfig: the parsed configuration. + """ + try: + with open(configPath) as configFile: + content = json.load(configFile) + except FileNotFoundError: + logging.debug(f"No configuration file 'config.json' was found at '{configPath}'.") + return PluginConfig(None, None, []) + except json.JSONDecodeError as err: + logging.error(f"Malformed JSON in the configuration file '{configPath}': {err}") + return PluginConfig(None, None, []) + except IOError as err: + logging.error(f"Error while accessing the configuration file '{configPath}': {err}") + return PluginConfig(None, None, []) + + if isinstance(content, list): + return PluginConfig(None, None, content) + + if not isinstance(content, dict): + logging.warning(f"Configuration file '{configPath}' must contain a list or an object, " + f"got {type(content).__name__}. Ignoring it.") + return PluginConfig(None, None, []) + + env = content.get("env", []) + if not isinstance(env, list): + logging.warning(f"'env' in configuration file '{configPath}' must be a list, " + f"got {type(env).__name__}. Ignoring it.") + env = [] + + return PluginConfig( + PluginConfig._sanitizeName(content.get("name"), configPath), + PluginConfig._sanitizeVersion(content.get("version"), configPath), + env, + ) + + @staticmethod + def _sanitizeName(name, configPath: Path) -> Optional[str]: + """ + Return "name" if it only contains letters and digits, None otherwise. + """ + if name is None: + return None + if not isinstance(name, str) or not _NAME_PATTERN.match(name): + logging.warning(f"Invalid 'name' in configuration file '{configPath}': {name!r}. " + f"Plugin names must only contain letters and digits. Ignoring it.") + return None + return name + + @staticmethod + def _sanitizeVersion(version, configPath: Path) -> Optional[str]: + """ + Return "version" if it only contains letters and digits, or follows "major.minor.micro", + None otherwise. + """ + if version is None: + return None + if not isinstance(version, str) or not _VERSION_PATTERN.match(version): + logging.warning(f"Invalid 'version' in configuration file '{configPath}': {version!r}. " + f"Versions must only contain letters and digits, or follow " + f"'major.minor.micro'. Ignoring it.") + return None + return version diff --git a/meshroom/core/plugins/env.py b/meshroom/core/plugins/env.py new file mode 100644 index 0000000000..af1951927e --- /dev/null +++ b/meshroom/core/plugins/env.py @@ -0,0 +1,232 @@ +from __future__ import annotations + +import glob +import logging +import os +import re +import sys + +from enum import Enum +from pathlib import Path + +from meshroom.common import BaseObject +from meshroom import _MESHROOM_ROOT +from meshroom.core.desc.node import _MESHROOM_COMPUTE_DEPS + + +def processEnvFactory(folder: str, configEnv: dict[str, str], pluginName: str, pluginSubPackage: str = None, + envType: str = "dirtree") -> ProcessEnv: + """ + Create the ProcessEnv matching "envType" for a plugin. + + Args: + folder: the source folder for the process. + configEnv: the dictionary containing the environment variables defined in a configuration file + for the process to run. + pluginName: the name of the plugin object. + pluginSubPackage: the dotted path, relative to the plugin's root, of the package containing + the node/submitter class this environment is built for, if any. + envType: "dirtree" to build a DirTreeProcessEnv, "rez" build a RezProcessEnv. + + Returns: + ProcessEnv: the created DirTreeProcessEnv or RezProcessEnv. + """ + if envType == "dirtree": + return DirTreeProcessEnv(folder, configEnv, pluginName, pluginSubPackage) + return RezProcessEnv(folder, configEnv, pluginName, pluginSubPackage) + + +class ProcessEnvType(Enum): + """ Supported process environments. """ + DIRTREE = "dirtree", + REZ = "rez" + + +class ProcessEnv(BaseObject): + """ + Describes the environment required by a node's process. + + Args: + folder: the source folder for the process. + configEnv: the dictionary containing the environment variables defined in a configuration file + for the process to run. + pluginName: the name of the plugin object. + pluginSubPackage: (optional) the dotted path, relative to the plugin's root, of the package + containing the node/submitter class this environment is built for. + envType: (optional) the type of process environment. + """ + + def __init__(self, folder: str, configEnv: dict[str, str], pluginName: str, pluginSubPackage: str = None, + envType: ProcessEnvType = ProcessEnvType.DIRTREE): + super().__init__() + self._folder: str = folder + self._configEnv: dict[str, str] = configEnv + self.pluginName: str = pluginName + self.pluginSubPackage: str = pluginSubPackage + self._processEnvType: ProcessEnvType = envType + self._env: dict = None + + def getEnvDict(self) -> dict: + """ Return the environment dictionary if it has been modified, None otherwise. """ + return self._env + + def getCommandPrefix(self) -> str: + """ Return the prefix to the command line that will be executed by the process. """ + return "" + + def getCommandSuffix(self) -> str: + """ Return the suffix to the command line that will be executed by the process. """ + return "" + + +class DirTreeProcessEnv(ProcessEnv): + """ + A ProcessEnv built from a plain directory tree: PYTHONPATH/LD_LIBRARY_PATH/PATH are assembled + from the plugin's "bin"/"lib"/"lib64" folders and, if present, its "venv" virtual environment. + """ + def __init__(self, folder: str, configEnv: dict[str, str], pluginName: str, pluginSubPackage: str): + super().__init__(folder, configEnv, pluginName, pluginSubPackage, envType=ProcessEnvType.DIRTREE) + + # If there is a virtual environment, it is expected to be named "venv". + # Beside the virtual environment, a standard "bin"/"lib"/"lib64" hierarchy at + # the top level of the plugin folder is expected. + venvFolder = Path(folder, "venv") + + # Find all the libs that are not directly at the "lib*"-level + envLibPaths = glob.glob(f'{folder}/lib*/python[0-9].[0-9]*/site-packages', + recursive=False) + venvLibPaths = glob.glob(f'{venvFolder}/lib*/python[0-9].[0-9]*/site-packages', + recursive=False) + + self.binPaths: list = [str(Path(folder, "bin")), str(Path(venvFolder, "bin"))] + self.libPaths: list = [str(Path(folder, "lib")), str(Path(folder, "lib64")), + str(Path(venvFolder, "lib")), str(Path(venvFolder, "lib64"))] + self.pythonPaths: list = [str(Path(folder)), str(Path(venvFolder))] + \ + self.binPaths + envLibPaths + venvLibPaths + + if sys.platform == "win32": + # For Windows platforms, try and include the content of the virtual env if it exists + # The virtual env is expected to be named "venv" + venvLibPath = Path(venvFolder, "Lib", "site-packages") + if venvLibPath.exists(): + self.pythonPaths.append(venvLibPath.as_posix()) + else: + # For Linux platforms, lib paths may need to be discovered recursively to be properly + # added to LD_LIBRARY_PATH + extraLibPaths = [] + regex = re.compile(r"^lib(\d{2})?$") + for envPath in envLibPaths + venvLibPaths: + for path, directories, _ in os.walk(envPath): + for directory in directories: + if re.match(regex, directory): + extraLibPaths.append(os.path.join(path, directory)) + self.libPaths = self.libPaths + extraLibPaths + + # Setup the environment dictionary + self._env = os.environ.copy() + self._env["PYTHONPATH"] = os.pathsep.join( + [f"{_MESHROOM_ROOT}"] + self.pythonPaths + [os.getenv('PYTHONPATH', '')]) + self._env["LD_LIBRARY_PATH"] = f"{os.pathsep.join(self.libPaths)}{os.pathsep}{os.getenv('LD_LIBRARY_PATH', '')}" + self._env["PATH"] = f"{os.pathsep.join(self.binPaths)}{os.pathsep}{os.getenv('PATH', '')}" + + for k, val in self._configEnv.items(): + # Preserve user-defined environment variables: + # manually set environment variable values take precedence over config file defaults. + if k in self._env: + continue + + self._env[k] = val + + +class RezProcessEnv(ProcessEnv): + """ + A ProcessEnv built by resolving a Rez environment for the plugin's subrequires, activated + through a "rez env" command prefix/suffix wrapped around the node's command line. + """ + + REZ_DELIMITER_PATTERN = re.compile(r"-|==|>=|>|<=|<") + + def __init__(self, folder: str, configEnv: dict[str, str], pluginName: str, pluginSubPackage: str): + if not pluginName: + raise RuntimeError("Missing name of the Rez environment needs to be provided.") + super().__init__(folder, configEnv, pluginName, pluginSubPackage, envType=ProcessEnvType.REZ) + + def resolveRezSubrequires(self) -> list[str]: + """ + Return the list of packages defined for the node execution. These execution packages are + named subrequires. + Note: If a package does not have a version number, the version is aligned with the main + Meshroom environment (if this package is defined). + """ + pluginNameUpper = self.pluginName.upper() + pluginSubPackageUpper = None + + if self.pluginSubPackage: + pluginSubPackageUpper = self.pluginSubPackage.split('.', 1)[0].upper() # first level sub package + + if pluginSubPackageUpper and os.getenv(f"{pluginNameUpper}_{pluginSubPackageUpper}_SUBREQUIRES"): + subrequires = os.environ.get(f"{pluginNameUpper}_{pluginSubPackageUpper}_SUBREQUIRES", "").split(os.pathsep) + else: + subrequires = os.environ.get(f"{pluginNameUpper}_SUBREQUIRES", "").split(os.pathsep) + if not subrequires: + return [] + + packages = [] + # Packages that are resolved in the current environment + currentEnvPackages = [] + resolvedVersions = {} + if "REZ_USED_RESOLVE" in os.environ: + resolvedPackages = os.getenv("REZ_USED_RESOLVE", "").split() + for package in resolvedPackages: + if package.startswith("~"): + continue + currentEnvPackages.append(package) + name, version = self.REZ_DELIMITER_PATTERN.split(package, maxsplit=1) + resolvedVersions[name] = version + logging.debug("Packages in the current environment: " + ", ".join(currentEnvPackages)) + + # Take packages with the set versions for those which have one, and try to take packages + # in the current environment (if they are resolved in it) + for package in subrequires: + packageTuple = self.REZ_DELIMITER_PATTERN.split(package, maxsplit=1) + if len(packageTuple) == 1: + # Only the package name in the subrequires. + # Search for a corresponding version in the parent environment. + packageName = packageTuple[0] + parentResolvedVersion = resolvedVersions.get(packageName) + if parentResolvedVersion: + packages.append(f"{packageName}=={parentResolvedVersion}") + else: + packages.append(package) + elif len(packageTuple) == 2: + # The subrequires ask for a specific version + packages.append(package) + + def extractPackageName(packageString: str) -> str: + return self.REZ_DELIMITER_PATTERN.split(packageString, maxsplit=1)[0] + packageNames = [extractPackageName(package) for package in packages] + + for package in _MESHROOM_COMPUTE_DEPS: + # For packages that are required by meshroom_compute, do not specify any version + # or align it with Meshroom's: the version will be found during the resolution of + # the environment based on the other packages. + # If any of these packages is already part of the environment a plugin's dependency, + # do not add it + if package not in packageNames: + packages.append(package) + + logging.debug("Packages for the execution environment: " + ", ".join(packages)) + return packages + + def getCommandPrefix(self): + # TODO: make Windows-compatible + + # Use the PYTHONPATH from the subrequires' environment (which will only be resolved once + # inside the execution environment) and add MESHROOM_ROOT and the plugin's folder itself + # to it + pythonPaths = f"{os.pathsep.join(['$PYTHONPATH', f'{_MESHROOM_ROOT}', f'{self._folder}'])}" + + return f"rez env {' '.join(self.resolveRezSubrequires())} -c 'PYTHONPATH={pythonPaths} " + + def getCommandSuffix(self): + return "'" \ No newline at end of file diff --git a/meshroom/core/plugins/loader.py b/meshroom/core/plugins/loader.py new file mode 100644 index 0000000000..0f97fbbaa3 --- /dev/null +++ b/meshroom/core/plugins/loader.py @@ -0,0 +1,387 @@ +from __future__ import annotations + +import importlib +import importlib.machinery +import importlib.util +import logging +import os +import sys +import traceback + +from types import ModuleType +from pathlib import Path +from typing import Optional + +from meshroom.core import desc +from meshroom.core.submitter import BaseSubmitter +from meshroom.core.plugins.base import Plugin, PluginType +from meshroom.core.plugins.config import PluginConfig + +# The virtual package all the imported plugins are nested in. +# Registered directly in sys.modules. +PLUGINS_ROOT_PACKAGE = "_meshroomPlugins" + + +class _LoadIssues: + """ + Collects the issues found while loading a single plugin. + """ + + def __init__(self): + self.loading: list[str] = [] + self.nodeDescProviders: list[str] = [] + self.submitterProviders: list[str] = [] + + def log(self, pluginName: str): + """ Log one consolidated message per issue type found while loading the plugin "pluginName". """ + if self.loading: + logging.warning(self._format(pluginName, self.loading)) + if self.nodeDescProviders: + logging.error(self._format(pluginName, self.nodeDescProviders)) + if self.submitterProviders: + logging.error(self._format(pluginName, self.submitterProviders)) + + @staticmethod + def _format(pluginName: str, messages: list[str]) -> str: + """ Format "messages" into a single, bulleted message for the plugin "pluginName". """ + return f"Plugin '{pluginName}' loading issue:" + "".join(f"\n{message}" for message in messages) + + +class PluginLoader: + """ + Loads/Unloads a plugin into a private virtual package. + + Every plugin is loaded under its own dotted name "PLUGINS_ROOT_PACKAGE.", + registered directly in sys.modules. + """ + + def loadPlugin(self, + pluginName: str, + pluginFolder: str, + pluginType: PluginType, + pluginVersion: Optional[str] = None, + isUserPlugin: bool = False, + hasMeshroomFolder: bool = True) -> Plugin: + """ + Load the plugin located in "pluginFolder" and return it. + + Python modules are expected in a "meshroom" folder, unless the plugin folder is itself the + folder. Every standalone Python file at its root, every direct subfolder that is a real + Python package, and every standalone Python file directly inside a plain (non-package) + subfolder is loaded. A NodeDescProvider is created for each node description that is found. + A SubmitterProvider for each submitter class, that is found. The templates and the configuration + file are read from "pluginFolder". + + The plugin's final name and version are resolved from its configuration file (config.json), + falling back to "pluginName"/"pluginVersion" when the configuration file does not provide them. + For a "pluginType" of PluginType.REZ, "pluginName" and "pluginVersion" are always used as-is, + the configuration file cannot override them. + + Args: + pluginName: the name of the plugin. Overridden by the "name" of its configuration file, + unless "pluginType" is PluginType.REZ. + pluginFolder: the plugin's root folder. + pluginType: the type of the plugin. + pluginVersion: the plugin's version. Overridden by the "version" of its configuration + file, unless "pluginType" is PluginType.REZ. + isUserPlugin: whether the plugin is a user plugin (not maintained by the core Meshroom team). + hasMeshroomFolder: whether "pluginFolder" directly contains the plugin's modules, instead of + gathering them in a "meshroom" folder. + + Returns: + Plugin: the loaded plugin, or None if its folders do not exist, if its name is already + used, or if it does not provide any node description, submitter, or template. + """ + if not os.path.isdir(pluginFolder): + logging.info(f"Plugin folder '{pluginFolder}' does not exist.") + return None + + # Case where the folder directly contains the plugin's modules, while the other plugins are + # expected to gather modules in a "meshroom" folder. + mrFolder = Path(pluginFolder) + + if hasMeshroomFolder: + mrFolder = Path(pluginFolder, "meshroom") + if not mrFolder.is_dir(): + logging.info(f"Plugin folder '{pluginFolder}' does not contain a 'meshroom' folder.") + return None + + # Resolve the plugin's final name/version from its configuration file. + # A Rez plugin name/version is definitive: config.json cannot override it. + pluginConfig = None + pluginConfigPath = mrFolder / "config.json" + if pluginConfigPath.is_file(): + pluginConfig = PluginConfig.load(pluginConfigPath) + if pluginType is not PluginType.REZ: + pluginName = pluginConfig.name or pluginName + pluginVersion = pluginConfig.version or pluginVersion + + # The plugin's name prefixes its modules. + # Two plugins shipping identically named files do not collide in sys.modules. + pluginPackage = f"{PLUGINS_ROOT_PACKAGE}.{pluginName}" + + # Reject a plugin whose name is already used. + if pluginPackage in sys.modules: + logging.warning(f"A plugin '{pluginName}' has already been loaded.") + return None + + # Initialize the plugin object. + plugin = Plugin(pluginName, pluginFolder, mrFolder, pluginType, pluginVersion, isUserPlugin, + pluginConfig) + + # Recursive load of modules. + issues = _LoadIssues() + self._loadRootFolder(plugin, pluginPackage, mrFolder, issues) + + # Log issues. + issues.log(pluginName) + + # Check if the plugin is empty. + if (len(plugin.nodeDescProviders) <= 0 + and len(plugin.submitterProviders) <= 0 + and len(plugin.templates) <= 0): + logging.debug(f"Plugin '{pluginName}' ({pluginFolder}) does not contain modules/templates.") + return None + + return plugin + + def unloadPlugin(self, pluginName: str): + """ + Remove from sys.modules the virtual package of the plugin named "pluginName", as well as + every module that has been loaded under it, so that the plugin can be loaded again. + + Args: + pluginName: the name of the plugin to unload. + """ + pluginPackage = f"{PLUGINS_ROOT_PACKAGE}.{pluginName}" + for moduleName in [name for name in sys.modules if name == pluginPackage or name.startswith(f"{pluginPackage}.")]: + del sys.modules[moduleName] + + def _loadRootFolder(self, plugin: Plugin, packageName: str, folderRootPath: Path, issues: _LoadIssues): + """ + Load plugin's root folder. The root itself is always treated as a flat folder, + even if it contains an "__init__.py" (which is ignored). Each direct subfolder + is loaded as a package folder if it contains an "init.py", or as a flat folder otherwise. + + Args: + plugin: the Plugin object to attach discovered node/submitter providers to. + packageName: the dotted name of the virtual package the folder stands for. + folderRootPath: the plugin's root folder to load. + issues: the collector for the issues found while loading the plugin. + """ + self._loadFlatFolder(plugin, packageName, folderRootPath, issues) + + for subFolderPath in sorted(p for p in folderRootPath.iterdir() + if p.is_dir() and not p.name.startswith(("__", "."))): + self._loadFolder(plugin, f"{packageName}.{subFolderPath.name}", subFolderPath, issues) + + def _loadFolder(self, plugin: Plugin, packageName: str, folderPath: Path, issues: _LoadIssues): + """ + Load "folderPath" as a package if it contains an "__init__.py", or as a flat, + file-by-file folder otherwise. + + Args: + plugin: the Plugin object to attach discovered node/submitter providers to. + packageName: the dotted name of the virtual package the folder stands for. + folderPath: the folder to load. + issues: the collector for the issues found while loading the plugin. + """ + if (folderPath / "__init__.py").is_file(): + self._loadPackageFolder(plugin, packageName, folderPath, issues) + else: + self._loadFlatFolder(plugin, packageName, folderPath, issues) + + def _loadPackageFolder(self, plugin: Plugin, packageName: str, folderPath: Path, issues: _LoadIssues): + """ + Load "folderPath" as a real Python package, executing its "__init__.py" instead of + faking one, so relative imports and package-level wiring between its modules behave + exactly as they would for a normally installed package. Every direct child of the + package, every standalone Python file, and every subfolder that is itself a package, + is then individually loaded and scanned, one level deep. + + Args: + plugin: the Plugin object to attach discovered node/submitter providers to. + packageName: the dotted name to load the package under. + folderPath: the package's folder, containing an "__init__.py". + issues: the collector for the issues found while loading the plugin. + """ + # Register package for sub-folders. + parentName = packageName.rpartition(".")[0] + if parentName: + self._registerPackage(parentName) + + # Load init module. + initModule = sys.modules.get(packageName) or self._execModule( + packageName, folderPath / "__init__.py", issues, submoduleSearchLocations=[str(folderPath)]) + if initModule is None: + return + + self._collectProviders(plugin, initModule, issues) + + # Every direct child is scanned too, regardless of whether "__init__.py" imports it. + for entryPath in sorted(folderPath.iterdir()): + if entryPath.name.startswith(("__", ".")): + continue + + if entryPath.is_dir(): + if not (entryPath / "__init__.py").is_file(): + # Not itself a package: not walked, matching the one-level rule for plain + # subfolders elsewhere in this loader. + continue + childName = f"{packageName}.{entryPath.name}" + childModule = sys.modules.get(childName) or self._execModule( + childName, entryPath / "__init__.py", issues, submoduleSearchLocations=[str(entryPath)]) + elif entryPath.suffix == ".py": + childName = f"{packageName}.{entryPath.stem}" + childModule = sys.modules.get(childName) or self._execModule(childName, entryPath, issues) + else: + continue + + if childModule is None: + continue + self._collectProviders(plugin, childModule, issues) + + def _loadFlatFolder(self, plugin: Plugin, packageName: str, folderPath: Path, issues: _LoadIssues): + """ + Load every Python file directly contained in "folderPath" as a module of the virtual + package "packageName", and collect the node/submitter providers these modules define. + + Args: + packageName: the dotted name of the virtual package the folder stands for. + folderPath: the folder containing the Python files to load. + issues: the collector for the issues found while loading the plugin. + """ + for filePath in sorted(folderPath.glob("*.py")): + # Skip special/dunder files like __init__.py + if filePath.stem.startswith("__"): + continue + + module = self._execModule(f"{packageName}.{filePath.stem}", filePath, issues) + if not module: + continue + + # Register the package now that the folder is known to provide modules. + self._registerPackage(packageName, folderPath) + self._collectProviders(plugin, module, issues) + + def _execModule(self, moduleName: str, filePath: Path, issues: _LoadIssues, + submoduleSearchLocations: list[str] = None) -> ModuleType: + """ + Load the Python file "filePath" as the module "moduleName" and register it in sys.modules. + + Nothing is left registered when the loading fails, so that a returned module and an entry in + sys.modules always come together. + + Args: + moduleName: the unique dotted name to load the module under. + filePath: the path of the Python file to load. + issues: the collector for the issues found while loading the plugin. + submoduleSearchLocations: if provided, "filePath" is treated as the "__init__.py" of a + real package whose submodules are looked up in these folders. + + Returns: + ModuleType: the loaded module, or None if it could not be loaded. + """ + spec = importlib.util.spec_from_file_location(moduleName, filePath, + submodule_search_locations=submoduleSearchLocations) + if spec is None or spec.loader is None: + issues.loading.append(f'Could not create the module spec for "{filePath}".') + return None + + module = importlib.util.module_from_spec(spec) + sys.modules[moduleName] = module + + try: + spec.loader.exec_module(module) + except Exception as exc: + # The module has been left partially initialized: unregister it, otherwise any + # subsequent import of that name would silently return the broken module. + sys.modules.pop(moduleName, None) + issues.loading.append(f'Failed to load the module "{moduleName}" from "{filePath}"' + f'{self._formatExceptionMessage(exc)}') + return None + + return module + + def _collectProviders(self, plugin: Plugin, module: ModuleType, issues: _LoadIssues): + """ + Add to the plugin a node provider for every node description, and a submitter provider for + every submitter class, that "module" defines. + + Args: + module: the module to scan for node description classes and submitter classes. + issues: the collector for the issues found while loading the plugin. + """ + for attrName in dir(module): + attr = getattr(module, attrName) + if not isinstance(attr, type) or attr.__module__ != module.__name__: + continue + + if issubclass(attr, desc.BaseNode): + try: + nodeDescProvider = plugin.addNodeDescProvider(attr) + if nodeDescProvider.error: + issues.nodeDescProviders.append(nodeDescProvider.error) + except Exception as exc: + issues.nodeDescProviders.append(f'Failed to create the node provider for "{attrName}" from ' + f'"{module.__file__}"{self._formatExceptionMessage(exc)}') + elif issubclass(attr, BaseSubmitter): + try: + submitterProvider = plugin.addSubmitterProvider(attr) + if submitterProvider.error: + issues.submitterProviders.append(submitterProvider.error) + except Exception as exc: + issues.submitterProviders.append(f'Failed to create the submitter provider for "{attrName}" from ' + f'"{module.__file__}"{self._formatExceptionMessage(exc)}') + + def _registerPackage(self, packageName: str, packagePath: Path = None): + """ + Register "packageName" in sys.modules as a virtual package, together with every parent + package it is nested in that is not registered yet, up to PLUGINS_ROOT_PACKAGE. + + Args: + packageName: the dotted name of the virtual package to register. + packagePath: the folder the package's modules are loaded from. It must be provided for + the packages directly containing modules that can be reloaded. + """ + if packageName in sys.modules: + return + + # A module is resolved through its parent package, so the whole chain has to be registered. + parentName = packageName.rpartition(".")[0] + if parentName: + self._registerPackage(parentName) + + # A spec without a loader but with search locations describes a package. + spec = importlib.machinery.ModuleSpec(packageName, None, is_package=True) + if packagePath: + spec.submodule_search_locations.append(str(packagePath)) + + sys.modules[packageName] = importlib.util.module_from_spec(spec) + + def _formatExceptionMessage(self, exc: Exception) -> str: + """ + Format an exception raised while loading a plugin into a message detailing where it comes from. + The location of the last call, the line of code that raised it and the full traceback are all + reported, as plugin authors need them to debug their nodes. + + Args: + exc: the exception to format. + + Returns: + str: the formatted error message, to be appended to a description of what failed. + """ + # Not using traceback.format_exception(exc): its single-argument form requires Python 3.10. + fullTraceback = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__)) + tb = traceback.extract_tb(exc.__traceback__) + if not tb: + return f" ({type(exc).__name__}): {exc}\n{fullTraceback}" + + lastCall = tb[-1] + return (f" ({type(exc).__name__}):\n{exc}\n" + # filename:lineNumber functionName + f"{lastCall.filename}:{lastCall.lineno} {lastCall.name}\n" + # line of code with the error + f"{lastCall.line}\n" + # Full traceback + f"{fullTraceback}") diff --git a/meshroom/core/plugins/manager.py b/meshroom/core/plugins/manager.py new file mode 100644 index 0000000000..5936f4a24a --- /dev/null +++ b/meshroom/core/plugins/manager.py @@ -0,0 +1,298 @@ +from __future__ import annotations + +import logging +import time + +from typing import Optional + +from meshroom.common import BaseObject +from meshroom.core.plugins.loader import PluginLoader +from meshroom.core.plugins.base import ( + Plugin, PluginType, NodeDescProvider, NodeDescProviderStatus, SubmitterProvider, SubmitterProviderStatus, +) + + +class PluginManager(BaseObject): + """ + Manager for all the loaded Plugin objects as well as the registered NodeDescProvider objects and + SubmitterProvider objects. + + Members: + pluginLoader: handle plugin loading in a common virtual package + plugins: dictionary containing all the loaded Plugins, with their name as the key + nodeDescProviders: dictionary containing all the NodeDescProviders that have been registered, + with their name as the key + submitterProviders: dictionary containing all the SubmitterProviders that have been registered, + with the name of the submitter as the key + """ + + def __init__(self): + super().__init__() + self._pluginLoader: PluginLoader = PluginLoader() # plugin loader in virtual package + self._plugins: dict[str, Plugin] = {} # loaded plugins + self._nodeDescProviders: dict[str, NodeDescProvider] = {} # registered node descriptor providers + self._submitterProviders: dict[str, SubmitterProvider] = {} # registered submitter providers + + def _addPlugin(self, + pluginName: str, + pluginFolder: str, + pluginType: PluginType, + pluginVersion: Optional[str] = None, + isUserPlugin: bool = False, + hasMeshroomFolder: bool = True, + registerProviders: bool = True): + """ + Add a Plugin object and register the valid node description and submitter providers it contains. + + A node description or submitter provider is not registered if it is invalid, or if its name is + already registered under another plugin: in that case, it remains part of "plugin" but is not + made available through the manager. + + Args: + pluginName: the name of the plugin. + pluginFolder: the plugin's root folder. + pluginType: the type of the plugin. + pluginVersion: the version of the plugin. + isUserPlugin: whether the plugin is a user plugin (not maintained by the core Meshroom team). + hasMeshroomFolder: whether "pluginFolder" directly contains the plugin's modules, instead of + gathering them in a "meshroom" folder. + registerProviders: True if all the valid providers from the plugin should be registered. + """ + startTime = time.perf_counter() + plugin = self._pluginLoader.loadPlugin(pluginName=pluginName, + pluginFolder=pluginFolder, + pluginType=pluginType, + pluginVersion=pluginVersion, + isUserPlugin=isUserPlugin, + hasMeshroomFolder=hasMeshroomFolder) + if plugin: + if self.getPlugin(plugin.name): + logging.warning(f"Plugin {plugin.name} is already registered.") + return + self._plugins[plugin.name] = plugin + + if registerProviders: + self.registerPluginProviders(plugin) + logging.debug(f"Plugin '{pluginName}' loaded in {time.perf_counter() - startTime:.3f}s") + + def addPluginFromRez(self, rezPackageName: str, rezPackageVersion: str, rezPackageFolder: str, + isUserPlugin: bool = False, registerProviders: bool = True): + """ + Load a plugin resolved through Rez and register its valid providers. + + The plugin's modules are expected in a "meshroom" folder inside "rezPackageFolder", and its + process environment is built by resolving a Rez environment for its subrequires. + + Args: + rezPackageName: the name of the Rez package, used as the plugin's name. + rezPackageVersion: the version of the Rez package, used as the plugin's version. + rezPackageFolder: the resolved root folder of the Rez package. + isUserPlugin: whether the plugin is a user plugin (not maintained by the core Meshroom team). + registerProviders: True if all the valid providers from the plugin should be registered. + """ + self._addPlugin(rezPackageName, rezPackageFolder, PluginType.REZ, pluginVersion=rezPackageVersion, + isUserPlugin=isUserPlugin, hasMeshroomFolder=True, registerProviders=registerProviders) + + def addPluginFromPath(self, defaultPluginName: str, pluginFolder: str, pluginVersion: Optional[str] = None, + isUserPlugin: bool = False, registerProviders: bool = True): + """ + Load a plugin located at an arbitrary path and register its valid providers. + + The plugin's modules are expected in a "meshroom" folder inside "pluginFolder", and its + process environment is built from that folder's directory tree ("bin"/"lib"/"lib64"/"venv"). + + Args: + defaultPluginName: the name to register the plugin under. + pluginFolder: the plugin's root folder. + pluginVersion: the plugin's version. + isUserPlugin: whether the plugin is a user plugin (not maintained by the core Meshroom team). + registerProviders: True if all the valid providers from the plugin should be registered. + """ + self._addPlugin(defaultPluginName, pluginFolder, PluginType.PATH, pluginVersion=pluginVersion, + isUserPlugin=isUserPlugin, hasMeshroomFolder=True, registerProviders=registerProviders) + + def addPluginFromBuiltInFolder(self, defaultPluginName: str, pluginFolder: str, + registerProviders: bool = True): + """ + Load a plugin from a built-in Meshroom folder and register its valid providers. + + "pluginFolder" is expected to directly contain the plugin's modules (no nested "meshroom" folder). + This is how Meshroom's own "nodes"/"submitters" folders are laid out. The plugin is never a user + plugin. + + Args: + defaultPluginName: the name to register the plugin under. + pluginFolder: the plugin's root folder, directly containing its modules. + registerProviders: True if all the valid providers from the plugin should be registered. + """ + self._addPlugin(defaultPluginName, pluginFolder, PluginType.BUILTIN, pluginVersion=None, + isUserPlugin=False, hasMeshroomFolder=False, registerProviders=registerProviders) + + def removePlugin(self, plugin: Plugin, unregisterProviders: bool = True, unloadPlugin: bool = True): + """ + Remove a loaded Plugin object. + + Args: + plugin: the Plugin to remove from the list of loaded plugins. + unregisterProviders: True if all the providers from the plugin should be unregistered. + unloadPlugin: True if the plugin virtual package should be unload. + """ + if self.getPlugin(plugin.name): + if unregisterProviders: + for name, nodeDescProvider in plugin.nodeDescProviders.items(): + if self._nodeDescProviders.get(name) is nodeDescProvider: + del self._nodeDescProviders[name] + for name, submitterProvider in plugin.submitterProviders.items(): + if self._submitterProviders.get(name) is submitterProvider: + del self._submitterProviders[name] + if unloadPlugin: + self._pluginLoader.unloadPlugin(plugin.name) + del self._plugins[plugin.name] + + def registerPluginProviders(self, plugin: Plugin): + """ + Register every valid node description and submitter provider "plugin" contains. + + Args: + plugin: the Plugin whose valid providers should be registered. + """ + for name, nodeDescProvider in plugin.nodeDescProviders.items(): + if nodeDescProvider.status != NodeDescProviderStatus.VALID: + continue + if name in self._nodeDescProviders: + existingProvider = self._nodeDescProviders[name] + if existingProvider != nodeDescProvider: + logging.warning( + f"Could not register node {name} ({nodeDescProvider.path}) " + f"because another node is already registered with this name ({existingProvider.path})" + ) + continue + self._nodeDescProviders[name] = nodeDescProvider + + for name, submitterProvider in plugin.submitterProviders.items(): + if submitterProvider.status != SubmitterProviderStatus.VALID: + continue + if name in self._submitterProviders: + existingProvider = self._submitterProviders[name] + if existingProvider != submitterProvider: + logging.warning( + f"Could not register submitter {name} ({submitterProvider.path}) " + f"because another submitter is already registered with this name ({existingProvider.path})" + ) + continue + self._submitterProviders[name] = submitterProvider + + def getPlugins(self) -> dict[str, Plugin]: + """ + Return a dictionary containing all the loaded Plugins, with {key, value} = + {name, Plugin}. + """ + return self._plugins + + def getPlugin(self, name: str) -> Plugin: + """ + Return the loaded Plugin object with "name". + + Args: + name: the unique name of the Plugin, used upon its loading. + + Returns: + Plugin | None: the loaded Plugin object if it exists, None otherwise. + """ + for plugin in self._plugins.values(): + if plugin.name == name: + return plugin + return None + + def getPluginFromNodeDesc(self, name: str) -> Plugin: + """ + Return the loaded Plugin that contains the node descriptor "name", independently + from whether it has been registered or not. + + Args: + name: the name of the node descriptor that needs to be searched for across + plugins. + + Returns: + Plugin | None: the Plugin the node belongs to if it exists, None otherwise. + """ + for plugin in self._plugins.values(): + if plugin.containsNodeDescProvider(name): + return plugin + return None + + def getPipelineTemplates(self) -> dict[str, str]: + """ + Return a dictionary combining the pipeline templates of every available Plugin, + with {key, value} = {template name, absolute path}. + + If several plugins provide a template with the same name, only the last one + encountered is kept. + + Returns: + dict: The combined templates of every available Plugin. + """ + templates = {} + for plugin in self._plugins.values(): + templates.update(plugin.templates) + return templates + + def isNodeDescRegistered(self, name: str) -> bool: + """ + Return whether the node descriptor provider has been registered already. + + Args: + name: the name of the node descriptor whose registration needs to be checked. + """ + return name in self._nodeDescProviders + + def getNodeDescProviders(self) -> dict[str, NodeDescProvider]: + """ + Return a dictionary containing all the registered NodeDescProviders, + with {key, value} = {name, NodeDescProvider}. + """ + return self._nodeDescProviders + + def getNodeDescProvider(self, name: str) -> NodeDescProvider: + """ + Return the NodeDescProvider object that has been registered under the name "name" if it exists. + + Args: + name: the name of the NodeDescProvider. + + Returns: + NodeDescProvider | None: the registered NodeDescProvider object if it exists, None otherwise. + """ + if self.isNodeDescRegistered(name): + return self._nodeDescProviders[name] + return None + + def isSubmitterRegistered(self, name: str) -> bool: + """ + Return whether the submitter provider has been registered already. + + Args: + name: the name of the submitter provider. + """ + return name in self._submitterProviders + + def getSubmitterProviders(self) -> dict[str, SubmitterProvider]: + """ + Return a dictionary containing all the registered SubmitterProvider, + with {key, value} = {name, SubmitterProvider}. + """ + return self._submitterProviders + + def getSubmitterProvider(self, name: str) -> SubmitterProvider: + """ + Return the SubmitterProvider object that has been registered under the name "name" if it exists. + + Args: + name: the name of the SubmitterProvider. + + Returns: + SubmitterProvider | None: the registered SubmitterProvider object if it exists, None otherwise. + """ + if self.isSubmitterRegistered(name): + return self._submitterProviders[name] + return None diff --git a/meshroom/core/test.py b/meshroom/core/test.py index f3402b984e..7aa2ea8550 100644 --- a/meshroom/core/test.py +++ b/meshroom/core/test.py @@ -30,11 +30,11 @@ def checkTemplateVersions(path: str, nodesAlreadyLoaded: bool = False) -> bool: for _, nodeData in graphData.items(): nodeType = nodeData["nodeType"] - if not meshroom.core.pluginManager.isRegistered(nodeType): + if not meshroom.core.pluginManager.isNodeDescRegistered(nodeType): print(f"'{nodeType}' in '{path}' is an unknown type.") return False - nodeDesc = meshroom.core.pluginManager.getRegisteredNodePlugin(nodeType).nodeDescriptor + nodeDesc = meshroom.core.pluginManager.getNodeDescProvider(nodeType).nodeDescClass currentNodeVersion = meshroom.core.nodeVersion(nodeDesc) inputs = nodeData.get("inputs", {}) @@ -63,8 +63,8 @@ def checkTemplateVersions(path: str, nodesAlreadyLoaded: bool = False) -> bool: finally: if not nodesAlreadyLoaded: - nodePlugins = meshroom.core.pluginManager.getRegisteredNodePlugins() - for node in nodePlugins: + nodeDescProviders = meshroom.core.pluginManager.getNodeDescProviders() + for node in nodeDescProviders: meshroom.core.pluginManager.unregisterNode(node) diff --git a/meshroom/ui/app.py b/meshroom/ui/app.py index 03b907b867..cb61c616a1 100644 --- a/meshroom/ui/app.py +++ b/meshroom/ui/app.py @@ -289,7 +289,7 @@ def __init__(self, inputArgs): self.engine.addImportPath(pyside6QmlPath) # expose available node types that can be instantiated - self.engine.rootContext().setContextProperty("_nodeTypes", {n: {"category": pluginManager.getRegisteredNodePlugins()[n].nodeDescriptor.category} for n in sorted(pluginManager.getRegisteredNodePlugins().keys())}) + self.engine.rootContext().setContextProperty("_nodeTypes", {n: {"category": pluginManager.getNodeDescProviders()[n].nodeDescClass.category} for n in sorted(pluginManager.getNodeDescProviders().keys())}) # instantiate the 3D Scene object self._undoStack = commands.UndoStack(self) diff --git a/meshroom/ui/scene.py b/meshroom/ui/scene.py index 6dfd328f4f..9a23d54fcf 100755 --- a/meshroom/ui/scene.py +++ b/meshroom/ui/scene.py @@ -19,7 +19,7 @@ from meshroom.core.node import Node, CompatibilityNode, Status, Position, CompatibilityIssue from meshroom.core.taskManager import TaskManager from meshroom.core.evaluation import MathEvaluator -from meshroom.core.plugins import NodePluginStatus +from meshroom.core.plugins.base import NodeDescProviderStatus from meshroom.ui import commands from meshroom.ui.graph import UIGraph @@ -417,7 +417,7 @@ def initActiveNodes(self): self._activeNodes.add(ActiveNode(category, parent=self)) # For all nodes declared to be accessed by the UI usedNodeTypes = {j for i in self.activeNodeCategories.values() for j in i} - allLoadedNodeTypes = set(meshroom.core.pluginManager.getRegisteredNodePlugins().keys()) + allLoadedNodeTypes = set(meshroom.core.pluginManager.getNodeDescProviders().keys()) allUiNodes = set(self.uiNodes) | usedNodeTypes | allLoadedNodeTypes for nodeType in allUiNodes: @@ -441,25 +441,25 @@ def reloadPlugins(self): def _reloadPlugins(self): """ - Reload all the NodePlugins from all the registered plugins. + Reload all the NodeDescProviders from all the registered plugins. The nodes in the graph will be updated to match the changes in the description, if there was any. """ reloadedNodes: list[str] = [] errorNodes: list[str] = [] for plugin in meshroom.core.pluginManager.getPlugins().values(): - for node in plugin.nodes.values(): + for node in plugin.nodeDescProviders.values(): if node.reload(): - reloadedNodes.append(node.nodeDescriptor.__name__) + reloadedNodes.append(node.nodeDescClass.__name__) else: - if node.status == NodePluginStatus.DESC_ERROR or node.status == NodePluginStatus.ERROR: - errorNodes.append(node.nodeDescriptor.__name__) + if node.status == NodeDescProviderStatus.DESC_ERROR or node.status == NodeDescProviderStatus.ERROR: + errorNodes.append(node.nodeDescClass.__name__) self.pluginsReloaded.emit(reloadedNodes, errorNodes) @Slot(list) def _onPluginsReloaded(self, reloadedNodes: list, errorNodes: list): - self._graph.reloadNodePlugins(reloadedNodes) + self._graph.reloadNodeDescProviders(reloadedNodes) if len(errorNodes) > 0: self.parent().showMessage(f"Some plugins failed to reload: {', '.join(errorNodes)}", "error") else: @@ -609,7 +609,7 @@ def setupTempCameraInit(self, node, attrName): if not sfmFile or not os.path.isfile(sfmFile): self.tempCameraInit = None return - nodeDesc = meshroom.core.pluginManager.getRegisteredNodePlugin("CameraInit").nodeDescriptor() + nodeDesc = meshroom.core.pluginManager.getNodeDescProvider("CameraInit").nodeDescClass() views, intrinsics = nodeDesc.readSfMData(sfmFile) tmpCameraInit = Node("CameraInit", viewpoints=views, intrinsics=intrinsics) tmpCameraInit.locked = True diff --git a/tests/__init__.py b/tests/__init__.py index 6ebeba20f3..ec311654bd 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,11 +1,9 @@ import os -from meshroom.core import loadAllNodes from meshroom.core import pluginManager -plugins = loadAllNodes(os.path.join(os.path.dirname(__file__), "nodes")) -for plugin in plugins: - pluginManager.addPlugin(plugin) + +pluginManager.addPluginFromBuiltInFolder("testNodes", os.path.join(os.path.dirname(__file__), "nodes")) if os.getenv("MESHROOM_PIPELINE_TEMPLATES_PATH", False): os.environ["MESHROOM_PIPELINE_TEMPLATES_PATH"] += os.pathsep + os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/plugins/meshroom/pluginB/__init__.py b/tests/plugins/meshroom/pluginB/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/plugins/meshroom/pluginC/__init__.py b/tests/plugins/meshroom/pluginC/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/plugins/meshroom/pluginSubmitter/__init__.py b/tests/plugins/meshroom/pluginSubmitter/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/plugins/meshroom/pluginA/PluginAInitInputNode.py b/tests/plugins/pluginA/meshroom/PluginAInitInputNode.py similarity index 100% rename from tests/plugins/meshroom/pluginA/PluginAInitInputNode.py rename to tests/plugins/pluginA/meshroom/PluginAInitInputNode.py diff --git a/tests/plugins/meshroom/pluginA/PluginAInitNode.py b/tests/plugins/pluginA/meshroom/PluginAInitNode.py similarity index 100% rename from tests/plugins/meshroom/pluginA/PluginAInitNode.py rename to tests/plugins/pluginA/meshroom/PluginAInitNode.py diff --git a/tests/plugins/meshroom/pluginA/PluginANodeA.py b/tests/plugins/pluginA/meshroom/PluginANodeA.py similarity index 100% rename from tests/plugins/meshroom/pluginA/PluginANodeA.py rename to tests/plugins/pluginA/meshroom/PluginANodeA.py diff --git a/tests/plugins/meshroom/pluginA/PluginANodeB.py b/tests/plugins/pluginA/meshroom/PluginANodeB.py similarity index 100% rename from tests/plugins/meshroom/pluginA/PluginANodeB.py rename to tests/plugins/pluginA/meshroom/PluginANodeB.py diff --git a/tests/plugins/meshroom/config.json b/tests/plugins/pluginA/meshroom/config.json similarity index 100% rename from tests/plugins/meshroom/config.json rename to tests/plugins/pluginA/meshroom/config.json diff --git a/tests/plugins/meshroom/sharedTemplate.mg b/tests/plugins/pluginA/meshroom/sharedTemplate.mg similarity index 100% rename from tests/plugins/meshroom/sharedTemplate.mg rename to tests/plugins/pluginA/meshroom/sharedTemplate.mg diff --git a/tests/plugins/meshroom/pluginB/PluginBNodeA.py b/tests/plugins/pluginB/meshroom/PluginBNodeA.py similarity index 100% rename from tests/plugins/meshroom/pluginB/PluginBNodeA.py rename to tests/plugins/pluginB/meshroom/PluginBNodeA.py diff --git a/tests/plugins/meshroom/pluginB/PluginBNodeB.py b/tests/plugins/pluginB/meshroom/PluginBNodeB.py similarity index 100% rename from tests/plugins/meshroom/pluginB/PluginBNodeB.py rename to tests/plugins/pluginB/meshroom/PluginBNodeB.py diff --git a/tests/plugins/meshroom/pluginC/PluginCNodeA.py b/tests/plugins/pluginC/meshroom/PluginCNodeA.py similarity index 100% rename from tests/plugins/meshroom/pluginC/PluginCNodeA.py rename to tests/plugins/pluginC/meshroom/PluginCNodeA.py diff --git a/tests/plugins/meshroom/pluginSubmitter/PluginSubmitter.py b/tests/plugins/pluginSubmitter/meshroom/PluginSubmitter.py similarity index 100% rename from tests/plugins/meshroom/pluginSubmitter/PluginSubmitter.py rename to tests/plugins/pluginSubmitter/meshroom/PluginSubmitter.py diff --git a/tests/test_compatibility.py b/tests/test_compatibility.py index 9e911a4c9a..ab7f995a03 100644 --- a/tests/test_compatibility.py +++ b/tests/test_compatibility.py @@ -8,7 +8,7 @@ import pytest from meshroom.core import desc, pluginManager -from meshroom.core.plugins import NodePlugin +from meshroom.core.plugins.base import NodeDescProvider from meshroom.core.exception import GraphCompatibilityError, NodeUpgradeError from meshroom.core.graph import Graph, loadGraph from meshroom.core.node import CompatibilityNode, CompatibilityIssue, Node @@ -195,7 +195,7 @@ class OutputTemplateNodeV2(desc.Node): def replaceNodeTypeDesc(nodeType: str, nodeDesc: Type[desc.Node]): """ Change the `nodeDesc` associated to `nodeType`. """ - pluginManager.getRegisteredNodePlugins()[nodeType] = NodePlugin(nodeDesc) + pluginManager.getNodeDescProviders()[nodeType] = NodeDescProvider(nodeDesc) def test_unknown_node_type(): @@ -239,7 +239,7 @@ def test_description_conflict(): Test compatibility behavior for conflicting node descriptions. """ # Copy registered node types to be able to restore them - originalNodeTypes = copy.deepcopy(pluginManager.getRegisteredNodePlugins()) + originalNodeTypes = copy.deepcopy(pluginManager.getNodeDescProviders()) nodeTypes = [SampleNodeV1, SampleNodeV2, SampleNodeV3, SampleNodeV4, SampleNodeV5] nodes = [] @@ -265,7 +265,7 @@ def test_description_conflict(): # Offset node types register to create description conflicts # Each node type name now reference the next one's implementation for i, nt in enumerate(nodeTypes[:-1]): - pluginManager.getRegisteredNodePlugins()[nt.__name__] = NodePlugin(nodeTypes[i + 1]) + pluginManager.getNodeDescProviders()[nt.__name__] = NodeDescProvider(nodeTypes[i + 1]) # Reload file g = loadGraph(graphFile) @@ -355,7 +355,7 @@ def test_description_conflict(): raise ValueError("Unexpected node type: " + srcNode.nodeType) # Restore original node types - pluginManager._nodePlugins = originalNodeTypes + pluginManager._nodeDescProviders = originalNodeTypes def test_upgradeAllNodes(tmp_path): @@ -377,10 +377,10 @@ def test_upgradeAllNodes(tmp_path): g.save(graphFile) # Replace SampleNodeV1 by SampleNodeV2 and SampleInitNodeV1 by SampleInitNodeV2 - pluginManager.getRegisteredNodePlugins()[SampleNodeV1.__name__] = \ - pluginManager.getRegisteredNodePlugin(SampleNodeV2.__name__) - pluginManager.getRegisteredNodePlugins()[SampleInitNodeV1.__name__] = \ - pluginManager.getRegisteredNodePlugin(SampleInitNodeV2.__name__) + pluginManager.getNodeDescProviders()[SampleNodeV1.__name__] = \ + pluginManager.getNodeDescProvider(SampleNodeV2.__name__) + pluginManager.getNodeDescProviders()[SampleInitNodeV1.__name__] = \ + pluginManager.getNodeDescProvider(SampleInitNodeV2.__name__) # Make SampleNodeV2 and SampleInitNodeV2 an unknown type unregisterNodeDesc(SampleNodeV2) unregisterNodeDesc(SampleInitNodeV2) @@ -420,8 +420,8 @@ def test_conformUpgrade(): g.save(graphFile) # Replace SampleNodeV5 by SampleNodeV6 - pluginManager.getRegisteredNodePlugins()[SampleNodeV5.__name__] = \ - pluginManager.getRegisteredNodePlugin(SampleNodeV6.__name__) + pluginManager.getNodeDescProviders()[SampleNodeV5.__name__] = \ + pluginManager.getNodeDescProvider(SampleNodeV6.__name__) # Reload file g = loadGraph(graphFile) diff --git a/tests/test_compute.py b/tests/test_compute.py index 3038cb5abf..b6caff771b 100644 --- a/tests/test_compute.py +++ b/tests/test_compute.py @@ -15,9 +15,8 @@ import logging from meshroom.core.graph import Graph, loadGraph -from meshroom.core import desc, pluginManager, loadClassesNodes +from meshroom.core import desc, pluginManager from meshroom.core.node import Status, ChunkIndex -from meshroom.core.plugins import Plugin from .utils import registerNodeDesc, unregisterNodeDesc LOGGER = logging.getLogger("TestCompute") @@ -200,7 +199,7 @@ def executeChunkCommandLine(chunk, cmd, env=None): node = SimpleNamespace( name="TestNode_1", graph=SimpleNamespace(filepath=graphFilepath.as_posix()), - nodeDesc=SimpleNamespace(pythonExecutable="python", plugin=plugin), + nodeDesc=SimpleNamespace(pythonExecutable="python", provider=plugin), getChunks=lambda: [object(), object()], ) chunk = SimpleNamespace( @@ -241,24 +240,17 @@ class TestLockUpdates: Tests for node locking behaviour during status transitions. Nodes should be properly locked when they undergo computation statuses and unlocked when their status is reset (through parameter changes, for example). """ - plugin = None @classmethod def setup_class(cls): - folder = os.path.join(os.path.dirname(__file__), "plugins", "meshroom") - package = "pluginA" - cls.plugin = Plugin(package, folder) - nodes = loadClassesNodes(folder, package, pluginUid=cls.plugin.uid) - for node in nodes: - cls.plugin.addNodePlugin(node) - pluginManager.addPlugin(cls.plugin) + folder = os.path.join(os.path.dirname(__file__), "plugins", "pluginA") + pluginManager.addPluginFromPath("pluginA", folder) @classmethod def teardown_class(cls): - for node in cls.plugin.nodes.values(): - pluginManager.unregisterNode(node) - pluginManager.removePlugin(cls.plugin) - cls.plugin = None + plugin = pluginManager.getPlugin("pluginA") + if plugin: + pluginManager.removePlugin(plugin) @staticmethod def checkNodeStatusAndLock(node, expectedStatus, expectedLock): diff --git a/tests/test_invalidation.py b/tests/test_invalidation.py index fbbc1ce14e..057ddaa36c 100644 --- a/tests/test_invalidation.py +++ b/tests/test_invalidation.py @@ -20,7 +20,7 @@ class SampleNode(desc.Node): def test_output_invalidation(): - registerNodeDesc(SampleNode) # Register standalone NodePlugin + registerNodeDesc(SampleNode) # Register standalone NodeDescProvider graph = Graph("") n1 = graph.addNewNode("SampleNode", input="/tmp") n2 = graph.addNewNode("SampleNode") @@ -53,7 +53,7 @@ def test_inputLinkInvalidation(): """ Input links should not change the invalidation. """ - registerNodeDesc(SampleNode) # Register standalone NodePlugin + registerNodeDesc(SampleNode) # Register standalone NodeDescProvider graph = Graph("") n1 = graph.addNewNode("SampleNode") n2 = graph.addNewNode("SampleNode") diff --git a/tests/test_nodeAttributeChangedCallback.py b/tests/test_nodeAttributeChangedCallback.py index 8812b21b2d..9b51c9b952 100644 --- a/tests/test_nodeAttributeChangedCallback.py +++ b/tests/test_nodeAttributeChangedCallback.py @@ -343,8 +343,8 @@ def processChunk(self, chunk): class TestAttributeCallbackBehaviorWithUpstreamDynamicOutputs: - # nodePluginAttributeChangedCallback = NodePlugin(NodeWithAttributeChangedCallback) - # nodePluginDynamicOutputValue = NodePlugin(NodeWithDynamicOutputValue) + # nodeDescProviderAttributeChangedCallback = NodeDescProvider(NodeWithAttributeChangedCallback) + # nodeDescProviderDynamicOutputValue = NodeDescProvider(NodeWithDynamicOutputValue) @classmethod def setup_class(cls): diff --git a/tests/test_nodeDynamicOutputs.py b/tests/test_nodeDynamicOutputs.py index a4a1a88f00..3a6ecb6676 100644 --- a/tests/test_nodeDynamicOutputs.py +++ b/tests/test_nodeDynamicOutputs.py @@ -4,7 +4,7 @@ from meshroom.core import pluginManager from meshroom.core.exception import UnknownNodeTypeError from meshroom.core.graph import Graph, loadGraph -from meshroom.core.plugins import NodePluginStatus +from meshroom.core.plugins.base import NodeDescProviderStatus from .utils import registerNodeDesc, unregisterNodeDesc @@ -227,16 +227,15 @@ def test_registerInitNodeWithDynamicOutputs(self): registerNodeDesc(InitNodeWithDynamicOutputs) # Check that the plugin has been correctly registered (there has been attempt to load it) - assert pluginManager.isRegistered(InitNodeWithDynamicOutputs.__name__) + assert pluginManager.isNodeDescRegistered(InitNodeWithDynamicOutputs.__name__) # Check that the plugin's status is DESC_ERROR, since the node description is invalid # Additionally, the list of errors should include an error about having a dynamic output in an InitNode - plugin = pluginManager.getRegisteredNodePlugin(InitNodeWithDynamicOutputs.__name__) + plugin = pluginManager.getNodeDescProvider(InitNodeWithDynamicOutputs.__name__) assert plugin - assert plugin.status == NodePluginStatus.DESC_ERROR - assert len(plugin.errors) == 1 - errType = plugin.errors[0][1] - assert errType == desc.ValueTypeErrors.DYNAMIC_OUTPUT + assert plugin.status == NodeDescProviderStatus.DESC_ERROR + assert plugin.error is not None + assert "Unsupported dynamic output" in plugin.error unregisterNodeDesc(InitNodeWithDynamicOutputs) diff --git a/tests/test_nodes.py b/tests/test_nodes.py index 495371bc70..85aefd1b54 100644 --- a/tests/test_nodes.py +++ b/tests/test_nodes.py @@ -7,10 +7,10 @@ from pathlib import Path import tempfile -from meshroom.core import desc, pluginManager, loadClassesNodes, initNodes +from meshroom.core import desc, pluginManager, initNodes from meshroom.core.node import Position, BaseNode from meshroom.core.graph import Graph, loadGraph -from meshroom.core.plugins import Plugin, ProcessEnv +from meshroom.core.plugins.env import ProcessEnv from meshroom.nodes.general.InputString import InputString from meshroom.nodes.general.GetMgSceneParams import GetMeshroomSceneParams @@ -43,33 +43,25 @@ def processNode(node: BaseNode): class TestNodeInfo: - plugin = None @classmethod def setup_class(cls): - cls.folder = os.path.join(os.path.dirname(__file__), "plugins", "meshroom") - package = "pluginC" - cls.plugin = Plugin(package, cls.folder) - nodes = loadClassesNodes(cls.folder, package, pluginUid=cls.plugin.uid) - for node in nodes: - cls.plugin.addNodePlugin(node) - pluginManager.addPlugin(cls.plugin) + cls.folder = os.path.join(os.path.dirname(__file__), "plugins", "pluginC") + pluginManager.addPluginFromPath("pluginC", cls.folder) @classmethod def teardown_class(cls): - for node in cls.plugin.nodes.values(): - pluginManager.unregisterNode(node) - pluginManager.removePlugin(cls.plugin) - cls.plugin = None + plugin = pluginManager.getPlugin("pluginC") + if plugin: + pluginManager.removePlugin(plugin) def test_loadedPlugin(self): assert len(pluginManager.getPlugins()) >= 1 - plugin = pluginManager.getPlugin("pluginC", uname=False) - pluginUName = plugin.uname - assert pluginUName.endswith("pluginC") - assert plugin == self.plugin - node = plugin.nodes["PluginCNodeA"] - nodeType = node.nodeDescriptor + plugin = pluginManager.getPlugin("pluginC") + assert plugin + assert plugin.name.endswith("pluginC") + node = plugin.nodeDescProviders["PluginCNodeA"] + nodeType = node.nodeDescClass g = Graph("") registerNodeDesc(nodeType) @@ -79,7 +71,7 @@ def test_loadedPlugin(self): assert nodeDocumentation == "PluginCNodeA" nodeInfo = {item["key"]: item["value"] for item in node.getNodeInfo()} assert nodeInfo["module"].endswith("pluginC.PluginCNodeA") - pluginPath = os.path.join(self.folder, "pluginC", "PluginCNodeA.py") + pluginPath = os.path.join(self.folder, "meshroom", "PluginCNodeA.py") assert nodeInfo["modulePath"] == Path(pluginPath).as_posix() # modulePath seems to follow Linux convention assert nodeInfo["author"] == "testAuthor" assert nodeInfo["license"] == "no-license" @@ -88,35 +80,31 @@ def test_loadedPlugin(self): class TestNodeVariables: - plugin = None @classmethod def setup_class(cls): - folder = os.path.join(os.path.dirname(__file__), "plugins", "meshroom") - package = "pluginA" - cls.plugin = Plugin(package, folder) - nodes = loadClassesNodes(folder, package, pluginUid=cls.plugin.uid) - for node in nodes: - cls.plugin.addNodePlugin(node) - pluginManager.addPlugin(cls.plugin) + folder = os.path.join(os.path.dirname(__file__), "plugins", "pluginA") + pluginManager.addPluginFromPath("pluginA", folder) @classmethod def teardown_class(cls): - for node in cls.plugin.nodes.values(): - pluginManager.unregisterNode(node) - pluginManager.removePlugin(cls.plugin) - cls.plugin = None + plugin = pluginManager.getPlugin("pluginA") + if plugin: + pluginManager.removePlugin(plugin) def test_staticVariables(self): g = Graph("") - for nodeName in self.plugin.nodes.keys(): + plugin = pluginManager.getPlugin("pluginA") + assert plugin + + for nodeName in plugin.nodeDescProviders.keys(): n = g.addNewNode(nodeName) assert nodeName == n._staticExpVars["nodeType"] assert n.sourceCodeFolder assert n.sourceCodeFolder == n._staticExpVars["nodeSourceCodeFolder"] - self.plugin.nodes[nodeName].reload() + plugin.nodeDescProviders[nodeName].reload() assert nodeName == n._staticExpVars["nodeType"] assert n.sourceCodeFolder @@ -125,7 +113,10 @@ def test_staticVariables(self): def test_expVariables(self): g = Graph("") - for nodeName in self.plugin.nodes.keys(): + plugin = pluginManager.getPlugin("pluginA") + assert plugin + + for nodeName in plugin.nodeDescProviders.keys(): n = g.addNewNode(nodeName) assert n._expVars["uid"] == n._uid assert n.internalFolder @@ -133,7 +124,7 @@ def test_expVariables(self): assert "node" in n._expVars assert n._expVars["node"] is n - self.plugin.nodes[nodeName].reload() + plugin.nodeDescProviders[nodeName].reload() assert n._expVars["uid"] == n._uid assert n.internalFolder @@ -143,24 +134,17 @@ def test_expVariables(self): class TestInputNode: - plugin = None @classmethod def setup_class(cls): - folder = os.path.join(os.path.dirname(__file__), "plugins", "meshroom") - package = "pluginA" - cls.plugin = Plugin(package, folder) - nodes = loadClassesNodes(folder, package, pluginUid=cls.plugin.uid) - for node in nodes: - cls.plugin.addNodePlugin(node) - pluginManager.addPlugin(cls.plugin) + folder = os.path.join(os.path.dirname(__file__), "plugins", "pluginA") + pluginManager.addPluginFromPath("pluginA", folder) @classmethod def teardown_class(cls): - for node in cls.plugin.nodes.values(): - pluginManager.unregisterNode(node) - pluginManager.removePlugin(cls.plugin) - cls.plugin = None + plugin = pluginManager.getPlugin("pluginA") + if plugin: + pluginManager.removePlugin(plugin) def test_inputNode(self): g = Graph("") @@ -187,8 +171,6 @@ def setup_class(cls): def teardown_class(cls): for plugin in pluginManager.getPlugins(): if plugin not in cls.loadedPlugins: - for node in plugin.nodes.values(): - pluginManager.unregisterNode(node) pluginManager.removePlugin(plugin) def test_backdropNode(self): @@ -617,8 +599,6 @@ def setup_class(cls): def teardown_class(cls): for plugin in pluginManager.getPlugins(): if plugin not in cls.loadedPlugins: - for node in plugin.nodes.values(): - pluginManager.unregisterNode(node) pluginManager.removePlugin(plugin) @staticmethod @@ -739,7 +719,7 @@ def test_GetMeshroomSceneParams(graphSavedOnDisk): print("graphFile", graphFile) graph.save(graphFile) # Process node - node.nodePlugin._processEnv = ProcessEnv("", {}, "test_plugin") + node.nodeDescProvider._processEnv = ProcessEnv("", {}, "test_plugin") processNode(node) # Check output outputJson = Path(node.internalFolder) / "values.json" diff --git a/tests/test_pluginLoader.py b/tests/test_pluginLoader.py new file mode 100644 index 0000000000..7624b6cc38 --- /dev/null +++ b/tests/test_pluginLoader.py @@ -0,0 +1,359 @@ +#!/usr/bin/env python +# coding:utf-8 + +import json +import logging +import sys + +from meshroom.core.plugins.base import PluginType, NodeDescProviderStatus, SubmitterProviderStatus +from meshroom.core.plugins.loader import PLUGINS_ROOT_PACKAGE, PluginLoader +from .utils import writeFile + + +def _nodeDescSource(className: str) -> str: + return ( + f"from meshroom.core import desc\n\n" + f"class {className}(desc.Node):\n" + f" pass\n" + ) + + +def _submitterSource(className: str, name: str = None) -> str: + name = name or className + return ( + "from meshroom.core.submitter import BaseSubmitter\n\n" + f"class {className}(BaseSubmitter):\n" + f" _name = \"{name}\"\n" + ) + + +def _brokenSource() -> str: + return "raise RuntimeError('broken')\n" + + +class TestPluginLoader: + + def test_flatRoot(self, tmp_path): + """ Standalone files directly in the plugin's root are discovered. """ + writeFile(tmp_path / "meshroom/MyNode.py", _nodeDescSource("MyNode")) + + loader = PluginLoader() + plugin = loader.loadPlugin("flatRootPlugin", str(tmp_path), PluginType.PATH) + + assert plugin is not None + assert list(plugin.nodeDescProviders.keys()) == ["MyNode"] + assert plugin.nodeDescProviders["MyNode"].status == NodeDescProviderStatus.VALID + + loader.unloadPlugin("flatRootPlugin") + + def test_rootInitIgnored(self, tmp_path): + """ The root's own "__init__.py" is ignored, even if it defines a node itself. """ + writeFile(tmp_path / "meshroom/__init__.py", _nodeDescSource("RootNode")) + writeFile(tmp_path / "meshroom/MyNode.py", _nodeDescSource("MyNode")) + + loader = PluginLoader() + plugin = loader.loadPlugin("rootInitIgnoredPlugin", str(tmp_path), PluginType.PATH) + + assert plugin is not None + assert "RootNode" not in plugin.nodeDescProviders + assert "MyNode" in plugin.nodeDescProviders + + loader.unloadPlugin("rootInitIgnoredPlugin") + + def test_packageSubfolder(self, tmp_path): + """ + A subfolder with an "__init__.py" is loaded as a real package: "__init__.py" and every + one of its direct children are scanned, whether or not "__init__.py" references them. + """ + writeFile(tmp_path / "meshroom/pkgA/__init__.py", "from .impl import ExposedNode\n") + writeFile(tmp_path / "meshroom/pkgA/impl.py", _nodeDescSource("ExposedNode")) + writeFile(tmp_path / "meshroom/pkgA/other.py", _nodeDescSource("OtherNode")) + + loader = PluginLoader() + plugin = loader.loadPlugin("packageSubfolderPlugin", str(tmp_path), PluginType.PATH) + + assert plugin is not None + assert set(plugin.nodeDescProviders.keys()) == {"ExposedNode", "OtherNode"} + + loader.unloadPlugin("packageSubfolderPlugin") + + def test_nestedPackageTwoLevelsNotDiscovered(self, tmp_path): + """ A package nested two levels from the root only has its "__init__.py" scanned. """ + writeFile(tmp_path / "meshroom/subpkgA/__init__.py") + writeFile(tmp_path / "meshroom/subpkgA/subpkgB/__init__.py") + writeFile(tmp_path / "meshroom/subpkgA/subpkgB/impl.py", _nodeDescSource("DeepNode")) + + loader = PluginLoader() + plugin = loader.loadPlugin("nestedPackagePlugin", str(tmp_path), PluginType.PATH) + + # Nothing was found: subpkgB's own children are never reached. + assert plugin is None + + loader.unloadPlugin("nestedPackagePlugin") + + def test_plainSubfolderOneLevel(self, tmp_path): + """ A plain (non-package) subfolder only yields its own direct files, one level deep. """ + writeFile(tmp_path / "meshroom/plain/Shallow.py", _nodeDescSource("Shallow")) + writeFile(tmp_path / "meshroom/plain/deeper/Deep.py", _nodeDescSource("Deep")) + + loader = PluginLoader() + plugin = loader.loadPlugin("plainSubfolderPlugin", str(tmp_path), PluginType.PATH) + + assert plugin is not None + assert list(plugin.nodeDescProviders.keys()) == ["Shallow"] + + loader.unloadPlugin("plainSubfolderPlugin") + + def test_nodeAndSubmitterTogether(self, tmp_path): + """ Node descriptors and submitters are discovered together on the same plugin. """ + writeFile(tmp_path / "meshroom/MyNode.py", _nodeDescSource("MyNode")) + writeFile(tmp_path / "meshroom/MySubmitter.py", _submitterSource("MySubmitter")) + + loader = PluginLoader() + plugin = loader.loadPlugin("mixedPlugin", str(tmp_path), PluginType.PATH) + + assert plugin is not None + assert list(plugin.nodeDescProviders.keys()) == ["MyNode"] + assert list(plugin.submitterProviders.keys()) == ["MySubmitter"] + assert plugin.submitterProviders["MySubmitter"].status == SubmitterProviderStatus.VALID + + loader.unloadPlugin("mixedPlugin") + + def test_brokenModuleDoesNotBlockOthers(self, tmp_path, caplog): + """ A module that fails to import is logged as an issue, but its siblings still load. """ + writeFile(tmp_path / "meshroom/Broken.py", _brokenSource()) + writeFile(tmp_path / "meshroom/Good.py", _nodeDescSource("GoodNode")) + + loader = PluginLoader() + with caplog.at_level(logging.WARNING): + plugin = loader.loadPlugin("brokenPlugin", str(tmp_path), PluginType.PATH) + + assert plugin is not None + assert list(plugin.nodeDescProviders.keys()) == ["GoodNode"] + assert "Broken" in caplog.text + + loader.unloadPlugin("brokenPlugin") + + def test_emptyPluginReturnsNone(self, tmp_path): + """ A plugin with no node, submitter, or template is not returned. """ + (tmp_path / "meshroom").mkdir() + + plugin = PluginLoader().loadPlugin("emptyPlugin", str(tmp_path), PluginType.PATH) + + assert plugin is None + + def test_missingMeshroomFolderReturnsNone(self, tmp_path): + """ A plugin folder without a "meshroom" subfolder is not loaded. """ + plugin = PluginLoader().loadPlugin("noMeshroomFolderPlugin", str(tmp_path), PluginType.BUILTIN) + + assert plugin is None + + def test_hasMeshroomFolderFalse(self, tmp_path): + """ With "hasMeshroomFolder=False", the plugin folder itself is the modules' root. """ + writeFile(tmp_path / "MyNode.py", _nodeDescSource("MyNode")) + + loader = PluginLoader() + plugin = loader.loadPlugin( + "flatFolderPlugin", str(tmp_path), PluginType.PATH, hasMeshroomFolder=False + ) + + assert plugin is not None + assert list(plugin.nodeDescProviders.keys()) == ["MyNode"] + + loader.unloadPlugin("flatFolderPlugin") + + def test_duplicatePluginNameRejected(self, tmp_path): + """ Loading a plugin under an already-loaded name is rejected. """ + pluginADir = tmp_path / "pluginA" + pluginBDir = tmp_path / "pluginB" + writeFile(pluginADir / "meshroom/MyNode.py", _nodeDescSource("MyNode")) + writeFile(pluginBDir / "meshroom/MyNode.py", _nodeDescSource("MyNode")) + + loader = PluginLoader() + first = loader.loadPlugin("duplicatePlugin", str(pluginADir), PluginType.PATH) + second = loader.loadPlugin("duplicatePlugin", str(pluginADir), PluginType.PATH) + third = loader.loadPlugin("duplicatePlugin", str(pluginBDir), PluginType.PATH) + + assert first is not None + assert second is None + assert third is None + + loader.unloadPlugin("duplicatePlugin") + + def test_unloadPluginAllowsReload(self, tmp_path): + """ "unloadPlugin" releases a plugin's name so it can be loaded again. """ + writeFile(tmp_path / "meshroom/MyNode.py", _nodeDescSource("MyNode")) + + loader = PluginLoader() + first = loader.loadPlugin("reloadablePlugin", str(tmp_path), PluginType.PATH) + assert first is not None + + loader.unloadPlugin("reloadablePlugin") + + second = loader.loadPlugin("reloadablePlugin", str(tmp_path), PluginType.PATH) + assert second is not None + assert second is not first + + loader.unloadPlugin("reloadablePlugin") + + def test_isUserPlugin(self, tmp_path): + """ "isUserPlugin" is propagated to the loaded Plugin object. """ + writeFile(tmp_path / "meshroom/MyNode.py", _nodeDescSource("MyNode")) + + loader = PluginLoader() + plugin = loader.loadPlugin("userPlugin", str(tmp_path), PluginType.PATH, isUserPlugin=True) + + assert plugin is not None + assert plugin.isUserPlugin is True + + loader.unloadPlugin("userPlugin") + + def test_templatesLoaded(self, tmp_path): + """ Pipeline templates (".mg" files) at the plugin's root are registered. """ + writeFile(tmp_path / "meshroom/MyNode.py", _nodeDescSource("MyNode")) + writeFile(tmp_path / "meshroom/myTemplate.mg", "{}") + + loader = PluginLoader() + plugin = loader.loadPlugin("templatePlugin", str(tmp_path), PluginType.PATH) + + assert plugin is not None + assert "myTemplate" in plugin.templates + + loader.unloadPlugin("templatePlugin") + + def test_uniqueNamespacePerPlugin(self, tmp_path): + """ Two plugins shipping identically-named files never collide in sys.modules. """ + pluginADir = tmp_path / "pluginA" + pluginBDir = tmp_path / "pluginB" + writeFile(pluginADir / "meshroom/SharedName.py", _nodeDescSource("NodeFromA")) + writeFile(pluginBDir / "meshroom/SharedName.py", _nodeDescSource("NodeFromB")) + + loader = PluginLoader() + pluginA = loader.loadPlugin("namespacePluginA", str(pluginADir), PluginType.PATH) + pluginB = loader.loadPlugin("namespacePluginB", str(pluginBDir), PluginType.PATH) + + assert pluginA is not None and pluginB is not None + assert list(pluginA.nodeDescProviders.keys()) == ["NodeFromA"] + assert list(pluginB.nodeDescProviders.keys()) == ["NodeFromB"] + + loader.unloadPlugin("namespacePluginA") + loader.unloadPlugin("namespacePluginB") + + def test_virtualPackageModuleNames(self, tmp_path): + """ + Every module a plugin provides is registered in sys.modules under the virtual + "PLUGINS_ROOT_PACKAGE." package, and its classes' "__module__" reflects + that same dotted name, not the real file path it was loaded from. + """ + writeFile(tmp_path / "meshroom/MyNodeA.py", _nodeDescSource("MyNodeA")) + writeFile(tmp_path / "meshroom/pkg/__init__.py") + writeFile(tmp_path / "meshroom/pkg/MyNodeB.py", _nodeDescSource("MyNodeB")) + writeFile(tmp_path / "meshroom/folder/MyNodeC.py", _nodeDescSource("MyNodeC")) + + loader = PluginLoader() + plugin = loader.loadPlugin("virtualPlugin", str(tmp_path), PluginType.PATH) + + assert plugin is not None + + rootModuleName = f"{PLUGINS_ROOT_PACKAGE}.virtualPlugin" + flatModuleName = f"{rootModuleName}.MyNodeA" + packageModuleName = f"{rootModuleName}.pkg" + packageChildModuleName = f"{rootModuleName}.pkg.MyNodeB" + folderChildModuleName = f"{rootModuleName}.folder.MyNodeC" + + # Every module is registered under the expected virtual dotted name. + assert PLUGINS_ROOT_PACKAGE in sys.modules + assert rootModuleName in sys.modules + assert flatModuleName in sys.modules + assert packageModuleName in sys.modules + assert packageChildModuleName in sys.modules + assert folderChildModuleName in sys.modules + + # Discovered classes report the virtual name, not their real file path. + assert plugin.nodeDescProviders["MyNodeA"].nodeDescClass.__module__ == flatModuleName + assert plugin.nodeDescProviders["MyNodeB"].nodeDescClass.__module__ == packageChildModuleName + assert plugin.nodeDescProviders["MyNodeC"].nodeDescClass.__module__ == folderChildModuleName + + loader.unloadPlugin("virtualPlugin") + + # Unloading releases every module registered under the plugin's virtual package. + # Leaves the shared root package itself (other plugins may still use it). + assert rootModuleName not in sys.modules + assert flatModuleName not in sys.modules + assert packageModuleName not in sys.modules + assert packageChildModuleName not in sys.modules + assert folderChildModuleName not in sys.modules + assert PLUGINS_ROOT_PACKAGE in sys.modules + + def test_configNameAndVersionOverride(self, tmp_path): + """ "name"/"version" in the configuration file override the given plugin name/version. """ + writeFile(tmp_path / "meshroom/MyNode.py", _nodeDescSource("MyNode")) + writeFile(tmp_path / "meshroom/config.json", json.dumps({ + "name": "overriddenName", + "version": "1.2.3", + "env": [{"key": "MY_VAR", "type": "string", "value": "myValue"}], + })) + + loader = PluginLoader() + plugin = loader.loadPlugin("originalName", str(tmp_path), PluginType.PATH, pluginVersion="0.0.1") + + assert plugin is not None + assert plugin.name == "overriddenName" + assert plugin.version == "1.2.3" + assert plugin.configEnv["MY_VAR"] == "myValue" + assert f"{PLUGINS_ROOT_PACKAGE}.overriddenName" in sys.modules + + loader.unloadPlugin("overriddenName") + + def test_configOverrideIgnoredForRezPlugin(self, tmp_path): + """ For a Rez plugin, "name"/"version" from the configuration file are ignored: the + Rez-resolved values take precedence. """ + writeFile(tmp_path / "meshroom/MyNode.py", _nodeDescSource("MyNode")) + writeFile(tmp_path / "meshroom/config.json", json.dumps({ + "name": "shouldBeIgnored", + "version": "9.9.9", + })) + + loader = PluginLoader() + plugin = loader.loadPlugin("rezPlugin", str(tmp_path), PluginType.REZ, pluginVersion="1.0.0") + + assert plugin is not None + assert plugin.name == "rezPlugin" + assert plugin.version == "1.0.0" + + loader.unloadPlugin("rezPlugin") + + def test_listConfigFormatWorks(self, tmp_path): + """ A "config.json" using the flat-list format that only sets environment variables. """ + writeFile(tmp_path / "meshroom/MyNode.py", _nodeDescSource("MyNode")) + writeFile(tmp_path / "meshroom/config.json", json.dumps( + [{"key": "MY_VAR", "type": "string", "value": "myValue"}] + )) + + loader = PluginLoader() + plugin = loader.loadPlugin("listConfigPlugin", str(tmp_path), PluginType.PATH, pluginVersion="1.0.0") + + assert plugin is not None + assert plugin.name == "listConfigPlugin" + assert plugin.version == "1.0.0" + assert plugin.configEnv["MY_VAR"] == "myValue" + + loader.unloadPlugin("listConfigPlugin") + + def test_invalidConfigNameAndVersionFallBack(self, tmp_path): + """ An invalid "name"/"version" in the configuration file is ignored, falling back to the + given values, with a warning logged. """ + writeFile(tmp_path / "meshroom/MyNode.py", _nodeDescSource("MyNode")) + writeFile(tmp_path / "meshroom/config.json", json.dumps({ + "name": "invalid-name", + "version": "not_a_version", + })) + + loader = PluginLoader() + plugin = loader.loadPlugin("fallbackPlugin", str(tmp_path), PluginType.PATH, pluginVersion="1.0.0") + + assert plugin is not None + assert plugin.name == "fallbackPlugin" + assert plugin.version == "1.0.0" + + loader.unloadPlugin("fallbackPlugin") diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 868d53ca0a..6ea340563e 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1,9 +1,9 @@ # coding:utf-8 -from meshroom.core import pluginManager, loadClassesNodes +from meshroom.core import pluginManager from meshroom.core.desc.node import NodeVersionType -from meshroom.core.plugins import NodePluginStatus, Plugin -from .utils import overrideOsEnvironmentVariables, registeredPlugins, registeredUserPlugins +from meshroom.core.plugins.base import NodeDescProviderStatus +from .utils import overrideOsEnvironmentVariables, registeredPlugin from pathlib import Path import os @@ -11,47 +11,36 @@ class TestPluginWithValidNodesOnly: - plugin = None @classmethod def setup_class(cls): - folder = os.path.join(os.path.dirname(__file__), "plugins", "meshroom") - package = "pluginA" - cls.plugin = Plugin(package, folder) - nodes = loadClassesNodes(folder, package, pluginUid=cls.plugin.uid) - for node in nodes: - cls.plugin.addNodePlugin(node) - pluginManager.addPlugin(cls.plugin) + cls.folder = os.path.join(os.path.dirname(__file__), "plugins", "pluginA") + pluginManager.addPluginFromPath("pluginA", cls.folder) @classmethod def teardown_class(cls): - for node in cls.plugin.nodes.values(): - pluginManager.unregisterNode(node) - pluginManager.removePlugin(cls.plugin) - cls.plugin = None + plugin = pluginManager.getPlugin("pluginA") + if plugin: + pluginManager.removePlugin(plugin) def test_getPlugin(self): # Assert that there are loaded plugins, and that "pluginA" is one of them assert len(pluginManager.getPlugins()) >= 1 # Get with name - plugin = pluginManager.getPlugin("pluginA", uname=False) - assert plugin == self.plugin - # Get with unique name - pluginUName = self.plugin.uname - assert pluginUName == f"{self.plugin.uid}_pluginA" - plugin = pluginManager.getPlugin(pluginUName, uname=True) - assert plugin == self.plugin + plugin = pluginManager.getPlugin("pluginA") + assert plugin + assert plugin.name == "pluginA" # Check path too - assert str(plugin.path) == os.path.join(os.path.dirname(__file__), "plugins", "meshroom") + assert str(plugin.path) == os.path.join(os.path.dirname(__file__), "plugins", "pluginA", "meshroom") def test_loadedPlugin(self): # Assert that there are loaded plugins, and that "pluginA" is one of them - plugin = pluginManager.getPlugin("pluginA", uname=False) + plugin = pluginManager.getPlugin("pluginA") # Assert that the nodes of pluginA have been successfully registered - assert len(pluginManager.getRegisteredNodePlugins()) >= 2 - for nodeName, nodePlugin in plugin.nodes.items(): - assert nodePlugin.status == NodePluginStatus.LOADED - assert pluginManager.isRegistered(nodeName) + assert len(pluginManager.getNodeDescProviders()) >= 2 + for nodeName, nodeDescProvider in plugin.nodeDescProviders.items(): + assert nodeDescProvider.status == NodeDescProviderStatus.VALID + assert pluginManager.isNodeDescRegistered(nodeName) # Assert the template has been loaded assert len(plugin.templates) == 1 @@ -59,209 +48,147 @@ def test_loadedPlugin(self): assert name == "sharedTemplate" assert plugin.templates[name] == os.path.join(str(plugin.path), "sharedTemplate.mg") - def test_unloadPlugin(self): - plugin = pluginManager.getPlugin("pluginA", uname=False) - assert plugin == self.plugin + def test_removePlugin(self): + plugin = pluginManager.getPlugin("pluginA") + assert plugin - # Unload the plugin without unregistering the nodes - pluginManager.removePlugin(plugin, unregisterNodePlugins=False) - - # Assert the plugin is not loaded anymore - assert pluginManager.getPlugin(plugin.name, uname=False) is None - - # Assert the nodes are still registered and belong to an unloaded plugin - for nodeName, nodePlugin in plugin.nodes.items(): - assert nodePlugin.status == NodePluginStatus.LOADED - assert pluginManager.isRegistered(nodeName) - assert pluginManager.belongsToPlugin(nodeName) is None - - # Re-add the plugin - pluginManager.addPlugin(plugin, registerNodePlugins=False) - assert pluginManager.getPlugin(plugin.name, uname=False) - - # Unload the plugin with a full unregistration of the nodes + # Remove the plugin pluginManager.removePlugin(plugin) # Assert the plugin is not loaded anymore - assert pluginManager.getPlugin(plugin.name, uname=False) is None + assert pluginManager.getPlugin(plugin.name) is None # Assert the nodes have been successfully unregistered - for nodeName, nodePlugin in plugin.nodes.items(): - assert nodePlugin.status == NodePluginStatus.NOT_LOADED - assert not pluginManager.isRegistered(nodeName) - - # Re-add the plugin and re-register the nodes - pluginManager.addPlugin(plugin) - assert pluginManager.getPlugin(plugin.name, uname=False) - for nodeName, nodePlugin in plugin.nodes.items(): - assert nodePlugin.status == NodePluginStatus.LOADED - assert pluginManager.isRegistered(nodeName) - - def test_updateRegisteredNodes(self): - nbRegisteredNodes = len(pluginManager.getRegisteredNodePlugins()) - plugin = pluginManager.getPlugin("pluginA", uname=False) - assert plugin == self.plugin - nodeA = pluginManager.getRegisteredNodePlugin("PluginANodeA") - nodeAName = nodeA.nodeDescriptor.__name__ - - # Unregister a node - assert nodeA - pluginManager.unregisterNode(nodeA) - - # Check that the node has been fully unregistered: - # - its status is "NOT_LOADED" - # - it is still part of pluginA - # - it is not in the list of registered plugins anymore (and returns None when requested) - assert nodeA.status == NodePluginStatus.NOT_LOADED - assert plugin.containsNodePlugin(nodeAName) - assert nodeA.plugin == plugin - - assert pluginManager.getRegisteredNodePlugin(nodeAName) is None - assert nodeAName not in pluginManager.getRegisteredNodePlugins() - assert len(pluginManager.getRegisteredNodePlugins()) == nbRegisteredNodes - 1 - - # Re-register the node - pluginManager.registerNode(nodeA) - - assert nodeA.status == NodePluginStatus.LOADED - assert pluginManager.getRegisteredNodePlugin(nodeAName) - assert len(pluginManager.getRegisteredNodePlugins()) == nbRegisteredNodes + for nodeName, nodeDescProvider in plugin.nodeDescProviders.items(): + assert not pluginManager.isNodeDescRegistered(nodeName) + + # Re-load the plugin and re-register the nodes + pluginManager.addPluginFromPath("pluginA", self.folder) + + # Assert the nodes have been successfully registered + assert pluginManager.getPlugin(plugin.name) + for nodeName, nodeDescProvider in plugin.nodeDescProviders.items(): + assert pluginManager.isNodeDescRegistered(nodeName) class TestPluginWithInvalidNodes: - plugin = None @classmethod def setup_class(cls): - folder = os.path.join(os.path.dirname(__file__), "plugins", "meshroom") - package = "pluginB" - cls.plugin = Plugin(package, folder) - nodes = loadClassesNodes(folder, package, pluginUid=cls.plugin.uid) - for node in nodes: - cls.plugin.addNodePlugin(node) - pluginManager.addPlugin(cls.plugin) + cls.folder = os.path.join(os.path.dirname(__file__), "plugins", "pluginB") + pluginManager.addPluginFromPath("pluginB", cls.folder) @classmethod def teardown_class(cls): - for node in cls.plugin.nodes.values(): - pluginManager.unregisterNode(node) - pluginManager.removePlugin(cls.plugin) - cls.plugin = None + plugin = pluginManager.getPlugin("pluginB") + if plugin: + pluginManager.removePlugin(plugin) def test_loadedPlugin(self): # Assert that there are loaded plugins, and that "pluginB" is one of them assert len(pluginManager.getPlugins()) >= 1 - plugin = pluginManager.getPlugin("pluginB", uname=False) - assert plugin == self.plugin - assert str(plugin.path) == os.path.join(os.path.dirname(__file__), "plugins", "meshroom") + plugin = pluginManager.getPlugin("pluginB") + assert plugin + assert str(plugin.path) == os.path.join(os.path.dirname(__file__), "plugins", "pluginB", "meshroom") # Assert that PluginBNodeA is successfully registered - assert pluginManager.isRegistered("PluginBNodeA") - assert plugin.nodes["PluginBNodeA"].status == NodePluginStatus.LOADED - assert plugin.nodes["PluginBNodeA"].plugin == plugin + assert pluginManager.isNodeDescRegistered("PluginBNodeA") + assert plugin.nodeDescProviders["PluginBNodeA"].status == NodeDescProviderStatus.VALID + assert plugin.nodeDescProviders["PluginBNodeA"].plugin == plugin # Assert that PluginBNodeB has not been registered (description error) - assert not pluginManager.isRegistered("PluginBNodeB") - assert plugin.nodes["PluginBNodeB"].status == NodePluginStatus.DESC_ERROR - assert plugin.nodes["PluginBNodeB"].plugin == plugin + assert not pluginManager.isNodeDescRegistered("PluginBNodeB") + assert plugin.nodeDescProviders["PluginBNodeB"].status == NodeDescProviderStatus.DESC_ERROR + assert plugin.nodeDescProviders["PluginBNodeB"].plugin == plugin - # Assert the template has been loaded - assert len(plugin.templates) == 1 - name = list(plugin.templates.keys())[0] - assert name == "sharedTemplate" - assert plugin.templates[name] == os.path.join(str(plugin.path), "sharedTemplate.mg") + # Assert no template has been loaded + assert len(plugin.templates) == 0 - def test_reloadNodePluginInvalidDescrpition(self): - plugin = pluginManager.getPlugin("pluginB", uname=False) - assert plugin == self.plugin - node = plugin.nodes["PluginBNodeB"] - nodeName = node.nodeDescriptor.__name__ + def test_reloadNodeDescProviderInvalidDescrpition(self): + plugin = pluginManager.getPlugin("pluginB") + assert plugin + nodeDescProvider = plugin.nodeDescProviders["PluginBNodeB"] # Check that the node has not been registered - assert node.status == NodePluginStatus.DESC_ERROR - assert not pluginManager.isRegistered(nodeName) - - # Check that the node cannot be registered - pluginManager.registerNode(node) - assert not pluginManager.isRegistered(nodeName) + assert nodeDescProvider.status == NodeDescProviderStatus.DESC_ERROR + assert not pluginManager.isNodeDescRegistered(nodeDescProvider.name) # Replace directly in the node file the line that fails the validation # on the description with a line that will pass originalFileContent = None - with open(node.path, "r") as f: + with open(nodeDescProvider.path, "r") as f: originalFileContent = f.read() replaceFileContent = originalFileContent.replace('"not an integer"', '1') - with open(node.path, "w") as f: + with open(nodeDescProvider.path, "w") as f: f.write(replaceFileContent) - # Reload the node and assert it is valid - node.reload() - assert node.status == NodePluginStatus.NOT_LOADED + # Reload the node desc provider and assert it is valid + nodeDescProvider.reload() + assert nodeDescProvider.status == NodeDescProviderStatus.VALID - # Attempt to register node plugin - pluginManager.registerNode(node) - assert pluginManager.isRegistered(nodeName) + # Attempt to register the node desc provider + pluginManager.registerPluginProviders(plugin) + assert pluginManager.isNodeDescRegistered(nodeDescProvider.name) # Reload the node again without any change - node.reload() - assert pluginManager.isRegistered(nodeName) + nodeDescProvider.reload() + assert pluginManager.isNodeDescRegistered(nodeDescProvider.name) # Hack to ensure that the timestamp of the file will be different after being rewritten # Without it, on some systems, the operation is too fast and the timestamp does not change, # cause the test to fail time.sleep(0.1) - # Restore the node file to its original state (with a description error) - with open(node.path, "w") as f: + # Restore the node desc file to its original state (with a description error) + with open(nodeDescProvider.path, "w") as f: f.write(originalFileContent) - timestampOr2 = os.path.getmtime(node.path) - print(f"New timestamp: {timestampOr2}") - print(os.stat(node.path)) - # Reload the node and assert it is invalid while still registered - node.reload() - assert node.status == NodePluginStatus.DESC_ERROR - assert pluginManager.isRegistered(nodeName) + nodeDescProvider.reload() + assert nodeDescProvider.status == NodeDescProviderStatus.DESC_ERROR + assert pluginManager.isNodeDescRegistered(nodeDescProvider.name) - # Unregister it - pluginManager.unregisterNode(node) - assert node.status == NodePluginStatus.DESC_ERROR # Not NOT_LOADED - assert not pluginManager.isRegistered(nodeName) + # Remove the plugin + pluginManager.removePlugin(plugin) + + # Re-add the plugin + pluginManager.addPluginFromPath("pluginB", self.folder) + nodeDescProvider = plugin.nodeDescProviders["PluginBNodeB"] + assert nodeDescProvider.status == NodeDescProviderStatus.DESC_ERROR + assert not pluginManager.isNodeDescRegistered(nodeDescProvider.name) - def test_reloadNodePluginSyntaxError(self): - plugin = pluginManager.getPlugin("pluginB", uname=False) - assert plugin == self.plugin - node = plugin.nodes["PluginBNodeA"] - nodeName = node.nodeDescriptor.__name__ + def test_reloadNodeDescProviderSyntaxError(self): + plugin = pluginManager.getPlugin("pluginB") + assert plugin + nodeDescProvider = plugin.nodeDescProviders["PluginBNodeA"] - # Check that the node has been registered - assert node.status == NodePluginStatus.LOADED - assert pluginManager.isRegistered(nodeName) + # Check that the node desc has been registered + assert nodeDescProvider.status == NodeDescProviderStatus.VALID + assert pluginManager.isNodeDescRegistered(nodeDescProvider.name) # Introduce a syntax error in the description originalFileContent = None - with open(node.path, "r") as f: + with open(nodeDescProvider.path, "r") as f: originalFileContent = f.read() replaceFileContent = originalFileContent.replace('name="input",', 'name="input"') - with open(node.path, "w") as f: + with open(nodeDescProvider.path, "w") as f: f.write(replaceFileContent) - # Reload the node and assert it is invalid but still registered - node.reload() - assert node.status == NodePluginStatus.DESC_ERROR - assert pluginManager.isRegistered(nodeName) + # Reload the node desc provider and assert it is invalid but still registered + nodeDescProvider.reload() + assert nodeDescProvider.status == NodeDescProviderStatus.DESC_ERROR + assert pluginManager.isNodeDescRegistered(nodeDescProvider.name) - # Restore the node file to its original state (with a description error) - with open(node.path, "w") as f: + # Restore the node desc file to its original state (with a description error) + with open(nodeDescProvider.path, "w") as f: f.write(originalFileContent) # Assert the status is correct and the node is still registered - node.reload() - assert node.status == NodePluginStatus.NOT_LOADED - assert pluginManager.isRegistered(nodeName) + nodeDescProvider.reload() + assert nodeDescProvider.status == NodeDescProviderStatus.VALID + assert pluginManager.isNodeDescRegistered(nodeDescProvider.name) class TestPluginsConfiguration: @@ -274,9 +201,9 @@ class TestPluginsConfiguration: def test_loadedConfig(self): # Check that the config.json file for the plugins in the "plugins" directory is # correctly loaded - folder = os.path.join(os.path.dirname(__file__), "plugins") - with registeredPlugins(folder): - plugin = pluginManager.getPlugin("pluginA", uname=False) + folder = os.path.join(os.path.dirname(__file__), "plugins", "pluginA") + with registeredPlugin("pluginA", folder): + plugin = pluginManager.getPlugin("pluginA") assert plugin # Check that the config file has been properly loaded @@ -311,10 +238,9 @@ def test_loadedConfigWithOnlyExistingKeys(self): self.ERRONEOUS_CONFIG_PATH[0]: self.ERRONEOUS_CONFIG_PATH[2], self.CONFIG_STRING[0]: self.CONFIG_STRING[2] } - - folder = os.path.join(os.path.dirname(__file__), "plugins") - with (overrideOsEnvironmentVariables(environment), registeredPlugins(folder)): - plugin = pluginManager.getPlugin("pluginA", uname=False) + folder = os.path.join(os.path.dirname(__file__), "plugins", "pluginA") + with (overrideOsEnvironmentVariables(environment), registeredPlugin("pluginA", folder)): + plugin = pluginManager.getPlugin("pluginA") assert plugin # Check that the config file has been properly loaded and read @@ -349,9 +275,9 @@ def test_loadedConfigWithSomeExistingKeys(self): self.CONFIG_STRING[0]: self.CONFIG_STRING[2] } - folder = os.path.join(os.path.dirname(__file__), "plugins") - with (overrideOsEnvironmentVariables(environment), registeredPlugins(folder)): - plugin = pluginManager.getPlugin("pluginA", uname=False) + folder = os.path.join(os.path.dirname(__file__), "plugins", "pluginA") + with (overrideOsEnvironmentVariables(environment), registeredPlugin("pluginA", folder)): + plugin = pluginManager.getPlugin("pluginA") assert plugin # Check that the config file has been properly loaded and read @@ -384,33 +310,33 @@ def test_loadedConfigWithSomeExistingKeys(self): class TestVersionPlugins: def test_nodeVersionType(self): - folder = os.path.join(os.path.dirname(__file__), "plugins") - with registeredPlugins(folder): - pluginA = pluginManager.getPlugin("pluginA", uname=False) + folder = os.path.join(os.path.dirname(__file__), "plugins", "pluginA") + with registeredPlugin("pluginA", folder): + pluginA = pluginManager.getPlugin("pluginA") assert pluginA - nodeA = pluginManager.getRegisteredNodePlugin("PluginANodeA") + nodeA = pluginManager.getNodeDescProvider("PluginANodeA") assert nodeA - assert nodeA.nodeDescriptor().nodeVersionType == NodeVersionType.RELEASED + assert nodeA.nodeDescClass().nodeVersionType == NodeVersionType.RELEASED - nodeB = pluginManager.getRegisteredNodePlugin("PluginANodeB") + nodeB = pluginManager.getNodeDescProvider("PluginANodeB") assert nodeB - assert nodeB.nodeDescriptor().nodeVersionType == NodeVersionType.BETA + assert nodeB.nodeDescClass().nodeVersionType == NodeVersionType.BETA - nodeInput = pluginManager.getRegisteredNodePlugin("PluginAInitNode") + nodeInput = pluginManager.getNodeDescProvider("PluginAInitNode") assert nodeInput - assert nodeInput.nodeDescriptor().nodeVersionType == NodeVersionType.UNKNOWN + assert nodeInput.nodeDescClass().nodeVersionType == NodeVersionType.UNKNOWN - with registeredUserPlugins(folder): - pluginA = pluginManager.getPlugin("pluginA", uname=False) + with registeredPlugin("pluginA", folder, isUserPlugin=True): + pluginA = pluginManager.getPlugin("pluginA") assert pluginA - nodeA = pluginManager.getRegisteredNodePlugin("PluginANodeA") + nodeA = pluginManager.getNodeDescProvider("PluginANodeA") assert nodeA - assert nodeA.nodeDescriptor().nodeVersionType == NodeVersionType.USER + assert nodeA.nodeDescClass().nodeVersionType == NodeVersionType.USER - nodeB = pluginManager.getRegisteredNodePlugin("PluginANodeB") + nodeB = pluginManager.getNodeDescProvider("PluginANodeB") assert nodeB - assert nodeB.nodeDescriptor().nodeVersionType == NodeVersionType.USER + assert nodeB.nodeDescClass().nodeVersionType == NodeVersionType.USER - nodeInput = pluginManager.getRegisteredNodePlugin("PluginAInitNode") + nodeInput = pluginManager.getNodeDescProvider("PluginAInitNode") assert nodeInput - assert nodeInput.nodeDescriptor().nodeVersionType == NodeVersionType.USER + assert nodeInput.nodeDescClass().nodeVersionType == NodeVersionType.USER diff --git a/tests/test_submit.py b/tests/test_submit.py index fb89b662e8..6329c54d48 100644 --- a/tests/test_submit.py +++ b/tests/test_submit.py @@ -11,9 +11,8 @@ from .utils import registerNodeDesc import meshroom -from meshroom.core import pluginManager, loadClassesNodes, loadSubmitters, registerSubmitter, meshroomFolder +from meshroom.core import pluginManager, meshroomFolder from meshroom.core.graph import Graph -from meshroom.core.plugins import Plugin from meshroom.core.node import Node, Status from meshroom.core.submitter import jobManager from meshroom.core.submitter import OrderedTask, OrderedTasks, OrderedTaskType @@ -37,8 +36,8 @@ def get_submitter() -> LocalFarmSubmitter: def getJobEnv(): - """ Required to have meshroom recognize plugins that were created here """ - pluginFolder = os.path.join(os.path.dirname(__file__), "plugins") + """ Required to have meshroom recognize the test plugin """ + pluginFolder = os.path.join(os.path.dirname(__file__), "plugins", "pluginSubmitter") return { "MESHROOM_PLUGINS_PATH": pluginFolder } @@ -128,30 +127,24 @@ class TestNodeSubmit: @classmethod def setup_class(cls): - submittersFolder = os.path.join(meshroomFolder, "submitters") - submitters = loadSubmitters(submittersFolder, "localFarm") - for submitter in submitters: - registerSubmitter(submitter()) - - cls.folder = os.path.join(os.path.dirname(__file__), "plugins", "meshroom") - package = "pluginSubmitter" - cls.plugin = Plugin(package, cls.folder) - nodes = loadClassesNodes(cls.folder, package, pluginUid=cls.plugin.uid) - for node in nodes: - cls.plugin.addNodePlugin(node) - pluginManager.addPlugin(cls.plugin) + localFarmFolder = os.path.join(meshroomFolder, "submitters", "localFarm") + pluginManager.addPluginFromBuiltInFolder("localFarm", localFarmFolder) + submitterProvider = pluginManager.getSubmitterProvider("LocalFarm") + meshroom.core.submitters[submitterProvider.name] = submitterProvider.instance + + cls.folder = os.path.join(os.path.dirname(__file__), "plugins", "pluginSubmitter") + pluginManager.addPluginFromBuiltInFolder("pluginSubmitter", cls.folder) @classmethod def teardown_class(cls): - for node in cls.plugin.nodes.values(): - pluginManager.unregisterNode(node) - pluginManager.removePlugin(cls.plugin) - cls.plugin = None + plugin = pluginManager.getPlugin("pluginSubmitter") + if plugin: + pluginManager.removePlugin(plugin) def registerNode(self, name): - plugin = pluginManager.getPlugin("pluginSubmitter", uname=False) - node = plugin.nodes[name] - nodeType = node.nodeDescriptor + plugin = pluginManager.getPlugin("pluginSubmitter") + node = plugin.nodeDescProviders[name] + nodeType = node.nodeDescClass registerNodeDesc(nodeType) return nodeType.__name__ diff --git a/tests/utils.py b/tests/utils.py index d91a725893..c1f2961d54 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,25 +1,18 @@ from contextlib import contextmanager from unittest.mock import patch +from pathlib import Path import meshroom -from meshroom.core import desc, pluginManager, loadPluginFolder -from meshroom.core.plugins import NodePlugin +from meshroom.core import desc, pluginManager +from meshroom.core.plugins.base import NodeDescProvider import os -@contextmanager -def registeredNodeTypes(nodeTypes: list[desc.Node]): - nodePluginsList = {} - for nodeType in nodeTypes: - nodePlugin = NodePlugin(nodeType) - pluginManager.registerNode(nodePlugin) - nodePluginsList[nodeType] = nodePlugin - - yield - - for nodeType in nodeTypes: - pluginManager.unregisterNode(nodePluginsList[nodeType]) +def writeFile(filePath: Path, content: str = "") -> Path: + filePath.parent.mkdir(parents=True, exist_ok=True) + filePath.write_text(content, encoding="utf-8") + return filePath @contextmanager @@ -34,35 +27,43 @@ def overrideNodeTypeVersion(nodeType: desc.Node, version: str): yield -def registerNodeDesc(nodeDesc: desc.Node): - name = nodeDesc.__name__ - if not pluginManager.isRegistered(name): - pluginManager._nodePlugins[name] = NodePlugin(nodeDesc) +@contextmanager +def registeredNodeTypes(nodeDescs: list[desc.Node]): + for nodeDesc in nodeDescs: + nodeType = nodeDesc.__name__ + if not pluginManager.isNodeDescRegistered(nodeType): + nodeDescProvider = NodeDescProvider(nodeDesc) + pluginManager._nodeDescProviders[nodeType] = nodeDescProvider + yield -def unregisterNodeDesc(nodeDesc: desc.Node): - name = nodeDesc.__name__ - if pluginManager.isRegistered(name): - del pluginManager._nodePlugins[name] + for nodeDesc in nodeDescs: + nodeType = nodeDesc.__name__ + if pluginManager.isNodeDescRegistered(nodeType): + del pluginManager._nodeDescProviders[nodeType] @contextmanager -def registeredPlugins(folder: str): - plugins = loadPluginFolder(folder) +def registeredPlugin(pluginName: str, pluginFolder: str, isUserPlugin: bool = False): + pluginManager.addPluginFromPath(pluginName, pluginFolder, isUserPlugin=isUserPlugin) yield - for plugin in plugins: + plugin = pluginManager.getPlugin(pluginName) + if plugin: pluginManager.removePlugin(plugin) -@contextmanager -def registeredUserPlugins(folder: str): - plugins = loadPluginFolder(folder, userPlugin=True) - yield +def registerNodeDesc(nodeDesc: desc.Node): + nodeType = nodeDesc.__name__ + if not pluginManager.isNodeDescRegistered(nodeType): + pluginManager._nodeDescProviders[nodeType] = NodeDescProvider(nodeDesc) - for plugin in plugins: - pluginManager.removePlugin(plugin) + +def unregisterNodeDesc(nodeDesc: desc.Node): + nodeType = nodeDesc.__name__ + if pluginManager.isNodeDescRegistered(nodeType): + del pluginManager._nodeDescProviders[nodeType] @contextmanager