diff --git a/CHANGES b/CHANGES index bb1dea6b..57a32efc 100644 --- a/CHANGES +++ b/CHANGES @@ -3,6 +3,13 @@ pyblish Changelog This contains all major version changes between pyblish releases. +Version 1.4.2 +------------- + +- Implemented #283, common processing loop +- Implemented #285, memory optimisation +- BACKWARD INCOMPATIBLE: `context != instance.context`, comparisons must now be made via `context.id` instead. + Version 1.4.1 ------------- diff --git a/README.md b/README.md index 70b3b26f..da9c13f4 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Test-driven content creation for collaborative, creative projects. Pyblish is a modular framework, consisting of many sub-projects. This project contains the primary API upon which all other projects build. -You may use this project as-is, or in conjunction with surrounding projects - such as [pyblish-maya][] for integration with Autodesk Maya, [pyblish-qml][] for a visual front-end and [pyblish-magenta][] for a starting point your publishing pipeline. +You may use this project as-is, or in conjunction with surrounding projects - such as [pyblish-maya][] for integration with Autodesk Maya, [pyblish-qml][] for a visual front-end and [pyblish-magenta][] for a starting point to your publishing pipeline. [pyblish-maya]: https://github.com/pyblish/pyblish-maya [pyblish-qml]: https://github.com/pyblish/pyblish-qml @@ -40,7 +40,7 @@ pyblish-base is avaialble on PyPI. $ pip install pyblish-base ``` -Like all other Pyblish projects, it may also be clone as-is via Git and added to your PYTHONPATH. +Like all other Pyblish projects, it may also be cloned as-is via Git and added to your PYTHONPATH. ```bash $ git clone https://github.com/pyblish/pyblish-base.git @@ -58,8 +58,8 @@ $ export PYTHONPATH=$(pwd)/pyblish-base Refer to the [getting started guide](http://learn.pyblish.com) for a gentle introduction to the framework and [the forums](http://forums.pyblish.com) for tips and tricks. -- [Learn Pyblish By Example](http://learn.pyblish.com) -- [Forums](http://forums.pyblish.com) +- [**learn**.pyblish.com](http://learn.pyblish.com) +- [**forums**.pyblish.com](http://forums.pyblish.com) [travis-image]: https://travis-ci.org/pyblish/pyblish-base.svg?branch=master [travis-link]: https://travis-ci.org/pyblish/pyblish-base @@ -68,8 +68,8 @@ Refer to the [getting started guide](http://learn.pyblish.com) for a gentle intr [cover-image]: https://coveralls.io/repos/pyblish/pyblish-base/badge.svg [cover-link]: https://coveralls.io/r/pyblish/pyblish-base -[pypi-image]: https://badge.fury.io/py/pyblish.svg -[pypi-link]: http://badge.fury.io/py/pyblish +[pypi-image]: https://badge.fury.io/py/pyblish-base.svg +[pypi-link]: http://badge.fury.io/py/pyblish-base [landscape-image]: https://landscape.io/github/pyblish/pyblish-base/master/landscape.png [landscape-repo]: https://landscape.io/github/pyblish/pyblish-base/master [gitter-image]: https://badges.gitter.im/Join%20Chat.svg diff --git a/pyblish/api.py b/pyblish/api.py index 3b743aa8..fd3a13e5 100644 --- a/pyblish/api.py +++ b/pyblish/api.py @@ -69,11 +69,6 @@ deregister_all_services, registered_services, - register_callback, - deregister_callback, - deregister_all_callbacks, - registered_callbacks, - sort as sort_plugins, registered_paths, @@ -85,9 +80,15 @@ log, time as __time, emit, - main_package_path as __main_package_path + main_package_path as __main_package_path, + + register_callback, + deregister_callback, + deregister_all_callbacks, + registered_callbacks, ) + from .logic import ( plugins_by_family, plugins_by_host, diff --git a/pyblish/engine.py b/pyblish/engine.py new file mode 100644 index 00000000..c8fa1bf2 --- /dev/null +++ b/pyblish/engine.py @@ -0,0 +1,505 @@ +"""An asynchronous, stateful scripting engine to Pyblish + +This engine is designed for graphical user interfaces +who need access to data with more diagnostic ability and +up-front information. + +Dependencies: + + _____________ + | | + | engine.py | + |_____________| + | + | + | uses + | + ________ v_________ + | | + | api.py | + |___________________| + | + | + | + | + ___________v____________ + | | + | implementation | + |________________________| + + +Usage: + >>> count = {"#": 0} + >>> def on_reset(): + ... count["#"] += 1 + ... + >>> engine = create_default() + >>> engine.was_reset.connect(on_reset) + >>> engine.reset() + >>> count["#"] + 1 + +""" + +import sys +import traceback + +from . import api, logic, plugin, lib + + +class TemplateSignal(object): + """Placeholder signal + + This signal is used as an indicator of where to install signals. + The instantiator of AbstractEngine later replaces these with + supplied signals, either DefaultSignal or QtCore.Signal. + + """ + + def __init__(self, *args): + self.args = args + + def connect(self, *args): + pass + + def disconnect(self, *args): + pass + + def emit(self, *args): + pass + + +class AbstractEngine(object): + """Asynchronous, stateful scripting engine""" + + # Emitted when the GUI is about to start processing; + # e.g. resetting, validating or publishing. + about_to_process = TemplateSignal(object, object) + + # Emitted for each process + was_processed = TemplateSignal(dict) # dict=result + + was_discovered = TemplateSignal() + was_reset = TemplateSignal() + was_collected = TemplateSignal() + was_validated = TemplateSignal() + was_extracted = TemplateSignal() + was_integrated = TemplateSignal() + was_published = TemplateSignal() + was_acted = TemplateSignal() + + # Emitted when processing has finished + was_finished = TemplateSignal() + + # Informational outlets for observers + warned = TemplateSignal(str) # str=message + logged = TemplateSignal(str) # str=message + + @property + def context(self): + return self._context + + @property + def plugins(self): + return self._plugins + + @property + def is_running(self): + return self._is_running + + @property + def current_error(self): + return self._current_error + + def dispatch(self, func, *args, **kwargs): + """External functionality + + This is an optional overridable for services that implement their own + client-side proxies to the supplied functionality. + + For example, in pyblish-qml, these functions are provided as a + wrapper to functionality running remotely. + + Arguments: + func (str): Name of external function + args (list, optional): Arguments passed to `func` + kwargs (dict, optional): Keyword arguments passed to `func` + + Raises: + KeyError on missing functionality and + whatever exception raised by target function. + + """ + + raise NotImplementedError + + def defer(self, delay, func): + """Append artificial delay to `func` + + This aids in keeping the GUI responsive, but complicates logic + when producing tests. To combat this, the environment variable ensures + that every operation is synchonous. + + This function is designed to be overridden in the implementation + of your graphical user engine. + + Arguments: + delay (float): Delay multiplier; default 1, 0 means no delay + func (callable): Any callable + + """ + + raise NotImplementedError + + def __init__(self): + super(AbstractEngine, self).__init__() + + self._is_running = False + + self._context = self.dispatch("Context") + self._plugins = list() + + # Transient state used during publishing. + self._pair_generator = None # Active producer of pairs + self._current_pair = (None, None) # Active pair + self._current_error = None + + # This is used to track whether or not to continue + # processing when, for example, validation has failed. + self._processing = { + "nextOrder": None, + "ordersWithError": set() + } + + # Signals + # + # / \ + # ( ( o ) ) + # \ | / + # | + # / \ + # / \ + # / \ + # / \ + # + # Install class attributes as instance attributes + for attr in dir(self): + signal = getattr(self, attr) + if isinstance(signal, DefaultSignal): + setattr(self, attr, DefaultSignal(*signal._args)) + + def stop(self): + self._is_running = False + + def reset(self): + """Discover plug-ins and run collection""" + self._context = self.dispatch("Context") + self._plugins[:] = self.dispatch("discover") + + self.was_discovered.emit() + + self._pair_generator = None + self._current_pair = (None, None) + self._current_error = None + + self._processing = { + "nextOrder": None, + "ordersWithError": set() + } + + self._load() + + self.was_reset.emit() + + def collect(self): + """Run until and including Collection""" + self._run(until=api.CollectorOrder, + on_finished=self.was_collected.emit) + + def validate(self): + """Run until and including Validation""" + self._run(until=api.ValidatorOrder, + on_finished=self.was_validated.emit) + + def extract(self): + """Run until and including Extraction""" + self._run(until=api.ExtractorOrder, + on_finished=self.was_extracted.emit) + + def integrate(self): + """Run until and including Integration""" + self._run(until=api.IntegratorOrder, + on_finished=self.was_integrated.emit) + + def publish(self): + """Run until there are no more plug-ins""" + self._run(on_finished=self.was_published.emit) + + def act(self, plug, action): + context = self._context + + def on_next(): + result = self.dispatch("process", plug, context, None, action.id) + self.was_processed.emit(result) + self.defer(500, on_finished) + + def on_finished(): + self.was_acted.emit() + self._is_running = False + + self._is_running = True + self.defer(100, on_next) + + def cleanup(self): + """Forcefully delete objects from memory + + In an ideal world, this shouldn't be necessary. Garbage + collection guarantees that anything without reference + is automatically removed. + + However, because this application is designed to be run + multiple times from the same interpreter process, extra + case must be taken to ensure there are no memory leaks. + + Explicitly deleting objects shines a light on where objects + may still be referenced in the form of an error. No errors + means this was uneccesary, but that's ok. + + """ + + while self._context: + self._context.pop(0) + + while self._plugins: + self._plugins.pop(0) + + self._context = [] + self._plugins = [] + + try: + self._pair_generator.close() + except AttributeError: + pass + + self._pair_generator = None + self._current_pair = () + self._current_error = None + + def _load(self): + """Initiate new generator and load first pair""" + self._is_running = True + self._pair_generator = self._iterator(self._plugins, self._context) + self._current_pair = next(self._pair_generator, (None, None)) + self._current_error = None + self._is_running = False + + def _run(self, until=float("inf"), on_finished=lambda: None): + """Process current pair and store next pair for next process + + Arguments: + until (api.Order, optional): Keep fetching next() + until this order, default value is infinity. + on_finished (callable, optional): What to do when finishing, + defaults to doing nothing. + + """ + + def on_next(): + if self._current_pair == (None, None): + return finished(100) + + # The magic number 0.5 is the range between + # the various CVEI processing stages; + order = self._current_pair[0].order + if order > (until + 0.5): + return finished(100) + + self.about_to_process.emit(*self._current_pair) + + self.defer(10, on_process) + + def on_process(): + try: + result = self._process(*self._current_pair) + if result["error"] is not None: + self._current_error = result["error"] + + self.was_processed.emit(result) + + except Exception as e: + stack = traceback.format_exc(e) + return self.defer( + 500, lambda: on_unexpected_error(error=stack)) + + # Now that processing has completed, and context potentially + # modified with new instances, produce the next pair. + # + # IMPORTANT: This *must* be done *after* processing of + # the current pair, otherwise data generated at that point + # will *not* be included. + try: + self._current_pair = next(self._pair_generator) + + except StopIteration: + # All pairs were processed successfully! + self._current_pair = (None, None) + return finished(500) + + except Exception as e: + # This is a bug + stack = traceback.format_exc(e) + self._current_pair = (None, None) + return self.defer( + 500, lambda: on_unexpected_error(error=stack)) + + self.defer(10, on_next) + + def on_unexpected_error(error): + self.warned.emit(str(error)) + return finished(500) + + def finished(delay): + self._is_running = False + self.was_finished.emit() + return self.defer(delay, on_finished) + + self._is_running = True + self.defer(10, on_next) + + def _iterator(self, plugins, context): + """Yield next plug-in and instance to process. + + Arguments: + plugins (list): Plug-ins to process + context (api.Context): Context to process + + """ + + test = logic.registered_test() + + for plug, instance in logic.Iterator(plugins, context): + if not plug.active: + continue + + if instance is not None and instance.data.get("publish") is False: + continue + + self._processing["nextOrder"] = plug.order + + if not self._is_running: + raise StopIteration("Stopped") + + if test(**self._processing): + raise StopIteration("Stopped due to %s" % test( + **self._processing)) + + yield plug, instance + + def _process(self, plug, instance=None): + """Produce `result` from `plugin` and `instance` + + :func:`_process` shares state with :func:`_iterator` such that + an instance/plugin pair can be fetched and processed in isolation. + + Arguments: + plug (api.Plugin): Produce result using plug-in + instance (optional, api.Instance): Process this instance, + if no instance is provided, context is processed. + + """ + + self._processing["nextOrder"] = plug.order + + try: + result = self.dispatch("process", plug, self._context, instance) + + except Exception as e: + raise Exception("Unknown error: %s" % e) + + else: + # Make note of the order at which the + # potential error occured. + has_error = result["error"] is not None + if has_error: + self._processing["ordersWithError"].add(plug.order) + + return result + + +def default_defer(self, delay, func): + """Synchronous, non-deferring""" + return func() + + +def default_dispatch(self, func, *args, **kwargs): + """Use local library""" + return { + "Context": api.Context, + "Instance": api.Instance, + "discover": api.discover, + "process": plugin.process, + }[func](*args, **kwargs) + + +class DefaultSignal(object): + """Simple, but capable signal + + Handles garbage collection of connected observers + via weak referencing. Runs in the emitting thread. + + """ + + def __init__(self, *args): + self._args = args + self._observers = list() + + def connect(self, func): + reference = lib.WeakRef(func) + if reference not in self._observers: + self._observers.append(reference) + + def disconnect(self, func): + self._observers.remove(lib.WeakRef(func)) + + def emit(self, *args): + for observer in self._observers: + try: + observer()(*args) + except ReferenceError: + pass + except Exception as e: + sys.stderr.write(str(e)) + + +def create(signal=DefaultSignal, + defer=default_defer, + base=object, + dispatch=default_dispatch): + """Instantiate an independent engine of supplied internals + + Arguments: + signal (class, optional): Must implement the interface + of TemplateSignal. Defaults to `DefaultSignal` + defer (callable, optional): Must implement the interface + of `default_defer`. Defaults to `default_defer` + base (class, optional): Baseclass of AbstractEngine, + defaults to `object`. + dispatch (callable, optional): Must implement the interface + of `default_dispatch`, defaults to `default_dispatch`. + + """ + + body = { + "defer": defer, + "dispatch": dispatch + } + + # Replace TemplateSignal with provided mechanism + for attr in dir(AbstractEngine): + prop = getattr(AbstractEngine, attr) + if isinstance(prop, TemplateSignal): + body[attr] = signal(*prop.args) + + cls = type("Engine", (AbstractEngine, base), body) + + return cls() diff --git a/pyblish/lib.py b/pyblish/lib.py index e40b2c25..f71b3f94 100644 --- a/pyblish/lib.py +++ b/pyblish/lib.py @@ -1,5 +1,7 @@ import os import sys +import types +import weakref import logging import datetime import warnings @@ -236,18 +238,24 @@ def emit(signal, **kwargs): Example: >>> import sys - >>> from . import plugin - >>> plugin.register_callback( - ... "mysignal", lambda data: sys.stdout.write(str(data))) + >>> def mycallback(data): + ... sys.stdout.write(str(data)) + ... + >>> register_callback("mysignal", mycallback) ... >>> emit("mysignal", data={"something": "cool"}) {'something': 'cool'} """ - for callback in _registered_callbacks.get(signal, []): + for callback in _registered_callbacks.get(signal, {}).values(): try: callback(**kwargs) + + except ReferenceError: + # Ignore end-of-life references + pass + except Exception: file = six.StringIO() traceback.print_exc(file=file) @@ -262,6 +270,74 @@ def emit(signal, **kwargs): # TODO(marcus): Make it prettier +def register_callback(signal, callback): + """Register a new callback + + Arguments: + signal (string): Name of signal to register the callback with. + callback (func): Function to execute when a signal is emitted. + + Raises: + ValueError if `callback` is not callable. + + """ + + if not hasattr(callback, "__call__"): + raise ValueError("%s must be callable" % callback) + + if signal not in _registered_callbacks: + # Need to store in a dictionary so as to + # enable removal via deregister_callback, + # since the actual function is not comparable + # to its weak reference equivalent. + + _registered_callbacks[signal] = weakref.WeakValueDictionary() + + name = callback.__name__ + callbacks = _registered_callbacks[signal] + + if name in callbacks: + raise ValueError( + "Callback by this name already registered: \"%s\"" % name + ) + + # Use weak reference such that connected callbacks + # can safely be garbage collected without interference + # from observers. + callbacks[name] = callback + + +def deregister_callback(signal, callback): + """Deregister a callback + + Arguments: + signal (string): Name of signal to deregister the callback with. + callback (func): Function to execute when a signal is emitted. + + Raises: + KeyError on missing signal or callback + + """ + + _registered_callbacks[signal].pop(callback.__name__) + + # Erase empty member + if not _registered_callbacks[signal]: + _registered_callbacks.pop(signal) + + +def deregister_all_callbacks(): + """Deregisters all callback""" + + _registered_callbacks.clear() + + +def registered_callbacks(): + """Returns registered callbacks""" + + return list(_registered_callbacks.keys()) + + def deprecated(func): """Deprecation decorator @@ -279,3 +355,116 @@ def wrapper(*args, **kwargs): lineno=func.func_code.co_firstlineno + 1) return func(*args, **kwargs) return wrapper + + +if sys.version_info < (3, 4): + class _WeakRef(object): + def __init__(self, func): + try: + if func.__self__ is not None: + self._instance = weakref.ref(func.__self__) + else: + # Unbound method + self._instance = None + + self._func = weakref.ref(func.__func__) + self._class = weakref.ref(func.__class__) + + except AttributeError: + # Not a method + self._instance = None + self._class = None + self._func = weakref.ref(func) + + def __call__(self): + if self._is_dead(): + return None + + if self._instance is None: + return self._func() + + return types.MethodType(self._func(), self._instance()) + + def _is_dead(self): + """Is the reference dead? + + Returns True if the referenced callable was a bound method and + the instance no longer exists. Otherwise, return False. + + Usage: + >>> class Object(object): + ... def func(self): + ... pass + ... + >>> o = Object() + >>> weak_func = WeakRef(o.func) + >>> weak_func._is_dead() + False + >>> del(o) + >>> weak_func._is_dead() + True + + """ + + return self._instance is not None and self._instance() is None + +else: + # Python 3.4 upwards implement weakref.WeakMethod + class _WeakRef: + def __init__(self, func): + try: + func.__self__ + self._func = weakref.WeakMethod(func) + except AttributeError: + self._func = weakref.ref(func) + + def __call__(self): + return self._func() + + +class WeakRef(_WeakRef): + """Alternative weak reference with support for instancemethods + + Usage: + >>> import weakref + >>> class MyClass(object): + ... def func(self): + ... pass + ... + >>> inst = MyClass() + >>> ref = weakref.ref(inst.func) + >>> ref() is None + True + >>> ref = WeakRef(inst.func) + >>> ref() is None + False + + """ + + def __eq__(self, other): + """Compare weak references against each other + + Example: + >>> def func_a(): + ... pass + ... + >>> def func_b(): + ... pass + ... + >>> weak_a1 = WeakRef(func_a) + >>> weak_a2 = WeakRef(func_a) + >>> weak_b = WeakRef(func_b) + >>> weak_a1 == weak_a2 + True + >>> weak_b == weak_a1 + False + + """ + + try: + return type(self) is type(other) and self() == other() + except: + return False + + def __ne__(self, other): + return not self == other diff --git a/pyblish/plugin.py b/pyblish/plugin.py index 47fd87fa..8ff35ec7 100644 --- a/pyblish/plugin.py +++ b/pyblish/plugin.py @@ -14,18 +14,18 @@ import os import sys import time +import uuid import types +import weakref import logging import inspect import warnings import contextlib -import uuid # Local library from . import ( __version__, version_info, - _registered_callbacks, _registered_services, _registered_plugins, _registered_hosts, @@ -726,7 +726,7 @@ def create_instance(self, name, **kwargs): """ - instance = Instance(name, parent=self) + instance = Instance(name, parent=weakref.proxy(self)) instance.data.update(kwargs) return instance @@ -835,54 +835,6 @@ def current_host(): return _registered_hosts[-1] if _registered_hosts else "unknown" -def register_callback(signal, callback): - """Register a new callback - - Arguments: - signal (string): Name of signal to register the callback with. - callback (func): Function to execute when a signal is emitted. - - Raises: - ValueError if `callback` is not callable. - - """ - - if not hasattr(callback, "__call__"): - raise ValueError("%s is not callable" % callback) - - if signal in _registered_callbacks: - _registered_callbacks[signal].append(callback) - else: - _registered_callbacks[signal] = [callback] - - -def deregister_callback(signal, callback): - """Deregister a callback - - Arguments: - signal (string): Name of signal to deregister the callback with. - callback (func): Function to execute when a signal is emitted. - - Raises: - KeyError on missing signal - ValueError on missing callback - """ - - _registered_callbacks[signal].remove(callback) - - -def deregister_all_callbacks(): - """Deregisters all callback""" - - _registered_callbacks.clear() - - -def registered_callbacks(): - """Returns registered callbacks""" - - return _registered_callbacks - - def register_plugin(plugin): """Register a new plug-in diff --git a/pyblish/util.py b/pyblish/util.py index 8e7e7f97..fe6017f7 100644 --- a/pyblish/util.py +++ b/pyblish/util.py @@ -1,18 +1,4 @@ -"""Conveinence function for Pyblish - -Attributes: - TAB: Number of spaces for a tab - LOG_TEMPATE: Template used for logging coming from - plug-ins - SCREEN_WIDTH: Default width at which logging and printing - will (attempt to) restrain to. - logging_handlers: Record of handlers at the start of - importing this module. This module will modify the - currently handlers and restore then once finished. - log: Current logger - intro_message: Message printed upon initiating a publish. - -""" +"""Conveinence functions for general publishing""" from __future__ import absolute_import @@ -26,7 +12,7 @@ log = logging.getLogger("pyblish.util") -def publish(context=None, plugins=None, **kwargs): +def publish(context=None, plugins=None): """Publish everything This function will process all available plugins of the @@ -41,7 +27,7 @@ def publish(context=None, plugins=None, **kwargs): Usage: >> context = plugin.Context() - >> publish(context) # Pass.. + >> publish(context) # Pass.. >> context = publish() # ..or receive a new """ @@ -114,44 +100,73 @@ def publish(context=None, plugins=None, **kwargs): return context -def collect(*args, **kwargs): - """Convenience function for collection""" - context = _convenience(0.5, *args, **kwargs) +def collect(context=None, plugins=None): + """Convenience function for collection + + _________ . . . . . . . . . . . . . . . . . . . + | | . . . . . . + | Collect |-->. Validate .-->. Extract .-->. Integrate . + |_________| . . . . . . . . . . . . . . . . . . . + + """ + + context = _convenience(0.5, context, plugins) api.emit("collected", context=context) return context -def validate(*args, **kwargs): - """Convenience function for validation""" - context = _convenience(1.5, *args, **kwargs) +def validate(context=None, plugins=None): + """Convenience function for collection through validation + + _________ __________ . . . . . . . . . . . . . + | | | | . . . . + | Collect |-->| Validate |-->. Extract .-->. Integrate . + |_________| |__________| . . . . . . . . . . . . . + + """ + + context = _convenience(1.5, context, plugins) api.emit("validated", context=context) return context -def extract(*args, **kwargs): - """Convenience function for extraction""" - context = _convenience(2.5, *args, **kwargs) +def extract(context=None, plugins=None): + """Convenience function for collection through extraction + + _________ __________ _________ . . . . . . . + | | | | | | . . + | Collect |-->| Validate |-->| Extract |-->. Integrate . + |_________| |__________| |_________| . . . . . . . + + """ + + context = _convenience(2.5, context, plugins) api.emit("extracted", context=context) return context -def integrate(*args, **kwargs): - """Convenience function for integration""" - context = _convenience(3.5, *args, **kwargs) +def integrate(context=None, plugins=None): + """Convenience function for collection through end + + _________ __________ _________ ___________ + | | | | | | | | + | Collect |-->| Validate |-->| Extract |-->| Integrate | + |_________| |__________| |_________| |___________| + + """ + + context = _convenience(float("inf"), context, plugins) api.emit("integrated", context=context) return context -def _convenience(order, *args, **kwargs): - plugins = [p for p in api.discover() - if p.order < order] +def _convenience(order, context=None, plugins=None): + plugins = list( + p for p in (api.discover() if plugins is None else plugins) + if p.order < order + ) - args = list(args) - if len(args) > 1: - args[1] = plugins - else: - kwargs["plugins"] = plugins - return publish(*args, **kwargs) + return publish(context, plugins) # Backwards compatibility diff --git a/pyblish/version.py b/pyblish/version.py index a82be524..2a3a3fa2 100644 --- a/pyblish/version.py +++ b/pyblish/version.py @@ -1,7 +1,7 @@ VERSION_MAJOR = 1 VERSION_MINOR = 4 -VERSION_PATCH = 1 +VERSION_PATCH = 2 version_info = (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH) version = '%i.%i.%i' % version_info diff --git a/tests/lib.py b/tests/lib.py index e4e6b70b..5001f3fd 100644 --- a/tests/lib.py +++ b/tests/lib.py @@ -5,15 +5,15 @@ import contextlib import pyblish +import pyblish.api import pyblish.cli -import pyblish.plugin from pyblish.vendor import six # Setup HOST = 'python' FAMILY = 'test.family' -REGISTERED = pyblish.plugin.registered_paths() +REGISTERED = pyblish.api.registered_paths() PACKAGEPATH = pyblish.lib.main_package_path() ENVIRONMENT = os.environ.get("PYBLISHPLUGINPATH", "") PLUGINPATH = os.path.join(PACKAGEPATH, '..', 'tests', 'plugins') @@ -21,24 +21,24 @@ def setup(): """Disable default plugins and only use test plugins""" - pyblish.plugin.deregister_all_paths() + pyblish.api.deregister_all_paths() def setup_empty(): """Disable all plug-ins""" setup() - pyblish.plugin.deregister_all_plugins() - pyblish.plugin.deregister_all_paths() - pyblish.plugin.deregister_all_hosts() - pyblish.plugin.deregister_all_callbacks() + pyblish.api.deregister_all_plugins() + pyblish.api.deregister_all_paths() + pyblish.api.deregister_all_hosts() + pyblish.api.deregister_all_callbacks() def teardown(): """Restore previously REGISTERED paths""" - pyblish.plugin.deregister_all_paths() + pyblish.api.deregister_all_paths() for path in REGISTERED: - pyblish.plugin.register_plugin_path(path) + pyblish.api.register_plugin_path(path) os.environ["PYBLISHPLUGINPATH"] = ENVIRONMENT pyblish.api.deregister_all_plugins() diff --git a/tests/test_engine.py b/tests/test_engine.py new file mode 100644 index 00000000..f366589c --- /dev/null +++ b/tests/test_engine.py @@ -0,0 +1,563 @@ +import sys +import weakref +import functools + +import pyblish.api +import pyblish.engine +from nose.tools import ( + with_setup, + assert_equals, +) + +from .lib import setup_empty + +self = sys.modules[__name__] + + +def setup(): + self.engine = pyblish.engine.create() + + +@with_setup(setup_empty) +def test_collection(): + """collect() works as expected.""" + + count = {"#": 0} + + class MyCollector(pyblish.api.ContextPlugin): + order = pyblish.api.CollectorOrder + + def process(self, context): + count["#"] += 1 + + def on_collection(): + count["#"] += 10 + + pyblish.api.register_plugin(MyCollector) + + self.engine.was_collected.connect(on_collection) + + self.engine.collect() + + count["#"] == 10, count + + +@with_setup(setup_empty) +def test_publish(): + """publish() works as expected""" + + count = {"#": 0} + + class MyCollector(pyblish.api.ContextPlugin): + order = pyblish.api.CollectorOrder + + def process(self, context): + instance = context.create_instance("myInstance") + instance.data["families"] = ["myFamily"] + count["#"] += 1 + + class MyValidator(pyblish.api.InstancePlugin): + order = pyblish.api.ValidatorOrder + + def process(self, instance): + count["#"] += 10 + + class MyExtractor(pyblish.api.InstancePlugin): + order = pyblish.api.ExtractorOrder + + def process(self, instance): + count["#"] += 100 + + class MyIntegrator(pyblish.api.InstancePlugin): + order = pyblish.api.IntegratorOrder + + def process(self, instance): + count["#"] += 1000 + + for Plugin in (MyCollector, + MyValidator, + MyExtractor, + MyIntegrator): + pyblish.api.register_plugin(Plugin) + + def on_published(): + """Emitted once, on completion""" + count["#"] += 10000 + + self.engine.was_published.connect(on_published) + + self.engine.reset() + + assert count["#"] == 0, count + + self.engine.publish() + + assert count["#"] == 11111, count + + +@with_setup(setup_empty) +def test_signals(): + """Signals are emitted as expected""" + + count = {"#": 0} + + def identity(): + return { + "was_processed": 0, + "was_discovered": 0, + "was_reset": 0, + "was_collected": 0, + "was_validated": 0, + "was_extracted": 0, + "was_integrated": 0, + "was_published": 0, + "was_acted": 0, + "was_finished": 0, + } + + emitted = identity() + + class MyCollector(pyblish.api.ContextPlugin): + order = pyblish.api.CollectorOrder + + def process(self, context): + context.create_instance("MyInstance") + count["#"] += 1 + + class MyValidator(pyblish.api.InstancePlugin): + order = pyblish.api.ValidatorOrder + + def process(self, instance): + count["#"] += 10 + + def on_signal(name, *args): + print("Emitting %s" % name) + emitted[name] += 1 + + pyblish.api.register_plugin(MyCollector) + pyblish.api.register_plugin(MyValidator) + + _funcs = list() + for signal in identity(): + func = functools.partial(on_signal, signal) + getattr(self.engine, signal).connect(func) + _funcs.append(func) + + # During reset, no plug-ins are run. + self.engine.reset() + + state = identity() + state["was_reset"] = 1 + state["was_discovered"] = 1 + assert_equals(emitted, state) + assert_equals(count["#"], 0) + + # Running up till and including collection + self.engine.collect() + state["was_collected"] = 1 + state["was_processed"] = 1 + state["was_finished"] = 1 + assert_equals(emitted, state) + assert_equals(count["#"], 1) + + # Up till and including validation, collection is *not* re-run. + self.engine.validate() + state["was_validated"] = 1 + state["was_processed"] = 2 + state["was_finished"] = 2 + assert_equals(emitted, state) + assert_equals(count["#"], 11) + + # Finish off publishing; at this point, there are no more plug-ins + # so count should remain the same. + self.engine.publish() + state["was_published"] = 1 + state["was_finished"] = 3 + assert_equals(emitted, state) + assert_equals(count["#"], 11) + + print("Disconnecting") + + +def test_engine_isolation(): + """One engine does not interfere with another engine in the same process + + An engine declares signals on a class-level, + but must not call upon signals declared in an + individual instance of a class. + + This is managed by the dynamic lookup of declared + signals at run-time, similar to what bindings of Qt + does with its signals and in fact exists due to + compatibility with such bindings. + + """ + + engine1 = pyblish.engine.create() + engine2 = pyblish.engine.create() + + count = {"#": 0} + + def increment(): + count["#"] += 1 + + engine1.was_reset.connect(increment) + + engine2.reset() + + assert count["#"] == 0 + + engine1.reset() + + assert count["#"] == 1 + + +def test_signals_to_instancemethod(): + """Signals to instancemethod works well. + + Default Python (2.x) weak references does not support making + references to methods to an instance of a class. + + """ + + count = {"#": 0} + + class MyClass(object): + def __init__(self): + engine = pyblish.engine.create() + engine.was_reset.connect(self.func) + + # Synchronous + engine.reset() + + def func(self): + count["#"] += 1 + + MyClass() + + assert count["#"] == 1, count + + +def test_default_signals_are_weakly_referenced(): + """Default signals are weakly referenced""" + + count = {"#": 0} + + class Class(object): + def method(self): + count["#"] += 1 + + obj = Class() + + signal = pyblish.engine.DefaultSignal(str) + signal.connect(obj.method) + + assert count["#"] == 0, count + signal.emit() + assert count["#"] == 1, count + + del(obj) + signal.emit() + + # Count does not increase, because the observer is dead + assert count["#"] == 1, count + + +@with_setup(setup_empty) +def test_engine_cleanup(): + """Engine cleans itself up on deletion""" + + class MyPlugin(pyblish.api.ContextPlugin): + def process(self, context): + context.create_instance("MyInstance") + + pyblish.api.register_plugin(MyPlugin) + + engine = pyblish.engine.create() + engine.reset() + engine.collect() + + instance = engine.context[0] + weak_instance = weakref.ref(instance) + + assert weak_instance() is not None + + engine.cleanup() + + del(instance) + + # TODO(marcus): This should pass + # assert weak_instance() is None, weak_instance + + # Cleaning up multiple times is harmless + engine.cleanup() + engine.cleanup() + + +def test_template_signal(): + """TemplateSignal fulfills basic interface""" + + signal = pyblish.engine.TemplateSignal(str) + + assert len(signal.args) == 1 + assert signal.args[0] == str + + def func(): + pass + + signal.connect(func) + signal.disconnect(func) + signal.emit("Hello") + + +@with_setup(setup_empty) +def test_cvei(): + """CVEI stages trigger plug-in in the appropriate order""" + + count = {"#": 0} + + class MyCollector(pyblish.api.ContextPlugin): + order = pyblish.api.CollectorOrder + + def process(self, context): + context.create_instance("MyInstance") + count["#"] += 1 + + class MyValidator(pyblish.api.InstancePlugin): + order = pyblish.api.ValidatorOrder + + def process(self, instance): + count["#"] += 10 + + class MyExtractor(pyblish.api.InstancePlugin): + order = pyblish.api.ExtractorOrder + + def process(self, instance): + count["#"] += 100 + + class MyIntegrator(pyblish.api.InstancePlugin): + order = pyblish.api.IntegratorOrder + + def process(self, instance): + count["#"] += 1000 + + for Plugin in (MyCollector, + MyValidator, + MyExtractor, + MyIntegrator): + pyblish.api.register_plugin(Plugin) + + engine = pyblish.engine.create() + engine.reset() + + assert count["#"] == 0 + + engine.collect() + + assert count["#"] == 1 + + engine.validate() + + assert count["#"] == 11 + + engine.extract() + + assert count["#"] == 111 + + engine.integrate() + + assert count["#"] == 1111 + + # No more plug-ins to run. + engine.publish() + + assert count["#"] == 1111 + + # When all has been said and done, the engine + # should no longer be running. + assert not engine.is_running + + +def test_defaultsignal_disconnect(): + """Observers may be disconnected""" + + count = {"#": 0} + signal = pyblish.engine.DefaultSignal(str) + + def func(): + count["#"] += 1 + + signal.connect(func) + + signal.emit() + assert count["#"] == 1 + + signal.disconnect(func) + + signal.emit() + assert count["#"] == 1 + + +@with_setup(setup_empty) +def test_act(): + """Running an action works as advertised""" + + count = {"#": 0} + + class MyAction(pyblish.api.Action): + def process(self, context, plugin): + count["#"] += 1 + + class MyPlugin(pyblish.api.ContextPlugin): + order = pyblish.api.CollectorOrder + actions = [MyAction] + + pyblish.api.register_plugin(MyPlugin) + + engine = pyblish.engine.create() + + def on_acted(): + count["#"] += 10 + + engine.was_acted.connect(on_acted) + + assert count["#"] == 0 + + engine.act(MyPlugin, MyAction) + + assert count["#"] == 11 + + assert not engine.is_running + + +@with_setup(setup_empty) +def test_current_error(): + """The exception raised by the last run plug-in is stored""" + + class MyPlugin(pyblish.api.ContextPlugin): + order = pyblish.api.CollectorOrder + + def process(self, context): + assert False + + pyblish.api.register_plugin(MyPlugin) + + engine = pyblish.engine.create() + + assert engine.current_error is None, engine.current_error + + engine.reset() + engine.collect() + + assert isinstance(engine.current_error, AssertionError) + + +@with_setup(setup_empty) +def test_inactive_instance(): + """An inactive instance shouldn't run""" + + count = {"#": 0} + + class MyCollector(pyblish.api.ContextPlugin): + order = pyblish.api.CollectorOrder + + def process(self, context): + active = context.create_instance("ActiveInstance") + active.data["publish"] = True + + inactive = context.create_instance("InactiveInstance") + inactive.data["publish"] = False + + count["#"] += 1 + + class MyValidator(pyblish.api.InstancePlugin): + order = pyblish.api.ValidatorOrder + + def process(self, instance): + count["#"] += 10 + + pyblish.api.register_plugin(MyCollector) + pyblish.api.register_plugin(MyValidator) + + engine = pyblish.engine.create() + engine.reset() + + assert count["#"] == 0 + + engine.publish() + + # Both plug-ins run, only one instance runs + assert count["#"] == 11, count + + +def test_inactive_plugin(): + """An inactive plug-in shouldn't run""" + + count = {"#": 0} + + class ActivePlugin(pyblish.api.ContextPlugin): + order = pyblish.api.CollectorOrder + active = True + + def process(self, context): + count["#"] += 1 + + class InactivePlugin(pyblish.api.ContextPlugin): + order = pyblish.api.CollectorOrder + active = False + + def process(self, context): + count["#"] += 10 + + pyblish.api.register_plugin(ActivePlugin) + pyblish.api.register_plugin(InactivePlugin) + + engine = pyblish.engine.create() + engine.reset() + + assert count["#"] == 0 + + engine.collect() + + assert count["#"] == 1 + + +@with_setup(setup_empty) +def test_processing_stops_at_validation(): + """Failed vaildation stops processing""" + + count = {"#": 0} + + class MyCollector(pyblish.api.ContextPlugin): + order = pyblish.api.CollectorOrder + + def process(self, context): + context.create_instance("MyInstance") + count["#"] += 1 + + class MyValidator(pyblish.api.InstancePlugin): + order = pyblish.api.ValidatorOrder + + def process(self, instance): + count["#"] += 10 + assert False + + class MyExtractor(pyblish.api.InstancePlugin): + order = pyblish.api.ExtractorOrder + + def process(self, instance): + count["#"] += 100 + + pyblish.api.register_plugin(MyCollector) + pyblish.api.register_plugin(MyValidator) + pyblish.api.register_plugin(MyExtractor) + + engine = pyblish.engine.create() + engine.reset() + + assert count["#"] == 0 + + # Validation fails + engine.publish() + + assert count["#"] == 11, count diff --git a/tests/test_lib.py b/tests/test_lib.py new file mode 100644 index 00000000..9ea6e9dc --- /dev/null +++ b/tests/test_lib.py @@ -0,0 +1,51 @@ +import sys +import functools + +import pyblish.lib + +from nose.tools import ( + with_setup, + assert_equals, +) + +from .lib import setup_empty + + +def test_weak_method(): + """Weak references work with instancemethod""" + + count = {"#": 0} + + class Object(object): + def func(self): + count["#"] += 1 + + o = Object() + wo = pyblish.lib.WeakRef(o.func) + + assert count["#"] == 0 + wo()() + assert count["#"] == 1 + + del(o) + + assert wo() is None, wo() + + +def test_weak_function(): + """WeakRef works with functions""" + + count = {"#": 0} + + def func(): + count["#"] += 1 + + fo = pyblish.lib.WeakRef(func) + + assert count["#"] == 0 + fo()() + assert count["#"] == 1, fo() + + del(func) + + assert fo() is None, fo diff --git a/tests/test_plugin.py b/tests/test_plugin.py index bbb1c75a..8532b9e0 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -53,7 +53,7 @@ def test_context_from_instance(): context = pyblish.plugin.Context() instance = context.create_instance("MyInstance") - assert_equals(context, instance.context) + assert_equals(context.id, instance.context.id) def test_legacy(): @@ -414,34 +414,49 @@ def my_callback(): def other_callback(data=None): pass - pyblish.plugin.register_callback("mySignal", my_callback) + pyblish.api.register_callback("mySignal", my_callback) - msg = "Registering a callback failed" - data = {"mySignal": [my_callback]} - assert "mySignal" in pyblish.plugin.registered_callbacks() == data, msg + assert "mySignal" in pyblish.api.registered_callbacks() - pyblish.plugin.deregister_callback("mySignal", my_callback) + pyblish.api.deregister_callback("mySignal", my_callback) - assert_raises( - ValueError, - pyblish.plugin.deregister_callback, - "mySignal", my_callback) + # The callback does not exist + assert_raises(KeyError, + pyblish.api.deregister_callback, + "mySignal", my_callback) - assert_raises( - KeyError, - pyblish.plugin.deregister_callback, - "notExist", my_callback) + # The signal does not exist + assert_raises(KeyError, + pyblish.api.deregister_callback, + "notExist", my_callback) - msg = "Deregistering a callback failed" - data = {"mySignal": []} - assert pyblish.plugin.registered_callbacks() == data, msg + assert_equals(pyblish.api.registered_callbacks(), []) - pyblish.plugin.register_callback("mySignal", my_callback) - pyblish.plugin.register_callback("otherSignal", other_callback) - pyblish.plugin.deregister_all_callbacks() + pyblish.api.register_callback("mySignal", my_callback) + pyblish.api.register_callback("otherSignal", other_callback) + pyblish.api.deregister_all_callbacks() - msg = "Deregistering all callbacks failed" - assert pyblish.plugin.registered_callbacks() == {}, msg + assert_equals(pyblish.api.registered_callbacks(), []) + + +def test_weak_callback(): + """Callbacks have weak references""" + + count = {"#": 0} + + def my_callback(): + count["#"] += 1 + + pyblish.api.register_callback("on_callback", my_callback) + pyblish.api.emit("on_callback") + assert count["#"] == 1 + + del(my_callback) + + pyblish.api.emit("on_callback") + + # No errors were thrown, count did not increase + assert count["#"] == 1 @with_setup(lib.setup_empty, lib.teardown) @@ -451,7 +466,7 @@ def test_emit_signal_wrongly(): def other_callback(an_argument=None): print("Ping from 'other_callback' with %s" % an_argument) - pyblish.plugin.register_callback("otherSignal", other_callback) + pyblish.api.register_callback("otherSignal", other_callback) with lib.captured_stderr() as stderr: pyblish.lib.emit("otherSignal", not_an_argument="") @@ -464,13 +479,13 @@ def other_callback(an_argument=None): @with_setup(lib.setup_empty, lib.teardown) def test_registering_invalid_callback(): """Can't register non-callables""" - pyblish.plugin.register_callback("invalid", None) + pyblish.api.register_callback("invalid", None) @raises(KeyError) def test_deregistering_nonexisting_callback(): """Can't deregister a callback that doesn't exist""" - pyblish.plugin.deregister_callback("invalid", lambda: "") + pyblish.api.deregister_callback("invalid", lambda: "") @raises(TypeError) diff --git a/tests/test_util.py b/tests/test_util.py index de5f72dd..135cdcc7 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1,12 +1,110 @@ from . import lib +import pyblish.api import pyblish.lib +import pyblish.util import pyblish.compat from nose.tools import ( with_setup ) +def test_convenience_plugins_argument(): + """util._convenience() `plugins` argument works + + Issue: #286 + + """ + + count = {"#": 0} + + class PluginA(pyblish.api.ContextPlugin): + order = pyblish.api.CollectorOrder + + def process(self, context): + count["#"] += 1 + + class PluginB(pyblish.api.ContextPlugin): + order = pyblish.api.CollectorOrder + + def process(self, context): + count["#"] += 10 + + assert count["#"] == 0 + + pyblish.api.register_plugin(PluginA) + pyblish.util._convenience(0.5, plugins=[PluginB]) + + assert count["#"] == 10, count + + +@with_setup(lib.setup, lib.teardown) +def test_convenience_functions(): + """convenience functions works as expected""" + + count = {"#": 0} + + class Collector(pyblish.plugin.ContextPlugin): + order = pyblish.plugin.CollectorOrder + + def process(self, context): + context.create_instance("MyInstance") + count["#"] += 1 + + class Validator(pyblish.plugin.InstancePlugin): + order = pyblish.plugin.ValidatorOrder + + def process(self, instance): + count["#"] += 10 + + class Extractor(pyblish.plugin.InstancePlugin): + order = pyblish.plugin.ExtractorOrder + + def process(self, instance): + count["#"] += 100 + + class Integrator(pyblish.plugin.ContextPlugin): + order = pyblish.plugin.IntegratorOrder + + def process(self, instance): + count["#"] += 1000 + + class PostIntegrator(pyblish.plugin.ContextPlugin): + order = pyblish.plugin.IntegratorOrder + 0.5 + + def process(self, instance): + count["#"] += 10000 + + assert count["#"] == 0 + + for Plugin in (Collector, + Validator, + Extractor, + Integrator, + PostIntegrator): + pyblish.api.register_plugin(Plugin) + + pyblish.util.collect() + + assert count["#"] == 1 + count["#"] = 0 + + pyblish.util.validate() + + assert count["#"] == 11 + + count["#"] = 0 + pyblish.util.extract() + + assert count["#"] == 111 + + # Integration runs integration, but also anything after + count["#"] = 0 + pyblish.util.integrate() + + assert count["#"] == 11111 + + @with_setup(lib.setup, lib.teardown) def test_multiple_instance_util_publish(): """Multiple instances work with util.publish() @@ -20,22 +118,22 @@ def test_multiple_instance_util_publish(): class MyContextCollector(pyblish.api.ContextPlugin): order = pyblish.api.CollectorOrder + def process(self, context): context.create_instance("A") context.create_instance("B") count["#"] += 1 - class MyInstancePluginCollector(pyblish.api.InstancePlugin): order = pyblish.api.CollectorOrder + 0.1 + def process(self, instance): count["#"] += 1 - pyblish.api.register_plugin(MyContextCollector) pyblish.api.register_plugin(MyInstancePluginCollector) # Ensure it runs without errors - context = pyblish.util.publish() + pyblish.util.publish() assert count["#"] == 3