From a4d0e27bcd5c87ce0b4c3d4eea85a8847431879c Mon Sep 17 00:00:00 2001 From: Reiley Yang Date: Thu, 21 Mar 2019 13:38:32 -0700 Subject: [PATCH] Introduce a generic context (#566) Introduced the generic RuntimeContext. --- context/opencensus-context/CHANGELOG.md | 5 + context/opencensus-context/README.rst | 7 + .../opencensus-context/examples/async_span.py | 55 ++++++ .../examples/explicit_threading.py | 48 +++++ context/opencensus-context/examples/py27.py | 34 ++++ context/opencensus-context/examples/py37.py | 51 +++++ context/opencensus-context/examples/span.py | 56 ++++++ .../examples/thread_pool.py | 49 +++++ .../opencensus-context/opencensus/__init__.py | 1 + .../opencensus/common/__init__.py | 1 + .../common/runtime_context/__init__.py | 174 ++++++++++++++++++ context/opencensus-context/setup.cfg | 2 + context/opencensus-context/setup.py | 48 +++++ .../tests/test_runtime_context.py | 35 ++++ context/opencensus-context/version.py | 15 ++ docs/requirements.txt | 1 + nox.py | 3 +- scripts/twine_upload.sh | 7 + 18 files changed, 591 insertions(+), 1 deletion(-) create mode 100644 context/opencensus-context/CHANGELOG.md create mode 100644 context/opencensus-context/README.rst create mode 100644 context/opencensus-context/examples/async_span.py create mode 100644 context/opencensus-context/examples/explicit_threading.py create mode 100644 context/opencensus-context/examples/py27.py create mode 100644 context/opencensus-context/examples/py37.py create mode 100644 context/opencensus-context/examples/span.py create mode 100644 context/opencensus-context/examples/thread_pool.py create mode 100644 context/opencensus-context/opencensus/__init__.py create mode 100644 context/opencensus-context/opencensus/common/__init__.py create mode 100644 context/opencensus-context/opencensus/common/runtime_context/__init__.py create mode 100644 context/opencensus-context/setup.cfg create mode 100644 context/opencensus-context/setup.py create mode 100644 context/opencensus-context/tests/test_runtime_context.py create mode 100644 context/opencensus-context/version.py diff --git a/context/opencensus-context/CHANGELOG.md b/context/opencensus-context/CHANGELOG.md new file mode 100644 index 000000000..805c3d79d --- /dev/null +++ b/context/opencensus-context/CHANGELOG.md @@ -0,0 +1,5 @@ +# Changelog + +## Unreleased + +- Add this changelog. diff --git a/context/opencensus-context/README.rst b/context/opencensus-context/README.rst new file mode 100644 index 000000000..a58c7e43f --- /dev/null +++ b/context/opencensus-context/README.rst @@ -0,0 +1,7 @@ +OpenCensus Runtime Context +============================================================================ + +|pypi| + +.. |pypi| image:: https://badge.fury.io/py/opencensus-context.svg + :target: https://pypi.org/project/opencensus-context/ diff --git a/context/opencensus-context/examples/async_span.py b/context/opencensus-context/examples/async_span.py new file mode 100644 index 000000000..a91460689 --- /dev/null +++ b/context/opencensus-context/examples/async_span.py @@ -0,0 +1,55 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +from opencensus.common.runtime_context import RuntimeContext + +RuntimeContext.register_slot('current_span', None) + + +class Span(object): + def __init__(self, name): + self.name = name + self.parent = RuntimeContext.current_span + + def __repr__(self): + return ('{}(name={}, parent={})' + .format( + type(self).__name__, + self.name, + self.parent, + )) + + async def __aenter__(self): + RuntimeContext.current_span = self + + async def __aexit__(self, exc_type, exc, tb): + RuntimeContext.current_span = self.parent + + +async def main(): + print(RuntimeContext) + async with Span('foo'): + print(RuntimeContext) + await asyncio.sleep(0.1) + async with Span('bar'): + print(RuntimeContext) + await asyncio.sleep(0.1) + print(RuntimeContext) + await asyncio.sleep(0.1) + print(RuntimeContext) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/context/opencensus-context/examples/explicit_threading.py b/context/opencensus-context/examples/explicit_threading.py new file mode 100644 index 000000000..ad6af8841 --- /dev/null +++ b/context/opencensus-context/examples/explicit_threading.py @@ -0,0 +1,48 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from threading import Thread +from opencensus.common.runtime_context import RuntimeContext + +RuntimeContext.register_slot('operation_id', '') + + +def work(name): + print('Entering worker:', RuntimeContext) + RuntimeContext.operation_id = name + print('Exiting worker:', RuntimeContext) + + +if __name__ == '__main__': + print('Main thread:', RuntimeContext) + RuntimeContext.operation_id = 'main' + + print('Main thread:', RuntimeContext) + + # by default context is not propagated to worker thread + thread = Thread(target=work, args=('foo',)) + thread.start() + thread.join() + + print('Main thread:', RuntimeContext) + + # user can propagate context explicitly + thread = Thread( + target=RuntimeContext.with_current_context(work), + args=('bar',), + ) + thread.start() + thread.join() + + print('Main thread:', RuntimeContext) diff --git a/context/opencensus-context/examples/py27.py b/context/opencensus-context/examples/py27.py new file mode 100644 index 000000000..7333a882f --- /dev/null +++ b/context/opencensus-context/examples/py27.py @@ -0,0 +1,34 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from opencensus.common.runtime_context import RuntimeContext + +RuntimeContext.register_slot('correlation_context', lambda: {}) + + +def hello(name): + correlation_context = RuntimeContext.correlation_context.copy() + correlation_context['name'] = name + RuntimeContext.correlation_context = correlation_context + + print(RuntimeContext) + + +if __name__ == '__main__': + print(RuntimeContext) + RuntimeContext.correlation_context['test'] = True + print(RuntimeContext) + hello('hello') + RuntimeContext.clear() + print(RuntimeContext) diff --git a/context/opencensus-context/examples/py37.py b/context/opencensus-context/examples/py37.py new file mode 100644 index 000000000..e21f67de6 --- /dev/null +++ b/context/opencensus-context/examples/py37.py @@ -0,0 +1,51 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio + +from opencensus.common.runtime_context import RuntimeContext + +RuntimeContext.register_slot('correlation_context', lambda: dict()) + + +async def hello(name): + correlation_context = RuntimeContext.correlation_context.copy() + correlation_context['name'] = name + RuntimeContext.correlation_context = correlation_context + + for i in range(3): + print('Hello {} {} {}'.format( + name, + i, + RuntimeContext, + )) + await asyncio.sleep(0.1) + + +async def main(): + print(RuntimeContext) + RuntimeContext.correlation_context['test'] = True + print(RuntimeContext) + await asyncio.gather( + hello('foo'), + hello('bar'), + hello('baz'), + ) + print(RuntimeContext) + RuntimeContext.clear() + print(RuntimeContext) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/context/opencensus-context/examples/span.py b/context/opencensus-context/examples/span.py new file mode 100644 index 000000000..5c7d58e29 --- /dev/null +++ b/context/opencensus-context/examples/span.py @@ -0,0 +1,56 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from opencensus.common.runtime_context import RuntimeContext + +RuntimeContext.register_slot('current_span', None) + + +class Span(object): + def __init__(self, name): + self.name = name + self.parent = RuntimeContext.current_span + + def __repr__(self): + return ('{}({})'.format(type(self).__name__, self.name)) + + def __enter__(self): + RuntimeContext.current_span = self + + def __exit__(self, type, value, traceback): + RuntimeContext.current_span = self.parent + + def start(self): + RuntimeContext.current_span = self + + def end(self): + RuntimeContext.current_span = self.parent + + +if __name__ == '__main__': + print(RuntimeContext) + with Span('foo'): + print(RuntimeContext) + with Span('bar'): + print(RuntimeContext) + print(RuntimeContext) + print(RuntimeContext) + + # explicit start/end span + span = Span('baz') + print(RuntimeContext) + span.start() + print(RuntimeContext) + span.end() + print(RuntimeContext) diff --git a/context/opencensus-context/examples/thread_pool.py b/context/opencensus-context/examples/thread_pool.py new file mode 100644 index 000000000..c36d5a304 --- /dev/null +++ b/context/opencensus-context/examples/thread_pool.py @@ -0,0 +1,49 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from multiprocessing.dummy import Pool as ThreadPool +import time +import threading +from opencensus.common.runtime_context import RuntimeContext + +RuntimeContext.register_slot('operation_id', '') +_console_lock = threading.Lock() + + +def println(msg): + with _console_lock: + print(msg) + + +def work(name): + println('Entering worker[{}]: {}'.format(name, RuntimeContext)) + RuntimeContext.operation_id = name + time.sleep(0.01) + println('Exiting worker[{}]: {}'.format(name, RuntimeContext)) + + +if __name__ == "__main__": + println('Main thread: {}'.format(RuntimeContext)) + RuntimeContext.operation_id = 'main' + pool = ThreadPool(2) # create a thread pool with 2 threads + pool.map(RuntimeContext.with_current_context(work), [ + 'bear', + 'cat', + 'dog', + 'horse', + 'rabbit', + ]) + pool.close() + pool.join() + println('Main thread: {}'.format(RuntimeContext)) diff --git a/context/opencensus-context/opencensus/__init__.py b/context/opencensus-context/opencensus/__init__.py new file mode 100644 index 000000000..69e3be50d --- /dev/null +++ b/context/opencensus-context/opencensus/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/context/opencensus-context/opencensus/common/__init__.py b/context/opencensus-context/opencensus/common/__init__.py new file mode 100644 index 000000000..69e3be50d --- /dev/null +++ b/context/opencensus-context/opencensus/common/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/context/opencensus-context/opencensus/common/runtime_context/__init__.py b/context/opencensus-context/opencensus/common/runtime_context/__init__.py new file mode 100644 index 000000000..ba2844e89 --- /dev/null +++ b/context/opencensus-context/opencensus/common/runtime_context/__init__.py @@ -0,0 +1,174 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import threading + +__all__ = ['RuntimeContext'] + + +class _RuntimeContext(object): + @classmethod + def clear(cls): + """Clear all slots to their default value.""" + + raise NotImplementedError # pragma: NO COVER + + @classmethod + def register_slot(cls, name, default=None): + """Register a context slot with an optional default value. + + :type name: str + :param name: The name of the context slot. + + :type default: object + :param name: The default value of the slot, can be a value or lambda. + + :returns: The registered slot. + """ + + raise NotImplementedError # pragma: NO COVER + + def apply(self, snapshot): + """Set the current context from a given snapshot dictionary""" + + for name in snapshot: + setattr(self, name, snapshot[name]) + + def snapshot(self): + """Return a dictionary of current slots by reference.""" + + return dict((n, self._slots[n].get()) for n in self._slots.keys()) + + def __repr__(self): + return ('{}({})'.format(type(self).__name__, self.snapshot())) + + def __getattr__(self, name): + if name not in self._slots: + raise AttributeError('{} is not a registered context slot' + .format(name)) + slot = self._slots[name] + return slot.get() + + def __setattr__(self, name, value): + if name not in self._slots: + raise AttributeError('{} is not a registered context slot' + .format(name)) + slot = self._slots[name] + slot.set(value) + + def with_current_context(self, func): + """Capture the current context and apply it to the provided func""" + + caller_context = self.snapshot() + + def call_with_current_context(*args, **kwargs): + try: + backup_context = self.snapshot() + self.apply(caller_context) + return func(*args, **kwargs) + finally: + self.apply(backup_context) + + return call_with_current_context + + +class _ThreadLocalRuntimeContext(_RuntimeContext): + _lock = threading.Lock() + _slots = {} + + class Slot(object): + _thread_local = threading.local() + + def __init__(self, name, default): + self.name = name + self.default = default if callable(default) else (lambda: default) + + def clear(self): + setattr(self._thread_local, self.name, self.default()) + + def get(self): + try: + return getattr(self._thread_local, self.name) + except AttributeError: + value = self.default() + self.set(value) + return value + + def set(self, value): + setattr(self._thread_local, self.name, value) + + @classmethod + def clear(cls): + with cls._lock: + for name in cls._slots: + slot = cls._slots[name] + slot.clear() + + @classmethod + def register_slot(cls, name, default=None): + with cls._lock: + if name in cls._slots: + raise ValueError('slot {} already registered'.format(name)) + slot = cls.Slot(name, default) + cls._slots[name] = slot + return slot + + +class _AsyncRuntimeContext(_RuntimeContext): + _lock = threading.Lock() + _slots = {} + + class Slot(object): + def __init__(self, name, default): + import contextvars + self.name = name + self.contextvar = contextvars.ContextVar(name) + self.default = default if callable(default) else (lambda: default) + + def clear(self): + self.contextvar.set(self.default()) + + def get(self): + try: + return self.contextvar.get() + except LookupError: + value = self.default() + self.set(value) + return value + + def set(self, value): + self.contextvar.set(value) + + @classmethod + def clear(cls): + with cls._lock: + for name in cls._slots: + slot = cls._slots[name] + slot.clear() + + @classmethod + def register_slot(cls, name, default=None): + with cls._lock: + if name in cls._slots: + raise ValueError('slot {} already registered'.format(name)) + slot = cls.Slot(name, default) + cls._slots[name] = slot + return slot + + +RuntimeContext = _ThreadLocalRuntimeContext() + +if sys.version_info >= (3, 7): + RuntimeContext = _AsyncRuntimeContext() diff --git a/context/opencensus-context/setup.cfg b/context/opencensus-context/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/context/opencensus-context/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/context/opencensus-context/setup.py b/context/opencensus-context/setup.py new file mode 100644 index 000000000..3f94b54f7 --- /dev/null +++ b/context/opencensus-context/setup.py @@ -0,0 +1,48 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from setuptools import find_packages +from setuptools import setup +from version import __version__ + +setup( + name='opencensus-context', + version=__version__, # noqa + author='OpenCensus Authors', + author_email='census-developers@googlegroups.com', + classifiers=[ + 'Intended Audience :: Developers', + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + ], + description='OpenCensus Runtime Context', + include_package_data=True, + long_description=open('README.rst').read(), + install_requires=[], + extras_require={}, + license='Apache-2.0', + packages=find_packages(exclude=('examples', 'tests',)), + namespace_packages=[], + url='https://github.com/census-instrumentation/opencensus-python/tree/master/contrib/opencensus-context', # noqa: E501 + zip_safe=False, +) diff --git a/context/opencensus-context/tests/test_runtime_context.py b/context/opencensus-context/tests/test_runtime_context.py new file mode 100644 index 000000000..7e2bc976e --- /dev/null +++ b/context/opencensus-context/tests/test_runtime_context.py @@ -0,0 +1,35 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from opencensus.common.runtime_context import RuntimeContext + + +class RuntimeContextTest(unittest.TestCase): + def test_register(self): + RuntimeContext.register_slot('foo') + self.assertIsNone(RuntimeContext.foo) + + RuntimeContext.foo = 123 + self.assertEqual(RuntimeContext.foo, 123) + + def test_register_with_default(self): + RuntimeContext.register_slot('bar', 123) + self.assertEqual(RuntimeContext.bar, 123) + + def test_register_duplicate(self): + self.assertRaises(ValueError, lambda: [ + RuntimeContext.register_slot('dup'), + RuntimeContext.register_slot('dup'), + ]) diff --git a/context/opencensus-context/version.py b/context/opencensus-context/version.py new file mode 100644 index 000000000..ff18aeb50 --- /dev/null +++ b/context/opencensus-context/version.py @@ -0,0 +1,15 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = '0.2.dev0' diff --git a/docs/requirements.txt b/docs/requirements.txt index 006c803eb..8d6fed0fc 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,6 @@ setuptools >= 36.4.0 sphinx >= 1.6.3 +context/opencensus-context contrib/opencensus-correlation . diff --git a/nox.py b/nox.py index 7a3fe29fe..254201771 100644 --- a/nox.py +++ b/nox.py @@ -19,6 +19,7 @@ def _install_dev_packages(session): + session.install('-e', 'context/opencensus-context') session.install('-e', 'contrib/opencensus-correlation') session.install('-e', '.') @@ -116,7 +117,7 @@ def lint(session): session.run( 'flake8', '--exclude=contrib/opencensus-ext-ocagent/opencensus/ext/ocagent/trace_exporter/gen/', - 'contrib/', 'opencensus/', 'tests/', 'examples/') + 'context/', 'contrib/', 'opencensus/', 'tests/', 'examples/') @nox.session diff --git a/scripts/twine_upload.sh b/scripts/twine_upload.sh index c6dab3d02..8bf486844 100755 --- a/scripts/twine_upload.sh +++ b/scripts/twine_upload.sh @@ -32,6 +32,13 @@ python3 -m pip install --upgrade twine wheel setuptools # Build the distribution. python3 setup.py bdist_wheel +for d in context/*/ ; do + pushd . + cd "$d" + python3 setup.py bdist_wheel --dist-dir "$BASEDIR/dist/" + popd +done + for d in contrib/*/ ; do pushd . cd "$d"