From 2869134877a661630f76cc03fe919500f5c98542 Mon Sep 17 00:00:00 2001 From: Richard Connon Date: Sun, 8 Oct 2023 01:00:52 +0100 Subject: [PATCH 1/7] Move dlopenflags handling into new library loader The C library is no longer dynamically loaded directly when `_lupa` is imported, so we need to move the special handling of dlopenflags into the `_import_newest_lib()` function in order to ensure they are set when the lupa C extension is `dlopen`'d. We also expose a context manager `eager_global_linking()` for users with special needs to get the same behaviour. --- lupa/__init__.py | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/lupa/__init__.py b/lupa/__init__.py index c9d868bc..783acb26 100644 --- a/lupa/__init__.py +++ b/lupa/__init__.py @@ -1,11 +1,15 @@ from __future__ import absolute_import +import sys + +from contextlib import contextmanager + +# Find the implementation with the latest Lua version available. +_newest_lib = None -# We need to enable global symbol visibility for lupa in order to -# support binary module loading in Lua. If we can enable it here, we -# do it temporarily. -def _try_import_with_global_library_symbols(): +@contextmanager +def eager_global_linking(): try: from os import RTLD_NOW, RTLD_GLOBAL except ImportError: @@ -14,23 +18,13 @@ def _try_import_with_global_library_symbols(): import sys old_flags = sys.getdlopenflags() + try: sys.setdlopenflags(dlopen_flags) - import lupa._lupa + yield finally: sys.setdlopenflags(old_flags) -try: - _try_import_with_global_library_symbols() -except: - pass - -del _try_import_with_global_library_symbols - - -# Find the implementation with the latest Lua version available. -_newest_lib = None - def _import_newest_lib(): global _newest_lib @@ -52,7 +46,12 @@ def _import_newest_lib(): raise RuntimeError("Failed to import Lupa binary module.") # prefer Lua over LuaJIT and high versions over low versions. module_name = max(modules, key=lambda m: (m[1] == 'lua', tuple(map(int, m[2] or '0')))) - _newest_lib = __import__(module_name[0], level=1, fromlist="*", globals=globals()) + + # We need to enable global symbol visibility for lupa in order to + # support binary module loading in Lua. If we can enable it here, we + # do it temporarily. + with eager_global_linking(): + _newest_lib = __import__(module_name[0], level=1, fromlist="*", globals=globals()) return _newest_lib @@ -81,7 +80,6 @@ def __getattr__(name): return attr -import sys if sys.version_info < (3, 7): # Module level "__getattr__" requires Py3.7 or later => import latest Lua now _import_newest_lib() From c8353f233508f8ca1cacc70142b11180a2a17ce0 Mon Sep 17 00:00:00 2001 From: scoder Date: Wed, 11 Oct 2023 09:24:10 +0200 Subject: [PATCH 2/7] Move 'sys' import down to keep it local to its usage. --- lupa/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lupa/__init__.py b/lupa/__init__.py index 783acb26..c3894094 100644 --- a/lupa/__init__.py +++ b/lupa/__init__.py @@ -1,7 +1,5 @@ from __future__ import absolute_import -import sys - from contextlib import contextmanager # Find the implementation with the latest Lua version available. @@ -80,6 +78,7 @@ def __getattr__(name): return attr +import sys if sys.version_info < (3, 7): # Module level "__getattr__" requires Py3.7 or later => import latest Lua now _import_newest_lib() From 92348a7b7ca258d5a2523558771d826eccbcc037 Mon Sep 17 00:00:00 2001 From: scoder Date: Wed, 11 Oct 2023 10:26:43 +0200 Subject: [PATCH 3/7] Disable dlopen-flags usage on MS-Windows This was previously handled by a plain try-except at module level. --- lupa/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lupa/__init__.py b/lupa/__init__.py index c3894094..2d140284 100644 --- a/lupa/__init__.py +++ b/lupa/__init__.py @@ -11,7 +11,13 @@ def eager_global_linking(): try: from os import RTLD_NOW, RTLD_GLOBAL except ImportError: - from DLFCN import RTLD_NOW, RTLD_GLOBAL # Py2.7 + try: + from DLFCN import RTLD_NOW, RTLD_GLOBAL # Py2.7 + except ImportError: + # MS-Windows does not have dlopen-flags. + yield + return + dlopen_flags = RTLD_NOW | RTLD_GLOBAL import sys From 50f1f0c1084b1baaf65a4c2a674c9fa99d8ab63c Mon Sep 17 00:00:00 2001 From: Richard Connon Date: Mon, 23 Oct 2023 22:07:34 +0100 Subject: [PATCH 4/7] Update lupa/__init__.py Co-authored-by: scoder --- lupa/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lupa/__init__.py b/lupa/__init__.py index 2d140284..41e091e9 100644 --- a/lupa/__init__.py +++ b/lupa/__init__.py @@ -7,7 +7,7 @@ @contextmanager -def eager_global_linking(): +def allow_lua_module_loading(): try: from os import RTLD_NOW, RTLD_GLOBAL except ImportError: From 6d1e87787407691b7ef1ad50b44d50c6f035fb91 Mon Sep 17 00:00:00 2001 From: Richard Connon Date: Wed, 1 Nov 2023 18:29:44 +0000 Subject: [PATCH 5/7] Conditionally enable module loading by default Enabling module loading by default doesn't work when there are multiple Lua modules because the symbols collide with each other when loaded with RTLD_GLOBAL. Make the default use of the dlopenflags conditional on there only being one available Lua module. --- lupa/__init__.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lupa/__init__.py b/lupa/__init__.py index 41e091e9..e2876390 100644 --- a/lupa/__init__.py +++ b/lupa/__init__.py @@ -51,10 +51,13 @@ def _import_newest_lib(): # prefer Lua over LuaJIT and high versions over low versions. module_name = max(modules, key=lambda m: (m[1] == 'lua', tuple(map(int, m[2] or '0')))) - # We need to enable global symbol visibility for lupa in order to - # support binary module loading in Lua. If we can enable it here, we - # do it temporarily. - with eager_global_linking(): + # Allowing module loading using dlopenflags by default doesn't work when there are multiple + # Lua modules because the symbols collide with each other when loaded with RTLD_GLOBAL. + # Enable this by default only if there is exactly one lua module available. + if len(modules) == 1: + with allow_lua_module_loading(): + _newest_lib = __import__(module_name[0], level=1, fromlist="*", globals=globals()) + else: _newest_lib = __import__(module_name[0], level=1, fromlist="*", globals=globals()) return _newest_lib From b082b23995a90c13fc9fdf95534cd101f5b551ce Mon Sep 17 00:00:00 2001 From: scoder Date: Thu, 2 Nov 2023 09:13:15 +0100 Subject: [PATCH 6/7] Do not allow module loading by default at all, add docstring. --- lupa/__init__.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/lupa/__init__.py b/lupa/__init__.py index e2876390..8b9edffc 100644 --- a/lupa/__init__.py +++ b/lupa/__init__.py @@ -8,6 +8,18 @@ @contextmanager def allow_lua_module_loading(): + """ + A context manager for enabling binary Lua module loading when importing Lua. + + This can only be used once within a Python runtime and must wrap the import of the + ``lupa.*`` Lua module, e.g.:: + + with lupa.allow_lua_module_loading() + from lupa import lua54 + + lua = lua54.LuaRuntime() + lua.require('cjson') + """ try: from os import RTLD_NOW, RTLD_GLOBAL except ImportError: @@ -51,15 +63,7 @@ def _import_newest_lib(): # prefer Lua over LuaJIT and high versions over low versions. module_name = max(modules, key=lambda m: (m[1] == 'lua', tuple(map(int, m[2] or '0')))) - # Allowing module loading using dlopenflags by default doesn't work when there are multiple - # Lua modules because the symbols collide with each other when loaded with RTLD_GLOBAL. - # Enable this by default only if there is exactly one lua module available. - if len(modules) == 1: - with allow_lua_module_loading(): - _newest_lib = __import__(module_name[0], level=1, fromlist="*", globals=globals()) - else: - _newest_lib = __import__(module_name[0], level=1, fromlist="*", globals=globals()) - + _newest_lib = __import__(module_name[0], level=1, fromlist="*", globals=globals()) return _newest_lib From c0f32e95ab76cdb2f317f96e61c106256d59abc4 Mon Sep 17 00:00:00 2001 From: scoder Date: Sun, 10 Dec 2023 07:31:59 +0100 Subject: [PATCH 7/7] Minor cleanups --- lupa/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lupa/__init__.py b/lupa/__init__.py index 8b9edffc..7b05b6e8 100644 --- a/lupa/__init__.py +++ b/lupa/__init__.py @@ -1,12 +1,12 @@ from __future__ import absolute_import -from contextlib import contextmanager +from contextlib import contextmanager as _contextmanager # Find the implementation with the latest Lua version available. _newest_lib = None -@contextmanager +@_contextmanager def allow_lua_module_loading(): """ A context manager for enabling binary Lua module loading when importing Lua. @@ -14,6 +14,7 @@ def allow_lua_module_loading(): This can only be used once within a Python runtime and must wrap the import of the ``lupa.*`` Lua module, e.g.:: + import lupa with lupa.allow_lua_module_loading() from lupa import lua54