From b76f8d884d3ed9367c69a521e33badfb194a20ee Mon Sep 17 00:00:00 2001 From: Dean Moldovan Date: Thu, 13 Jun 2024 16:54:56 +0200 Subject: [PATCH] Provide an alternative to `embedded_python_tools.symlink_import()` Until now, we've primarily been symlinking the Python dir into the build folder. However, that has a couple of issues: 1. Occasionally the symlink goes wrong and needs to be deleted manually. This is especially problematic on Windows. 2. The `conanfile.py` syntax is surprising. Even more so with Conan v2 where it requires a manual `sys.path.append()` to work. `symlink_import()` was essentially creating a symlink from `bin/python` to `/embedded_python`. The project executable would point `PyConfig::home` to `bin/python`. This commit provides an alternative that simply writes that directory path to a file called `bin/.embedded_python.home`. The executable can read that file on startup and point `PyConfig::home` there. For now, both methods are valid. If the home file works out, we can deprecate `symlink_import()` and remove it down the line. --- changelog.md | 1 + core/conanfile.py | 1 + core/embedded_python-core.cmake | 13 +++++++++++ core/test_package/conanfile.py | 29 +++++++----------------- core/test_package/src/main.cpp | 18 ++++++++++++++- embedded_python.cmake | 16 +++++++++++--- test_package/conanfile.py | 39 ++++++++++++--------------------- 7 files changed, 67 insertions(+), 50 deletions(-) diff --git a/changelog.md b/changelog.md index 03fbe5b..ec69eb9 100644 --- a/changelog.md +++ b/changelog.md @@ -3,6 +3,7 @@ ## v1.9.1 | In development - Fixed an issue where calling CMake with `-DPython_EXECUTABLE=` created conflicts with the embedded Python (either a loud version error, or silently passing the wrong library paths). Some IDEs would pass this flag implicitly and it would hijack the `find_package(Python)` call used internally by this recipe. Now, we specifically protect against this since there should be no traces of system Python in a project that wishes to embed it. +- Provided an alternative to `embedded_python_tools.symlink_import()`. For dev builds, it's now possible to point `PyConfig::home` to the contents of `bin/.embedded_python(-core).home` to avoid needing to copy the entire Python environment into the build tree every time the project is reconfigured. ## v1.9.0 | 2024-05-03 diff --git a/core/conanfile.py b/core/conanfile.py index d13063a..71153f8 100644 --- a/core/conanfile.py +++ b/core/conanfile.py @@ -28,6 +28,7 @@ class EmbeddedPythonCore(ConanFile): "zip_stdlib": "stored", } exports_sources = "embedded_python_tools.py", "embedded_python-core.cmake" + package_type = "shared-library" def validate(self): minimum_python = "3.11.5" diff --git a/core/embedded_python-core.cmake b/core/embedded_python-core.cmake index 448ebf8..1fcaf39 100644 --- a/core/embedded_python-core.cmake +++ b/core/embedded_python-core.cmake @@ -14,3 +14,16 @@ else() endif() find_package(Python ${self.pyversion} EXACT REQUIRED GLOBAL COMPONENTS Interpreter Development) + +# For development, we want avoid copying all of Python's `lib` and `site-packages` into our +# build tree every time we re-configure the project. Instead, we can point `PyConfig::home` +# to the contents of this file to gain access to all the Python packages. +# For release/deployment, the entire `Python_ROOT_DIR` should be copied into the app's `bin` +# folder and `PyConfig::home` should point to that. +set(home_file ".embedded_python-core.home") +if(DEFINED CMAKE_RUNTIME_OUTPUT_DIRECTORY) + set(home_file ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${home_file}) +endif() +file(GENERATE OUTPUT ${home_file} CONTENT "${Python_ROOT_DIR}") + +unset(home_file) diff --git a/core/test_package/conanfile.py b/core/test_package/conanfile.py index 4872959..da5ca97 100644 --- a/core/test_package/conanfile.py +++ b/core/test_package/conanfile.py @@ -3,14 +3,14 @@ import subprocess import conan from conan import ConanFile -from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout +from conan.tools.cmake import CMake, cmake_layout # noinspection PyUnresolvedReferences class TestEmbeddedPythonCore(ConanFile): name = "test_embedded_python" settings = "os", "compiler", "build_type", "arch" - generators = "CMakeDeps", "VirtualRunEnv" + generators = "CMakeToolchain", "CMakeDeps", "VirtualRunEnv" test_type = "explicit" def layout(self): @@ -19,18 +19,7 @@ def layout(self): def requirements(self): self.requires(self.tested_reference_str) - def generate(self): - build_type = self.settings.build_type.value - tc = CMakeToolchain(self) - tc.variables[f"CMAKE_RUNTIME_OUTPUT_DIRECTORY_{build_type.upper()}"] = "bin" - tc.generate() - def build(self): - sys.path.append(str(self._core_package_path)) - - import embedded_python_tools - - embedded_python_tools.symlink_import(self, dst="bin/python") cmake = CMake(self) cmake.configure( variables={ @@ -42,13 +31,6 @@ def build(self): ) cmake.build() - @property - def _py_exe(self): - if self.settings.os == "Windows": - return pathlib.Path(self.build_folder, "bin/python/python.exe") - else: - return pathlib.Path(self.build_folder, "bin/python/bin/python3") - @property def _core_package_path(self): if conan.__version__.startswith("2"): @@ -56,6 +38,11 @@ def _core_package_path(self): else: return pathlib.Path(self.deps_cpp_info["embedded_python-core"].rootpath) + @property + def _py_exe(self): + exe = "python.exe" if sys.platform == "win32" else "python3" + return self._core_package_path / "embedded_python" / exe + def _test_stdlib(self): """Ensure that Python runs and built the optional stdlib modules""" self.run(f'{self._py_exe} -c "import sys; print(sys.version);"') @@ -78,7 +65,7 @@ def _test_libpython_path(self): def _test_embed(self): """Ensure that everything is available to compile and link to the embedded Python""" - self.run(pathlib.Path("bin", "test_package"), env="conanrun") + self.run(pathlib.Path(self.cpp.build.bindir, "test_package").absolute(), env="conanrun") def _test_licenses(self): """Ensure that the license file is included""" diff --git a/core/test_package/src/main.cpp b/core/test_package/src/main.cpp index 9bac7c2..911012e 100644 --- a/core/test_package/src/main.cpp +++ b/core/test_package/src/main.cpp @@ -1,13 +1,29 @@ #include #include +#include #include +std::string find_python_home(std::filesystem::path bin) { + const auto local_home = bin / "python"; + if (std::filesystem::exists(local_home)) { + return local_home.string(); + } + + auto home_file = bin / ".embedded_python.home"; + if (!std::filesystem::exists(home_file)) { + home_file = bin / ".embedded_python-core.home"; + } + auto stream = std::ifstream(home_file); + return std::string(std::istreambuf_iterator(stream), + std::istreambuf_iterator()); +} + int main(int argc, const char* argv[]) { auto config = PyConfig{}; PyConfig_InitIsolatedConfig(&config); const auto bin = std::filesystem::path(argv[0]).parent_path(); - const auto python_home = (bin / "python").string(); + const auto python_home = find_python_home(bin); if (auto status = PyConfig_SetBytesString(&config, &config.home, python_home.c_str()); PyStatus_Exception(status)) { PyConfig_Clear(&config); diff --git a/embedded_python.cmake b/embedded_python.cmake index 33d2104..9363289 100644 --- a/embedded_python.cmake +++ b/embedded_python.cmake @@ -9,9 +9,19 @@ # library. On top of that, `embedded_python.cmake` adds `EmbeddedPython_EXECUTABLE` which is aware # of the full environment with `pip` packages. Note that we do no provide any include or lib dirs # since those are already provided by `core`. - +set(EmbeddedPython_ROOT_DIR "${CMAKE_CURRENT_LIST_DIR}/embedded_python" CACHE STRING "" FORCE) if(WIN32) - set(EmbeddedPython_EXECUTABLE "${CMAKE_CURRENT_LIST_DIR}/embedded_python/python.exe" CACHE STRING "" FORCE) + set(EmbeddedPython_EXECUTABLE "${EmbeddedPython_ROOT_DIR}/python.exe" CACHE STRING "" FORCE) else() - set(EmbeddedPython_EXECUTABLE "${CMAKE_CURRENT_LIST_DIR}/embedded_python/python3" CACHE STRING "" FORCE) + set(EmbeddedPython_EXECUTABLE "${EmbeddedPython_ROOT_DIR}/python3" CACHE STRING "" FORCE) +endif() + +# See the matching file in `embedded_python-core`. It's up to the user to pick if they want to +# point the `-core` package (no `pip` package) or the full embedded environment. +set(home_file ".embedded_python.home") +if(DEFINED CMAKE_RUNTIME_OUTPUT_DIRECTORY) + set(home_file ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${home_file}) endif() +file(GENERATE OUTPUT ${home_file} CONTENT "${EmbeddedPython_ROOT_DIR}") + +unset(home_file) diff --git a/test_package/conanfile.py b/test_package/conanfile.py index 82d967c..4e0e19f 100644 --- a/test_package/conanfile.py +++ b/test_package/conanfile.py @@ -3,7 +3,7 @@ import subprocess import conan from conan import ConanFile -from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout +from conan.tools.cmake import CMake, cmake_layout project_root = pathlib.Path(__file__).parent @@ -17,12 +17,13 @@ def _read_env(name): class TestEmbeddedPython(ConanFile): name = "test_embedded_python" settings = "os", "compiler", "build_type", "arch" - generators = "CMakeDeps", "VirtualRunEnv" + generators = "CMakeToolchain", "CMakeDeps", "VirtualRunEnv" options = {"env": [None, "ANY"]} default_options = { "env": None, "embedded_python-core/*:version": "3.11.5", } + package_type = "shared-library" @property def _core_package_path(self): @@ -38,6 +39,11 @@ def _package_path(self): else: return pathlib.Path(self.deps_cpp_info["embedded_python"].rootpath) + @property + def _py_exe(self): + exe = "python.exe" if sys.platform == "win32" else "python3" + return self._package_path / "embedded_python" / exe + def layout(self): cmake_layout(self) @@ -48,19 +54,7 @@ def configure(self): if self.options.env: self.options["embedded_python"].packages = _read_env(self.options.env) - def generate(self): - build_type = self.settings.build_type.value - tc = CMakeToolchain(self) - tc.variables[f"CMAKE_RUNTIME_OUTPUT_DIRECTORY_{build_type.upper()}"] = "bin" - tc.generate() - def build(self): - sys.path.append(str(self._package_path)) - - import embedded_python_tools - - embedded_python_tools.symlink_import(self, dst="bin/python") - cmake = CMake(self) cmake.configure( variables={ @@ -75,22 +69,17 @@ def build(self): def _test_env(self): """Ensure that Python runs and finds the installed environment""" - if self.settings.os == "Windows": - python_exe = str(pathlib.Path("./bin/python/python").resolve()) - else: - python_exe = str(pathlib.Path("./bin/python/bin/python3").resolve()) - - self.run(f'{python_exe} -c "import sys; print(sys.version);"') - + self.run(f'{self._py_exe} -c "import sys; print(sys.version);"') name = str(self.options.env) if self.options.env else "baseline" - self.run(f"{python_exe} {project_root / name / 'test.py'}", env="conanrun") + self.run(f"{self._py_exe} {project_root / name / 'test.py'}", env="conanrun") def _test_libpython_path(self): if self.settings.os != "Macos": return - python_exe = str(pathlib.Path("./bin/python/bin/python3").resolve()) - p = subprocess.run(["otool", "-L", python_exe], check=True, text=True, capture_output=True) + p = subprocess.run( + ["otool", "-L", self._py_exe], check=True, text=True, capture_output=True + ) lines = str(p.stdout).strip().split("\n")[1:] libraries = [line.split()[0] for line in lines] candidates = [lib for lib in libraries if "libpython" in lib] @@ -101,7 +90,7 @@ def _test_libpython_path(self): def _test_embed(self): """Ensure that everything is available to compile and link to the embedded Python""" - self.run(pathlib.Path("bin", "test_package"), env="conanrun") + self.run(pathlib.Path(self.cpp.build.bindir, "test_package").absolute(), env="conanrun") def _test_licenses(self): """Ensure that the licenses have been gathered"""