From 5c284f657e4cd98bc1cf14bd76e9dadfa8080b70 Mon Sep 17 00:00:00 2001 From: Joris Vaillant Date: Tue, 21 May 2024 20:27:49 +0200 Subject: [PATCH 1/4] core: Use template instantiation to avoid to build all exposeType in the same translation unit --- CMakeLists.txt | 2 ++ src/cppad.cpp | 10 +++++++++- src/expose-type-double-row-major.cpp | 9 +++++++++ src/expose-type-double.cpp | 9 +++++++++ 4 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 src/expose-type-double-row-major.cpp create mode 100644 src/expose-type-double.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index bf08bb1..acd52ed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -67,6 +67,8 @@ SET(${PROJECT_NAME}_HEADERS SET(${PROJECT_NAME}_SOURCES src/cppad.cpp + src/expose-type-double.cpp + src/expose-type-double-row-major.cpp ) IF(BUILD_WITH_CPPAD_CODEGEN_BINDINGS) diff --git a/src/cppad.cpp b/src/cppad.cpp index a50e064..4db01a7 100644 --- a/src/cppad.cpp +++ b/src/cppad.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2021 INRIA + * Copyright 2021-2024 INRIA */ #ifdef PYCPPAD_WITH_CPPAD_CODEGEN_BINDINGS @@ -9,6 +9,14 @@ #include "pycppad/cppad.hpp" #include "pycppad/cppad-scalar.hpp" + +// Use explicit template instantiation to build these +// function in different translation unit and avoid consuming +// too much memory on Windows +typedef ::CppAD::AD ADScalar; +extern template void eigenpy::exposeType(); +extern template void eigenpy::exposeType(); + namespace pycppad { diff --git a/src/expose-type-double-row-major.cpp b/src/expose-type-double-row-major.cpp new file mode 100644 index 0000000..12793bd --- /dev/null +++ b/src/expose-type-double-row-major.cpp @@ -0,0 +1,9 @@ +/* + * Copyright 2024 INRIA + */ + +#include "pycppad/cppad.hpp" +#include "pycppad/cppad-scalar.hpp" + +typedef ::CppAD::AD ADScalar; +template void eigenpy::exposeType(); diff --git a/src/expose-type-double.cpp b/src/expose-type-double.cpp new file mode 100644 index 0000000..df30a53 --- /dev/null +++ b/src/expose-type-double.cpp @@ -0,0 +1,9 @@ +/* + * Copyright 2024 INRIA + */ + +#include "pycppad/cppad.hpp" +#include "pycppad/cppad-scalar.hpp" + +typedef ::CppAD::AD ADScalar; +template void eigenpy::exposeType(); From 474bf7ca342c809695ece4b91d9cdbd02662ffd7 Mon Sep 17 00:00:00 2001 From: Joris Vaillant Date: Tue, 21 May 2024 20:29:00 +0200 Subject: [PATCH 2/4] python: Fix Windows pyd directory and manage new python 3.8 rule to import a library on Windows --- python/CMakeLists.txt | 3 ++ python/pycppad/__init__.py | 27 +++++++++++ python/pycppad/windows_dll_manager.py | 65 +++++++++++++++++++++++++++ 3 files changed, 95 insertions(+) create mode 100644 python/pycppad/windows_dll_manager.py diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index ff9f5e8..276b4e5 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -36,6 +36,8 @@ SET_TARGET_PROPERTIES(${PYWRAP} SUFFIX ${PYTHON_EXT_SUFFIX} OUTPUT_NAME "${PYWRAP}" LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/python/${PROJECT_NAME}" + # On Windows, shared library are treated as binary + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/python/${PROJECT_NAME}" ) IF(UNIX AND NOT APPLE) @@ -47,6 +49,7 @@ INSTALL(TARGETS ${PYWRAP} DESTINATION ${${PYWRAP}_INSTALL_DIR}) # --- INSTALL SCRIPTS SET(PYTHON_FILES __init__.py + windows_dll_manager.py ) FOREACH(python ${PYTHON_FILES}) diff --git a/python/pycppad/__init__.py b/python/pycppad/__init__.py index 2a85989..7d76833 100644 --- a/python/pycppad/__init__.py +++ b/python/pycppad/__init__.py @@ -4,3 +4,30 @@ from .pycppad_pywrap import * from .pycppad_pywrap import __version__, __raw_version__ + +# On Windows, if pycppad.dll is not in the same directory than +# the .pyd, it will not be loaded. +# We first try to load pycppad, then, if it fail and we are on Windows: +# 1. We add all paths inside PYCPPAD_WINDOWS_DLL_PATH to DllDirectory +# 2. If PYCPPAD_WINDOWS_DLL_PATH we add the relative path from the +# package directory to the bin directory to DllDirectory +# This solution is inspired from: +# - https://github.com/PixarAnimationStudios/OpenUSD/pull/1511/files +# - https://stackoverflow.com/questions/65334494/python-c-extension-packaging-dll-along-with-pyd +# More resources on https://github.com/diffpy/pyobjcryst/issues/33 +try: + from .pycppad_pywrap import * + from .pycppad_pywrap import __version__, __raw_version__ +except ImportError: + import platform + + if platform.system() == "Windows": + from .windows_dll_manager import get_dll_paths, build_directory_manager + + with build_directory_manager() as dll_dir_manager: + for p in get_dll_paths(): + dll_dir_manager.add_dll_directory(p) + from .pycppad_pywrap import * + from .pycppad_pywrap import __version__, __raw_version__ + else: + raise diff --git a/python/pycppad/windows_dll_manager.py b/python/pycppad/windows_dll_manager.py new file mode 100644 index 0000000..07ef0fe --- /dev/null +++ b/python/pycppad/windows_dll_manager.py @@ -0,0 +1,65 @@ +import os +import sys +import contextlib + + +def get_dll_paths(): + pycppad_paths = os.getenv("PYCPPAD_WINDOWS_DLL_PATH") + if pycppad_paths is None: + # From https://peps.python.org/pep-0250/#implementation + # lib/python-version/site-packages/package + RELATIVE_DLL_PATH1 = "..\\..\\..\\..\\bin" + # lib/site-packages/package + RELATIVE_DLL_PATH2 = "..\\..\\..\\bin" + # For unit test + RELATIVE_DLL_PATH3 = "..\\..\\bin" + return [ + os.path.join(os.path.dirname(__file__), RELATIVE_DLL_PATH1), + os.path.join(os.path.dirname(__file__), RELATIVE_DLL_PATH2), + os.path.join(os.path.dirname(__file__), RELATIVE_DLL_PATH3), + ] + else: + return pycppad_paths.split(os.pathsep) + + +class PathManager(contextlib.AbstractContextManager): + """Restore PATH state after importing Python module""" + + def add_dll_directory(self, dll_dir: str): + os.environ["PATH"] += os.pathsep + dll_dir + + def __enter__(self): + self.old_path = os.environ["PATH"] + return self + + def __exit__(self, *exc_details): + os.environ["PATH"] = self.old_path + + +class DllDirectoryManager(contextlib.AbstractContextManager): + """Restore DllDirectory state after importing Python module""" + + def add_dll_directory(self, dll_dir: str): + # add_dll_directory can fail on relative path and non + # existing path. + # Since we don't know all the fail criterion we just ignore + # thrown exception + try: + self.dll_dirs.append(os.add_dll_directory(dll_dir)) + except OSError: + pass + + def __enter__(self): + self.dll_dirs = [] + return self + + def __exit__(self, *exc_details): + for d in self.dll_dirs: + d.close() + + +def build_directory_manager(): + if sys.version_info >= (3, 8): + return DllDirectoryManager() + else: + return PathManager() From 3b7e0a48064cfb6b6f168e441a948acc54f70338 Mon Sep 17 00:00:00 2001 From: Joris Vaillant Date: Tue, 21 May 2024 20:29:18 +0200 Subject: [PATCH 3/4] cmake: Update jrl-cmakemodules --- cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake b/cmake index 1f6b9eb..5275edf 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 1f6b9ebae6c0e16c6eedd869d459ef6a0da41151 +Subproject commit 5275edfd1a8c6d45e0f90177edad93ef15b2008f From e666894f0101d96be25b116143869def63c3a701 Mon Sep 17 00:00:00 2001 From: Joris Vaillant Date: Tue, 21 May 2024 20:29:38 +0200 Subject: [PATCH 4/4] ci: Modernize CI --- .github/workflows/ci-ubuntu-macos.yml | 39 ++++---- .github/workflows/ci-windows-clang.yml | 103 ---------------------- .github/workflows/ci-windows-v142.yml | 74 ---------------- .github/workflows/ci-windows.yml | 70 +++++++++++++++ .github/workflows/conda/conda-env-win.yml | 8 -- .github/workflows/conda/conda-env.yml | 8 +- 6 files changed, 94 insertions(+), 208 deletions(-) delete mode 100644 .github/workflows/ci-windows-clang.yml delete mode 100644 .github/workflows/ci-windows-v142.yml create mode 100644 .github/workflows/ci-windows.yml delete mode 100644 .github/workflows/conda/conda-env-win.yml diff --git a/.github/workflows/ci-ubuntu-macos.yml b/.github/workflows/ci-ubuntu-macos.yml index 8240ecd..cf13f65 100644 --- a/.github/workflows/ci-ubuntu-macos.yml +++ b/.github/workflows/ci-ubuntu-macos.yml @@ -22,42 +22,39 @@ jobs: os: ["ubuntu-latest", "macos-latest"] steps: - - uses: actions/checkout@v2 - - - name: Checkout submodules - run: | - git submodule update --init + - uses: actions/checkout@v4 + with: + submodules: recursive - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 with: activate-environment: pycppad auto-update-conda: true environment-file: .github/workflows/conda/conda-env.yml python-version: 3.8 - - - name: Install cmake and update conda - shell: bash -l {0} - run: | - conda activate pycppad - conda install cmake -c main + auto-activate-base: false - name: Build PyCppAD - shell: bash -l {0} + shell: bash -el {0} run: | - conda activate pycppad + conda list echo $CONDA_PREFIX mkdir build cd build - cmake .. -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DPYTHON_EXECUTABLE=$(which python3) -DBUILD_WITH_CPPAD_CODEGEN_BINDINGS=ON - make - export CTEST_OUTPUT_ON_FAILURE=1 - make test - make install + cmake .. \ + -G "Ninja" \ + -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \ + -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} \ + -DPYTHON_EXECUTABLE=$(which python3) \ + -DBUILD_WITH_CPPAD_CODEGEN_BINDINGS=ON + ninja -j1 + ctest --output-on-failure -C Release -V + ninja install - name: Uninstall PyCppAD - shell: bash -l {0} + shell: bash -el {0} run: | cd build - make uninstall + ninja uninstall diff --git a/.github/workflows/ci-windows-clang.yml b/.github/workflows/ci-windows-clang.yml deleted file mode 100644 index deb5ca2..0000000 --- a/.github/workflows/ci-windows-clang.yml +++ /dev/null @@ -1,103 +0,0 @@ -name: PyCppAD CI for Windows - Clang -on: - pull_request: - push: - -env: - # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) - BUILD_TYPE: Release - -jobs: - build: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: true - matrix: - name: [windows-latest-clang-cl] - - include: - - name: windows-latest-clang-cl - os: windows-2019 - compiler: clang-cl - - steps: - - uses: actions/checkout@v2 - - name: Checkout submodules - run: | - git submodule update --init - - uses: goanpeca/setup-miniconda@v1 - env: - ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true' - with: - activate-environment: pycppad - environment-file: .github/workflows/conda/conda-env-win.yml - python-version: 3.7 - - name: Install cmake and update conda - run: | - conda install cmake -c main - - name: Build PyCppAD - shell: cmd /C CALL {0} - env: - ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true' - run: | - :: unset extra Boost envs - set Boost_ROOT= - set BOOST_ROOT_1_69_0= - set BOOST_ROOT_1_72_0= - set PATH=%PATH:C:\hostedtoolcache\windows\Boost\1.72.0;=% - - call "%programfiles(x86)%\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" amd64 - - :: Install CppAD - git clone --recursive https://github.com/jcarpent/CppAD - cd CppAD - git checkout topic/windows - mkdir build - pushd build - cmake ^ - -G "Visual Studio 16 2019" -T "ClangCl" -DCMAKE_GENERATOR_PLATFORM=x64 ^ - -DCMAKE_INSTALL_PREFIX=%CONDA_PREFIX%\Library ^ - -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} ^ - .. - cmake --build ${{github.workspace}}/CppAD/build --config ${{env.BUILD_TYPE}} --target install - - cd ${{github.workspace}} - - :: Install CppADCodeGen - git clone --recursive https://github.com/joaoleal/CppADCodeGen - cd CppADCodeGen - mkdir build - pushd build - cmake ^ - -G "Visual Studio 16 2019" -T "ClangCl" -DCMAKE_GENERATOR_PLATFORM=x64 ^ - -DCMAKE_INSTALL_PREFIX=%CONDA_PREFIX%\Library ^ - -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} ^ - -DGOOGLETEST_GIT=ON ^ - .. - cmake --build ${{github.workspace}}/CppADCodeGen/build --config ${{env.BUILD_TYPE}} --target install - - cd ${{github.workspace}} - - mkdir build - pushd build - - :: Configure - cmake ^ - -G "Visual Studio 16 2019" -T "ClangCl" -DCMAKE_GENERATOR_PLATFORM=x64 ^ - -DCMAKE_INSTALL_PREFIX=%CONDA_PREFIX%\Library ^ - -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} ^ - -DPYTHON_SITELIB=%CONDA_PREFIX%\Lib\site-packages ^ - -DPYTHON_EXECUTABLE=%CONDA_PREFIX%\python.exe ^ - -DBUILD_WITH_CPPAD_CODEGEN_BINDINGS=ON ^ - .. - - :: Build - cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} --target install - - :: Testing - set PATH=%PATH%;%CONDA_PREFIX%\Lib\site-packages\pycppad - ctest --output-on-failure -C Release -V - - :: Test Python import - cd .. - python -c "import pycppad" diff --git a/.github/workflows/ci-windows-v142.yml b/.github/workflows/ci-windows-v142.yml deleted file mode 100644 index d93e549..0000000 --- a/.github/workflows/ci-windows-v142.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: PyCppAD CI for Windows - (v142) -on: - pull_request: - push: - -env: - # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) - BUILD_TYPE: Release - -jobs: - build: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: true - matrix: - name: [windows-latest] - - include: - - name: windows-latest - os: windows-2019 - - steps: - - uses: actions/checkout@v2 - - name: Checkout submodules - run: | - git submodule update --init - - uses: goanpeca/setup-miniconda@v1 - env: - ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true' - with: - activate-environment: pycppad - environment-file: .github/workflows/conda/conda-env.yml - python-version: 3.7 - - name: Install cmake and update conda - run: | - conda install cmake -c main - conda install cppadcodegen -c conda-forge - - - name: Build PyCppAD - shell: cmd /C CALL {0} - env: - ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true' - run: | - :: unset extra Boost envs - set Boost_ROOT= - set BOOST_ROOT_1_69_0= - set BOOST_ROOT_1_72_0= - set PATH=%PATH:C:\hostedtoolcache\windows\Boost\1.72.0;=% - - call "%programfiles(x86)%\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" amd64 - - :: Create build directory - mkdir build - pushd build - - :: Configure - cmake ^ - -G "Visual Studio 16 2019" -T "v142" -DCMAKE_GENERATOR_PLATFORM=x64 ^ - -DCMAKE_INSTALL_PREFIX=%CONDA_PREFIX%\Library ^ - -DPYTHON_SITELIB=%CONDA_PREFIX%\Lib\site-packages ^ - -DPYTHON_EXECUTABLE=%CONDA_PREFIX%\python.exe ^ - -DBUILD_WITH_CPPAD_CODEGEN_BINDINGS=OFF ^ - .. - - :: Build - cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} --target install - - :: Testing - set PATH=%PATH%;%CONDA_PREFIX%\Lib\site-packages\pycppad - ctest --output-on-failure -C Release -V - - :: Test Python import - cd .. - python -c "import pycppad" diff --git a/.github/workflows/ci-windows.yml b/.github/workflows/ci-windows.yml new file mode 100644 index 0000000..a011dbf --- /dev/null +++ b/.github/workflows/ci-windows.yml @@ -0,0 +1,70 @@ +name: PyCppAD CI for Windows +on: + pull_request: + push: + +env: + # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) + BUILD_TYPE: Release + +jobs: + build: + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: true + matrix: + os: [windows-2019] + compiler: + - cmd: cl + cppad_codegen: OFF + - cmd: clang-cl + cppad_codegen: ON + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: pycppad + auto-update-conda: true + environment-file: .github/workflows/conda/conda-env.yml + python-version: 3.8 + auto-activate-base: false + + - name: Build PyCppAD + shell: cmd /C CALL {0} + run: | + :: Set compiler to use. + :: This must be done here and not in env since + :: cxx-compiler activation script set this variable + :: and is called before. + set CC=${{ matrix.compiler.cmd }} + set CXX=${{ matrix.compiler.cmd }} + + :: Create build directory + mkdir build + pushd build + + :: Configure + cmake ^ + -G "Ninja" ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DCMAKE_INSTALL_PREFIX=%CONDA_PREFIX%\Library ^ + -DPYTHON_SITELIB=%CONDA_PREFIX%\Lib\site-packages ^ + -DPYTHON_EXECUTABLE=%CONDA_PREFIX%\python.exe ^ + -DBUILD_WITH_CPPAD_CODEGEN_BINDINGS=${{ matrix.compiler.cppad_codegen }} ^ + .. + + :: Build + ninja -j1 + + :: Testing + ctest --output-on-failure -C Release -V + + :: Test Python import + ninja install + cd .. + python -c "import pycppad" diff --git a/.github/workflows/conda/conda-env-win.yml b/.github/workflows/conda/conda-env-win.yml deleted file mode 100644 index d8f75b1..0000000 --- a/.github/workflows/conda/conda-env-win.yml +++ /dev/null @@ -1,8 +0,0 @@ -name: pycppad -channels: - - conda-forge - - nodefaults -dependencies: - - boost - - eigenpy - - python diff --git a/.github/workflows/conda/conda-env.yml b/.github/workflows/conda/conda-env.yml index 731fd9a..1f0cbde 100644 --- a/.github/workflows/conda/conda-env.yml +++ b/.github/workflows/conda/conda-env.yml @@ -1,10 +1,14 @@ name: pycppad channels: - conda-forge - - nodefaults dependencies: - boost - eigenpy - python - cppad - - cppadcodegen \ No newline at end of file + - cppadcodegen + - ninja + - cmake + - pkg-config + - ninja + - cxx-compiler