diff --git a/.github/conan_dockerfile/Dockerfile b/.github/conan_dockerfile/Dockerfile index 16c15cdd..c4393f77 100644 --- a/.github/conan_dockerfile/Dockerfile +++ b/.github/conan_dockerfile/Dockerfile @@ -11,10 +11,11 @@ RUN sudo -E apt-get update \ libncursesw5-dev xz-utils libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev # setup python environment -ARG PY_VERSION=3.8 -RUN pyenv install -s $PY_VERSION \ +ARG PY_VERSION=3.12 +RUN pyenv update \ + && pyenv install -s $PY_VERSION \ && pyenv global $PY_VERSION -RUN pip install "conan==1.59.0" catkin_pkg "numpy<2.0" wheel auditwheel cmake +RUN pip install "conan~=2.8.0" catkin_pkg "numpy<2.0" wheel auditwheel cmake # install patchelf RUN wget https://github.com/NixOS/patchelf/releases/download/0.17.2/patchelf-0.17.2-x86_64.tar.gz \ @@ -22,10 +23,6 @@ RUN wget https://github.com/NixOS/patchelf/releases/download/0.17.2/patchelf-0.1 && sudo -E ln -s $HOME/bin/patchelf /bin/patchelf \ && patchelf --version -# setup conan for python dependencies via bincrafters remote -RUN conan remote add bincrafters https://bincrafters.jfrog.io/artifactory/api/conan/public-conan \ - && conan config set general.revisions_enabled=1 - FROM lanelet2_conan_deps as lanelet2_conan_src @@ -38,28 +35,18 @@ FROM lanelet2_conan_src as lanelet2_conan # compile ARG CONAN_ARGS="" +ARG PLATFORM="manylinux_2_31_x86_64" WORKDIR /home/conan/src/lanelet2 -RUN conan create . lanelet2/stable --build=missing ${CONAN_ARGS} \ - && export LANELET2_VERSION=$(conan inspect . --raw version) \ - && echo "Lanelet2 version: $LANELET2_VERSION" \ - && conan install lanelet2/$LANELET2_VERSION@lanelet2/stable --build=missing -g virtualenv +RUN conan profile detect \ + && conan create . --format=json --build=missing -o "&:build_wheel=True" -o "&:platform=${PLATFORM}" ${CONAN_ARGS} > conaninfo.json + +# obtain wheel FROM lanelet2_conan as lanelet2_conan_with_pip_wheel SHELL ["/bin/bash", "-c"] WORKDIR /home/conan -RUN source /home/conan/src/lanelet2/activate.sh \ - && LANELET2_PACKAGE_DIR=$(python -c "import lanelet2; from pathlib import Path; print(Path(lanelet2.__file__).parent)") \ - && cp -r $LANELET2_PACKAGE_DIR . -RUN export LANELET2_VERSION=$(conan inspect ./src/lanelet2 --raw version) \ - && echo "Lanelet2 version: $LANELET2_VERSION" \ - && sed 's/{{ version }}/'"$LANELET2_VERSION"'/' /home/conan/src/lanelet2/lanelet2_python/setup.py.template > /home/conan/setup.py - -ARG PLATFORM="manylinux_2_31_x86_64" -RUN source /home/conan/src/lanelet2/activate.sh \ - && pip wheel -w broken-dist/ . \ - && auditwheel repair -w dist/ --plat ${PLATFORM} broken-dist/*.whl +RUN LANELET2_PACKAGE_DIR=$(python3 -c "import json; f=open('src/lanelet2/conaninfo.json'); data=json.load(f); ll2=next(d for d in data['graph']['nodes'].values() if 'lanelet2' in d['ref']); print(f'{ll2[\"package_folder\"]}/wheel')") \ + && cp -r $LANELET2_PACKAGE_DIR dist # to extract the wheel manually run: -# $ docker run -it -v /path/to/some/local/folder:/dist /bin/bash -# then inside the container: -# $ sudo cp dist/lanelet2-<...>.whl /dist/. +# $ docker run --rm -v /path/to/some/local/folder:/dist /bin/bash -c 'sudo cp dist/lanelet2-<...>.whl /dist/.' \ No newline at end of file diff --git a/.github/workflows/cd.yaml b/.github/workflows/cd.yaml index e8665b84..027d6128 100644 --- a/.github/workflows/cd.yaml +++ b/.github/workflows/cd.yaml @@ -19,19 +19,18 @@ jobs: python-version: ["3.8", "3.9", "3.10", "3.11"] include: - python-version: "3.8" - boost-version: "1.75.0" platform-version: manylinux_2_27_x86_64 image-version: conanio/gcc7:latest - python-version: "3.9" - boost-version: "1.75.0" platform-version: manylinux_2_27_x86_64 image-version: conanio/gcc7:latest - python-version: "3.10" - boost-version: "1.81.0" platform-version: manylinux_2_31_x86_64 image-version: conanio/gcc10:latest - python-version: "3.11" - boost-version: "1.81.0" + platform-version: manylinux_2_31_x86_64 + image-version: conanio/gcc10:latest + - python-version: "3.12" platform-version: manylinux_2_31_x86_64 image-version: conanio/gcc10:latest runs-on: ubuntu-latest @@ -52,7 +51,6 @@ jobs: FROM=${{ matrix.image-version }} PY_VERSION=${{ matrix.python-version }} PLATFORM=${{ matrix.platform-version }} - CONAN_ARGS=--require-override=boost/${{ matrix.boost-version }} - name: Create output directory run: mkdir -p ${{ github.workspace }}/output @@ -82,8 +80,8 @@ jobs: strategy: matrix: # test only on currently supported version - python-version: ["3.8", "3.9", "3.10", "3.11"] - os: ["ubuntu-22.04", "ubuntu-20.04"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + os: ["ubuntu-20.04", "ubuntu-22.04", "ubuntu-24.04"] runs-on: ${{ matrix.os }} steps: - name: Restore wheel @@ -95,8 +93,13 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + - name: Install wheel 24.04 + # use dist/ directory as package source instead of pypi.org + if: ${{ matrix.os == 'ubuntu-24.04' }} + run: pip install lanelet2 --break-system-packages --no-index --find-links dist/ - name: Install wheel # use dist/ directory as package source instead of pypi.org + if: ${{ matrix.os != 'ubuntu-24.04' }} run: pip install lanelet2 --no-index --find-links dist/ - name: Test wheel run: python -c "import lanelet2; assert lanelet2.core.Point2d(0, 0, 0) is not None" @@ -124,7 +127,7 @@ jobs: password: ${{ secrets.PYPI_API_TOKEN }} deploy-pages: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 if: github.ref == 'refs/heads/master' concurrency: deploy-${{ github.ref }} steps: diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index be918adb..46567d0e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -187,19 +187,18 @@ jobs: python-version: ["3.8", "3.9", "3.10", "3.11"] include: - python-version: "3.8" - boost-version: "1.75.0" platform-version: manylinux_2_27_x86_64 image-version: conanio/gcc7:latest - python-version: "3.9" - boost-version: "1.75.0" platform-version: manylinux_2_27_x86_64 image-version: conanio/gcc7:latest - python-version: "3.10" - boost-version: "1.81.0" platform-version: manylinux_2_31_x86_64 image-version: conanio/gcc10:latest - python-version: "3.11" - boost-version: "1.81.0" + platform-version: manylinux_2_31_x86_64 + image-version: conanio/gcc10:latest + - python-version: "3.12" platform-version: manylinux_2_31_x86_64 image-version: conanio/gcc10:latest runs-on: ubuntu-latest @@ -220,4 +219,3 @@ jobs: FROM=${{ matrix.image-version }} PY_VERSION=${{ matrix.python-version }} PLATFORM=${{ matrix.platform-version }} - CONAN_ARGS=--require-override=boost/${{ matrix.boost-version }} diff --git a/README.md b/README.md index a7049829..92604f6b 100644 --- a/README.md +++ b/README.md @@ -136,26 +136,23 @@ catkin build If unsure, see the [Dockerfile](Dockerfile) or the [travis build log](https://travis-ci.org/fzi-forschungszentrum-informatik/Lanelet2). It shows the full installation process, with subsequent build and test based on a docker image with a clean Ubuntu installation. ### Manual, experimental installation using conan +**Note: Updated instructions for conan2!** For non-catkin users, we also offer a conan based install process. Its experimental and might not work on all platforms, especially Windows. Since conan handles installing all C++ dependencies, all you need is a cloned repository, conan itself and a few python dependencies: ```bash pip install conan catkin_pkg numpy -conan remote add bincrafters https://bincrafters.jfrog.io/artifactory/api/conan/public-conan # required for python bindings -conan config set general.revisions_enabled=1 # requried to use bincrafters remote git clone https://github.com/fzi-forschungszentrum-informatik/lanelet2.git cd lanelet2 ``` From here, just use the default conan build/install procedure, e.g.: ```bash -conan source . -conan create . lanelet2/stable --build=missing +conan create . --build=missing ``` Different from the conan defaults, we build lanelet2 and boost as shared libraries, because otherwise the lanelet2's plugin mechanisms as well as boost::python will fail. E.g. loading maps will not be possible and the python API will not be usable. -To be able to use the python bindings, you have to make conan export the PYTHONPATH for lanelet2: +To be able to use the python bindings, you have to make conan export the PYTHONPATH for lanelet2 after `conan create`: ```bash -conan install lanelet2/0.0.0@lanelet2/stable --build=missing -g virtualenv # replace 0.0.0 with the version shown by conan source activate.sh python -c "import lanelet2" # or whatever you want to do source deactivate.sh diff --git a/conanfile.py b/conanfile.py index 0160043b..edf3ec92 100644 --- a/conanfile.py +++ b/conanfile.py @@ -1,30 +1,52 @@ import os import sys import xml.etree.ElementTree as ET -from conans import ConanFile, CMake, tools from distutils.sysconfig import get_python_lib +from io import StringIO +from pathlib import Path -find_mrt_cmake=""" +from conan import ConanFile +from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout +from conan.tools.files import copy + +find_mrt_cmake = """ set(mrt_cmake_modules_FOUND True) include(${CMAKE_CURRENT_LIST_DIR}/mrt_cmake_modules-extras.cmake) """ -cmake_lists=""" +cmake_lists = """ cmake_minimum_required(VERSION 3.5) project(lanelet2) if(POLICY CMP0079) cmake_policy(SET CMP0079 NEW) # allows to do target_link_libraries on targets from subdirs endif() set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}) -set(BoostPython_FOUND Yes) -include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake) -conan_basic_setup(SKIP_STD) +# mrt_cmake_modules inofficially support conan, but only conan1. So we have to set these: +set(CONAN_PACKAGE_NAME lanelet2) +macro(conan_define_targets) + add_library(${PROJECT_NAME}_conan_deps INTERFACE) + target_include_directories(${PROJECT_NAME}_conan_deps INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/include) + set(CONAN_TARGETS ${PROJECT_NAME}_conan_deps) +endmacro() # hint to gtest set(GOOGLETEST_VERSION 1.0.0) set(MRT_GTEST_DIR ${CMAKE_CURRENT_LIST_DIR}) enable_testing() +# find thirdparty +find_package(mrt_cmake_modules REQUIRED) +find_package(Boost REQUIRED) +find_package(BoostPython REQUIRED) +find_package(GeographicLib REQUIRED) +find_package(Eigen3 REQUIRED) +find_package(pugixml REQUIRED) +add_library(conan INTERFACE IMPORTED) +set_target_properties( + conan + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${BoostPython_INCLUDE_DIRS}") +target_link_libraries(conan INTERFACE ${BoostPython_LIBRARIES} boost::boost pugixml::pugixml GeographicLib::GeographicLib Eigen3::Eigen) + # declare dependencies include_directories(lanelet2_core/include lanelet2_io/include lanelet2_projection/include lanelet2_traffic_rules/include lanelet2_routing/include lanelet2_validation/include) @@ -39,6 +61,7 @@ add_subdirectory(lanelet2_maps) add_subdirectory(lanelet2_matching) # declare dependencies +target_link_libraries(lanelet2_core PUBLIC conan) target_link_libraries(lanelet2_io PUBLIC lanelet2_core) target_link_libraries(lanelet2_projection PUBLIC lanelet2_core) target_link_libraries(lanelet2_traffic_rules PUBLIC lanelet2_core) @@ -49,30 +72,46 @@ target_link_libraries(lanelet2_python_compiler_flags INTERFACE lanelet2_core lanelet2_io lanelet2_routing lanelet2_traffic_rules lanelet2_projection lanelet2_matching) """ + def read_version(): - package = ET.parse('lanelet2_core/package.xml') - return package.find('version').text + package = ET.parse("lanelet2_core/package.xml") + return package.find("version").text + def get_py_version(): return "{}.{}".format(sys.version_info.major, sys.version_info.minor) -def get_py_exec(): - return "python3" if sys.version_info.major == 3 else "python" class Lanelet2Conan(ConanFile): name = "lanelet2" version = read_version() settings = "os", "compiler", "build_type", "arch" - generators = "cmake" license = "BSD" url = "https://github.com/fzi-forschungszentrum-informatik/lanelet2" description = "Map handling framework for automated driving" - options = {"shared": [True, False], "fPIC": [True]} - default_options = {"shared": True, "fPIC": True, "boost:shared": True, "boost:python_version": get_py_version(), "boost:without_python": False, "python_dev_config:python": get_py_exec()} + options = { + "shared": [True, False], + "fPIC": [True], + "python": ["ANY"], + "build_wheel": [True, False], + "platform": ["ANY"], + } + generators = "CMakeDeps", "VirtualRunEnv" + default_options = { + "shared": True, + "fPIC": True, + "build_wheel": False, + "python": sys.executable, + "platform": "manylinux_2_31_x86_64", + "boost/*:shared": True, + "boost/*:python_version": get_py_version(), + "boost/*:without_python": False, + } + + virtualrunenv = True requires = ( - "python_dev_config/0.6@bincrafters/stable", - "boost/[>=1.75.0 <=1.81.0]", + "boost/1.81.0" if sys.version_info.minor > 9 else "boost/1.75.0", "eigen/3.4.0", "geographiclib/1.52", "pugixml/1.13", @@ -82,31 +121,63 @@ class Lanelet2Conan(ConanFile): exports = "lanelet2_core/package.xml" proj_list = [ - 'lanelet2_core', - 'lanelet2_io', - 'lanelet2_matching', - 'lanelet2_projection', - 'lanelet2_traffic_rules', - 'lanelet2_routing', - 'lanelet2_validation' + "lanelet2_core", + "lanelet2_io", + "lanelet2_matching", + "lanelet2_projection", + "lanelet2_traffic_rules", + "lanelet2_routing", + "lanelet2_validation", ] + def layout(self): + cmake_layout(self) + + def generate(self): + # This generates "conan_toolchain.cmake" in self.generators_folder + tc = CMakeToolchain(self) + output = StringIO() + py_exec = str(self.options.python) + output = StringIO() + self.run( + "{0} -c \"from sys import *; print('%d.%d' % (version_info[0],version_info[1]))\"".format( + py_exec + ), + stdout=output, + ) + py_version = output.getvalue().strip() + tc.variables["PYTHON_VERSION"] = py_version + tc.variables["PYTHON_EXECUTABLE"] = py_exec + tc.variables["MRT_CMAKE_ENV"] = "sh env PYTHONPATH=" + py_exec + tc.generate() + self._set_env(self.runenv) + def _configure_cmake(self): cmake = CMake(self) - cmake.definitions["PYTHON_VERSION"] = get_py_version() - cmake.definitions["MRT_CMAKE_ENV"] = "sh;env;PYTHONPATH=" + os.path.join(self.package_folder, self._pythonpath()) - cmake.configure() + mrt_env = "sh;env;PYTHONPATH=" + os.path.join( + self.package_folder, self._pythonpath() + ) + cmake.configure( + variables={"PYTHON_VERSION": get_py_version(), "MRT_CMAKE_ENV": mrt_env} + ) return cmake def _pythonpath(self): - return os.path.relpath(get_python_lib(prefix=self.package_folder), start=self.package_folder) + return os.path.relpath( + get_python_lib(prefix=self.package_folder), start=self.package_folder + ) def source(self): if not os.path.exists("mrt_cmake_modules"): self.run("git clone https://github.com/KIT-MRT/mrt_cmake_modules.git") mrt_cmake_dir = os.path.join(os.getcwd(), "mrt_cmake_modules") with open("mrt_cmake_modules/cmake/mrt_cmake_modules-extras.cmake.in") as f: - extras = f.read().replace("@DEVELSPACE@", "True").replace("@PROJECT_SOURCE_DIR@", mrt_cmake_dir).replace("@CMAKE_CURRENT_SOURCE_DIR@", mrt_cmake_dir) + extras = ( + f.read() + .replace("@DEVELSPACE@", "True") + .replace("@PROJECT_SOURCE_DIR@", mrt_cmake_dir) + .replace("@CMAKE_CURRENT_SOURCE_DIR@", mrt_cmake_dir) + ) with open("mrt_cmake_modules-extras.cmake", "w") as f: f.write(extras) with open("Findmrt_cmake_modules.cmake", "w") as f: @@ -117,22 +188,52 @@ def source(self): self.run("git clone https://github.com/google/googletest.git") def build(self): - cmake = self._configure_cmake() + cmake = CMake(self) + cmake.configure() cmake.build() - cmake.test() # not working as long as the pythonpath is not adapted first - cmake.install() + cmake.test( + cli_args=["-v"] + ) # not working as long as the pythonpath is not adapted first + if self.options.build_wheel: + with ( + Path(self.source_folder) / "lanelet2_python" / "setup.py.template" + ).open() as f: + setup_template = f.read() + setup_py = setup_template.replace("{{ version }}", self.version) + with (Path(self.build_folder) / "setup.py").open("w") as f: + f.write(setup_py) def package(self): - cmake = self._configure_cmake() + cmake = CMake(self) cmake.install() + if self.options.build_wheel: + whl_tmp = os.path.join(self.package_folder, "wheel-incomplete") + whl_out = os.path.join(self.package_folder, "wheel") + copy(self, "setup.py", self.build_folder, whl_tmp) + copy( + self, + "*", + os.path.join(self.package_folder, self._pythonpath()), + whl_tmp, + ) + self.run(f"pip wheel -w {whl_tmp} {whl_tmp}") + self.run( + f"export LD_LIBRARY_PATH={os.path.join(self.package_folder, self.cpp_info.libdir)}:$LD_LIBRARY_PATH && auditwheel repair -w {whl_out} --plat {self.options.platform} {whl_tmp}/*.whl", + scope="conanrun", + ) + + def _set_env(self, env_info): + if not self.package_folder: + return + execs = ("lanelet2_examples", "lanelet2_validation", "lanelet2_python") + env_info.define_path( + "PYTHONPATH", os.path.join(self.package_folder, self._pythonpath()) + ) + for libname in execs: + env_info.append_path( + "PATH", os.path.join(self.package_folder, "lib", libname) + ) def package_info(self): self.cpp_info.libs = list(reversed(self.proj_list)) - libpath = os.path.join(self.package_folder, "lib") - boost_libpaths = self.deps_cpp_info["boost"].lib_paths - execs = ("lanelet2_examples", "lanelet2_validation", "lanelet2_python") - binpaths = [os.path.join(self.package_folder, "lib", libname) for libname in execs] - self.env_info.PYTHONPATH.append(os.path.join(self.package_folder, self._pythonpath())) - self.env_info.LD_LIBRARY_PATH += [libpath] + boost_libpaths - self.env_info.DYLD_LIBRARY_PATH += [libpath] + boost_libpaths - self.env_info.PATH += binpaths + self._set_env(self.runenv_info) diff --git a/lanelet2_python/setup.py.template b/lanelet2_python/setup.py.template index d5d2b7e7..57effa9c 100644 --- a/lanelet2_python/setup.py.template +++ b/lanelet2_python/setup.py.template @@ -11,22 +11,44 @@ DESCRIPTION = "Map handling framework for automated driving." MAINTAINER = "Fabian Immel" MAINTAINER_EMAIL = "fabian.immel@kit.edu" URL = "https://github.com/fzi-forschungszentrum-informatik/Lanelet2" -LICENSE = "BSD" DOWNLOAD_URL = "https://github.com/fzi-forschungszentrum-informatik/Lanelet2/releases/tag/{{ version }}" VERSION = "{{ version }}" -this_directory = Path(__file__).parent -long_description = (this_directory /"src" / "lanelet2" / "README.md").read_text() +long_description = """ +Lanelet2 is a C++-based framework for handling map data in the context of automated driving. It is designed to utilize high-definition map data in order to efficiently handle the challenges posed to a vehicle in complex traffic scenarios. Flexibility and extensibility are some of the core principles to handle the upcoming challenges of future maps. + +Features: +- **2D and 3D** support +- **Consistent modification**: if one point is modified, all owning objects see the change +- Supports **lane changes**, routing through areas, etc. +- **Separated routing** for pedestrians, vehicles, bikes, etc. +- Many **customization points** to add new traffic rules, routing costs, parsers, etc. +- **Simple convenience functions** for common tasks when handling maps +- **Accurate Projection** between the lat/lon geographic world and local metric coordinates +- **IO Interface** for reading and writing e.g. *osm* data formats (this does not mean it can deal with *osm maps*) +- **Python** bindings for the whole C++ interface +- **Boost Geometry** support for all thinkable kinds of geometry calculations on map primitives +- Released under the **BSD 3-Clause license** +""" + +classifiers = [ + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: BSD License", + "Operating System :: Unix", + "Programming Language :: Python :: 3", + "Programming Language :: C++", +] + class ExtModules(list): def __bool__(self): return True setup(name=DISTNAME, + classifiers=classifiers, description=DESCRIPTION, maintainer=MAINTAINER, maintainer_email=MAINTAINER_EMAIL, url=URL, - license=LICENSE, download_url=DOWNLOAD_URL, version=VERSION, packages=["lanelet2"],