From ff5ddc3bb34a36aa37c7c9ad298c7655c46f42b9 Mon Sep 17 00:00:00 2001 From: offbeat Date: Wed, 12 Jun 2024 13:54:37 +0400 Subject: [PATCH 01/16] Fixed dependencies and made easier docker build --- Dockerfile | 7 ++++--- docker-compose.yaml | 16 ++++++++++++++++ opensfm/src/CMakeLists.txt | 4 ++++ requirements.txt | 5 +++-- setup.py | 37 +++++++++++++++++++++++++++++++++---- 5 files changed, 60 insertions(+), 9 deletions(-) create mode 100644 docker-compose.yaml diff --git a/Dockerfile b/Dockerfile index 75de8b3cf..cf8de36fe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,7 @@ RUN apt-get update \ COPY . /source/OpenSfM WORKDIR /source/OpenSfM - -RUN pip3 install -r requirements.txt && \ - python3 setup.py build +RUN rm -rf cmake_build +RUN pip3 install -r requirements.txt +RUN python3 setup.py build +RUN python3 setup.py bdist_wheel \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 000000000..ddbc62c1a --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,16 @@ +version: '3.9' + +services: + opensfm: + platform: linux/amd64 + build: + context: . + dockerfile: Dockerfile + args: + - TWINE_USERNAME + - TWINE_PASSWORD + image: artichoke/opensfm + environment: + - TWINE_USERNAME + - TWINE_PASSWORD + diff --git a/opensfm/src/CMakeLists.txt b/opensfm/src/CMakeLists.txt index c85aa6fb4..bd8e29799 100644 --- a/opensfm/src/CMakeLists.txt +++ b/opensfm/src/CMakeLists.txt @@ -36,6 +36,10 @@ endif() # For compiling VLFeat add_definitions(-DVL_DISABLE_AVX) +if(CMAKE_SYSTEM_PROCESSOR MATCHES "arm.*" OR CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") + # message(STATUS "Detected ARM architecture") + add_definitions( -DVL_DISABLE_SSE2 ) +endif() # Use the version of vlfeat in ./src/third_party/vlfeat add_definitions(-DINPLACE_VLFEAT) diff --git a/requirements.txt b/requirements.txt index 0d50b21ef..a084a615f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,9 +8,9 @@ networkx==2.5 numpy>=1.19 Pillow>=8.1.1 pyproj>=1.9.5.1 -pytest==3.0.7 +pytest==7.0.0 python-dateutil>=2.7 -pyyaml==5.4 +pyyaml==6.0 scipy>=1.10.0 Sphinx==4.2.0 six @@ -18,3 +18,4 @@ xmltodict==0.10.2 wheel opencv-python==4.5.1.48 ; sys_platform == "win32" opencv-python ; sys_platform == "linux" +twine diff --git a/setup.py b/setup.py index 96620d3b2..7ac91958d 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 - +import platform import multiprocessing import os import subprocess @@ -9,7 +9,8 @@ from sphinx.setup_command import BuildDoc from wheel.bdist_wheel import bdist_wheel -VERSION = (0, 5, 2) + +VERSION = (0, 5, 2, "post6") def version_str(version): @@ -35,6 +36,25 @@ def configure_c_extension(): "../opensfm/src", "-DPYTHON_EXECUTABLE=" + sys.executable, ] + + # if platform.system() == 'Darwin': + # # Проверяем архитектуру процессора + # if platform.machine() == 'arm64': + # print('Detected Apple Silicon processor') + # + # # Define the compilers + # c_compiler = os.path.abspath('/opt/homebrew/Cellar/gcc@12/12.3.0/bin/gcc-12') + # cxx_compiler = os.path.abspath('/opt/homebrew/Cellar/gcc@12/12.3.0/bin/g++-12') + # c_compiler_arg = f'-DCMAKE_C_COMPILER={c_compiler}' + # cxx_compiler_arg = f'-DCMAKE_CXX_COMPILER={cxx_compiler}' + # + # cmake_command += [ + # c_compiler_arg, + # cxx_compiler_arg, + # ] + # print(cmake_command) + # raise RuntimeError() + if sys.platform == "win32": cmake_command += [ "-DVCPKG_TARGET_TRIPLET=x64-windows", @@ -52,13 +72,20 @@ def build_c_extension(): ) else: subprocess.check_call( - ["make", "-j" + str(multiprocessing.cpu_count())], cwd="cmake_build" + ["make", "-s", "-j" + str(multiprocessing.cpu_count())], cwd="cmake_build" ) configure_c_extension() build_c_extension() +lib_folder = os.path.dirname(os.path.realpath(__file__)) +requirement_path = f"{lib_folder}/requirements.txt" +install_requires = [] # Here we'll add: ["gunicorn", "docutils>=0.3", "lxml==0.5a7"] +if os.path.isfile(requirement_path): + with open(requirement_path) as f: + install_requires = f.read().splitlines() + setuptools.setup( name="opensfm", version=version_str(VERSION), @@ -75,6 +102,7 @@ def build_c_extension(): scripts=[ "bin/opensfm_run_all", "bin/opensfm", + "bin/opensfm_main.py" ], package_data={ "opensfm": [ @@ -106,4 +134,5 @@ def build_c_extension(): "build_dir": ("setup.py", "build/doc"), } }, -) + install_requires=install_requires, +) \ No newline at end of file From 7176df43ec572bf13507a91242408026d6e111de Mon Sep 17 00:00:00 2001 From: offbeat Date: Fri, 14 Jun 2024 15:58:14 +0400 Subject: [PATCH 02/16] Added Dockerfile to workon with manylinux and added gflags submodule --- .dockerignore | 1 + .gitmodules | 3 + Dockerfile | 187 ++++++++++++++++++++++++++++----- build_wheel.sh | 38 +++++++ docker-compose.yaml | 1 - opensfm/src/third_party/gflags | 1 + requirements.txt | 4 +- setup.py | 40 ++----- 8 files changed, 217 insertions(+), 58 deletions(-) create mode 100644 build_wheel.sh create mode 160000 opensfm/src/third_party/gflags diff --git a/.dockerignore b/.dockerignore index 83f5d8131..6c93da5bb 100644 --- a/.dockerignore +++ b/.dockerignore @@ -16,6 +16,7 @@ !setup.cfg !setup.py !viewer +!build_wheel.sh # Picking exceptions doc/build diff --git a/.gitmodules b/.gitmodules index 219b530e7..dc47b6ee3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "opensfm/src/third_party/pybind11"] path = opensfm/src/third_party/pybind11 url = https://github.com/pybind/pybind11.git +[submodule "opensfm/src/third_party/gflags"] + path = opensfm/src/third_party/gflags + url = https://github.com/gflags/gflags.git diff --git a/Dockerfile b/Dockerfile index cf8de36fe..55d93cecb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,32 +1,165 @@ -FROM ubuntu:20.04 +FROM quay.io/pypa/manylinux2014_x86_64 +# FROM ubuntu:20.04 ARG DEBIAN_FRONTEND=noninteractive +ARG TWINE_USERNAME +ARG TWINE_PASSWORD +ENV TWINE_USERNAME=$TWINE_USERNAME +ENV TWINE_PASSWORD=$TWINE_PASSWORD + +#RUN apt-get update \ +# && apt-get install -y \ +# build-essential \ +# cmake \ +# git \ +# libeigen3-dev \ +# libopencv-dev \ +# libceres-dev \ +# python3-dev \ +# python3-numpy \ +# python3-opencv \ +# python3-pip \ +# python3-pyproj \ +# python3-scipy \ +# python3-yaml \ +# curl \ +# && apt-get clean \ +# && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ # Install apt-getable dependencies -RUN apt-get update \ - && apt-get install -y \ - build-essential \ - cmake \ - git \ - libeigen3-dev \ - libopencv-dev \ - libceres-dev \ - python3-dev \ - python3-numpy \ - python3-opencv \ - python3-pip \ - python3-pyproj \ - python3-scipy \ - python3-yaml \ - curl \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - - -COPY . /source/OpenSfM - -WORKDIR /source/OpenSfM +#RUN yum install -y \ +# build-essential \ +# cmake \ +# git \ +# libeigen3-dev \ +# libopencv-dev \ +# libceres-dev \ +# python3-dev \ +# python3-numpy \ +# opencv \ +# python3-pip \ +# python3-pyproj \ +# python3-scipy \ +# python3-yaml \ +# curl \ +# && yum clean all \ + +## Установим Miniconda +#RUN curl -sSL https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -o miniconda.sh && \ +# bash miniconda.sh -b -p /opt/conda && \ +# rm miniconda.sh +# +## Добавим conda в PATH +#ENV PATH=/opt/conda/bin:$PATH +# +## Установим Ceres Solver 2.1 или 2.2 через conda +#RUN conda install -c conda-forge ceres-solver=2.2 opencv +#ENV CMAKE_PREFIX_PATH="/opt/conda/pkgs/ceres-solver-2.2.0-hfae76b8_3/lib/cmake/Ceres/" + +#RUN /opt/python/cp39-cp39/bin/pip install --upgrade pip && \ +# /opt/python/cp39-cp39/bin/pip install auditwheel +# +##RUN yum install -y opencv opencv-devel \ +## ceres-solver ceres-solver-devel \ +## eigen3 \ +## glog-devel \ +## gflags-devel \ +## suitesparse-devel \ +## blas-devel \ +## lapack-devel \ +## openblas-devel +# +RUN yum install -y opencv opencv-devel \ + blas-devel \ + lapack-devel \ + metis-devel \ + tbb-devel \ + wget \ + openblas-devel \ + atlas-devel + +RUN mkdir /source +WORKDIR /source +RUN git clone https://github.com/gflags/gflags.git +WORKDIR /source/gflags/ +RUN git checkout tags/v2.2.2 +RUN mkdir /source/gflags/build/ +WORKDIR /source/gflags/build/ +RUN cmake -DCMAKE_CXX_FLAGS="-fPIC" .. +RUN make -j8 +RUN make install + +WORKDIR /source +RUN git clone https://github.com/google/glog.git +WORKDIR /source/glog +RUN git checkout tags/v0.6.0 +RUN mkdir /source/glog/build/ +WORKDIR /source/glog/build/ +RUN cmake .. +RUN make -j8 +RUN make install + +WORKDIR /source +RUN git clone https://gitlab.com/libeigen/eigen.git +WORKDIR /source/eigen/ +RUN git checkout tags/3.4.0 +RUN mkdir build +WORKDIR /source/eigen/build/ +RUN cmake .. +RUN make install + +RUN yum install -y lzip +WORKDIR /source +RUN wget https://gmplib.org/download/gmp/gmp-6.3.0.tar.lz +RUN tar -xf gmp-6.3.0.tar.lz +WORKDIR /source/gmp-6.3.0 +RUN ./configure +RUN make +RUN make install + +WORKDIR /source +RUN wget https://www.mpfr.org/mpfr-current/mpfr-4.2.1.tar.xz +RUN tar -xf mpfr-4.2.1.tar.xz +WORKDIR /source/mpfr-4.2.1 +RUN ./configure +RUN make +RUN make install + +#WORKDIR /source +#RUN git clone https://github.com/DrTimothyAldenDavis/SuiteSparse.git +#WORKDIR /source/SuiteSparse +#RUN git checkout tags/v7.7.0 +#WORKDIR /source/SuiteSparse/build/ +#RUN cmake .. -DBLAS_LIBRARIES=/usr/lib64/libopenblas.so -DLAPACK_LIBRARIES=/usr/lib64/liblapack.so +#RUN cmake --build . +#RUN cmake install + +#WORKDIR /source +#RUN git clone https://github.com/google/googletest.git +#WORKDIR /source/googletest +#RUN git checkout tags/release-1.8.1 +#RUN mkdir build +#WORKDIR /source/googletest/build +#RUN cmake .. +#RUN make -j4 +#RUN make install + +RUN yum install -y suitesparse-devel +WORKDIR /source +RUN git clone https://github.com/ceres-solver/ceres-solver.git +WORKDIR /source/ceres-solver +RUN git checkout tags/2.0.0 +RUN mkdir build +WORKDIR /source/ceres-solver/build/ +RUN cmake .. +RUN make -j8 +RUN make install + + +ENV WHEEL_DIR=/source/wheelhouse +ENV SFM_DIR=/source/OpenSfM +COPY . $SFM_DIR + +WORKDIR $SFM_DIR RUN rm -rf cmake_build -RUN pip3 install -r requirements.txt -RUN python3 setup.py build -RUN python3 setup.py bdist_wheel \ No newline at end of file +RUN sh /source/OpenSfM/build_wheel.sh \ No newline at end of file diff --git a/build_wheel.sh b/build_wheel.sh new file mode 100644 index 000000000..8371346b5 --- /dev/null +++ b/build_wheel.sh @@ -0,0 +1,38 @@ +!/bin/bash +set -e -u -x + +function repair_wheel { + wheel="$1" + if ! auditwheel show "$wheel"; then + echo "Skipping non-platform wheel $wheel" + else + auditwheel repair "$wheel" --plat "$PLAT" -w $WHEEL_DIR + fi +} + +# Compile wheels +for PYBIN in /opt/python/*/bin; do + if [[ "${PYBIN}" != *cp311* ]] && [[ "${PYBIN}" != *cp312* ]] && [[ "${PYBIN}" != *cp313* ]] && [[ "${PYBIN}" != *cp36* ]] && [[ "${PYBIN}" != *cp37* ]] && [[ "${PYBIN}" != *py37* ]] && [[ "${PYBIN}" != *cp38* ]] && [[ "${PYBIN}" != *py38* ]] && [[ "${PYBIN}" != *py39* ]] && [[ "${PYBIN}" != *py310* ]]; then + "${PYBIN}/pip" install --upgrade pip + "${PYBIN}/pip" install -r ${SFM_DIR}/requirements.txt + "${PYBIN}/pip" wheel $SFM_DIR --no-deps -w $WHEEL_DIR + fi +done + +# Bundle external shared libraries into the wheels +for whl in $WHEEL_DIR/*.whl; do + repair_wheel "$whl" +done + +# Install packages and test +for PYBIN in /opt/python/*/bin; do + if [[ "${PYBIN}" != *cp311* ]] && [[ "${PYBIN}" != *cp312* ]] && [[ "${PYBIN}" != *cp313* ]] && [[ "${PYBIN}" != *cp36* ]] && [[ "${PYBIN}" != *cp37* ]] && [[ "${PYBIN}" != *py37* ]] && [[ "${PYBIN}" != *cp38* ]] && [[ "${PYBIN}" != *py38* ]] && [[ "${PYBIN}" != *py39* ]] && [[ "${PYBIN}" != *py310* ]]; then + "${PYBIN}/pip" install opensfm --no-index -f $WHEEL_DIR +# opensfm --help + fi +done + + +# Need to update this code +/opt/python/cp39-cp39/bin/python -m twine upload --repository-url "http://pypi.artichoke-labs.ai" $WHEEL_DIR/opensfm-0.5.2.post6-cp39-cp39-linux_x86_64.whl +/opt/python/cp310-cp310/bin/python -m twine upload --repository-url "http://pypi.artichoke-labs.ai" $WHEEL_DIR/opensfm-0.5.2.post6-cp310-cp310-linux_x86_64.whl \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index ddbc62c1a..e4ba520cb 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -13,4 +13,3 @@ services: environment: - TWINE_USERNAME - TWINE_PASSWORD - diff --git a/opensfm/src/third_party/gflags b/opensfm/src/third_party/gflags new file mode 160000 index 000000000..46f73f88b --- /dev/null +++ b/opensfm/src/third_party/gflags @@ -0,0 +1 @@ +Subproject commit 46f73f88b18aee341538c0dfc22b1710a6abedef diff --git a/requirements.txt b/requirements.txt index a084a615f..59c5fc771 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,10 +12,12 @@ pytest==7.0.0 python-dateutil>=2.7 pyyaml==6.0 scipy>=1.10.0 -Sphinx==4.2.0 +sphinx==4.2.0 six xmltodict==0.10.2 wheel opencv-python==4.5.1.48 ; sys_platform == "win32" opencv-python ; sys_platform == "linux" twine +sphinx +auditwheel \ No newline at end of file diff --git a/setup.py b/setup.py index 7ac91958d..6af856f1f 100644 --- a/setup.py +++ b/setup.py @@ -1,16 +1,16 @@ #!/usr/bin/env python3 -import platform import multiprocessing import os import subprocess import sys +from pathlib import Path import setuptools from sphinx.setup_command import BuildDoc from wheel.bdist_wheel import bdist_wheel -VERSION = (0, 5, 2, "post6") +VERSION = (0, 5, 2, "post7") def version_str(version): @@ -37,24 +37,6 @@ def configure_c_extension(): "-DPYTHON_EXECUTABLE=" + sys.executable, ] - # if platform.system() == 'Darwin': - # # Проверяем архитектуру процессора - # if platform.machine() == 'arm64': - # print('Detected Apple Silicon processor') - # - # # Define the compilers - # c_compiler = os.path.abspath('/opt/homebrew/Cellar/gcc@12/12.3.0/bin/gcc-12') - # cxx_compiler = os.path.abspath('/opt/homebrew/Cellar/gcc@12/12.3.0/bin/g++-12') - # c_compiler_arg = f'-DCMAKE_C_COMPILER={c_compiler}' - # cxx_compiler_arg = f'-DCMAKE_CXX_COMPILER={cxx_compiler}' - # - # cmake_command += [ - # c_compiler_arg, - # cxx_compiler_arg, - # ] - # print(cmake_command) - # raise RuntimeError() - if sys.platform == "win32": cmake_command += [ "-DVCPKG_TARGET_TRIPLET=x64-windows", @@ -78,6 +60,8 @@ def build_c_extension(): configure_c_extension() build_c_extension() +cmake_dir = Path(__file__).parent.joinpath("cmake_build") +data_files = [p.as_posix() for p in cmake_dir.glob("*.so")] lib_folder = os.path.dirname(os.path.realpath(__file__)) requirement_path = f"{lib_folder}/requirements.txt" @@ -106,21 +90,19 @@ def build_c_extension(): ], package_data={ "opensfm": [ - "pybundle.*", - "pygeo.*", - "pygeometry.*", - "pyrobust.*", - "pyfeatures.*", - "pydense.*", - "pysfm.*", - "pyfoundation.*", - "pymap.*", "data/sensor_data.json", "data/camera_calibration.yaml", "data/bow/bow_hahog_root_uchar_10000.npz", "data/bow/bow_hahog_root_uchar_64.npz", ] }, + # ext_modules=[ + # setuptools.Extension('pybundle', ['source/OpenSfM/opensfm/src/pybundle.cpython-310-x86_64-linux-gnu.so']), + # setuptools.Extension('pydense', ['source/OpenSfM/opensfm/src/pydense.cpython-310-x86_64-linux-gnu.so']), + # setuptools.Extension('pyfeatures', ['source/OpenSfM/opensfm/src/pyfeatures.cpython-310-x86_64-linux-gnu.so']), + # # Add all your shared libraries here + # ], + data_files=[('lib', data_files)], cmdclass={ "bdist_wheel": platform_bdist_wheel, "build_doc": BuildDoc, From 83b998d12f0dbd6a0235e6f98304a9f4c53ffeae Mon Sep 17 00:00:00 2001 From: offbeat Date: Thu, 20 Jun 2024 13:45:59 +0400 Subject: [PATCH 03/16] Added building image --- .dockerignore | 1 + .gitmodules | 9 +++++++ Dockerfile | 36 +++++++++++++++------------- build_wheel.sh | 21 ++++------------ opensfm/src/third_party/ceres-solver | 1 + opensfm/src/third_party/eigen | 1 + opensfm/src/third_party/gflags | 2 +- opensfm/src/third_party/glog | 1 + requirements.txt | 2 +- setup.py | 16 ++++--------- test_and_upload_wheel.sh | 15 ++++++++++++ 11 files changed, 58 insertions(+), 47 deletions(-) create mode 160000 opensfm/src/third_party/ceres-solver create mode 160000 opensfm/src/third_party/eigen create mode 160000 opensfm/src/third_party/glog create mode 100644 test_and_upload_wheel.sh diff --git a/.dockerignore b/.dockerignore index 6c93da5bb..0d83bade3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -17,6 +17,7 @@ !setup.py !viewer !build_wheel.sh +!test_and_upload_wheel.sh # Picking exceptions doc/build diff --git a/.gitmodules b/.gitmodules index dc47b6ee3..7d34e2b6a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,12 @@ [submodule "opensfm/src/third_party/gflags"] path = opensfm/src/third_party/gflags url = https://github.com/gflags/gflags.git +[submodule "opensfm/src/third_party/glog"] + path = opensfm/src/third_party/glog + url = https://github.com/google/glog.git +[submodule "opensfm/src/third_party/eigen"] + path = opensfm/src/third_party/eigen + url = https://gitlab.com/libeigen/eigen.git +[submodule "opensfm/src/third_party/ceres-solver"] + path = opensfm/src/third_party/ceres-solver + url = https://github.com/ceres-solver/ceres-solver.git diff --git a/Dockerfile b/Dockerfile index 55d93cecb..89af995cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ FROM quay.io/pypa/manylinux2014_x86_64 # FROM ubuntu:20.04 +ENV PLAT=manylinux_2_17_x86_64 ARG DEBIAN_FRONTEND=noninteractive ARG TWINE_USERNAME ARG TWINE_PASSWORD @@ -108,22 +109,22 @@ WORKDIR /source/eigen/build/ RUN cmake .. RUN make install -RUN yum install -y lzip -WORKDIR /source -RUN wget https://gmplib.org/download/gmp/gmp-6.3.0.tar.lz -RUN tar -xf gmp-6.3.0.tar.lz -WORKDIR /source/gmp-6.3.0 -RUN ./configure -RUN make -RUN make install - -WORKDIR /source -RUN wget https://www.mpfr.org/mpfr-current/mpfr-4.2.1.tar.xz -RUN tar -xf mpfr-4.2.1.tar.xz -WORKDIR /source/mpfr-4.2.1 -RUN ./configure -RUN make -RUN make install +#RUN yum install -y lzip +#WORKDIR /source +#RUN wget https://gmplib.org/download/gmp/gmp-6.3.0.tar.lz +#RUN tar -xf gmp-6.3.0.tar.lz +#WORKDIR /source/gmp-6.3.0 +#RUN ./configure +#RUN make +#RUN make install +# +#WORKDIR /source +#RUN wget https://www.mpfr.org/mpfr-current/mpfr-4.2.1.tar.xz +#RUN tar -xf mpfr-4.2.1.tar.xz +#WORKDIR /source/mpfr-4.2.1 +#RUN ./configure +#RUN make +#RUN make install #WORKDIR /source #RUN git clone https://github.com/DrTimothyAldenDavis/SuiteSparse.git @@ -162,4 +163,5 @@ COPY . $SFM_DIR WORKDIR $SFM_DIR RUN rm -rf cmake_build -RUN sh /source/OpenSfM/build_wheel.sh \ No newline at end of file +RUN sh /source/OpenSfM/build_wheel.sh +# RUN sh /source/OpenSfM/test_and_upload_wheel.sh diff --git a/build_wheel.sh b/build_wheel.sh index 8371346b5..c9832ea6b 100644 --- a/build_wheel.sh +++ b/build_wheel.sh @@ -3,11 +3,7 @@ set -e -u -x function repair_wheel { wheel="$1" - if ! auditwheel show "$wheel"; then - echo "Skipping non-platform wheel $wheel" - else - auditwheel repair "$wheel" --plat "$PLAT" -w $WHEEL_DIR - fi + /opt/_internal/cpython-3.9.19/bin/auditwheel repair "$wheel" --plat "$PLAT" -w $WHEEL_DIR } # Compile wheels @@ -15,6 +11,8 @@ for PYBIN in /opt/python/*/bin; do if [[ "${PYBIN}" != *cp311* ]] && [[ "${PYBIN}" != *cp312* ]] && [[ "${PYBIN}" != *cp313* ]] && [[ "${PYBIN}" != *cp36* ]] && [[ "${PYBIN}" != *cp37* ]] && [[ "${PYBIN}" != *py37* ]] && [[ "${PYBIN}" != *cp38* ]] && [[ "${PYBIN}" != *py38* ]] && [[ "${PYBIN}" != *py39* ]] && [[ "${PYBIN}" != *py310* ]]; then "${PYBIN}/pip" install --upgrade pip "${PYBIN}/pip" install -r ${SFM_DIR}/requirements.txt + "${PYBIN}/pip" install auditwheel==2.0.0 + "${PYBIN}/pip" install auditwheel==1.4.0 "${PYBIN}/pip" wheel $SFM_DIR --no-deps -w $WHEEL_DIR fi done @@ -24,15 +22,4 @@ for whl in $WHEEL_DIR/*.whl; do repair_wheel "$whl" done -# Install packages and test -for PYBIN in /opt/python/*/bin; do - if [[ "${PYBIN}" != *cp311* ]] && [[ "${PYBIN}" != *cp312* ]] && [[ "${PYBIN}" != *cp313* ]] && [[ "${PYBIN}" != *cp36* ]] && [[ "${PYBIN}" != *cp37* ]] && [[ "${PYBIN}" != *py37* ]] && [[ "${PYBIN}" != *cp38* ]] && [[ "${PYBIN}" != *py38* ]] && [[ "${PYBIN}" != *py39* ]] && [[ "${PYBIN}" != *py310* ]]; then - "${PYBIN}/pip" install opensfm --no-index -f $WHEEL_DIR -# opensfm --help - fi -done - - -# Need to update this code -/opt/python/cp39-cp39/bin/python -m twine upload --repository-url "http://pypi.artichoke-labs.ai" $WHEEL_DIR/opensfm-0.5.2.post6-cp39-cp39-linux_x86_64.whl -/opt/python/cp310-cp310/bin/python -m twine upload --repository-url "http://pypi.artichoke-labs.ai" $WHEEL_DIR/opensfm-0.5.2.post6-cp310-cp310-linux_x86_64.whl \ No newline at end of file +cd ${WHEEL_DIR} && rm -rf *-linux*whl diff --git a/opensfm/src/third_party/ceres-solver b/opensfm/src/third_party/ceres-solver new file mode 160000 index 000000000..399cda773 --- /dev/null +++ b/opensfm/src/third_party/ceres-solver @@ -0,0 +1 @@ +Subproject commit 399cda773035d99eaf1f4a129a666b3c4df9d1b1 diff --git a/opensfm/src/third_party/eigen b/opensfm/src/third_party/eigen new file mode 160000 index 000000000..3147391d9 --- /dev/null +++ b/opensfm/src/third_party/eigen @@ -0,0 +1 @@ +Subproject commit 3147391d946bb4b6c68edd901f2add6ac1f31f8c diff --git a/opensfm/src/third_party/gflags b/opensfm/src/third_party/gflags index 46f73f88b..e171aa2d1 160000 --- a/opensfm/src/third_party/gflags +++ b/opensfm/src/third_party/gflags @@ -1 +1 @@ -Subproject commit 46f73f88b18aee341538c0dfc22b1710a6abedef +Subproject commit e171aa2d15ed9eb17054558e0b3a6a413bb01067 diff --git a/opensfm/src/third_party/glog b/opensfm/src/third_party/glog new file mode 160000 index 000000000..b33e3bad4 --- /dev/null +++ b/opensfm/src/third_party/glog @@ -0,0 +1 @@ +Subproject commit b33e3bad4c46c8a6345525fd822af355e5ef9446 diff --git a/requirements.txt b/requirements.txt index 59c5fc771..8d654c395 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,4 +20,4 @@ opencv-python==4.5.1.48 ; sys_platform == "win32" opencv-python ; sys_platform == "linux" twine sphinx -auditwheel \ No newline at end of file +auditwheel==1.4.0 \ No newline at end of file diff --git a/setup.py b/setup.py index 6af856f1f..5a797d62f 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ from wheel.bdist_wheel import bdist_wheel -VERSION = (0, 5, 2, "post7") +VERSION = (0, 5, 2, "post16") def version_str(version): @@ -60,8 +60,8 @@ def build_c_extension(): configure_c_extension() build_c_extension() -cmake_dir = Path(__file__).parent.joinpath("cmake_build") -data_files = [p.as_posix() for p in cmake_dir.glob("*.so")] +opensfm_dir = Path(__file__).parent.joinpath("opensfm") +lib_files = [p.as_posix() for p in opensfm_dir.rglob("*.so")] lib_folder = os.path.dirname(os.path.realpath(__file__)) requirement_path = f"{lib_folder}/requirements.txt" @@ -82,7 +82,7 @@ def build_c_extension(): }, author="Mapillary", license="BSD", - packages=setuptools.find_packages(), + packages=setuptools.find_packages(exclude=['*.so']), scripts=[ "bin/opensfm_run_all", "bin/opensfm", @@ -94,15 +94,9 @@ def build_c_extension(): "data/camera_calibration.yaml", "data/bow/bow_hahog_root_uchar_10000.npz", "data/bow/bow_hahog_root_uchar_64.npz", + "*", ] }, - # ext_modules=[ - # setuptools.Extension('pybundle', ['source/OpenSfM/opensfm/src/pybundle.cpython-310-x86_64-linux-gnu.so']), - # setuptools.Extension('pydense', ['source/OpenSfM/opensfm/src/pydense.cpython-310-x86_64-linux-gnu.so']), - # setuptools.Extension('pyfeatures', ['source/OpenSfM/opensfm/src/pyfeatures.cpython-310-x86_64-linux-gnu.so']), - # # Add all your shared libraries here - # ], - data_files=[('lib', data_files)], cmdclass={ "bdist_wheel": platform_bdist_wheel, "build_doc": BuildDoc, diff --git a/test_and_upload_wheel.sh b/test_and_upload_wheel.sh new file mode 100644 index 000000000..c69dd2327 --- /dev/null +++ b/test_and_upload_wheel.sh @@ -0,0 +1,15 @@ +!/bin/bash +set -e -u -x + +# Install packages and test +for PYBIN in /opt/python/*/bin; do + if [[ "${PYBIN}" != *cp311* ]] && [[ "${PYBIN}" != *cp312* ]] && [[ "${PYBIN}" != *cp313* ]] && [[ "${PYBIN}" != *cp36* ]] && [[ "${PYBIN}" != *cp37* ]] && [[ "${PYBIN}" != *py37* ]] && [[ "${PYBIN}" != *cp38* ]] && [[ "${PYBIN}" != *py38* ]] && [[ "${PYBIN}" != *py39* ]] && [[ "${PYBIN}" != *py310* ]]; then + "${PYBIN}/pip" install opensfm --no-index -f $WHEEL_DIR + "${PYBIN}/python" -c "import opensfm" + fi +done + + +for WHL in $WHEEL_DIR/*.whl; do + /opt/python/cp39-cp39/bin/python -m twine upload --repository-url "http://pypi.artichoke-labs.ai" $WHL +done From db68070ecc05e3ea0316d85c4aeefbcc25591596 Mon Sep 17 00:00:00 2001 From: offbeat Date: Fri, 21 Jun 2024 18:05:09 +0400 Subject: [PATCH 04/16] Done for linux, working on macos --- .gitmodules | 3 + Dockerfile | 183 ++++++++++--------- README.md | 2 + build_wheel.sh | 10 +- opensfm/src/third_party/SuiteSparse | 1 + opensfm/src/third_party/ceres-solver | 2 +- opensfm/src/third_party/gflags | 2 +- requirements.txt | 2 +- setup.py | 262 ++++++++++++++++++++++++--- test_and_upload_wheel.sh | 4 +- 10 files changed, 350 insertions(+), 121 deletions(-) create mode 160000 opensfm/src/third_party/SuiteSparse diff --git a/.gitmodules b/.gitmodules index 7d34e2b6a..82a34c1b5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,3 +13,6 @@ [submodule "opensfm/src/third_party/ceres-solver"] path = opensfm/src/third_party/ceres-solver url = https://github.com/ceres-solver/ceres-solver.git +[submodule "opensfm/src/third_party/SuiteSparse"] + path = opensfm/src/third_party/SuiteSparse + url = https://github.com/DrTimothyAldenDavis/SuiteSparse.git diff --git a/Dockerfile b/Dockerfile index 89af995cb..a46d01b36 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ -FROM quay.io/pypa/manylinux2014_x86_64 -# FROM ubuntu:20.04 +#FROM quay.io/pypa/manylinux2014_x86_64 +FROM ubuntu:20.04 -ENV PLAT=manylinux_2_17_x86_64 +ENV PLAT=manylinux_2_31_x86_64 ARG DEBIAN_FRONTEND=noninteractive ARG TWINE_USERNAME ARG TWINE_PASSWORD @@ -27,87 +27,65 @@ ENV TWINE_PASSWORD=$TWINE_PASSWORD # && apt-get clean \ # && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ -# Install apt-getable dependencies -#RUN yum install -y \ -# build-essential \ -# cmake \ -# git \ -# libeigen3-dev \ -# libopencv-dev \ -# libceres-dev \ -# python3-dev \ -# python3-numpy \ -# opencv \ -# python3-pip \ -# python3-pyproj \ -# python3-scipy \ -# python3-yaml \ -# curl \ -# && yum clean all \ -## Установим Miniconda -#RUN curl -sSL https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -o miniconda.sh && \ -# bash miniconda.sh -b -p /opt/conda && \ -# rm miniconda.sh -# -## Добавим conda в PATH -#ENV PATH=/opt/conda/bin:$PATH -# -## Установим Ceres Solver 2.1 или 2.2 через conda -#RUN conda install -c conda-forge ceres-solver=2.2 opencv -#ENV CMAKE_PREFIX_PATH="/opt/conda/pkgs/ceres-solver-2.2.0-hfae76b8_3/lib/cmake/Ceres/" +#RUN yum install -y opencv opencv-devel \ +# blas-devel \ +# lapack-devel \ +# metis-devel \ +# tbb-devel \ +# wget \ +# openblas-devel \ +# atlas-devel \ +# suitesparse-devel -#RUN /opt/python/cp39-cp39/bin/pip install --upgrade pip && \ -# /opt/python/cp39-cp39/bin/pip install auditwheel +RUN apt-get update \ + && apt-get install -y \ + python3-dev \ + python3-numpy \ + python3-opencv \ + python3-pip \ + libblas-dev \ + liblapack-dev \ + libmetis-dev \ + libtbb-dev \ + wget \ + libopenblas-dev \ + libatlas-base-dev \ + git \ + cmake \ + build-essential \ +# libsuitesparse-dev \ + curl # -##RUN yum install -y opencv opencv-devel \ -## ceres-solver ceres-solver-devel \ -## eigen3 \ -## glog-devel \ -## gflags-devel \ -## suitesparse-devel \ -## blas-devel \ -## lapack-devel \ -## openblas-devel +#RUN mkdir /source +#WORKDIR /source +#RUN git clone https://github.com/gflags/gflags.git +#WORKDIR /source/gflags/ +#RUN git checkout tags/v2.2.2 +#RUN mkdir /source/gflags/build/ +#WORKDIR /source/gflags/build/ +#RUN cmake -DCMAKE_CXX_FLAGS="-fPIC" .. +#RUN make -j8 +#RUN make install # -RUN yum install -y opencv opencv-devel \ - blas-devel \ - lapack-devel \ - metis-devel \ - tbb-devel \ - wget \ - openblas-devel \ - atlas-devel - -RUN mkdir /source -WORKDIR /source -RUN git clone https://github.com/gflags/gflags.git -WORKDIR /source/gflags/ -RUN git checkout tags/v2.2.2 -RUN mkdir /source/gflags/build/ -WORKDIR /source/gflags/build/ -RUN cmake -DCMAKE_CXX_FLAGS="-fPIC" .. -RUN make -j8 -RUN make install - -WORKDIR /source -RUN git clone https://github.com/google/glog.git -WORKDIR /source/glog -RUN git checkout tags/v0.6.0 -RUN mkdir /source/glog/build/ -WORKDIR /source/glog/build/ -RUN cmake .. -RUN make -j8 -RUN make install - -WORKDIR /source -RUN git clone https://gitlab.com/libeigen/eigen.git -WORKDIR /source/eigen/ -RUN git checkout tags/3.4.0 -RUN mkdir build -WORKDIR /source/eigen/build/ -RUN cmake .. -RUN make install +#WORKDIR /source +#RUN git clone https://github.com/google/glog.git +#WORKDIR /source/glog +#RUN git checkout tags/v0.6.0 +#RUN mkdir /source/glog/build/ +#WORKDIR /source/glog/build/ +#RUN cmake .. +#RUN make -j8 +#RUN make install +# +#WORKDIR /source +#RUN git clone https://gitlab.com/libeigen/eigen.git +#WORKDIR /source/eigen/ +#RUN git checkout tags/3.4.0 +#RUN mkdir build +#WORKDIR /source/eigen/build/ +#RUN cmake .. +#RUN make install #RUN yum install -y lzip #WORKDIR /source @@ -145,23 +123,48 @@ RUN make install #RUN make -j4 #RUN make install -RUN yum install -y suitesparse-devel +#WORKDIR /source +#RUN git clone https://github.com/ceres-solver/ceres-solver.git +#WORKDIR /source/ceres-solver +#RUN git checkout tags/2.0.0 +#RUN mkdir build +#WORKDIR /source/ceres-solver/build/ +#RUN cmake .. +#RUN make -j8 +#RUN make install + WORKDIR /source -RUN git clone https://github.com/ceres-solver/ceres-solver.git -WORKDIR /source/ceres-solver -RUN git checkout tags/2.0.0 -RUN mkdir build -WORKDIR /source/ceres-solver/build/ -RUN cmake .. -RUN make -j8 +RUN git clone https://github.com/NixOS/patchelf.git +WORKDIR /source/patchelf/ +RUN git checkout tags/0.16.0 +RUN ./bootstrap.sh +RUN ./configure +RUN make RUN make install +RUN apt-get install -y liblzma-dev libssl-dev libopencv-dev +RUN apt-get install -y libncurses-dev libffi-dev libreadline6-dev libbz2-dev libsqlite3-dev + +WORKDIR $SFM_DIR +RUN curl https://pyenv.run | bash +RUN /root/.pyenv/bin/pyenv install 3.9.9 +RUN /root/.pyenv/bin/pyenv global 3.9.9 +ENV PATH=$PATH:/root/.pyenv/versions/3.9.9/bin/ +RUN python -m pip install --upgrade pip + ENV WHEEL_DIR=/source/wheelhouse ENV SFM_DIR=/source/OpenSfM COPY . $SFM_DIR WORKDIR $SFM_DIR RUN rm -rf cmake_build -RUN sh /source/OpenSfM/build_wheel.sh -# RUN sh /source/OpenSfM/test_and_upload_wheel.sh +RUN /root/.pyenv/versions/3.9.9/bin/pip install -r requirements.txt +RUN /root/.pyenv/versions/3.9.9/bin/pip wheel $SFM_DIR --no-deps -w $WHEEL_DIR +RUN ls /source/wheelhouse/*.whl | xargs -n 1 -I {} auditwheel repair {} --plat $PLAT -w $WHEEL_DIR +RUN cd ${WHEEL_DIR} && rm -rf *-linux*whl +RUN /root/.pyenv/versions/3.9.9/bin/pip install opensfm --no-index -f $WHEEL_DIR +RUN python -c "import opensfm" +RUN ls /source/wheelhouse/*.whl | xargs -n 1 -I {} python -m twine upload --repository-url "http://pypi.artichoke-labs.ai" {} +##RUN sh /source/OpenSfM/build_wheel.sh +##RUN sh /source/OpenSfM/test_and_upload_wheel.sh diff --git a/README.md b/README.md index 727f3af23..61b9cbe3a 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ OpenSfM is a Structure from Motion library written in Python. The library serves Checkout this [blog post with more demos](http://blog.mapillary.com/update/2014/12/15/sfm-preview.html) +## MacOS Install + `brew install opencv` ## Getting Started diff --git a/build_wheel.sh b/build_wheel.sh index c9832ea6b..43d7dab58 100644 --- a/build_wheel.sh +++ b/build_wheel.sh @@ -3,16 +3,18 @@ set -e -u -x function repair_wheel { wheel="$1" - /opt/_internal/cpython-3.9.19/bin/auditwheel repair "$wheel" --plat "$PLAT" -w $WHEEL_DIR + if ! /opt/_internal/cpython-3.9.19/bin/auditwheel show "$wheel"; then + echo "Skipping non-platform wheel $wheel" + else + /opt/_internal/cpython-3.9.19/bin/auditwheel repair "$wheel" --plat "$PLAT" -w $WHEEL_DIR + fi } # Compile wheels for PYBIN in /opt/python/*/bin; do - if [[ "${PYBIN}" != *cp311* ]] && [[ "${PYBIN}" != *cp312* ]] && [[ "${PYBIN}" != *cp313* ]] && [[ "${PYBIN}" != *cp36* ]] && [[ "${PYBIN}" != *cp37* ]] && [[ "${PYBIN}" != *py37* ]] && [[ "${PYBIN}" != *cp38* ]] && [[ "${PYBIN}" != *py38* ]] && [[ "${PYBIN}" != *py39* ]] && [[ "${PYBIN}" != *py310* ]]; then + if [[ "${PYBIN}" != *cp311* ]] && [[ "${PYBIN}" != *cp312* ]] && [[ "${PYBIN}" != *cp313* ]] && [[ "${PYBIN}" != *cp36* ]] && [[ "${PYBIN}" != *cp37* ]] && [[ "${PYBIN}" != *py37* ]] && [[ "${PYBIN}" != *cp38* ]] && [[ "${PYBIN}" != *py38* ]] && [[ "${PYBIN}" != *py39* ]] && [[ "${PYBIN}" != *cp310* ]] && [[ "${PYBIN}" != *py310* ]]; then "${PYBIN}/pip" install --upgrade pip "${PYBIN}/pip" install -r ${SFM_DIR}/requirements.txt - "${PYBIN}/pip" install auditwheel==2.0.0 - "${PYBIN}/pip" install auditwheel==1.4.0 "${PYBIN}/pip" wheel $SFM_DIR --no-deps -w $WHEEL_DIR fi done diff --git a/opensfm/src/third_party/SuiteSparse b/opensfm/src/third_party/SuiteSparse new file mode 160000 index 000000000..13806726c --- /dev/null +++ b/opensfm/src/third_party/SuiteSparse @@ -0,0 +1 @@ +Subproject commit 13806726cbf470914d012d132a85aea1aff9ee77 diff --git a/opensfm/src/third_party/ceres-solver b/opensfm/src/third_party/ceres-solver index 399cda773..3d9b6140e 160000 --- a/opensfm/src/third_party/ceres-solver +++ b/opensfm/src/third_party/ceres-solver @@ -1 +1 @@ -Subproject commit 399cda773035d99eaf1f4a129a666b3c4df9d1b1 +Subproject commit 3d9b6140e8c946212f5b3fce03d3a250e15f82dc diff --git a/opensfm/src/third_party/gflags b/opensfm/src/third_party/gflags index e171aa2d1..eae848a49 160000 --- a/opensfm/src/third_party/gflags +++ b/opensfm/src/third_party/gflags @@ -1 +1 @@ -Subproject commit e171aa2d15ed9eb17054558e0b3a6a413bb01067 +Subproject commit eae848a497303f2e9d9c3e597dfed56060b8e1e8 diff --git a/requirements.txt b/requirements.txt index 8d654c395..0bdf77e9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,4 +20,4 @@ opencv-python==4.5.1.48 ; sys_platform == "win32" opencv-python ; sys_platform == "linux" twine sphinx -auditwheel==1.4.0 \ No newline at end of file +auditwheel==6.0.0 \ No newline at end of file diff --git a/setup.py b/setup.py index 5a797d62f..267658b22 100644 --- a/setup.py +++ b/setup.py @@ -1,17 +1,54 @@ #!/usr/bin/env python3 + import multiprocessing import os +import platform import subprocess import sys +from shutil import rmtree from pathlib import Path import setuptools +from setuptools.command.install import install from sphinx.setup_command import BuildDoc from wheel.bdist_wheel import bdist_wheel -VERSION = (0, 5, 2, "post16") +VERSION = (0, 5, 2, "post19") + +THIRD_PARTY_INSTALL_DIR = Path(__file__).parent / "third_party_install" +LIB_DIR = THIRD_PARTY_INSTALL_DIR / "usr/local/lib" +INCLUDE_DIR = THIRD_PARTY_INSTALL_DIR / "usr/local/include" +SHARE_DIR = THIRD_PARTY_INSTALL_DIR / "usr/local/share" +SHARED_CMAKE_DIR = THIRD_PARTY_INSTALL_DIR / "usr/local/lib/cmake/" + +CMAKE_PREFIX_PATH = ( + f'-DCMAKE_PREFIX_PATH={SHARED_CMAKE_DIR};' + f'{SHARE_DIR};' + f'{INCLUDE_DIR};' + f'{THIRD_PARTY_INSTALL_DIR}' +) + +os.environ['LDFLAGS'] = f'-L/opt/homebrew/Cellar/libomp/18.1.6/lib' +os.environ['C_INCLUDE_PATH'] = f'/opt/homebrew/Cellar/libomp/18.1.6/include/' +os.environ['CPLUS_INCLUDE_PATH'] = f'/opt/homebrew/Cellar/libomp/18.1.6/include/' +# +# def _is_apple_silicon() -> bool: +# +# if platform.system() == 'Darwin': +# if platform.machine() == 'arm64': +# return True +# return False +# +# if _is_apple_silicon(): +# print('Detected Apple Silicon processor') +# # Define the compilers +# C_COMPILER_PATH = os.path.abspath('/opt/homebrew/Cellar/gcc@12/12.3.0/bin/gcc-12') +# CXX_COMPILER_PATH = os.path.abspath('/opt/homebrew/Cellar/gcc@12/12.3.0/bin/g++-12') +# C_COMPILER_ARG = f'-DCMAKE_C_COMPILER={C_COMPILER_PATH}' +# CXX_COMPILER_ARG = f'-DCMAKE_CXX_COMPILER={CXX_COMPILER_PATH}' +# def version_str(version): return ".".join(map(str, version)) @@ -25,24 +62,182 @@ def finalize_options(self): self.root_is_pure = False -def configure_c_extension(): - """Configure cmake project to C extension.""" - print( - f"Configuring for python {sys.version_info.major}.{sys.version_info.minor}..." +class InstallPlatlib(install): + def finalize_options(self): + install.finalize_options(self) + self.install_lib = self.install_platlib + + +def install_gflag(install_dir: Path): + gflags_build_dir = Path(__file__).parent.joinpath("opensfm/src/third_party/gflags/build") + + if gflags_build_dir.exists(): + rmtree(gflags_build_dir) + gflags_build_dir.mkdir() + cmake_command = [ + 'cmake', + '-DCMAKE_CXX_FLAGS="-fPIC"', + gflags_build_dir.parent.as_posix() + ] + # if _is_apple_silicon(): + # cmake_command += [ + # C_COMPILER_ARG, + # CXX_COMPILER_ARG, + # ] + subprocess.check_call( + cmake_command, + cwd=gflags_build_dir.as_posix(), + env=os.environ, + ) + + subprocess.check_call(["make", "-j8"], cwd=gflags_build_dir.as_posix()) + subprocess.check_call(["make", f"DESTDIR={install_dir}", "install"], cwd=gflags_build_dir.as_posix()) + + +def install_glog(install_dir: Path): + glog_path = Path(__file__).parent.joinpath("opensfm/src/third_party/glog/build") + + if glog_path.exists(): + rmtree(glog_path) + glog_path.mkdir() + + cmake_command = [ + 'cmake', + CMAKE_PREFIX_PATH, + glog_path.parent.as_posix() + ] + + # if _is_apple_silicon(): + # cmake_command += [ + # C_COMPILER_ARG, + # CXX_COMPILER_ARG, + # ] + + subprocess.check_call( + cmake_command, + cwd=glog_path.as_posix(), + env=os.environ, + ) + subprocess.check_call(["make", "-j8"], cwd=glog_path.as_posix()) + subprocess.check_call(["make", f"DESTDIR={install_dir}", "install"], cwd=glog_path.as_posix()) + + +def install_eigen(install_dir: Path): + eigen_path = Path(__file__).parent.joinpath("opensfm/src/third_party/eigen/build") + + if eigen_path.exists(): + rmtree(eigen_path) + + eigen_path.mkdir() + cmake_command = [ + 'cmake', + CMAKE_PREFIX_PATH, + eigen_path.parent.as_posix() + ] + + # if _is_apple_silicon(): + # cmake_command += [ + # C_COMPILER_ARG, + # CXX_COMPILER_ARG, + # ] + subprocess.check_call( + cmake_command, + cwd=eigen_path.as_posix(), + env=os.environ, + ) + subprocess.check_call(["make", f"DESTDIR={install_dir}", "install"], cwd=eigen_path.as_posix()) + + +def install_suitesparse(install_dir: Path): + suitesparse_path = Path(__file__).parent.joinpath("opensfm/src/third_party/SuiteSparse/build") + if suitesparse_path.exists(): + rmtree(suitesparse_path) + suitesparse_path.mkdir() + cmake_command = [ + 'cmake', + CMAKE_PREFIX_PATH, + suitesparse_path.parent.as_posix(), + ] + # if _is_apple_silicon(): + # cmake_command += [ + # C_COMPILER_ARG, + # CXX_COMPILER_ARG, + # ] + subprocess.check_call( + cmake_command, + cwd=suitesparse_path.as_posix(), + env=os.environ, + ) + subprocess.check_call(["cmake", "--build", suitesparse_path.as_posix()], cwd=suitesparse_path.as_posix()) + subprocess.check_call(["make", f"DESTDIR={install_dir}", "install"], cwd=suitesparse_path.as_posix()) + + +def install_ceres(install_dir: Path): + ceres_path = Path(__file__).parent.joinpath("opensfm/src/third_party/ceres-solver/build") + if ceres_path.exists(): + rmtree(ceres_path) + ceres_path.mkdir() + cmake_command = [ + 'cmake', + CMAKE_PREFIX_PATH, + f'SUITESPARSE_LIBRARY_DIR_HINTS={install_dir}', + ceres_path.parent.as_posix(), + ] + # if _is_apple_silicon(): + # cmake_command += [ + # C_COMPILER_ARG, + # CXX_COMPILER_ARG, + # ] + subprocess.check_call( + cmake_command, + cwd=ceres_path.as_posix() ) - os.makedirs("cmake_build", exist_ok=True) + subprocess.check_call(["make", "-j8"], cwd=ceres_path.as_posix()) + subprocess.check_call(["make", f"DESTDIR={install_dir}", "install"], cwd=ceres_path.as_posix(), env=os.environ) + + +def install_opensfm(install_dir: Path): + + if install_dir.exists(): + rmtree(install_dir) + install_dir.mkdir() + cmake_command = [ "cmake", + CMAKE_PREFIX_PATH, "../opensfm/src", "-DPYTHON_EXECUTABLE=" + sys.executable, ] - + # if _is_apple_silicon(): + # cmake_command += [ + # C_COMPILER_ARG, + # CXX_COMPILER_ARG, + # ] if sys.platform == "win32": cmake_command += [ "-DVCPKG_TARGET_TRIPLET=x64-windows", "-DCMAKE_TOOLCHAIN_FILE=../vcpkg/scripts/buildsystems/vcpkg.cmake", ] - subprocess.check_call(cmake_command, cwd="cmake_build") + subprocess.check_call( + cmake_command, + cwd=install_dir.as_posix(), + # env=os.environ + ) + + +def configure_c_extension(): + """Configure cmake project to C extension.""" + print( + f"Configuring for python {sys.version_info.major}.{sys.version_info.minor}..." + ) + + # Third party install + install_gflag(THIRD_PARTY_INSTALL_DIR) + install_glog(THIRD_PARTY_INSTALL_DIR) + install_eigen(THIRD_PARTY_INSTALL_DIR) + install_suitesparse(THIRD_PARTY_INSTALL_DIR) + install_ceres(THIRD_PARTY_INSTALL_DIR) + install_opensfm(Path(__file__).parent / "cmake_build") def build_c_extension(): @@ -54,21 +249,16 @@ def build_c_extension(): ) else: subprocess.check_call( - ["make", "-s", "-j" + str(multiprocessing.cpu_count())], cwd="cmake_build" + ["make", "-j" + str(multiprocessing.cpu_count())], cwd="cmake_build" ) configure_c_extension() build_c_extension() -opensfm_dir = Path(__file__).parent.joinpath("opensfm") -lib_files = [p.as_posix() for p in opensfm_dir.rglob("*.so")] -lib_folder = os.path.dirname(os.path.realpath(__file__)) -requirement_path = f"{lib_folder}/requirements.txt" -install_requires = [] # Here we'll add: ["gunicorn", "docutils>=0.3", "lxml==0.5a7"] -if os.path.isfile(requirement_path): - with open(requirement_path) as f: - install_requires = f.read().splitlines() +install_requires = [] +with open("requirements.txt") as f: + install_requires = f.read().splitlines() setuptools.setup( name="opensfm", @@ -82,24 +272,52 @@ def build_c_extension(): }, author="Mapillary", license="BSD", - packages=setuptools.find_packages(exclude=['*.so']), + packages=setuptools.find_packages(), scripts=[ + "bin/import_colmap.py", + "bin/opensfm.bat", + "bin/plot_features", + "bin/plot_tracks", + "bin/create_calibrtion_pattern", + "bin/import_video", + "bin/opensfm_main.py", + "bin/plot_gcp.py", + "bin/run_bundler", + "bin/export_geojson", + "bin/iterative_self_calibration", "bin/opensfm_run_all", + "bin/plot_inliers", + "bin/update_geotag", + "bin/export_gps", + "bin/migrate_undistort.sh", + "bin/opensfm_run_all.bat", + "bin/plot_matches.py", + "bin/import_bundler", "bin/opensfm", - "bin/opensfm_main.py" + "bin/plot_depthmaps", + "bin/plot_submodels_gps", ], package_data={ "opensfm": [ + "pybundle.*", + "pygeo.*", + "pygeometry.*", + "pyrobust.*", + "pyfeatures.*", + "pydense.*", + "pysfm.*", + "pyfoundation.*", + "pymap.*", "data/sensor_data.json", "data/camera_calibration.yaml", "data/bow/bow_hahog_root_uchar_10000.npz", "data/bow/bow_hahog_root_uchar_64.npz", - "*", ] }, cmdclass={ "bdist_wheel": platform_bdist_wheel, "build_doc": BuildDoc, + "install": InstallPlatlib, }, command_options={ "build_doc": { @@ -110,5 +328,5 @@ def build_c_extension(): "build_dir": ("setup.py", "build/doc"), } }, - install_requires=install_requires, -) \ No newline at end of file + install_requires=install_requires +) diff --git a/test_and_upload_wheel.sh b/test_and_upload_wheel.sh index c69dd2327..485b763c0 100644 --- a/test_and_upload_wheel.sh +++ b/test_and_upload_wheel.sh @@ -3,9 +3,9 @@ set -e -u -x # Install packages and test for PYBIN in /opt/python/*/bin; do - if [[ "${PYBIN}" != *cp311* ]] && [[ "${PYBIN}" != *cp312* ]] && [[ "${PYBIN}" != *cp313* ]] && [[ "${PYBIN}" != *cp36* ]] && [[ "${PYBIN}" != *cp37* ]] && [[ "${PYBIN}" != *py37* ]] && [[ "${PYBIN}" != *cp38* ]] && [[ "${PYBIN}" != *py38* ]] && [[ "${PYBIN}" != *py39* ]] && [[ "${PYBIN}" != *py310* ]]; then + if [[ "${PYBIN}" != *cp311* ]] && [[ "${PYBIN}" != *cp312* ]] && [[ "${PYBIN}" != *cp313* ]] && [[ "${PYBIN}" != *cp36* ]] && [[ "${PYBIN}" != *cp37* ]] && [[ "${PYBIN}" != *py37* ]] && [[ "${PYBIN}" != *cp38* ]] && [[ "${PYBIN}" != *py38* ]] && [[ "${PYBIN}" != *py39* ]] && [[ "${PYBIN}" != *cp310* ]] && [[ "${PYBIN}" != *py310* ]]; then "${PYBIN}/pip" install opensfm --no-index -f $WHEEL_DIR - "${PYBIN}/python" -c "import opensfm" + cd / && "${PYBIN}/python" -c "import opensfm" fi done From 1d703c52ba37001b8e8e2c327b3b8646068e119e Mon Sep 17 00:00:00 2001 From: offbeat Date: Fri, 21 Jun 2024 18:14:22 +0400 Subject: [PATCH 05/16] Done for linux, working on macos --- opensfm/src/third_party/ceres-solver | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensfm/src/third_party/ceres-solver b/opensfm/src/third_party/ceres-solver index 3d9b6140e..399cda773 160000 --- a/opensfm/src/third_party/ceres-solver +++ b/opensfm/src/third_party/ceres-solver @@ -1 +1 @@ -Subproject commit 3d9b6140e8c946212f5b3fce03d3a250e15f82dc +Subproject commit 399cda773035d99eaf1f4a129a666b3c4df9d1b1 From 6c6b3059e2c030744eacd2265827f4668bab5247 Mon Sep 17 00:00:00 2001 From: offbeat Date: Fri, 21 Jun 2024 18:17:37 +0400 Subject: [PATCH 06/16] Done for linux, working on macos --- opensfm/src/third_party/gflags | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensfm/src/third_party/gflags b/opensfm/src/third_party/gflags index eae848a49..e171aa2d1 160000 --- a/opensfm/src/third_party/gflags +++ b/opensfm/src/third_party/gflags @@ -1 +1 @@ -Subproject commit eae848a497303f2e9d9c3e597dfed56060b8e1e8 +Subproject commit e171aa2d15ed9eb17054558e0b3a6a413bb01067 From 47094bb8530f42cabb595b967d47ffc9c942e424 Mon Sep 17 00:00:00 2001 From: offbeat Date: Fri, 21 Jun 2024 19:01:57 +0400 Subject: [PATCH 07/16] Done for linux, working on macos --- .gitignore | 3 +++ setup.py | 74 +++++++++++++++++++++++++++++------------------------- 2 files changed, 43 insertions(+), 34 deletions(-) diff --git a/.gitignore b/.gitignore index a739a68fd..367fdfe6e 100644 --- a/.gitignore +++ b/.gitignore @@ -10,12 +10,15 @@ launch.json # Ignore generated files /build /cmake_build +/third_party_install +/third_party_build /doc/build /dist /OpenSfM.egg-info .cache .pytest_cache eval +/.env # Viewer node_modules diff --git a/setup.py b/setup.py index 267658b22..08bac767c 100644 --- a/setup.py +++ b/setup.py @@ -17,6 +17,8 @@ VERSION = (0, 5, 2, "post19") THIRD_PARTY_INSTALL_DIR = Path(__file__).parent / "third_party_install" +THIRD_PARTY_BUILD_DIR = Path(__file__).parent / "third_party_build" +THIRD_PARTY_BUILD_DIR.mkdir(exist_ok=True) LIB_DIR = THIRD_PARTY_INSTALL_DIR / "usr/local/lib" INCLUDE_DIR = THIRD_PARTY_INSTALL_DIR / "usr/local/include" SHARE_DIR = THIRD_PARTY_INSTALL_DIR / "usr/local/share" @@ -69,15 +71,15 @@ def finalize_options(self): def install_gflag(install_dir: Path): - gflags_build_dir = Path(__file__).parent.joinpath("opensfm/src/third_party/gflags/build") - + gflags_build_dir = THIRD_PARTY_BUILD_DIR / "gflags" + gflag_source_dir = Path(__file__).parent.joinpath("opensfm/src/third_party/gflags/") if gflags_build_dir.exists(): rmtree(gflags_build_dir) gflags_build_dir.mkdir() cmake_command = [ 'cmake', '-DCMAKE_CXX_FLAGS="-fPIC"', - gflags_build_dir.parent.as_posix() + gflag_source_dir ] # if _is_apple_silicon(): # cmake_command += [ @@ -95,16 +97,17 @@ def install_gflag(install_dir: Path): def install_glog(install_dir: Path): - glog_path = Path(__file__).parent.joinpath("opensfm/src/third_party/glog/build") + glog_build_dir = THIRD_PARTY_BUILD_DIR / "glog" + glog_source_dir = Path(__file__).parent.joinpath("opensfm/src/third_party/glog") - if glog_path.exists(): - rmtree(glog_path) - glog_path.mkdir() + if glog_build_dir.exists(): + rmtree(glog_build_dir) + glog_build_dir.mkdir() cmake_command = [ 'cmake', CMAKE_PREFIX_PATH, - glog_path.parent.as_posix() + glog_source_dir.as_posix() ] # if _is_apple_silicon(): @@ -115,24 +118,25 @@ def install_glog(install_dir: Path): subprocess.check_call( cmake_command, - cwd=glog_path.as_posix(), + cwd=glog_build_dir.as_posix(), env=os.environ, ) - subprocess.check_call(["make", "-j8"], cwd=glog_path.as_posix()) - subprocess.check_call(["make", f"DESTDIR={install_dir}", "install"], cwd=glog_path.as_posix()) + subprocess.check_call(["make", "-j8"], cwd=glog_build_dir.as_posix()) + subprocess.check_call(["make", f"DESTDIR={install_dir}", "install"], cwd=glog_build_dir.as_posix()) def install_eigen(install_dir: Path): - eigen_path = Path(__file__).parent.joinpath("opensfm/src/third_party/eigen/build") + eigen_build_dir = THIRD_PARTY_BUILD_DIR / "eigen" + eigen_source_dir = Path(__file__).parent.joinpath("opensfm/src/third_party/eigen") - if eigen_path.exists(): - rmtree(eigen_path) + if eigen_build_dir.exists(): + rmtree(eigen_build_dir) - eigen_path.mkdir() + eigen_build_dir.mkdir() cmake_command = [ 'cmake', CMAKE_PREFIX_PATH, - eigen_path.parent.as_posix() + eigen_source_dir.as_posix(), ] # if _is_apple_silicon(): @@ -142,21 +146,22 @@ def install_eigen(install_dir: Path): # ] subprocess.check_call( cmake_command, - cwd=eigen_path.as_posix(), + cwd=eigen_build_dir.as_posix(), env=os.environ, ) - subprocess.check_call(["make", f"DESTDIR={install_dir}", "install"], cwd=eigen_path.as_posix()) + subprocess.check_call(["make", f"DESTDIR={install_dir}", "install"], cwd=eigen_build_dir.as_posix()) def install_suitesparse(install_dir: Path): - suitesparse_path = Path(__file__).parent.joinpath("opensfm/src/third_party/SuiteSparse/build") - if suitesparse_path.exists(): - rmtree(suitesparse_path) - suitesparse_path.mkdir() + suitesparse_build_dir = THIRD_PARTY_BUILD_DIR / "SuiteSparse" + suitesparse_source_dir = Path(__file__).parent.joinpath("opensfm/src/third_party/SuiteSparse") + if suitesparse_build_dir.exists(): + rmtree(suitesparse_build_dir) + suitesparse_build_dir.mkdir() cmake_command = [ 'cmake', CMAKE_PREFIX_PATH, - suitesparse_path.parent.as_posix(), + suitesparse_source_dir.as_posix(), ] # if _is_apple_silicon(): # cmake_command += [ @@ -165,23 +170,24 @@ def install_suitesparse(install_dir: Path): # ] subprocess.check_call( cmake_command, - cwd=suitesparse_path.as_posix(), + cwd=suitesparse_build_dir.as_posix(), env=os.environ, ) - subprocess.check_call(["cmake", "--build", suitesparse_path.as_posix()], cwd=suitesparse_path.as_posix()) - subprocess.check_call(["make", f"DESTDIR={install_dir}", "install"], cwd=suitesparse_path.as_posix()) + subprocess.check_call(["cmake", "--build", suitesparse_build_dir.as_posix()], cwd=suitesparse_build_dir.as_posix()) + subprocess.check_call(["make", f"DESTDIR={install_dir}", "install"], cwd=suitesparse_build_dir.as_posix()) def install_ceres(install_dir: Path): - ceres_path = Path(__file__).parent.joinpath("opensfm/src/third_party/ceres-solver/build") - if ceres_path.exists(): - rmtree(ceres_path) - ceres_path.mkdir() + ceres_build_dir = THIRD_PARTY_BUILD_DIR / "ceres_solver" + ceres_source_dir = Path(__file__).parent.joinpath("opensfm/src/third_party/ceres-solver") + if ceres_build_dir.exists(): + rmtree(ceres_build_dir) + ceres_build_dir.mkdir() cmake_command = [ 'cmake', CMAKE_PREFIX_PATH, f'SUITESPARSE_LIBRARY_DIR_HINTS={install_dir}', - ceres_path.parent.as_posix(), + ceres_source_dir.as_posix() ] # if _is_apple_silicon(): # cmake_command += [ @@ -190,10 +196,10 @@ def install_ceres(install_dir: Path): # ] subprocess.check_call( cmake_command, - cwd=ceres_path.as_posix() + cwd=ceres_build_dir.as_posix() ) - subprocess.check_call(["make", "-j8"], cwd=ceres_path.as_posix()) - subprocess.check_call(["make", f"DESTDIR={install_dir}", "install"], cwd=ceres_path.as_posix(), env=os.environ) + subprocess.check_call(["make", "-j8"], cwd=ceres_build_dir.as_posix()) + subprocess.check_call(["make", f"DESTDIR={install_dir}", "install"], cwd=ceres_build_dir.as_posix(), env=os.environ) def install_opensfm(install_dir: Path): From 717174c407b010e27a2b14f91d67e86291b2e604 Mon Sep 17 00:00:00 2001 From: offbeat Date: Fri, 21 Jun 2024 20:00:53 +0400 Subject: [PATCH 08/16] Done for linux, working on macos --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 08bac767c..bb6cac700 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ f'-DCMAKE_PREFIX_PATH={SHARED_CMAKE_DIR};' f'{SHARE_DIR};' f'{INCLUDE_DIR};' + f'{LIB_DIR};' f'{THIRD_PARTY_INSTALL_DIR}' ) From c678b3b806f1ecd1aa1061acb23f25cc22d0183e Mon Sep 17 00:00:00 2001 From: offbeat Date: Fri, 21 Jun 2024 23:45:21 +0400 Subject: [PATCH 09/16] Failed for MacOS --- .gitmodules | 3 + Dockerfile | 40 ++++--- README.md | 2 +- opensfm/src/third_party/SuiteSparse | 2 +- opensfm/src/third_party/tbb | 1 + setup.py | 165 ++++++++++++++++------------ 6 files changed, 122 insertions(+), 91 deletions(-) create mode 160000 opensfm/src/third_party/tbb diff --git a/.gitmodules b/.gitmodules index 82a34c1b5..2b488bf4b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,3 +16,6 @@ [submodule "opensfm/src/third_party/SuiteSparse"] path = opensfm/src/third_party/SuiteSparse url = https://github.com/DrTimothyAldenDavis/SuiteSparse.git +[submodule "opensfm/src/third_party/tbb"] + path = opensfm/src/third_party/tbb + url = https://github.com/oneapi-src/oneTBB.git diff --git a/Dockerfile b/Dockerfile index a46d01b36..0c2129676 100644 --- a/Dockerfile +++ b/Dockerfile @@ -87,22 +87,22 @@ RUN apt-get update \ #RUN cmake .. #RUN make install -#RUN yum install -y lzip -#WORKDIR /source -#RUN wget https://gmplib.org/download/gmp/gmp-6.3.0.tar.lz -#RUN tar -xf gmp-6.3.0.tar.lz -#WORKDIR /source/gmp-6.3.0 -#RUN ./configure -#RUN make -#RUN make install -# -#WORKDIR /source -#RUN wget https://www.mpfr.org/mpfr-current/mpfr-4.2.1.tar.xz -#RUN tar -xf mpfr-4.2.1.tar.xz -#WORKDIR /source/mpfr-4.2.1 -#RUN ./configure -#RUN make -#RUN make install +RUN apt install -y lzip +WORKDIR /source +RUN wget https://gmplib.org/download/gmp/gmp-6.3.0.tar.lz +RUN tar -xf gmp-6.3.0.tar.lz +WORKDIR /source/gmp-6.3.0 +RUN ./configure +RUN make +RUN make install + +WORKDIR /source +RUN wget https://www.mpfr.org/mpfr-current/mpfr-4.2.1.tar.xz +RUN tar -xf mpfr-4.2.1.tar.xz +WORKDIR /source/mpfr-4.2.1 +RUN ./configure +RUN make +RUN make install #WORKDIR /source #RUN git clone https://github.com/DrTimothyAldenDavis/SuiteSparse.git @@ -153,11 +153,17 @@ RUN /root/.pyenv/bin/pyenv global 3.9.9 ENV PATH=$PATH:/root/.pyenv/versions/3.9.9/bin/ RUN python -m pip install --upgrade pip +WORKDIR /source +RUN apt remove -y cmake +RUN wget https://github.com/Kitware/CMake/releases/download/v3.30.0-rc3/cmake-3.30.0-rc3-linux-x86_64.tar.gz +RUN tar xvzf cmake-3.30.0-rc3-linux-x86_64.tar.gz --strip-components=1 -C /usr/local + ENV WHEEL_DIR=/source/wheelhouse ENV SFM_DIR=/source/OpenSfM COPY . $SFM_DIR WORKDIR $SFM_DIR + RUN rm -rf cmake_build RUN /root/.pyenv/versions/3.9.9/bin/pip install -r requirements.txt RUN /root/.pyenv/versions/3.9.9/bin/pip wheel $SFM_DIR --no-deps -w $WHEEL_DIR @@ -166,5 +172,3 @@ RUN cd ${WHEEL_DIR} && rm -rf *-linux*whl RUN /root/.pyenv/versions/3.9.9/bin/pip install opensfm --no-index -f $WHEEL_DIR RUN python -c "import opensfm" RUN ls /source/wheelhouse/*.whl | xargs -n 1 -I {} python -m twine upload --repository-url "http://pypi.artichoke-labs.ai" {} -##RUN sh /source/OpenSfM/build_wheel.sh -##RUN sh /source/OpenSfM/test_and_upload_wheel.sh diff --git a/README.md b/README.md index 61b9cbe3a..ccd58c6b2 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ OpenSfM is a Structure from Motion library written in Python. The library serves Checkout this [blog post with more demos](http://blog.mapillary.com/update/2014/12/15/sfm-preview.html) ## MacOS Install - `brew install opencv` + `brew install qt pyqt boost` ## Getting Started diff --git a/opensfm/src/third_party/SuiteSparse b/opensfm/src/third_party/SuiteSparse index 13806726c..24e8e2bdd 160000 --- a/opensfm/src/third_party/SuiteSparse +++ b/opensfm/src/third_party/SuiteSparse @@ -1 +1 @@ -Subproject commit 13806726cbf470914d012d132a85aea1aff9ee77 +Subproject commit 24e8e2bdddc43b98d66268b6d97f21b3229f5374 diff --git a/opensfm/src/third_party/tbb b/opensfm/src/third_party/tbb new file mode 160000 index 000000000..427c252e0 --- /dev/null +++ b/opensfm/src/third_party/tbb @@ -0,0 +1 @@ +Subproject commit 427c252e0bb9e191767a62d8a744b21950c343f6 diff --git a/setup.py b/setup.py index bb6cac700..8e7fc97de 100644 --- a/setup.py +++ b/setup.py @@ -5,8 +5,8 @@ import platform import subprocess import sys -from shutil import rmtree from pathlib import Path +from shutil import copytree, rmtree import setuptools from setuptools.command.install import install @@ -18,9 +18,11 @@ THIRD_PARTY_INSTALL_DIR = Path(__file__).parent / "third_party_install" THIRD_PARTY_BUILD_DIR = Path(__file__).parent / "third_party_build" +THIRD_PARTY_SOURCE_DIR = Path(__file__).parent / "opensfm/src/third_party" THIRD_PARTY_BUILD_DIR.mkdir(exist_ok=True) LIB_DIR = THIRD_PARTY_INSTALL_DIR / "usr/local/lib" INCLUDE_DIR = THIRD_PARTY_INSTALL_DIR / "usr/local/include" + SHARE_DIR = THIRD_PARTY_INSTALL_DIR / "usr/local/share" SHARED_CMAKE_DIR = THIRD_PARTY_INSTALL_DIR / "usr/local/lib/cmake/" @@ -29,20 +31,24 @@ f'{SHARE_DIR};' f'{INCLUDE_DIR};' f'{LIB_DIR};' - f'{THIRD_PARTY_INSTALL_DIR}' ) os.environ['LDFLAGS'] = f'-L/opt/homebrew/Cellar/libomp/18.1.6/lib' os.environ['C_INCLUDE_PATH'] = f'/opt/homebrew/Cellar/libomp/18.1.6/include/' os.environ['CPLUS_INCLUDE_PATH'] = f'/opt/homebrew/Cellar/libomp/18.1.6/include/' -# -# def _is_apple_silicon() -> bool: -# -# if platform.system() == 'Darwin': -# if platform.machine() == 'arm64': -# return True -# return False + +def _is_apple_silicon() -> bool: + + if platform.system() == 'Darwin': + if platform.machine() == 'arm64': + return True + return False + + +if _is_apple_silicon(): + os.environ['arch'] = 'arm64' + # # if _is_apple_silicon(): # print('Detected Apple Silicon processor') @@ -74,9 +80,8 @@ def finalize_options(self): def install_gflag(install_dir: Path): gflags_build_dir = THIRD_PARTY_BUILD_DIR / "gflags" gflag_source_dir = Path(__file__).parent.joinpath("opensfm/src/third_party/gflags/") - if gflags_build_dir.exists(): - rmtree(gflags_build_dir) - gflags_build_dir.mkdir() + if not gflags_build_dir.exists(): + gflags_build_dir.mkdir() cmake_command = [ 'cmake', '-DCMAKE_CXX_FLAGS="-fPIC"', @@ -99,11 +104,10 @@ def install_gflag(install_dir: Path): def install_glog(install_dir: Path): glog_build_dir = THIRD_PARTY_BUILD_DIR / "glog" - glog_source_dir = Path(__file__).parent.joinpath("opensfm/src/third_party/glog") + glog_source_dir = THIRD_PARTY_SOURCE_DIR / "glog" - if glog_build_dir.exists(): - rmtree(glog_build_dir) - glog_build_dir.mkdir() + if not glog_build_dir.exists(): + glog_build_dir.mkdir() cmake_command = [ 'cmake', @@ -111,12 +115,6 @@ def install_glog(install_dir: Path): glog_source_dir.as_posix() ] - # if _is_apple_silicon(): - # cmake_command += [ - # C_COMPILER_ARG, - # CXX_COMPILER_ARG, - # ] - subprocess.check_call( cmake_command, cwd=glog_build_dir.as_posix(), @@ -128,23 +126,18 @@ def install_glog(install_dir: Path): def install_eigen(install_dir: Path): eigen_build_dir = THIRD_PARTY_BUILD_DIR / "eigen" - eigen_source_dir = Path(__file__).parent.joinpath("opensfm/src/third_party/eigen") + eigen_source_dir = THIRD_PARTY_SOURCE_DIR / "eigen" - if eigen_build_dir.exists(): - rmtree(eigen_build_dir) + if not eigen_build_dir.exists(): + eigen_build_dir.mkdir() - eigen_build_dir.mkdir() cmake_command = [ 'cmake', CMAKE_PREFIX_PATH, eigen_source_dir.as_posix(), ] - # if _is_apple_silicon(): - # cmake_command += [ - # C_COMPILER_ARG, - # CXX_COMPILER_ARG, - # ] + subprocess.check_call( cmake_command, cwd=eigen_build_dir.as_posix(), @@ -153,48 +146,82 @@ def install_eigen(install_dir: Path): subprocess.check_call(["make", f"DESTDIR={install_dir}", "install"], cwd=eigen_build_dir.as_posix()) +def install_tbb(): + tbb_source_dir = THIRD_PARTY_SOURCE_DIR / "tbb" + tbb_build_dir = THIRD_PARTY_BUILD_DIR / "tbb" + tbb_build_lib = tbb_build_dir / 'lib' + tbb_build_include = tbb_build_dir / 'include' + + make_command = [] + make_command.extend(['make']) + subprocess.check_call( + make_command, + cwd=tbb_source_dir.as_posix(), + env=os.environ, + ) + tbb_potential_dirs = list(tbb_source_dir.rglob("libtbb*")) + if len(tbb_potential_dirs) == 0: + raise RuntimeError('Could not find tbb lib') + tbb_libb_dir = tbb_potential_dirs[0].parent + print(f'Found tbb lib dir: {tbb_libb_dir}') + if tbb_build_lib.exists(): + rmtree(tbb_build_lib) + if tbb_build_include.exists(): + rmtree(tbb_build_include) + + copytree(tbb_libb_dir, tbb_build_lib) + copytree(tbb_source_dir.joinpath("include"), tbb_build_include) + os.environ['TBB_ROOT'] = tbb_build_dir.as_posix() + os.environ['LD_LIBRARY_PATH'] = f"{tbb_build_lib.as_posix()}:{os.environ.get('LD_LIBRARY_PATH', '')}" + os.environ['LIBRARY_PATH'] = f"{tbb_build_lib.as_posix()}:{os.environ.get('LD_LIBRARY_PATH', '')}" + os.environ['C_INCLUDE_PATH'] = f"{tbb_build_include}:{os.environ.get('C_INCLUDE_PATH', '')}" + os.environ['CPLUS_INCLUDE_PATH'] = f"{tbb_build_include}:{os.environ.get('CPLUS_INCLUDE_PATH', '')}" + + def install_suitesparse(install_dir: Path): suitesparse_build_dir = THIRD_PARTY_BUILD_DIR / "SuiteSparse" - suitesparse_source_dir = Path(__file__).parent.joinpath("opensfm/src/third_party/SuiteSparse") - if suitesparse_build_dir.exists(): - rmtree(suitesparse_build_dir) - suitesparse_build_dir.mkdir() - cmake_command = [ - 'cmake', - CMAKE_PREFIX_PATH, - suitesparse_source_dir.as_posix(), - ] - # if _is_apple_silicon(): - # cmake_command += [ - # C_COMPILER_ARG, - # CXX_COMPILER_ARG, - # ] + suitesparse_source_dir = THIRD_PARTY_SOURCE_DIR / "SuiteSparse" + + if not suitesparse_build_dir.exists(): + suitesparse_build_dir.mkdir() + subprocess.check_call( - cmake_command, - cwd=suitesparse_build_dir.as_posix(), - env=os.environ, + [ + "make", + '-j8' + ], + cwd=suitesparse_source_dir.as_posix() ) - subprocess.check_call(["cmake", "--build", suitesparse_build_dir.as_posix()], cwd=suitesparse_build_dir.as_posix()) - subprocess.check_call(["make", f"DESTDIR={install_dir}", "install"], cwd=suitesparse_build_dir.as_posix()) + subprocess.check_call( + [ + "make", + 'local' + ], + cwd=suitesparse_source_dir.as_posix() + ) + subprocess.check_call(["make", f"DESTDIR={install_dir}", "install"], cwd=suitesparse_source_dir.as_posix()) def install_ceres(install_dir: Path): ceres_build_dir = THIRD_PARTY_BUILD_DIR / "ceres_solver" - ceres_source_dir = Path(__file__).parent.joinpath("opensfm/src/third_party/ceres-solver") - if ceres_build_dir.exists(): - rmtree(ceres_build_dir) - ceres_build_dir.mkdir() + ceres_source_dir = THIRD_PARTY_SOURCE_DIR / "ceres-solver" + if not ceres_build_dir.exists(): + ceres_build_dir.mkdir() + + del os.environ['arch'] + # del os.environ['OMP_NUM_THREADS'] + # del os.environ['OMP_PROC_BIND'] + # del os.environ['OMP_SCHEDULE'] + cmake_command = [ 'cmake', CMAKE_PREFIX_PATH, + "-Dopenmp=OFF", f'SUITESPARSE_LIBRARY_DIR_HINTS={install_dir}', + f'-DTBB_ROOT={os.environ["TBB_ROOT"]}', + ceres_source_dir.as_posix() ] - # if _is_apple_silicon(): - # cmake_command += [ - # C_COMPILER_ARG, - # CXX_COMPILER_ARG, - # ] subprocess.check_call( cmake_command, cwd=ceres_build_dir.as_posix() @@ -205,9 +232,8 @@ def install_ceres(install_dir: Path): def install_opensfm(install_dir: Path): - if install_dir.exists(): - rmtree(install_dir) - install_dir.mkdir() + if not install_dir.exists(): + install_dir.mkdir() cmake_command = [ "cmake", @@ -215,11 +241,7 @@ def install_opensfm(install_dir: Path): "../opensfm/src", "-DPYTHON_EXECUTABLE=" + sys.executable, ] - # if _is_apple_silicon(): - # cmake_command += [ - # C_COMPILER_ARG, - # CXX_COMPILER_ARG, - # ] + if sys.platform == "win32": cmake_command += [ "-DVCPKG_TARGET_TRIPLET=x64-windows", @@ -239,10 +261,11 @@ def configure_c_extension(): ) # Third party install - install_gflag(THIRD_PARTY_INSTALL_DIR) - install_glog(THIRD_PARTY_INSTALL_DIR) - install_eigen(THIRD_PARTY_INSTALL_DIR) - install_suitesparse(THIRD_PARTY_INSTALL_DIR) + # install_gflag(THIRD_PARTY_INSTALL_DIR) + # install_glog(THIRD_PARTY_INSTALL_DIR) + # install_eigen(THIRD_PARTY_INSTALL_DIR) + install_tbb() + # install_suitesparse(THIRD_PARTY_INSTALL_DIR) install_ceres(THIRD_PARTY_INSTALL_DIR) install_opensfm(Path(__file__).parent / "cmake_build") From e2ef2728aaf6e695dadcb6896d42b299aadee77b Mon Sep 17 00:00:00 2001 From: offbeat Date: Fri, 21 Jun 2024 23:51:31 +0400 Subject: [PATCH 10/16] Failed for MacOS --- setup.py | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/setup.py b/setup.py index 8e7fc97de..05432d2e6 100644 --- a/setup.py +++ b/setup.py @@ -50,14 +50,7 @@ def _is_apple_silicon() -> bool: os.environ['arch'] = 'arm64' # -# if _is_apple_silicon(): -# print('Detected Apple Silicon processor') -# # Define the compilers -# C_COMPILER_PATH = os.path.abspath('/opt/homebrew/Cellar/gcc@12/12.3.0/bin/gcc-12') -# CXX_COMPILER_PATH = os.path.abspath('/opt/homebrew/Cellar/gcc@12/12.3.0/bin/g++-12') -# C_COMPILER_ARG = f'-DCMAKE_C_COMPILER={C_COMPILER_PATH}' -# CXX_COMPILER_ARG = f'-DCMAKE_CXX_COMPILER={CXX_COMPILER_PATH}' -# + def version_str(version): return ".".join(map(str, version)) @@ -87,11 +80,6 @@ def install_gflag(install_dir: Path): '-DCMAKE_CXX_FLAGS="-fPIC"', gflag_source_dir ] - # if _is_apple_silicon(): - # cmake_command += [ - # C_COMPILER_ARG, - # CXX_COMPILER_ARG, - # ] subprocess.check_call( cmake_command, cwd=gflags_build_dir.as_posix(), @@ -209,9 +197,6 @@ def install_ceres(install_dir: Path): ceres_build_dir.mkdir() del os.environ['arch'] - # del os.environ['OMP_NUM_THREADS'] - # del os.environ['OMP_PROC_BIND'] - # del os.environ['OMP_SCHEDULE'] cmake_command = [ 'cmake', @@ -261,11 +246,11 @@ def configure_c_extension(): ) # Third party install - # install_gflag(THIRD_PARTY_INSTALL_DIR) - # install_glog(THIRD_PARTY_INSTALL_DIR) - # install_eigen(THIRD_PARTY_INSTALL_DIR) + install_gflag(THIRD_PARTY_INSTALL_DIR) + install_glog(THIRD_PARTY_INSTALL_DIR) + install_eigen(THIRD_PARTY_INSTALL_DIR) install_tbb() - # install_suitesparse(THIRD_PARTY_INSTALL_DIR) + install_suitesparse(THIRD_PARTY_INSTALL_DIR) install_ceres(THIRD_PARTY_INSTALL_DIR) install_opensfm(Path(__file__).parent / "cmake_build") From 046fae207402a4f6b0cbb82ba0d8f5bb53d1d127 Mon Sep 17 00:00:00 2001 From: offbeat Date: Sat, 22 Jun 2024 00:05:28 +0400 Subject: [PATCH 11/16] Removed cmake option --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 05432d2e6..71311ac06 100644 --- a/setup.py +++ b/setup.py @@ -201,9 +201,8 @@ def install_ceres(install_dir: Path): cmake_command = [ 'cmake', CMAKE_PREFIX_PATH, - "-Dopenmp=OFF", f'SUITESPARSE_LIBRARY_DIR_HINTS={install_dir}', - f'-DTBB_ROOT={os.environ["TBB_ROOT"]}', + # f'-DTBB_ROOT={os.environ["TBB_ROOT"]}', ceres_source_dir.as_posix() ] From 2ba64e1f615434da8604b298922056bb22e53dae Mon Sep 17 00:00:00 2001 From: offbeat Date: Sat, 22 Jun 2024 00:11:14 +0400 Subject: [PATCH 12/16] Removed cmake option --- setup.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup.py b/setup.py index 71311ac06..f86a17222 100644 --- a/setup.py +++ b/setup.py @@ -196,8 +196,6 @@ def install_ceres(install_dir: Path): if not ceres_build_dir.exists(): ceres_build_dir.mkdir() - del os.environ['arch'] - cmake_command = [ 'cmake', CMAKE_PREFIX_PATH, From fdd5331869be0b259960fe726e8c2b8bfe516cc0 Mon Sep 17 00:00:00 2001 From: offbeat Date: Mon, 24 Jun 2024 10:24:33 +0400 Subject: [PATCH 13/16] Finishing --- setup.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index f86a17222..7531bbabe 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ from wheel.bdist_wheel import bdist_wheel -VERSION = (0, 5, 2, "post19") +VERSION = (0, 5, 2, "post20") THIRD_PARTY_INSTALL_DIR = Path(__file__).parent / "third_party_install" THIRD_PARTY_BUILD_DIR = Path(__file__).parent / "third_party_build" @@ -29,13 +29,17 @@ CMAKE_PREFIX_PATH = ( f'-DCMAKE_PREFIX_PATH={SHARED_CMAKE_DIR};' f'{SHARE_DIR};' + f'{SHARE_DIR}/eigen3;' f'{INCLUDE_DIR};' + f'{INCLUDE_DIR}/eigen3;' f'{LIB_DIR};' + f'{THIRD_PARTY_BUILD_DIR};' + f'{THIRD_PARTY_BUILD_DIR / "SuiteSparse"};' ) os.environ['LDFLAGS'] = f'-L/opt/homebrew/Cellar/libomp/18.1.6/lib' os.environ['C_INCLUDE_PATH'] = f'/opt/homebrew/Cellar/libomp/18.1.6/include/' -os.environ['CPLUS_INCLUDE_PATH'] = f'/opt/homebrew/Cellar/libomp/18.1.6/include/' +os.environ['CPLUS_INCLUDE_PATH'] = f'/opt/homebrew/Cellar/libomp/18.1.6/include/:/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX14.5.sdk/usr/include/c++/v1/' def _is_apple_silicon() -> bool: @@ -169,6 +173,8 @@ def install_tbb(): def install_suitesparse(install_dir: Path): suitesparse_build_dir = THIRD_PARTY_BUILD_DIR / "SuiteSparse" suitesparse_source_dir = THIRD_PARTY_SOURCE_DIR / "SuiteSparse" + suitesparse_build_lib = suitesparse_build_dir / 'lib' + suitesparse_build_include = suitesparse_build_dir / 'include' if not suitesparse_build_dir.exists(): suitesparse_build_dir.mkdir() @@ -189,6 +195,14 @@ def install_suitesparse(install_dir: Path): ) subprocess.check_call(["make", f"DESTDIR={install_dir}", "install"], cwd=suitesparse_source_dir.as_posix()) + if suitesparse_build_lib.exists(): + rmtree(suitesparse_build_lib) + if suitesparse_build_include.exists(): + rmtree(suitesparse_build_include) + + copytree(suitesparse_source_dir / "lib", suitesparse_build_lib) + copytree(suitesparse_source_dir.joinpath("include"), suitesparse_build_include) + def install_ceres(install_dir: Path): ceres_build_dir = THIRD_PARTY_BUILD_DIR / "ceres_solver" @@ -199,7 +213,10 @@ def install_ceres(install_dir: Path): cmake_command = [ 'cmake', CMAKE_PREFIX_PATH, - f'SUITESPARSE_LIBRARY_DIR_HINTS={install_dir}', + f'-DSUITESPARSE_LIBRARY_DIR_HINTS={install_dir}', + f'-DSUITESPARSE_INCLUDE_DIRS={THIRD_PARTY_BUILD_DIR / "SuiteSparse/include/"}', + f'-DSUITESPARSE_LIBRARY_DIRS={THIRD_PARTY_BUILD_DIR / "SuiteSparse/lib/"}', + f'-DSUITESPARSE_LIBRARIES={THIRD_PARTY_BUILD_DIR / "SuiteSparse/lib/"}', # f'-DTBB_ROOT={os.environ["TBB_ROOT"]}', ceres_source_dir.as_posix() @@ -220,6 +237,7 @@ def install_opensfm(install_dir: Path): cmake_command = [ "cmake", CMAKE_PREFIX_PATH, + f'-DEigen3_DIR={INCLUDE_DIR}/eigen3', "../opensfm/src", "-DPYTHON_EXECUTABLE=" + sys.executable, ] From e57fd25397fd85c13acc1f52d7d81a30479faa20 Mon Sep 17 00:00:00 2001 From: offbeat Date: Mon, 24 Jun 2024 11:50:53 +0400 Subject: [PATCH 14/16] Working on Mac --- Dockerfile | 112 ++++++++++++++++------------------ opensfm/src/third_party/eigen | 2 +- setup.py | 28 +++++---- 3 files changed, 72 insertions(+), 70 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0c2129676..381dbc823 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,56 +54,56 @@ RUN apt-get update \ git \ cmake \ build-essential \ -# libsuitesparse-dev \ + libsuitesparse-dev \ curl -# -#RUN mkdir /source -#WORKDIR /source -#RUN git clone https://github.com/gflags/gflags.git -#WORKDIR /source/gflags/ -#RUN git checkout tags/v2.2.2 -#RUN mkdir /source/gflags/build/ -#WORKDIR /source/gflags/build/ -#RUN cmake -DCMAKE_CXX_FLAGS="-fPIC" .. -#RUN make -j8 -#RUN make install -# -#WORKDIR /source -#RUN git clone https://github.com/google/glog.git -#WORKDIR /source/glog -#RUN git checkout tags/v0.6.0 -#RUN mkdir /source/glog/build/ -#WORKDIR /source/glog/build/ -#RUN cmake .. -#RUN make -j8 -#RUN make install -# -#WORKDIR /source -#RUN git clone https://gitlab.com/libeigen/eigen.git -#WORKDIR /source/eigen/ -#RUN git checkout tags/3.4.0 -#RUN mkdir build -#WORKDIR /source/eigen/build/ -#RUN cmake .. -#RUN make install -RUN apt install -y lzip +RUN mkdir /source WORKDIR /source -RUN wget https://gmplib.org/download/gmp/gmp-6.3.0.tar.lz -RUN tar -xf gmp-6.3.0.tar.lz -WORKDIR /source/gmp-6.3.0 -RUN ./configure -RUN make +RUN git clone https://github.com/gflags/gflags.git +WORKDIR /source/gflags/ +RUN git checkout tags/v2.2.2 +RUN mkdir /source/gflags/build/ +WORKDIR /source/gflags/build/ +RUN cmake -DCMAKE_CXX_FLAGS="-fPIC" .. +RUN make -j8 RUN make install WORKDIR /source -RUN wget https://www.mpfr.org/mpfr-current/mpfr-4.2.1.tar.xz -RUN tar -xf mpfr-4.2.1.tar.xz -WORKDIR /source/mpfr-4.2.1 -RUN ./configure -RUN make +RUN git clone https://github.com/google/glog.git +WORKDIR /source/glog +RUN git checkout tags/v0.6.0 +RUN mkdir /source/glog/build/ +WORKDIR /source/glog/build/ +RUN cmake .. +RUN make -j8 +RUN make install + +WORKDIR /source +RUN git clone https://gitlab.com/libeigen/eigen.git +WORKDIR /source/eigen/ +RUN git checkout tags/3.4.0 +RUN mkdir build +WORKDIR /source/eigen/build/ +RUN cmake .. RUN make install +#RUN yum install -y lzip +#WORKDIR /source +#RUN wget https://gmplib.org/download/gmp/gmp-6.3.0.tar.lz +#RUN tar -xf gmp-6.3.0.tar.lz +#WORKDIR /source/gmp-6.3.0 +#RUN ./configure +#RUN make +#RUN make install +# +#WORKDIR /source +#RUN wget https://www.mpfr.org/mpfr-current/mpfr-4.2.1.tar.xz +#RUN tar -xf mpfr-4.2.1.tar.xz +#WORKDIR /source/mpfr-4.2.1 +#RUN ./configure +#RUN make +#RUN make install + #WORKDIR /source #RUN git clone https://github.com/DrTimothyAldenDavis/SuiteSparse.git #WORKDIR /source/SuiteSparse @@ -123,15 +123,15 @@ RUN make install #RUN make -j4 #RUN make install -#WORKDIR /source -#RUN git clone https://github.com/ceres-solver/ceres-solver.git -#WORKDIR /source/ceres-solver -#RUN git checkout tags/2.0.0 -#RUN mkdir build -#WORKDIR /source/ceres-solver/build/ -#RUN cmake .. -#RUN make -j8 -#RUN make install +WORKDIR /source +RUN git clone https://github.com/ceres-solver/ceres-solver.git +WORKDIR /source/ceres-solver +RUN git checkout tags/2.0.0 +RUN mkdir build +WORKDIR /source/ceres-solver/build/ +RUN cmake .. +RUN make -j8 +RUN make install WORKDIR /source RUN git clone https://github.com/NixOS/patchelf.git @@ -153,17 +153,11 @@ RUN /root/.pyenv/bin/pyenv global 3.9.9 ENV PATH=$PATH:/root/.pyenv/versions/3.9.9/bin/ RUN python -m pip install --upgrade pip -WORKDIR /source -RUN apt remove -y cmake -RUN wget https://github.com/Kitware/CMake/releases/download/v3.30.0-rc3/cmake-3.30.0-rc3-linux-x86_64.tar.gz -RUN tar xvzf cmake-3.30.0-rc3-linux-x86_64.tar.gz --strip-components=1 -C /usr/local - ENV WHEEL_DIR=/source/wheelhouse ENV SFM_DIR=/source/OpenSfM COPY . $SFM_DIR WORKDIR $SFM_DIR - RUN rm -rf cmake_build RUN /root/.pyenv/versions/3.9.9/bin/pip install -r requirements.txt RUN /root/.pyenv/versions/3.9.9/bin/pip wheel $SFM_DIR --no-deps -w $WHEEL_DIR @@ -171,4 +165,6 @@ RUN ls /source/wheelhouse/*.whl | xargs -n 1 -I {} auditwheel repair {} --plat RUN cd ${WHEEL_DIR} && rm -rf *-linux*whl RUN /root/.pyenv/versions/3.9.9/bin/pip install opensfm --no-index -f $WHEEL_DIR RUN python -c "import opensfm" -RUN ls /source/wheelhouse/*.whl | xargs -n 1 -I {} python -m twine upload --repository-url "http://pypi.artichoke-labs.ai" {} +#RUN ls /source/wheelhouse/*.whl | xargs -n 1 -I {} python -m twine upload --repository-url "http://pypi.artichoke-labs.ai" {} +#RUN sh /source/OpenSfM/build_wheel.sh +#RUN sh /source/OpenSfM/test_and_upload_wheel.sh diff --git a/opensfm/src/third_party/eigen b/opensfm/src/third_party/eigen index 3147391d9..eeac81b8c 160000 --- a/opensfm/src/third_party/eigen +++ b/opensfm/src/third_party/eigen @@ -1 +1 @@ -Subproject commit 3147391d946bb4b6c68edd901f2add6ac1f31f8c +Subproject commit eeac81b8c067763e811d1dd7f19fe91640834f82 diff --git a/setup.py b/setup.py index 7531bbabe..0b4b34019 100644 --- a/setup.py +++ b/setup.py @@ -34,8 +34,9 @@ f'{INCLUDE_DIR}/eigen3;' f'{LIB_DIR};' f'{THIRD_PARTY_BUILD_DIR};' - f'{THIRD_PARTY_BUILD_DIR / "SuiteSparse"};' + f'{THIRD_PARTY_BUILD_DIR / "SuiteSparse"}; ' ) +CMAKE_OSX_SYSROOT = '-DCMAKE_OSX_SYSROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/' os.environ['LDFLAGS'] = f'-L/opt/homebrew/Cellar/libomp/18.1.6/lib' os.environ['C_INCLUDE_PATH'] = f'/opt/homebrew/Cellar/libomp/18.1.6/include/' @@ -80,10 +81,11 @@ def install_gflag(install_dir: Path): if not gflags_build_dir.exists(): gflags_build_dir.mkdir() cmake_command = [ - 'cmake', - '-DCMAKE_CXX_FLAGS="-fPIC"', - gflag_source_dir - ] + 'cmake', + CMAKE_OSX_SYSROOT, + '-DCMAKE_CXX_FLAGS="-fPIC"', + gflag_source_dir + ] subprocess.check_call( cmake_command, cwd=gflags_build_dir.as_posix(), @@ -103,6 +105,7 @@ def install_glog(install_dir: Path): cmake_command = [ 'cmake', + CMAKE_OSX_SYSROOT, CMAKE_PREFIX_PATH, glog_source_dir.as_posix() ] @@ -125,6 +128,7 @@ def install_eigen(install_dir: Path): cmake_command = [ 'cmake', + CMAKE_OSX_SYSROOT, CMAKE_PREFIX_PATH, eigen_source_dir.as_posix(), ] @@ -212,6 +216,7 @@ def install_ceres(install_dir: Path): cmake_command = [ 'cmake', + CMAKE_OSX_SYSROOT, CMAKE_PREFIX_PATH, f'-DSUITESPARSE_LIBRARY_DIR_HINTS={install_dir}', f'-DSUITESPARSE_INCLUDE_DIRS={THIRD_PARTY_BUILD_DIR / "SuiteSparse/include/"}', @@ -236,6 +241,7 @@ def install_opensfm(install_dir: Path): cmake_command = [ "cmake", + CMAKE_OSX_SYSROOT, CMAKE_PREFIX_PATH, f'-DEigen3_DIR={INCLUDE_DIR}/eigen3', "../opensfm/src", @@ -261,12 +267,12 @@ def configure_c_extension(): ) # Third party install - install_gflag(THIRD_PARTY_INSTALL_DIR) - install_glog(THIRD_PARTY_INSTALL_DIR) - install_eigen(THIRD_PARTY_INSTALL_DIR) - install_tbb() - install_suitesparse(THIRD_PARTY_INSTALL_DIR) - install_ceres(THIRD_PARTY_INSTALL_DIR) + # install_gflag(THIRD_PARTY_INSTALL_DIR) + # install_glog(THIRD_PARTY_INSTALL_DIR) + # install_eigen(THIRD_PARTY_INSTALL_DIR) + # install_tbb() + # install_suitesparse(THIRD_PARTY_INSTALL_DIR) + # install_ceres(THIRD_PARTY_INSTALL_DIR) install_opensfm(Path(__file__).parent / "cmake_build") From 06cf9934a4ad6e72ec547bec1aa7f1b67edcfaff Mon Sep 17 00:00:00 2001 From: offbeat Date: Thu, 4 Jul 2024 15:43:51 +0400 Subject: [PATCH 15/16] Finished updating dependencies --- .dockerignore | 2 -- Dockerfile | 4 +--- requirements.txt | 22 +++++++++++----------- setup.py | 18 ++++++++++-------- 4 files changed, 22 insertions(+), 24 deletions(-) diff --git a/.dockerignore b/.dockerignore index 0d83bade3..83f5d8131 100644 --- a/.dockerignore +++ b/.dockerignore @@ -16,8 +16,6 @@ !setup.cfg !setup.py !viewer -!build_wheel.sh -!test_and_upload_wheel.sh # Picking exceptions doc/build diff --git a/Dockerfile b/Dockerfile index 381dbc823..7bba4c59e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -165,6 +165,4 @@ RUN ls /source/wheelhouse/*.whl | xargs -n 1 -I {} auditwheel repair {} --plat RUN cd ${WHEEL_DIR} && rm -rf *-linux*whl RUN /root/.pyenv/versions/3.9.9/bin/pip install opensfm --no-index -f $WHEEL_DIR RUN python -c "import opensfm" -#RUN ls /source/wheelhouse/*.whl | xargs -n 1 -I {} python -m twine upload --repository-url "http://pypi.artichoke-labs.ai" {} -#RUN sh /source/OpenSfM/build_wheel.sh -#RUN sh /source/OpenSfM/test_and_upload_wheel.sh +RUN ls /source/wheelhouse/*.whl | xargs -n 1 -I {} python -m twine upload --repository-url "http://pypi.artichoke-labs.ai" {} diff --git a/requirements.txt b/requirements.txt index 0bdf77e9e..0e1d26748 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,22 +1,22 @@ -cloudpickle==0.4.0 -exifread==2.1.2 -flask==2.3.2 -fpdf2==2.4.6 -joblib==0.14.1 +cloudpickle>=0.4.0 +exifread>=2.1.2 +flask>=2.3.2 +fpdf2>=2.4.6 +joblib>=0.14.1 matplotlib networkx==2.5 -numpy>=1.19 +numpy Pillow>=8.1.1 pyproj>=1.9.5.1 -pytest==7.0.0 +pytest>=3.0.7 python-dateutil>=2.7 -pyyaml==6.0 -scipy>=1.10.0 +pyyaml>=6.0 +scipy sphinx==4.2.0 six -xmltodict==0.10.2 +xmltodict>=0.10.2 wheel -opencv-python==4.5.1.48 ; sys_platform == "win32" +opencv-python>=4.10.0 ; sys_platform == "win32" opencv-python ; sys_platform == "linux" twine sphinx diff --git a/setup.py b/setup.py index 0b4b34019..f7b44ed4f 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ from wheel.bdist_wheel import bdist_wheel -VERSION = (0, 5, 2, "post20") +VERSION = (0, 5, 2, "post22") THIRD_PARTY_INSTALL_DIR = Path(__file__).parent / "third_party_install" THIRD_PARTY_BUILD_DIR = Path(__file__).parent / "third_party_build" @@ -266,13 +266,15 @@ def configure_c_extension(): f"Configuring for python {sys.version_info.major}.{sys.version_info.minor}..." ) - # Third party install - # install_gflag(THIRD_PARTY_INSTALL_DIR) - # install_glog(THIRD_PARTY_INSTALL_DIR) - # install_eigen(THIRD_PARTY_INSTALL_DIR) - # install_tbb() - # install_suitesparse(THIRD_PARTY_INSTALL_DIR) - # install_ceres(THIRD_PARTY_INSTALL_DIR) + if platform.system() == 'Darwin': + # Third party install + install_gflag(THIRD_PARTY_INSTALL_DIR) + install_glog(THIRD_PARTY_INSTALL_DIR) + install_eigen(THIRD_PARTY_INSTALL_DIR) + install_tbb() + install_suitesparse(THIRD_PARTY_INSTALL_DIR) + install_ceres(THIRD_PARTY_INSTALL_DIR) + install_opensfm(Path(__file__).parent / "cmake_build") From 5eabbd8806b93cb73112147aa3af22da44bf36bd Mon Sep 17 00:00:00 2001 From: offbeat Date: Thu, 4 Jul 2024 15:44:40 +0400 Subject: [PATCH 16/16] deleted redundant files --- build_wheel.sh | 27 --------------------------- test_and_upload_wheel.sh | 15 --------------- 2 files changed, 42 deletions(-) delete mode 100644 build_wheel.sh delete mode 100644 test_and_upload_wheel.sh diff --git a/build_wheel.sh b/build_wheel.sh deleted file mode 100644 index 43d7dab58..000000000 --- a/build_wheel.sh +++ /dev/null @@ -1,27 +0,0 @@ -!/bin/bash -set -e -u -x - -function repair_wheel { - wheel="$1" - if ! /opt/_internal/cpython-3.9.19/bin/auditwheel show "$wheel"; then - echo "Skipping non-platform wheel $wheel" - else - /opt/_internal/cpython-3.9.19/bin/auditwheel repair "$wheel" --plat "$PLAT" -w $WHEEL_DIR - fi -} - -# Compile wheels -for PYBIN in /opt/python/*/bin; do - if [[ "${PYBIN}" != *cp311* ]] && [[ "${PYBIN}" != *cp312* ]] && [[ "${PYBIN}" != *cp313* ]] && [[ "${PYBIN}" != *cp36* ]] && [[ "${PYBIN}" != *cp37* ]] && [[ "${PYBIN}" != *py37* ]] && [[ "${PYBIN}" != *cp38* ]] && [[ "${PYBIN}" != *py38* ]] && [[ "${PYBIN}" != *py39* ]] && [[ "${PYBIN}" != *cp310* ]] && [[ "${PYBIN}" != *py310* ]]; then - "${PYBIN}/pip" install --upgrade pip - "${PYBIN}/pip" install -r ${SFM_DIR}/requirements.txt - "${PYBIN}/pip" wheel $SFM_DIR --no-deps -w $WHEEL_DIR - fi -done - -# Bundle external shared libraries into the wheels -for whl in $WHEEL_DIR/*.whl; do - repair_wheel "$whl" -done - -cd ${WHEEL_DIR} && rm -rf *-linux*whl diff --git a/test_and_upload_wheel.sh b/test_and_upload_wheel.sh deleted file mode 100644 index 485b763c0..000000000 --- a/test_and_upload_wheel.sh +++ /dev/null @@ -1,15 +0,0 @@ -!/bin/bash -set -e -u -x - -# Install packages and test -for PYBIN in /opt/python/*/bin; do - if [[ "${PYBIN}" != *cp311* ]] && [[ "${PYBIN}" != *cp312* ]] && [[ "${PYBIN}" != *cp313* ]] && [[ "${PYBIN}" != *cp36* ]] && [[ "${PYBIN}" != *cp37* ]] && [[ "${PYBIN}" != *py37* ]] && [[ "${PYBIN}" != *cp38* ]] && [[ "${PYBIN}" != *py38* ]] && [[ "${PYBIN}" != *py39* ]] && [[ "${PYBIN}" != *cp310* ]] && [[ "${PYBIN}" != *py310* ]]; then - "${PYBIN}/pip" install opensfm --no-index -f $WHEEL_DIR - cd / && "${PYBIN}/python" -c "import opensfm" - fi -done - - -for WHL in $WHEEL_DIR/*.whl; do - /opt/python/cp39-cp39/bin/python -m twine upload --repository-url "http://pypi.artichoke-labs.ai" $WHL -done