From b51c26e0341ef6e6ae79a6ba1495c7090bef24e3 Mon Sep 17 00:00:00 2001 From: Mikhail Zakharov Date: Sun, 12 Jun 2022 21:27:00 -0400 Subject: [PATCH] lapack/scipy: support NDK r21e, x86/64 archs --- ci/makefiles/android.mk | 16 ++++++----- pythonforandroid/recipes/lapack/__init__.py | 25 ++++++++++------- pythonforandroid/recipes/numpy/__init__.py | 1 - .../numpy/patches/fix-missing-threads.h.patch | 26 ------------------ pythonforandroid/recipes/scipy/__init__.py | 27 ++++++++++--------- 5 files changed, 40 insertions(+), 55 deletions(-) delete mode 100644 pythonforandroid/recipes/numpy/patches/fix-missing-threads.h.patch diff --git a/ci/makefiles/android.mk b/ci/makefiles/android.mk index bb29c3ea17..6550daa0ba 100644 --- a/ci/makefiles/android.mk +++ b/ci/makefiles/android.mk @@ -2,7 +2,7 @@ # Those android NDK/SDK variables can be override when running the file ANDROID_NDK_VERSION ?= 23b -ANDROID_NDK_VERSION_LEGACY ?= 19c +ANDROID_NDK_VERSION_LEGACY ?= 21e ANDROID_SDK_TOOLS_VERSION ?= 6514223 ANDROID_SDK_BUILD_TOOLS_VERSION ?= 29.0.3 ANDROID_HOME ?= $(HOME)/.android @@ -93,12 +93,14 @@ extract_android_ndk_legacy: && rm -f $(ANDROID_NDK_ARCHIVE_LEGACY) extract_android_ndk_gfortran: - rm -rf $(ANDROID_NDK_HOME_LEGACY)/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/ - mkdir $(ANDROID_NDK_HOME_LEGACY)/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/ - tar -xvf $(ANDROID_NDK_GFORTRAN_ARCHIVE_ARM64) -C $(ANDROID_NDK_HOME_LEGACY)/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/ --strip-components 1 - rm -rf $(ANDROID_NDK_HOME_LEGACY)/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/ - mkdir $(ANDROID_NDK_HOME_LEGACY)/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/ - tar -xvf $(ANDROID_NDK_GFORTRAN_ARCHIVE_ARM) -C $(ANDROID_NDK_HOME_LEGACY)/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/ --strip-components 1 + rm -rf $(ANDROID_NDK_HOME_LEGACY)/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/ \ + && mkdir $(ANDROID_NDK_HOME_LEGACY)/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/ \ + && tar -xf $(ANDROID_NDK_GFORTRAN_ARCHIVE_ARM64) -C $(ANDROID_NDK_HOME_LEGACY)/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/ --strip-components 1 \ + && rm -f $(ANDROID_NDK_GFORTRAN_ARCHIVE_ARM64) \ + && rm -rf $(ANDROID_NDK_HOME_LEGACY)/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/ \ + && mkdir $(ANDROID_NDK_HOME_LEGACY)/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/ \ + && tar -xf $(ANDROID_NDK_GFORTRAN_ARCHIVE_ARM) -C $(ANDROID_NDK_HOME_LEGACY)/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/ --strip-components 1 \ + && rm -f $(ANDROID_NDK_GFORTRAN_ARCHIVE_ARM) diff --git a/pythonforandroid/recipes/lapack/__init__.py b/pythonforandroid/recipes/lapack/__init__.py index 078946bd36..0e2729781b 100644 --- a/pythonforandroid/recipes/lapack/__init__.py +++ b/pythonforandroid/recipes/lapack/__init__.py @@ -1,5 +1,5 @@ ''' -known to build with cmake version 3.19.2 and NDK r19c. +known to build with cmake version 3.23.2 and NDK r21e. See https://gitlab.kitware.com/cmake/cmake/-/issues/18739 ''' @@ -15,10 +15,16 @@ arch_to_sysroot = {'armeabi': 'arm', 'armeabi-v7a': 'arm', 'arm64-v8a': 'arm64'} +def arch_to_toolchain(arch): + if 'arm' in arch.arch: + return arch.command_prefix + return arch.arch + + class LapackRecipe(Recipe): name = 'lapack' - version = 'v3.9.0' + version = 'v3.10.1' url = 'https://github.com/Reference-LAPACK/lapack/archive/{version}.tar.gz' libdir = 'build/install/lib' built_libraries = {'libblas.so': libdir, 'liblapack.so': libdir, 'libcblas.so': libdir} @@ -28,14 +34,14 @@ def get_recipe_env(self, arch): ndk_dir = environ.get("LEGACY_NDK") if ndk_dir is None: - raise BuildInterruptingException("Please set the environment variable 'LEGACY_NDK' to point to a NDK location with gcc/gfortran support (last tested NDK version was 'r19c')") + raise BuildInterruptingException("Please set the environment variable 'LEGACY_NDK' to point to a NDK location with gcc/gfortran support (supported NDK version: 'r21e')") GCC_VER = '4.9' HOST = build_platform sysroot_suffix = arch_to_sysroot.get(arch.arch, arch.arch) sysroot = f"{ndk_dir}/platforms/{env['NDK_API']}/arch-{sysroot_suffix}" - FC = f"{ndk_dir}/toolchains/{arch.command_prefix}-{GCC_VER}/prebuilt/{HOST}/bin/{arch.command_prefix}-gfortran" + FC = f"{ndk_dir}/toolchains/{arch_to_toolchain(arch)}-{GCC_VER}/prebuilt/{HOST}/bin/{arch.command_prefix}-gfortran" env['FC'] = f'{FC} --sysroot={sysroot}' if sh.which(FC) is None: raise BuildInterruptingException(f"{FC} not found. See https://github.com/mzakharo/android-gfortran") @@ -51,19 +57,20 @@ def build_arch(self, arch): env = self.get_recipe_env(arch) ndk_dir = environ["LEGACY_NDK"] shprint(sh.rm, '-rf', 'CMakeFiles/', 'CMakeCache.txt', _env=env) - shprint(sh.cmake, source_dir, + opts = [ '-DCMAKE_SYSTEM_NAME=Android', '-DCMAKE_POSITION_INDEPENDENT_CODE=1', '-DCMAKE_ANDROID_ARCH_ABI={arch}'.format(arch=arch.arch), '-DCMAKE_ANDROID_NDK=' + ndk_dir, + '-DCMAKE_ANDROID_API={api}'.format(api=self.ctx.ndk_api), '-DCMAKE_BUILD_TYPE=Release', '-DCMAKE_INSTALL_PREFIX={}'.format(install_target), - '-DANDROID_ABI={arch}'.format(arch=arch.arch), - '-DANDROID_ARM_NEON=ON', - '-DENABLE_NEON=ON', '-DCBLAS=ON', '-DBUILD_SHARED_LIBS=ON', - _env=env) + ] + if arch.arch == 'armeabi-v7a': + opts.append('-DCMAKE_ANDROID_ARM_NEON=ON') + shprint(sh.cmake, source_dir, *opts, _env=env) shprint(sh.make, '-j' + str(cpu_count()), _env=env) shprint(sh.make, 'install', _env=env) diff --git a/pythonforandroid/recipes/numpy/__init__.py b/pythonforandroid/recipes/numpy/__init__.py index 8ac811958f..6570134498 100644 --- a/pythonforandroid/recipes/numpy/__init__.py +++ b/pythonforandroid/recipes/numpy/__init__.py @@ -19,7 +19,6 @@ class NumpyRecipe(CompiledComponentsPythonRecipe): patches = [ join("patches", "remove-default-paths.patch"), join("patches", "add_libm_explicitly_to_build.patch"), - join("patches", "fix-missing-threads.h.patch"), ] def get_recipe_env(self, arch=None, with_flags_in_cc=True): diff --git a/pythonforandroid/recipes/numpy/patches/fix-missing-threads.h.patch b/pythonforandroid/recipes/numpy/patches/fix-missing-threads.h.patch deleted file mode 100644 index ef7d66133e..0000000000 --- a/pythonforandroid/recipes/numpy/patches/fix-missing-threads.h.patch +++ /dev/null @@ -1,26 +0,0 @@ -From: Mikhail Zakharov -Date: Sun, 5 Jun 2022 11:14:23 -0400 -Subject: [PATCH] fix missing threads.h - ---- - numpy/f2py/cfuncs.py | 3 +-- - 1 file changed, 1 insertion(+), 2 deletions(-) - - -diff --git a/numpy/f2py/cfuncs.py b/numpy/f2py/cfuncs.py -index bdd27ad..39cb470 100644 ---- a/numpy/f2py/cfuncs.py -+++ b/numpy/f2py/cfuncs.py -@@ -586,8 +586,7 @@ cppmacros["F2PY_THREAD_LOCAL_DECL"] = """\ - so `!defined(__STDC_NO_THREADS__)` may give false positive for the existence - of `threads.h` when using an older release of glibc 2.12 - See gh-19437 for details on OpenBSD */ --#include --#define F2PY_THREAD_LOCAL_DECL thread_local -+#define F2PY_THREAD_LOCAL_DECL __thread - #elif defined(__GNUC__) \\ - && (__GNUC__ > 4 || (__GNUC__ == 4 && (__GNUC_MINOR__ >= 4))) - #define F2PY_THREAD_LOCAL_DECL __thread --- -2.25.1 - diff --git a/pythonforandroid/recipes/scipy/__init__.py b/pythonforandroid/recipes/scipy/__init__.py index 6ddfd4b13a..6302d9c235 100644 --- a/pythonforandroid/recipes/scipy/__init__.py +++ b/pythonforandroid/recipes/scipy/__init__.py @@ -4,7 +4,11 @@ from os import environ from pythonforandroid.util import build_platform -arch_to_sysroot = {'armeabi': 'arm', 'armeabi-v7a': 'arm', 'arm64-v8a': 'arm64'} + +def arch_to_toolchain(arch): + if 'arm' in arch.arch: + return arch.command_prefix + return arch.arch class ScipyRecipe(CompiledComponentsPythonRecipe): @@ -35,16 +39,12 @@ def get_recipe_env(self, arch): suffix = '64' if '64' in arch.arch else '' prefix = arch.command_prefix - sysroot_suffix = arch_to_sysroot.get(arch.arch, arch.arch) - sysroot = f"{ndk_dir}/platforms/{env['NDK_API']}/arch-{sysroot_suffix}" - sysroot_include = f'{ndk_dir}/toolchains/llvm/prebuilt/{HOST}/sysroot/usr/include' CLANG_BIN = f'{ndk_dir}/toolchains/llvm/prebuilt/{HOST}/bin/' - GCC = f'{ndk_dir}/toolchains/{prefix}-{GCC_VER}/prebuilt/{HOST}' + GCC = f'{ndk_dir}/toolchains/{arch_to_toolchain(arch)}-{GCC_VER}/prebuilt/{HOST}' libgfortran = f'{GCC}/{prefix}/lib{suffix}' - numpylib = self.ctx.get_python_install_dir(arch.arch) + '/numpy' - LDSHARED_opts = env['LDSHARED'].split('clang')[1] arch_cflags = ' '.join(arch.arch_cflags) + LDSHARED_opts = f'-target {arch.target} {arch_cflags} ' + ' '.join(arch.common_ldshared) # TODO: add pythran support env['SCIPY_USE_PYTHRAN'] = '0' @@ -58,13 +58,16 @@ def get_recipe_env(self, arch): env['F90'] = f'{GCC}/bin/{prefix}-gfortran' env['CC'] = f'{CLANG_BIN}/clang -target {arch.target} {arch_cflags}' env['CXX'] = f'{CLANG_BIN}/clang++ -target {arch.target} {arch_cflags}' + + # scipy expects ldshared to be a single executable without options env['LDSHARED'] = f'{CLANG_BIN}/clang' - # flags - env['CPPFLAGS'] = f'-DANDROID -I{sysroot_include}/{prefix} --sysroot={sysroot} -I{sysroot_include}/c++/v1 -I{sysroot_include}' - env['LDFLAGS'] += f' {LDSHARED_opts} --sysroot={sysroot} -L{libgfortran} -L{numpylib}/core/lib -L{numpylib}/random/lib' - env['LDFLAGS'] += f' -l{self.stl_lib_name} ' - env['LDFLAGS'] += f' -L{ndk_dir}/sources/cxx-stl/llvm-libc++/libs/{arch.arch}/' # for arm32 - unwind + # erase the default NDK C++ include options + env['CPPFLAGS'] = '-DANDROID' + + # configure linker + env['LDFLAGS'] += f' {LDSHARED_opts} -L{libgfortran} -L{numpylib}/core/lib -L{numpylib}/random/lib' + env['LDFLAGS'] += f' -l{self.stl_lib_name}' return env