From 1ed3a3932ca87690202b5cdb3896da9c68e845a2 Mon Sep 17 00:00:00 2001 From: Zack Singer Date: Fri, 17 Nov 2023 13:28:56 -0500 Subject: [PATCH 01/20] Add inverse_distortion speedups --- .github/workflows/package.yml | 2 +- .github/workflows/test.yml | 33 ++-- hexrd/distortion/ge_41rt.py | 178 ++++-------------- hexrd/transforms/cpp_sublibrary/Makefile | 18 ++ .../cpp_sublibrary/src/inverse_distortion.cpp | 31 +++ install_build_dependencies.py | 72 +++++++ setup.py | 58 ++++++ tests/test_inverse_distortion.py | 86 +++++++++ 8 files changed, 316 insertions(+), 162 deletions(-) create mode 100644 hexrd/transforms/cpp_sublibrary/Makefile create mode 100644 hexrd/transforms/cpp_sublibrary/src/inverse_distortion.cpp create mode 100644 install_build_dependencies.py create mode 100644 tests/test_inverse_distortion.py diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index a728867b6..735dae606 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -2,7 +2,7 @@ name: conda-package on: push: - branches: [ master ] + branches: [ master, inverse-distortion ] tags: ["*"] pull_request: branches: [ master ] diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ea3cd94ca..4a8c24d02 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,10 +2,13 @@ name: test on: push: - branches: [ master ] + branches: [ master, inverse-distortion ] pull_request: branches: [ master ] +env: + CONDA_PREFIX: ${{ github.workspace }}/miniconda3/envs/hexrd + jobs: pytest: name: ${{ matrix.config.name }} @@ -15,21 +18,14 @@ jobs: matrix: python-version: [3.11] config: - - { - name: "Linux", - os: ubuntu-latest - } - - { - name: "MacOSX", - os: macos-latest - } - - { - name: "Windows", - os: windows-latest - } + - {name: "Linux", os: ubuntu-latest} + - {name: "MacOSX", os: macos-latest} + - {name: "Windows", os: windows-latest} + defaults: run: shell: bash + steps: - name: Set up Python ${{ matrix.python-version }} @@ -60,9 +56,9 @@ jobs: pip install fabio==0.12 if: ${{ matrix.config.os == 'macos-latest'}} - - name: Install HEXRD + - name: Install Build Dependencies run: | - pip install . + python install_build_dependencies.py working-directory: hexrd - name: Install requirements-dev.txt @@ -70,9 +66,14 @@ jobs: pip install -r tests/requirements-dev.txt working-directory: hexrd + - name: Install HEXRD + run: | + pip install . + working-directory: hexrd + - name: Run tests env: HEXRD_EXAMPLE_REPO_PATH: ${{ github.workspace }}/examples run: | pytest -s tests/ - working-directory: hexrd + working-directory: hexrd \ No newline at end of file diff --git a/hexrd/distortion/ge_41rt.py b/hexrd/distortion/ge_41rt.py index 0cad7646e..9b1ee67f9 100644 --- a/hexrd/distortion/ge_41rt.py +++ b/hexrd/distortion/ge_41rt.py @@ -2,151 +2,39 @@ import numpy as np from hexrd import constants as cnst -from hexrd.constants import USE_NUMBA -if USE_NUMBA: - import numba +import numba from .distortionabc import DistortionABC from .registry import _RegisterDistortionClass from .utils import newton +from hexrd.extensions import inverse_distortion RHO_MAX = 204.8 # max radius in mm for ge detector -if USE_NUMBA: - @numba.njit(nogil=True, cache=True) - def _ge_41rt_inverse_distortion(out, in_, rhoMax, params): - maxiter = 100 - prec = cnst.epsf - - p0, p1, p2, p3, p4, p5 = params[0:6] - rxi = 1.0/rhoMax - for el in range(len(in_)): - xi, yi = in_[el, 0:2] - ri = np.sqrt(xi*xi + yi*yi) - if ri < cnst.sqrt_epsf: - ri_inv = 0.0 - else: - ri_inv = 1.0/ri - sinni = yi*ri_inv - cosni = xi*ri_inv - ro = ri - cos2ni = cosni*cosni - sinni*sinni - sin2ni = 2*sinni*cosni - cos4ni = cos2ni*cos2ni - sin2ni*sin2ni - # newton solver iteration - for i in range(maxiter): - ratio = ri*rxi - fx = (p0*ratio**p3*cos2ni + - p1*ratio**p4*cos4ni + - p2*ratio**p5 + 1)*ri - ro # f(x) - fxp = (p0*ratio**p3*cos2ni*(p3+1) + - p1*ratio**p4*cos4ni*(p4+1) + - p2*ratio**p5*(p5+1) + 1) # f'(x) - - delta = fx/fxp - ri = ri - delta - # convergence check for newton - if np.abs(delta) <= prec*np.abs(ri): - break - - xi = ri*cosni - yi = ri*sinni - out[el, 0] = xi - out[el, 1] = yi - - return out - - @numba.njit(nogil=True, cache=True) - def _ge_41rt_distortion(out, in_, rhoMax, params): - p0, p1, p2, p3, p4, p5 = params[0:6] - rxi = 1.0/rhoMax - - for el in range(len(in_)): - xi, yi = in_[el, 0:2] - ri = np.sqrt(xi*xi + yi*yi) - if ri < cnst.sqrt_epsf: - ri_inv = 0.0 - else: - ri_inv = 1.0/ri - sinni = yi*ri_inv - cosni = xi*ri_inv - cos2ni = cosni*cosni - sinni*sinni - sin2ni = 2*sinni*cosni - cos4ni = cos2ni*cos2ni - sin2ni*sin2ni - ratio = ri*rxi - - ri = (p0*ratio**p3*cos2ni - + p1*ratio**p4*cos4ni - + p2*ratio**p5 - + 1)*ri - xi = ri*cosni - yi = ri*sinni - out[el, 0] = xi - out[el, 1] = yi - - return out -else: - # non-numba versions for the direct and inverse distortion - def _ge_41rt_inverse_distortion(out, in_, rhoMax, params): - maxiter = 100 - prec = cnst.epsf - - p0, p1, p2, p3, p4, p5 = params[0:6] - rxi = 1.0/rhoMax - - xi, yi = in_[:, 0], in_[:, 1] +@numba.njit(nogil=True, cache=True, fastmath=True) +def _ge_41rt_inverse_distortion(inputs, rhoMax, params): + radii = np.hypot(inputs[:, 0], inputs[:, 1]) + inverted_radii = np.reciprocal(radii) + cosines = inputs[:, 0]*inverted_radii + cosine_double_angles = 2*np.square(cosines)-1 + cosine_quadruple_angles = 2*np.square(cosine_double_angles)-1 + sqrt_p_is = rhoMax / np.sqrt(-(params[0]*cosine_double_angles+params[1]*cosine_quadruple_angles+params[2])) + solutions = (2/np.sqrt(3))*sqrt_p_is*np.cos(np.arccos(((-3*np.sqrt(3)/2)*radii/sqrt_p_is))/3+4*np.pi/3) + + return solutions[:, None] * inputs * inverted_radii[:, None] + +@numba.njit(nogil=True, cache=True) +def _ge_41rt_distortion(out, in_, rhoMax, params): + p0, p1, p2, p3, p4, p5 = params[0:6] + rxi = 1.0/rhoMax + + for el in range(len(in_)): + xi, yi = in_[el, 0:2] ri = np.sqrt(xi*xi + yi*yi) - # !!! adding fix TypeError when processings list of coords - zfix = [] - if np.any(ri) < cnst.sqrt_epsf: - zfix = ri < cnst.sqrt_epsf - ri[zfix] = 1.0 - ri_inv = 1.0/ri - ri_inv[zfix] = 0. - - sinni = yi*ri_inv - cosni = xi*ri_inv - ro = ri - cos2ni = cosni*cosni - sinni*sinni - sin2ni = 2*sinni*cosni - cos4ni = cos2ni*cos2ni - sin2ni*sin2ni - - # newton solver iteration - # - # FIXME: looks like we hae a problem here, - # should iterate over single coord pairs? - for i in range(maxiter): - ratio = ri*rxi - fx = (p0*ratio**p3*cos2ni + - p1*ratio**p4*cos4ni + - p2*ratio**p5 + 1)*ri - ro # f(x) - fxp = (p0*ratio**p3*cos2ni*(p3+1) + - p1*ratio**p4*cos4ni*(p4+1) + - p2*ratio**p5*(p5+1) + 1) # f'(x) - - delta = fx/fxp - ri = ri - delta - - # convergence check for newton - if np.max(np.abs(delta/ri)) <= prec: - break - - out[:, 0] = ri*cosni - out[:, 1] = ri*sinni - - return out - - def _ge_41rt_distortion(out, in_, rhoMax, params): - p0, p1, p2, p3, p4, p5 = params[0:6] - rxi = 1.0/rhoMax - - xi, yi = in_[:, 0], in_[:, 1] - - # !!! included fix on ValueError for array--like in_ - ri = np.sqrt(xi*xi + yi*yi) - ri[ri < cnst.sqrt_epsf] = np.inf - ri_inv = 1.0/ri - + if ri < cnst.sqrt_epsf: + ri_inv = 0.0 + else: + ri_inv = 1.0/ri sinni = yi*ri_inv cosni = xi*ri_inv cos2ni = cosni*cosni - sinni*sinni @@ -158,11 +46,12 @@ def _ge_41rt_distortion(out, in_, rhoMax, params): + p1*ratio**p4*cos4ni + p2*ratio**p5 + 1)*ri - out[:, 0] = ri*cosni - out[:, 1] = ri*sinni - - return out + xi = ri*cosni + yi = ri*sinni + out[el, 0] = xi + out[el, 1] = yi + return out def _rho_scl_func_inv(ri, ni, ro, rx, p): retval = (p[0]*(ri/rx)**p[3] * np.cos(2.0 * ni) + @@ -222,8 +111,7 @@ def apply_inverse(self, xy_in): return xy_in else: xy_in = np.asarray(xy_in, dtype=float) - xy_out = np.empty_like(xy_in) - _ge_41rt_inverse_distortion( - xy_out, xy_in, float(RHO_MAX), np.asarray(self.params) + xy_out = inverse_distortion.ge_41rt_inverse_distortion( + xy_in, float(RHO_MAX), np.asarray(self.params[:3]) ) - return xy_out + return xy_out \ No newline at end of file diff --git a/hexrd/transforms/cpp_sublibrary/Makefile b/hexrd/transforms/cpp_sublibrary/Makefile new file mode 100644 index 000000000..7ce236b58 --- /dev/null +++ b/hexrd/transforms/cpp_sublibrary/Makefile @@ -0,0 +1,18 @@ +CXX = c++ +CXXFLAGS = -O3 -Wall -shared -fPIC -funroll-loops -Wno-maybe-uninitialized -fopenmp +INCLUDES = -I/${CONDA_PREFIX}/include/eigen3/ -I/${CONDA_PREFIX}/include/xsimd -I/${CONDA_PREFIX}/include -I/${CONDA_PREFIX}/include/python3.11/ +SRCS_INVERSE_DISTORTION = src/inverse_distortion.cpp +TARGET_DIR = ../../extensions/ +TARGET_NAME_INVERSE_DISTORTION = inverse_distortion +EXTENSION_SUFFIX = $(shell python3-config --extension-suffix) + +all: inverse_distortion + +inverse_distortion: $(TARGET_DIR)$(TARGET_NAME_INVERSE_DISTORTION)$(EXTENSION_SUFFIX) + +$(TARGET_DIR)$(TARGET_NAME_INVERSE_DISTORTION)$(EXTENSION_SUFFIX): $(SRCS_INVERSE_DISTORTION) + $(CXX) $(CXXFLAGS) $(PYBIND_INCLUDES) $< -o $@ $(INCLUDES) + +.PHONY: clean inverse_distortion +clean: + rm -f $(TARGET_DIR)$(TARGET_NAME_INVERSE_DISTORTION)$(EXTENSION_SUFFIX) \ No newline at end of file diff --git a/hexrd/transforms/cpp_sublibrary/src/inverse_distortion.cpp b/hexrd/transforms/cpp_sublibrary/src/inverse_distortion.cpp new file mode 100644 index 000000000..21a0001b5 --- /dev/null +++ b/hexrd/transforms/cpp_sublibrary/src/inverse_distortion.cpp @@ -0,0 +1,31 @@ +#include +#include +#include + +namespace py = pybind11; +constexpr double FOUR_THIRDS_PI = 4.1887902; +constexpr double N_THREE_HALVES_SQRT_3 = -2.59807621; +constexpr double TWO_OVER_SQRT_THREE = 1.154700538; + +Eigen::ArrayXXd ge_41rt_inverse_distortion(const Eigen::ArrayXXd& inputs, const double rhoMax, const Eigen::ArrayXd& params) { + Eigen::ArrayXd radii = inputs.matrix().rowwise().norm(); + if (radii.maxCoeff() < 1e-10) { + return Eigen::ArrayXXd::Zero(inputs.rows(), inputs.cols()); + } + Eigen::ArrayXd inverted_radii = radii.cwiseInverse(); + Eigen::ArrayXd cosines = inputs.col(0) * inverted_radii; + Eigen::ArrayXd cosine_double_angles = 2*cosines.square() - 1; + Eigen::ArrayXd cosine_quadruple_angles = 2*cosine_double_angles.square() - 1; + Eigen::ArrayXd sqrt_p_is = rhoMax / (-params[0]*cosine_double_angles - params[1]*cosine_quadruple_angles - params[2]).sqrt(); + Eigen::ArrayXd solutions = TWO_OVER_SQRT_THREE*sqrt_p_is*(acos(N_THREE_HALVES_SQRT_3*radii/sqrt_p_is)/3 + FOUR_THIRDS_PI).cos(); + Eigen::ArrayXXd results = solutions.rowwise().replicate(inputs.cols()).array() * inputs * inverted_radii.rowwise().replicate(inputs.cols()).array(); + + return results; +} + +PYBIND11_MODULE(inverse_distortion, m) +{ + m.doc() = "HEXRD inverse distribution plugin"; + + m.def("ge_41rt_inverse_distortion", &ge_41rt_inverse_distortion, "Inverse distortion for ge_41rt"); +} diff --git a/install_build_dependencies.py b/install_build_dependencies.py new file mode 100644 index 000000000..38c8a9f73 --- /dev/null +++ b/install_build_dependencies.py @@ -0,0 +1,72 @@ +import hashlib +import os +from pathlib import Path +import shutil +import tarfile +import tempfile +import urllib.request + +# Define where to place the libraries +CONDA_INCLUDE_PATH = os.path.join(os.getenv('CONDA_PREFIX'), 'include') + +def get_file_md5(filepath): + with open(filepath, 'rb') as rf: + return hashlib.md5(rf.read()).hexdigest() + +def download_and_extract_tgz(url, md5sum, path): + temp_dir = tempfile.gettempdir() + temp_file = Path(temp_dir) / '_hexrd_temp_file' + if temp_file.exists(): + temp_file.unlink() + + urllib.request.urlretrieve(url, temp_file) + + file_md5sum = get_file_md5(temp_file) + if file_md5sum != md5sum: + raise Exception( + f'md5sum "{file_md5sum}" of file from "{url}" ' + f'does not match expected md5sum "{md5sum}"' + ) + + os.makedirs(path, exist_ok=True) + with tarfile.open(temp_file, 'r:gz') as tarball: + tarball.extractall(path) + + temp_file.unlink() + +def download_xsimd(path): + url = 'https://github.com/xtensor-stack/xsimd/archive/refs/tags/7.6.0.tar.gz' + md5sum = '6e52c2af8b3cb4688993a0e70825f4e8' + out_dir_name = 'xsimd-7.6.0' + + with tempfile.TemporaryDirectory() as temp_dir: + download_and_extract_tgz(url, md5sum, temp_dir) + + target_path = Path(path) / 'xsimd' + if target_path.exists(): + shutil.rmtree(target_path) + + os.makedirs(path, exist_ok=True) + shutil.move(str(Path(temp_dir) / out_dir_name / 'include/xsimd'), str(Path(path) / 'xsimd/xsimd')) + + return str(target_path) + + +def download_eigen(path): + url = 'https://gitlab.com/libeigen/eigen/-/archive/3.4.0/eigen-3.4.0.tar.gz' + md5sum = '4c527a9171d71a72a9d4186e65bea559' + out_dir_name = 'eigen-3.4.0' + + download_and_extract_tgz(url, md5sum, path) + + target_path = Path(path) / 'eigen3' + if target_path.exists(): + shutil.rmtree(target_path) + + shutil.move(str(Path(path) / out_dir_name), str(Path(path) / 'eigen3')) + + return str(target_path) + +# Call the functions to download and place the libraries +download_eigen(CONDA_INCLUDE_PATH) +download_xsimd(CONDA_INCLUDE_PATH) diff --git a/setup.py b/setup.py index 317ce8869..6543e961e 100644 --- a/setup.py +++ b/setup.py @@ -3,9 +3,11 @@ import os from pathlib import Path from setuptools import setup, find_packages, Extension +import sysconfig import sys import numpy +import pybind11 np_include_dir = numpy.get_include() install_reqs = [ @@ -36,6 +38,15 @@ else: compiler_optimize_flags = [] +# This a hack to get around the fact that scikit-image on conda-forge doesn't install +# dist info so setuptools can't find it, even though its there, which results in +# pkg_resources.DistributionNotFound, even though the package is available. So we +# only added it if we aren't building with conda. +# appdirs has the same issue. +if os.environ.get('CONDA_BUILD') != '1': + install_reqs.append('scikit-image') + install_reqs.append('appdirs') + # extension for convolution from astropy def get_convolution_extensions(): @@ -56,6 +67,52 @@ def get_convolution_extensions(): return [_convolve_ext] +def get_include_path(library_name): + env_var_hint = os.getenv(f"{library_name.upper()}_INCLUDE_DIR") + if env_var_hint is not None and os.path.exists(env_var_hint): + return env_var_hint + conda_include_dir = os.getenv('CONDA_PREFIX') + if conda_include_dir: + conda_include_dir = os.path.join(conda_include_dir, 'include') + full_path = os.path.join(conda_include_dir, library_name) + if os.path.exists(full_path): + return full_path + possible_directories = [ + sysconfig.get_path('include'), + "/usr/include", + "/usr/local/include", + ] + for directory in possible_directories: + full_path = os.path.join(directory, library_name) + if os.path.exists(full_path): + return full_path + raise Exception(f"The {library_name} library was not found in any known directory.") + +def get_cpp_extensions(): + cpp_transform_pkgdir = Path('hexrd') / 'transforms/cpp_sublibrary' + src_files = [str(cpp_transform_pkgdir / 'src/inverse_distortion.cpp')] + + extra_compile_args = ['-O3', '-Wall', '-shared', '-std=c++11', '-funroll-loops'] + if not sys.platform.startswith('win'): + extra_compile_args.append('-fPIC') + + # Define include directories + include_dirs = [ + sysconfig.get_path('include'), + get_include_path('eigen3'), + get_include_path('xsimd'), + pybind11.get_include(), + numpy.get_include(), + ] + + inverse_distortion_ext = Extension(name='hexrd.extensions.inverse_distortion', + sources=[src_files[1]], + extra_compile_args=extra_compile_args, + include_dirs=include_dirs, + language='c++') + + return [inverse_distortion_ext] + def get_old_xfcapi_extension_modules(): # for transforms @@ -88,6 +145,7 @@ def get_extension_modules(): get_old_xfcapi_extension_modules(), get_new_xfcapi_extension_modules(), get_convolution_extensions(), + get_cpp_extensions(), ) for item in sublist] diff --git a/tests/test_inverse_distortion.py b/tests/test_inverse_distortion.py new file mode 100644 index 000000000..147f4fc6c --- /dev/null +++ b/tests/test_inverse_distortion.py @@ -0,0 +1,86 @@ +import numpy as np +import sys +sys.path.append('..') +from hexrd.distortion.ge_41rt import _ge_41rt_inverse_distortion + +RHO_MAX = 204.8 +params = [-2.277777438488093e-05, -8.763805995117837e-05, -0.00047451698761967085] + +big_test_in = np.array([[ 47.031483 , -5.2170362], + [ 60.0171 , 27.218563 ], + [ 60.697784 , 25.48354 ], + [ 56.90082 , -35.88738 ], + [ 55.631718 , -37.62758 ], + [ 41.258152 , -63.237328 ], + [ 78.00906 , -8.576369 ], + [ 77.7207 , -10.315189 ], + [ 73.20562 , 32.426453 ], + [ 92.81449 , -13.717096 ], + [101.26153 , 18.790167 ], + [ 33.428936 , 99.08392 ], + [101.51741 , 17.049505 ], + [ 85.12195 , 59.84272 ], + [-44.172375 , 11.69572 ], + [-57.850574 , -19.013659 ], + [-57.169926 , -20.749386 ], + [-52.772987 , 44.117252 ], + [-54.043003 , 42.37685 ], + [-74.88209 , 16.800816 ], + [-70.36986 , -25.959877 ], + [-38.385185 , 69.727234 ], + [-75.17097 , 15.061297 ], + [-89.98957 , 20.207851 ], + [-98.708176 , -10.574363 ], + [-82.30401 , -53.390797 ], + [-98.45244 , -12.316093 ]], dtype=np.float32) + +big_test_out = np.array([[ 47.03288205, -5.21719147], + [ 60.02002476, 27.21988891], + [ 60.70080301, 25.48480692], + [ 56.90341105, -35.88901181], + [ 55.63418288, -37.62924612], + [ 41.26039938, -63.24077218], + [ 78.01561783, -8.57708985], + [ 77.72717551, -10.31604838], + [ 73.21095917, 32.42881771], + [ 92.82553762, -13.71872887], + [101.27584945, 18.79282435], + [ 33.43310185, 99.09627097], + [101.5318596 , 17.05193195], + [ 85.13101706, 59.84909563], + [-44.17351296, 11.69602109], + [-57.85318107, -19.01451522], + [-57.17243759, -20.75029751], + [-52.77530392, 44.11918895], + [-54.04540365, 42.37873249], + [-74.88783308, 16.80210463], + [-70.37458323, -25.96162026], + [-38.38762237, 69.73166096], + [-75.17679039, 15.06246417], + [-89.99957766, 20.21009857], + [-98.72150605, -10.57579081], + [-82.31199676, -53.39597868], + [-98.46565139, -12.31774635]]) + +def test_known_values(): + xy_in = np.array([[140.40087891, 117.74253845]]) + expected_output = np.array([[140.44540352, 117.77987754]]) + xy_out = _ge_41rt_inverse_distortion(xy_in, RHO_MAX, params) + assert np.allclose(xy_out, expected_output) + +def test_big_input(): + xy_out = _ge_41rt_inverse_distortion(big_test_in, RHO_MAX, params) + assert np.allclose(xy_out, big_test_out) + +def test_large_input(): + xy_in = np.array([[1e5, 1e5]]) + xy_out = _ge_41rt_inverse_distortion(xy_in, RHO_MAX, params) + # No specific expected output here, just ensure it doesn't fail + assert xy_out.shape == xy_in.shape + +def test_random_values(): + np.random.seed(42) # Set seed for reproducibility + xy_in = np.random.rand(10, 2) * 200 + xy_out = _ge_41rt_inverse_distortion(xy_in, RHO_MAX, params) + # Verify function does not raise any exception + assert xy_out.shape == xy_in.shape From 6c2128c73ff19f8032637c84e306773b5c39a320 Mon Sep 17 00:00:00 2001 From: Zack Singer Date: Fri, 17 Nov 2023 13:39:35 -0500 Subject: [PATCH 02/20] Include pybind11 in the build pipeline --- install_build_dependencies.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/install_build_dependencies.py b/install_build_dependencies.py index 38c8a9f73..d9fc0eba7 100644 --- a/install_build_dependencies.py +++ b/install_build_dependencies.py @@ -67,6 +67,22 @@ def download_eigen(path): return str(target_path) +def download_pybind11(path): + url = 'https://github.com/pybind/pybind11/archive/refs/tags/v2.11.0.tar.gz' + md5sum = '90c4946e87c64d8d8fc0ae4edf35d780' + out_dir_name = 'pybind11-2.11.0' + + download_and_extract_tgz(url, md5sum, path) + + target_path = Path(path) / 'pybind11' + if target_path.exists(): + shutil.rmtree(target_path) + + shutil.move(str(Path(path) / out_dir_name), str(Path(path) / 'pybind11')) + + return str(target_path) + # Call the functions to download and place the libraries download_eigen(CONDA_INCLUDE_PATH) download_xsimd(CONDA_INCLUDE_PATH) +download_pybind11(CONDA_INCLUDE_PATH) From 188657cce4e22d49a0092dd6433a541363daf235 Mon Sep 17 00:00:00 2001 From: Zack Singer Date: Fri, 17 Nov 2023 13:45:51 -0500 Subject: [PATCH 03/20] Add pybind11 to meta.yaml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index dc9d6a44b..f08833fec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,2 +1,2 @@ [build-system] -requires = ["setuptools", "wheel", "numpy<1.27", "setuptools_scm[toml]"] +requires = ["setuptools", "wheel", "numpy<1.27", "setuptools_scm[toml]", "pybind11>=2.11.0"] From 74c5cf7be872b67ea3062d731dfe347bbf443697 Mon Sep 17 00:00:00 2001 From: Zack Singer Date: Fri, 17 Nov 2023 13:48:49 -0500 Subject: [PATCH 04/20] Fix off-by-1 error in setup file --- pyproject.toml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f08833fec..dc9d6a44b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,2 +1,2 @@ [build-system] -requires = ["setuptools", "wheel", "numpy<1.27", "setuptools_scm[toml]", "pybind11>=2.11.0"] +requires = ["setuptools", "wheel", "numpy<1.27", "setuptools_scm[toml]"] diff --git a/setup.py b/setup.py index 6543e961e..17c92fccd 100644 --- a/setup.py +++ b/setup.py @@ -106,7 +106,7 @@ def get_cpp_extensions(): ] inverse_distortion_ext = Extension(name='hexrd.extensions.inverse_distortion', - sources=[src_files[1]], + sources=[src_files[0]], extra_compile_args=extra_compile_args, include_dirs=include_dirs, language='c++') From 5e53d43e12e3e88422c97341bc426195a09bec79 Mon Sep 17 00:00:00 2001 From: Zack Singer Date: Fri, 17 Nov 2023 13:50:13 -0500 Subject: [PATCH 05/20] Re-add pybind11 to toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index dc9d6a44b..f08833fec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,2 +1,2 @@ [build-system] -requires = ["setuptools", "wheel", "numpy<1.27", "setuptools_scm[toml]"] +requires = ["setuptools", "wheel", "numpy<1.27", "setuptools_scm[toml]", "pybind11>=2.11.0"] From e6bf94380592dcd686788c4d709bd25312bb91c0 Mon Sep 17 00:00:00 2001 From: Zack Singer Date: Fri, 17 Nov 2023 14:58:07 -0500 Subject: [PATCH 06/20] Include unit tests for inverse distortion --- conda.recipe/meta.yaml | 1 + hexrd/distortion/ge_41rt.py | 11 ++++++++--- tests/test_inverse_distortion.py | 27 +++++++++++++++++++++------ 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index ae2011a40..606a0ca57 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -20,6 +20,7 @@ requirements: - numpy >=1.20 # [build_platform != target_platform] # Numba is only here to make sure we use a version of numpy that is compatible - numba # [build_platform != target_platform] + - pybind11 # [build_platform != target_platform] host: - python {{ python }} - numpy >=1.20 diff --git a/hexrd/distortion/ge_41rt.py b/hexrd/distortion/ge_41rt.py index 9b1ee67f9..cc9c3e585 100644 --- a/hexrd/distortion/ge_41rt.py +++ b/hexrd/distortion/ge_41rt.py @@ -1,18 +1,23 @@ """GE41RT Detector Distortion""" -import numpy as np +from typing import List -from hexrd import constants as cnst +import numpy as np import numba from .distortionabc import DistortionABC from .registry import _RegisterDistortionClass from .utils import newton + +from hexrd import constants as cnst from hexrd.extensions import inverse_distortion RHO_MAX = 204.8 # max radius in mm for ge detector +# NOTE: Deprecated in favor of inverse_distortion.ge_41rt_inverse_distortion @numba.njit(nogil=True, cache=True, fastmath=True) -def _ge_41rt_inverse_distortion(inputs, rhoMax, params): +def _ge_41rt_inverse_distortion(inputs: np.ndarray[np.float64, np.float64], + rhoMax: float, + params: List[float]): radii = np.hypot(inputs[:, 0], inputs[:, 1]) inverted_radii = np.reciprocal(radii) cosines = inputs[:, 0]*inverted_radii diff --git a/tests/test_inverse_distortion.py b/tests/test_inverse_distortion.py index 147f4fc6c..11f4dc53e 100644 --- a/tests/test_inverse_distortion.py +++ b/tests/test_inverse_distortion.py @@ -1,7 +1,9 @@ +import pickle + import numpy as np import sys sys.path.append('..') -from hexrd.distortion.ge_41rt import _ge_41rt_inverse_distortion +from hexrd.extensions import inverse_distortion RHO_MAX = 204.8 params = [-2.277777438488093e-05, -8.763805995117837e-05, -0.00047451698761967085] @@ -65,22 +67,35 @@ def test_known_values(): xy_in = np.array([[140.40087891, 117.74253845]]) expected_output = np.array([[140.44540352, 117.77987754]]) - xy_out = _ge_41rt_inverse_distortion(xy_in, RHO_MAX, params) + xy_out = inverse_distortion.ge_41rt_inverse_distortion(xy_in, RHO_MAX, params) assert np.allclose(xy_out, expected_output) def test_big_input(): - xy_out = _ge_41rt_inverse_distortion(big_test_in, RHO_MAX, params) + xy_out = inverse_distortion.ge_41rt_inverse_distortion(big_test_in, RHO_MAX, params) assert np.allclose(xy_out, big_test_out) def test_large_input(): xy_in = np.array([[1e5, 1e5]]) - xy_out = _ge_41rt_inverse_distortion(xy_in, RHO_MAX, params) + xy_out = inverse_distortion.ge_41rt_inverse_distortion(xy_in, RHO_MAX, params) # No specific expected output here, just ensure it doesn't fail assert xy_out.shape == xy_in.shape +def test_logged_data(): + # Load logged data + with open('data/inverse_distortion_in_out.pkl', 'rb') as f: + logged_data = pickle.load(f) + + logged_inputs = logged_data['inputs'] + logged_outputs = logged_data['outputs'] + logged_params = logged_data['params'] + + for xy_in, xy_out_expected, params in zip(logged_inputs, logged_outputs, logged_params): + xy_out = inverse_distortion.ge_41rt_inverse_distortion(xy_in, RHO_MAX, params) + assert np.allclose(xy_out, xy_out_expected, atol=1e-6) + def test_random_values(): - np.random.seed(42) # Set seed for reproducibility + np.random.seed(42) xy_in = np.random.rand(10, 2) * 200 - xy_out = _ge_41rt_inverse_distortion(xy_in, RHO_MAX, params) + xy_out = inverse_distortion.ge_41rt_inverse_distortion(xy_in, RHO_MAX, params) # Verify function does not raise any exception assert xy_out.shape == xy_in.shape From f92af5b9cb4342ad369a23c81ea51e88b1821b94 Mon Sep 17 00:00:00 2001 From: Zack Singer Date: Fri, 17 Nov 2023 15:08:52 -0500 Subject: [PATCH 07/20] Fix pickle pathing --- tests/data/inverse_distortion_in_out.pkl | Bin 0 -> 11303 bytes tests/test_inverse_distortion.py | 9 +++++---- 2 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 tests/data/inverse_distortion_in_out.pkl diff --git a/tests/data/inverse_distortion_in_out.pkl b/tests/data/inverse_distortion_in_out.pkl new file mode 100644 index 0000000000000000000000000000000000000000..0899ed9a9be516cb528a852da42473883b72f07c GIT binary patch literal 11303 zcmeI&XH-+&wlH82kd8``DouJ*dWRW84}=s#f`Al7L8Vv4u7FBW=_-nXii#Zsq%Io@ zf>K10jtvB*2r5Xs>%8Zjam#nd`}_NG_mAv7MzWKPJ?CC)&S&j|T-h>OY=3S0veZ&n z28V{nL`P-$;A7#?n2_+@I$Odb19k9WU}W%?tW(tj8zY(Qrc-a(tblDd)2~~f0_2F6M=oqs_)`@hcQk$@+$k6jzAr(Bn&j_$ zB?p{=SDn|SCfNzzZ|k(uSnq;b4ro{2t)#>J)yUT;8Delny}&ngfB^>nzd7$|5aGZX z*}MMPR`4>~#zWPHfi@*yP|i5V1Wx1TmFeDgaAjVoPJ1g2jH2&~B@9uJhEg8q+!0S` z{&ZQ)SWg@od`TC3``8xMHzn?rOV&bc4L$BNTQyMb2JIv51rD$`vAWrr+XXFkJiedQ zU67OE-~d*zg=1`0_ei(R24^xS)ocIZn6 zko-&S&kcK^m$klP#R@j)miMdGdv8%un!D5R^dTB#rE>ihf4tusy*oS*2Rn7eHj}L5 z$C9&XS!yJXRCbFjGMVhWjJLmDNNmwr0i<=L6-imC?8K~S{IMXe|4a~{FpUI@Jw&(| z#%0{jX@aij3vD2j5@4cgdFb_a0{AK3qLwOCU^AE>_fnIE@U|f7G)*eld&ZK&htz;O z1x926I_j*K=xbsLFtt>i{)q;n>!;5=P@#jS@rLa$%AL@yz55*Ue5hb(*ybd0 znF4b5t79(xWJ12iie&rqM0jWMxhiLx1s~2TxKy|3qoZrGMFl)aKq^a8aCh(k^_22M zY5X?mjW(@u-9anJOssw^W(SDeLVIQ}wxG%ossGd(>57 z;kR(W8QiON_neQRAh>n=`^jE$q(g5za^Z;`YUz!&*py2F&5Z{Nt?Ma>XQuvsVh0gK zSOQ`do6S%~LarxVJoN@Lywr`1d@OV?_>dA$9QVhgc=py#0>L zoS`n1Oz6LO!|RHSW67ghyv}fzO(yhCzAZ>;TGySmafZTQlQ-1eY$0-3Yv{5C1!A8J zwd9l}dPKRk}k)^WrTCnTXb+4BY1vIPfxILRO@+rq=;F#-)`6e|f-pt~Hs5H5}oJ^%B=FK05F(4=x^h zMTMq?E^33JJ(3k?$z`q*LOag4_mc7*k(&IpLA$a`3SoXG93dqB@}I zbg>pv5iS0dx1Iu7y7TvZ)!k5c>js|tS!TfVLTo5rzzxMy#H)=rn!~XkbZ^TW8lvQu z5_J4!k=M6I4ihUU#J=_Gt6y8l@Wxd#r16U*@?Bdp`tcqaXj4DaD&Er&4?(TYXIdH! z6n;o07I>g&{@vGzIm&3F>51(;!2>DF6`Rga*rH90n4md|Eh6r#{FwHF39q9CG*1)~ zkwaC2%*_cVw5Y3qDZ>NZ@p}=aS!nxL=w<(LpZ{m**;f2>8jz@8l+NuTfKc6%JO8pd zS`{G`@LQD#@qvsTotv!TrTo1c<9fzm%~|~=hv^30``6u!T1A1TyyShGY*b)f*|%v? zH!56waZS-Jt;YJQ#s$8SCVP0#6h|uRj!4SWqvBMJj3_5MIDd+I zS$0PsdKeRC!{B%A2b+ObHDf>BwzV4ubpgZLCwYq}Wc&K3d4<>A?+MnQmiwLvRmqZ?yJAs&f z>SW7f4=|FY4=_SJp=Rt_#)y?7S|Hy0HT;l(eyl$(dg+4+I(XBy==Lr^s3YW=rX>Lm zSL8>YzE4K^nMSWwF9@UKN-2v+_#DvY8=ICxF4?2j;FEk@-gYRHo@7+Hodk1vzn`r> zLqWaO3JU7GB|-6AN0?+A0Xa*y-#0R(pdvGNu})_yT7Cc8h;u#x?jKd_Qr5OawD~LF z`p(fH{=iV(15Fw#ixJat{ALP`mR+KPSv2%{iXnP2L<2oryq()BOGE4tW9#d4oY7Fk z)O`JG3SwNTxu?E|0+)HOzUd%2B9$}YQ=%#i^eZh+$@C!$eK&56<1yO+!$TSeE?Y5? zF!v1ytH^(j3iAJZY9#chXA3Dfz@&ffXd77pS#wN3veB{u&5LXbhZ-G`Ozi3C+3y4} zB#57j)o}s0;2o#$Y;^6{W<=!8 z`mvyYSBJ((CLO}o+@mb%QGvi1Hoc82kTvtKY;SguQI$;HiPBvLsBZt5*ixqr>WppG zXD_uwJcUm}$+kKmEkT0Q#|oPwED@3%q)}`AKz#3o_U_ z&bVuD4zi!bOpDJ`(5%eduf-5w(rZk(Pna%BHg&V0To@+nB^RgwP zPK7%P5~)ZlL_Q*>jsVPPjkf}9boApiCH`!e6y#OM^`33DNA?|mbfa(uG8xpAxkJY*=@ zl?Cx@)lKTHnMmZ`(SU2kM9?H=+!fqIKq(?MJN`WJ_-oYoAE~kNU!%s*c|pNqnjNI~ z`kN$h>!D#SDN7wwbNF@l;`gzSw!ka1ef)v66VxUuCQYZ=pa_c6Y`TUU9CPiuVsOL& z5@$_>R<)7A{j5g$zSl;e@0`3(su~Z}P^RM6=z>BM1oL%-NpPZVkbj7S3_RLJv{U7D zIJhOU`Lg3<>n;O#)lxZUcv@K2`}`6KrI@$Ju6phSOr~OOgSs2MG1n-0&&B}LCT?Sn z2SgMdSbN;`s55*&^6B#{MR%ak8o#~waEBa}hpXA?sI2X|^}bD#;82I8 z>NYu|_v2E}#3x9I?joC^rO79e&rYya)2XA@jnHQ3}pwSS(c74As9DToYby|cu zP!26yDwdeTwRP2~_aFfcr%*2a$#es2g3MvwB4xB=|KhTTrQ?6#lM}QeROIk&t@9Zdg!LmGsS=cb^#u{3yx?}P(9<-UQamw z>u~zdJ`FhjBfq37ln${iJFAQ~Qz4w-(B;`|g4nh_3ENU<3m-#@a>`Z{(cWa~QjIwS z^sy`4)M$qt`eLa%dYqt)n&y@lbyUgd*p)&yRaaf$u`e}#6G%cuI~zN?f(QtnRi^!% zAtU=Mq`(9CIwG@p4drHKw666`)9nd5bf;NKbO_N9E1iS6dB^~DWgi#rpU3C#i%hz) z3)evX=XyE2XsEZO?tmCs6N;>E^c2TYP|Ji&H2*tWFi>r*cRS{UdTis*PcPEJ?)G^v zzBoJ@joTO4+^|QcBR-oZ+8HQF_0PcROcQiYtB6hDh8+^^9cxaGWkO1__$+%P3(0@u zPgd%XMzw#o?<80-5OqO7OprzcYECKkn@!xv8v=FmnRF& z_^yf;*0x8HzeIlOKVFSijL8tz&!5=T zove%=Pvwm6SZNC-KW7J2=G>5`SHOY&cc`Fzcg@B-6E+Yx)4HwKf(~VkBD+F%J0ZsK z&9Qyo+~E;N@|T-d4xpVMyDOK-0>4bfUiGb(=m$%EE;!N?e%lG}{nMol!c%K)INeB~ zm5~)SSmKTnM9T)po$-X?*`$%?tq)dtA~uD%s~=bRf+lRNK>OT=rSeuH^2n(B*ypZ? zZZG^$w~aJHK4Dib%uJABRz}NJ64xv@*U1v);}szKSS@#W8W~ZFWh6JbYk;tOwNOH! zEn1y^=iT@(M|4o5Z1wj!3M$`Yvvp69HIfq_EU_rXW#(+^9}?(-zR{P8%)M0PsFioS z^N$&tKGd2~lte>z55tqtLl@{uD*D`E?2hWhPx6m&*rF#}FNB*6IHBrygxOh=2gu45 zD5&PLkXUnyL|TJ7ik&E74yH3vuBPl=g&zdyKRs_&P{%@ciNO@X&+>oO7XPCo`FH5` z4t-22^Rov7o>ed2zt9JbiKgc1SWD=CUSl`h4M=Y`Y%lR98I&$I>IoR=qExbrPs$Gp zzT2WZc8RJW%kYxSl{;OaDSZEmZKie*rbMmcU!=nM+uL+6-yp!I(T<~gaR-5723>63 z0j2mkoA(*hA(HP@f8ekIe6vt6`SZXL9udCr7vQ;c|NIHRtE)WVq}hB&qLB@VUvlBC zc;*6@i7gVRE2yaC*3-N6J)UsCT=34f^O|rtBIs%4ItDlk`KVL1NXYEf%tfIyo*=() zLb8MVbN$N6-YX+gR&eaK$i`#3Y|*M`nxfnnHBghH-Ui=1GxUlq))gmV z3(E{cs=l==YG>qWsN6M%UfHSooF!+JC@pkkVUd7}l?IBuEU+_7IR{B3=|Th4-$&+jIlk%q{BS}BX8-?ER+=frht(l1D+!@ zayN2dD!oHjTCq^p;)XK;3+3)pyT9JSLK)l~EA`3Rs^%QJ0B@Qssh;Wg`~Ky<>X58j@uA z8h0Z1C>F}gwl9sujyb_=b4~YDER?G%FE+?Oq(dcV8#k|q7!1VfEpZMqVDEcVc0)}f zjAp1Gmdde$uq)Fo#aJj;>NnN9VRMeDj~#pxZ%sRog)-%ELirFD%KGfSj81 z%9-aC6*f!wf8FEKtt%L;{ISoOF(=P z7D@&FR>vGHl#BK?9|*TxkdQ^;%RDTU4fnToJ4-sE!I!}f)mSL2l!K%qG#Myvv}gG0 zfHivXCiDRBeRC8=?LErz!2_JKS{j;LSx9Z0ysb)>8W))Nlq{W92Ak zt{1k(o`~T0U$Hf2$Hv{7#MUTH6;phLt+CLp>eO3ojr!3!wcXen<4^0ju6XSLoMN0V z7qB&&KFPhUgU!%NBBC^~$rQ}%N((mXJ3xO^k)hV4E2`xFNa@GcXmRmkKoz#e@$du2 zW7rx`{J8gL1fQ4sDdB~@DjhZzhEP})PH5+Idea=XM)#|{)iKx_zdffZZU4oD0M9t@ z5p0b{i#B6%Gc1U(db~39l|Bk-O|NUl)@XUdS7i!YqaX20&{b@WH!k!II$&#D`fcT} zhOLput*e^}$xvCA>o8d<#D4?kgR^sjhcP4=@#7Tzj%oDMoei{_EEer%1)`?>AL zu{CzuYw!GttWSDI9T!?xbYW}E`$UYF_ccQg7x|vs;CtP@_e*U# zw#H3&%Q=K-`iT0oLN9{N9UYpC7NxIp1e#2+Q7^Ve*Xn(fHP{*>hlE#^Vr#TsHu^D$ zt&w<_HMoqeF~P@IJ;&7y3RZ|t% z8cC_!{_w{sLCXQsBXuKPn79#pT!qgS8AR=vjKw$IsXrMc=#$pgsrhR$|a-;TjNnr{ax}=B&7bEP;mlV zqrX(HW%73oux&fD&gcXkxUa;1p(tsAv!KBPwOh_$zniS5qfA0aijF#Lc4vahYl)lU zcDk_s{r;W+Q9F2fPEIb3;(@Y%Y%ZL`)<|P>8;|aDfXxYSf@`rg)~s)ru+Vgb?wOJ0 zD1JIL3euZ?V{81-oaZu*t#MaRk9i}u#<%BI-A%;SSo`S74i{{V^ze{7f3P){`yT!L z2wP*!TDdzvu{Bz0j;pp|YYfr+kk@d^$A;J#eQ$@9Bhqx=hlfm z!`2u$*WmvWTjQJhfy!KLjTbK|@$A@22H#8d+G@j&$fE9gIqNJ zr%*&h+1b}kw@xzQSHR=p_t+Y9JQ{}T3;)&{wWc!7Zewe#WNXOWfUPmR``Q^%H6qCD z9=71}vIhA}b|1|2jUm@hdteb;-oLe4_o8d8X;)L)~G+X zyGH4yJ@mU4tlf^S@lJR0)mdzfdg2uC1Z<7bqAa-{Y>nDeh5Kah>cfiAisR;17>pjS zjGe^Rcy8#+Wg0sT4pi#M8&+GO*mt#4Td_6r@+Y&;V{6owrFg%>l|pl`NMBx=E7I({ z6KL^<21ggKs-0+ah4}3OQS@mhY}pWXo8vYS9yg0eCR8|qLqsrFz-k2ys`I_HxXOoneAT%=dm@$DSYnPj;+xv z;DmM^w#HM6v&zNT8XtINY_!GJDE*ya!5c&Z^Hi>VbZm|1^4=ajiLFuK1-HQ_Y>oK? zPtHeRYmDKPP>?54(I$Tfk6dhx3NbXZTi6afcJ9YX9CEYmc@^nqzBRnAF`jjjfSq zgNS7awnjA*f^apq#_6#a>6+sN=y-ejdjqz{(ba{ca)EA8uq8~DtO($*H1l*ETjLRt z$W`&%1mLBlSNn$mxz8kFYb@xqymP3{1GaDNDte2pu~W)dpN~$2;&5}_^xGbY!?6C44_<7l zAw+)piLEjGK>5@SY>guEd8JkUEO=j-Te~BR4(%QEl@a>b8hv;#YT#PpdI)=E=o>P! zil|O$#@0CYn?qUbtqqb14)~;ut+At}^V}h9jfUY)fhVvvHrVj4u*cSTk{(mEjIGh` z{quXju{GMAy2`#f(*U!`9eQ^N8>cTjPaG_RCCcjqkQeT0O?rD4O?tCLCL1IqBuN>U$|5_$$Q>V{7cD zx3BNSwFD_ws_;3sMpsIVAMpVlO)~11V`V&$q`}>oozmuz7UR134!4p^K5G6?ur(@& zmp9Re2*@Y${g?xp1?ugLi6~&AYt}95glj~&R35gJgsqVx!5y0S^lz%&^{(78DwOA{n#q#;N5Rj zCE$duaj=siB<~C>0vpI`g(P%^jbl6pTjQ^=vV0r-JJnn7Z1ZJjKt@Z+y@S{qzv;F- z?8Md>s?>DQZaua}wH14OnC|f0W6xtI$rUXJj-hMV8U-w3RJUGm1uAdk>w6bG!AirM zs#+ujvMuhXN^!riX1FHFLf~utNUVp`<_34*6Q4Vh zabc->-OW(*cyK_Q^r?#Ny4dL(cG)N9MRglmvf@AHIkRF zs2{-A*jZ=%b0_YP4#(Y&VPI>_dwFFz30vbZw`aw5Y>g%Fo{e3=)+mxZu#YV1g#L`| zDk9*Pa`%<(O(nR0NS$lr9uQdvCne-}d#%Pcd6)U27%v|PReaKxadJlm&z0GAQb_20 zK(~(RB|0M7*>fA&Q{YseYOnQecl1!ww~w{e8C^J>=TSY%LZ{=6Cqym&)*8P&w_vJw z*@J%n;l&YbjU|IKuNtv6&h1E9s>Ie9rd0fyh>dK^XHoti*cu0hD}U_Da)a=hfVWI+ zjXN#4kFM9Hz~ZvCxXK1Y^zm%!zHDrbGOxR2PGW0pI^A0*curz#lso-_%@14SWJ>Dm!lyK-csaH(imlOj_Qn?TAXjLwH~l$) zt16ugqE=*%=E`%Dx3D!XM3|SI#@6WKO1t7s@JxFQf? zYjiog`lrPL3H50DFc+~k5;IDF2V!gFTeVn$?Xt%I6_FU6HY81x= zgSl_PgV-9iG*5%<8)&qZvF?0kpCny@wYO+`O)&9*?>Vjqbzur|7-TVn`jaxpnhpZyv z^-RU{>~5lT&^h$9Z-uHc6x$HBc3d+BR-M&qTO&H?bteWTU~5#qr}5LTd@aOgX!m4b zYb?~B?yAAo_>j*~>Jo_wV@PZN5VpqEt=@M|o%95YN0Qms^97NJ_pRus*cuh@au&$5 zQ;~>~aPjGW7Gy0yaj7&hL}pgOOa6GBbfZgiL>jin(1!)W^Vk}{9y=@Z0$XFwn1#6+ zw#NM2OimiMMyZVx{i}QE#9-6{;0u~Yo`{w zaSE4xn`raF*7&scl0!GP#=dv@GAY;^{dRSRZOHXNYp$-AufW!LqVeY)Aq^H{%=j1z zRGNd6wTwxeB@?YT{bAINtx@WW!FWOs3n``E64&wnTWbucBz6a5YZNFQzuSPVkr_w6 zRgbOla>=!A>DU?_JflAd%2OcxK;PqJY>nbQSyx5rrtno_q3|HKMs~GoAwFj+T+L~j z`HZb`@Uw|=54OgN=Z~!;ur>aO;)-_#)=gFv*)li%3eOO z;Wf5Kn;R>#xh8FZnO&0SXh{dti|Y3lur*$#HfnNycZZ?@4t^VJ2MF79l_$`a1v0in zql$r+=xwh_t!I=cbauTh7wOi99$B9GTi6<7-FJQSyXlTPD_^F+Cwaoskv*;8qYu|P zcG9Y`HP%MF%a$ZsVVJte*MY6kdg$=m-`En%xXXmtkJEFq!2L<|nDCimYuki(JjUNYJ zo1DYeXzTDxzX)4nMA_VtL2QkCo0>FT=FN~8!^oa58G}`6?^E9@7Z9uG%O;uN2UxbA zP2S9Di%zZRi~okLF}tBeF3ZIOp0@5PTgKMNEf6Uk{#+foyj4)nKgmQs798jFQ0+zV`t6kn?%e{79nDIYm+U~60#=sfF-tubCO?)xunjrz{h zT1~ivIISvuEfrhi*g(-Kq6uDfIa06PiLG&M-TQE#az}VqqV6+*=ThO4um`Z(0|tbh zEV+zrplA)1$Em>u_^o7@N3k_ZzRo;ko8Sqmr)w&vE@(nThG`|6ECZId`@iGXCZT;M zgDW`CdcxHTrP6$!&-JRI`C8AUt>8l&|HJT|Oe820P|r1Oi-hCjMRl-SPLjM^jeMghu?KUUtVo@o?BaADHJy)ENNZ;ueafU?*BnqD--|A<23*G SMgMn=aFYE0kw<|sI{yI)_p6@( literal 0 HcmV?d00001 diff --git a/tests/test_inverse_distortion.py b/tests/test_inverse_distortion.py index 11f4dc53e..5d7d94b9e 100644 --- a/tests/test_inverse_distortion.py +++ b/tests/test_inverse_distortion.py @@ -1,10 +1,11 @@ +import os import pickle import numpy as np -import sys -sys.path.append('..') from hexrd.extensions import inverse_distortion +test_dir = os.path.dirname(os.path.abspath(__file__)) + RHO_MAX = 204.8 params = [-2.277777438488093e-05, -8.763805995117837e-05, -0.00047451698761967085] @@ -82,7 +83,7 @@ def test_large_input(): def test_logged_data(): # Load logged data - with open('data/inverse_distortion_in_out.pkl', 'rb') as f: + with open(os.path.join(test_dir, "data", "inverse_distortion_in_out.pkl"), 'rb') as f: logged_data = pickle.load(f) logged_inputs = logged_data['inputs'] @@ -91,7 +92,7 @@ def test_logged_data(): for xy_in, xy_out_expected, params in zip(logged_inputs, logged_outputs, logged_params): xy_out = inverse_distortion.ge_41rt_inverse_distortion(xy_in, RHO_MAX, params) - assert np.allclose(xy_out, xy_out_expected, atol=1e-6) + assert np.allclose(xy_out, xy_out_expected, atol=1e-7) def test_random_values(): np.random.seed(42) From 30bf00ccfee32f944754ea97bfdff4eb66e734f6 Mon Sep 17 00:00:00 2001 From: Zack Singer Date: Fri, 17 Nov 2023 15:15:01 -0500 Subject: [PATCH 08/20] Add installation procedures to package.yml instructions --- .github/workflows/package.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 735dae606..bc28e7daa 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -52,6 +52,7 @@ jobs: conda activate hexrd conda install --override-channels -c conda-forge anaconda-client conda-build conda + python install_build_dependencies.py # This is need to ensure ~/.profile or ~/.bashrc are used so the activate # command works. shell: bash -l {0} From 309bbc3404f1663d1083a3388c48e2c335451f87 Mon Sep 17 00:00:00 2001 From: Zack Singer Date: Fri, 17 Nov 2023 15:52:27 -0500 Subject: [PATCH 09/20] Add pybind11 to package.yml installs --- .github/workflows/package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index bc28e7daa..4c81d5d7c 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -50,7 +50,7 @@ jobs: conda config --set solver libmamba conda activate hexrd - conda install --override-channels -c conda-forge anaconda-client conda-build conda + conda install --override-channels -c conda-forge anaconda-client conda-build conda pybind11 python install_build_dependencies.py # This is need to ensure ~/.profile or ~/.bashrc are used so the activate From 8ad311417ae86e39b6d8f06595a73077464fd223 Mon Sep 17 00:00:00 2001 From: Zack Singer Date: Fri, 17 Nov 2023 15:54:23 -0500 Subject: [PATCH 10/20] Remove push trigger for github action --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4a8c24d02..c5fd898ff 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,7 @@ name: test on: push: - branches: [ master, inverse-distortion ] + branches: [ master ] pull_request: branches: [ master ] From d70f84272f7e2e5fcd2b2656dfc833ab0920c644 Mon Sep 17 00:00:00 2001 From: Zack Singer Date: Fri, 17 Nov 2023 15:55:08 -0500 Subject: [PATCH 11/20] Remove action trigger for this branch in package --- .github/workflows/package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 4c81d5d7c..bdfebab8c 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -2,7 +2,7 @@ name: conda-package on: push: - branches: [ master, inverse-distortion ] + branches: [ master ] tags: ["*"] pull_request: branches: [ master ] From f7632feefacdf10da30c44874c9d1465e80ca5e3 Mon Sep 17 00:00:00 2001 From: Patrick Avery Date: Mon, 20 Nov 2023 18:57:44 -0600 Subject: [PATCH 12/20] A few minor updates Signed-off-by: Patrick Avery --- .github/workflows/package.yml | 3 +- .github/workflows/test.yml | 31 +++--- install_build_dependencies.py | 88 --------------- scripts/install/install_build_dependencies.py | 100 ++++++++++++++++++ setup.py | 48 ++++----- 5 files changed, 139 insertions(+), 131 deletions(-) delete mode 100644 install_build_dependencies.py create mode 100755 scripts/install/install_build_dependencies.py diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index bdfebab8c..a728867b6 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -50,9 +50,8 @@ jobs: conda config --set solver libmamba conda activate hexrd - conda install --override-channels -c conda-forge anaconda-client conda-build conda pybind11 + conda install --override-channels -c conda-forge anaconda-client conda-build conda - python install_build_dependencies.py # This is need to ensure ~/.profile or ~/.bashrc are used so the activate # command works. shell: bash -l {0} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c5fd898ff..ea3cd94ca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,9 +6,6 @@ on: pull_request: branches: [ master ] -env: - CONDA_PREFIX: ${{ github.workspace }}/miniconda3/envs/hexrd - jobs: pytest: name: ${{ matrix.config.name }} @@ -18,14 +15,21 @@ jobs: matrix: python-version: [3.11] config: - - {name: "Linux", os: ubuntu-latest} - - {name: "MacOSX", os: macos-latest} - - {name: "Windows", os: windows-latest} - + - { + name: "Linux", + os: ubuntu-latest + } + - { + name: "MacOSX", + os: macos-latest + } + - { + name: "Windows", + os: windows-latest + } defaults: run: shell: bash - steps: - name: Set up Python ${{ matrix.python-version }} @@ -56,9 +60,9 @@ jobs: pip install fabio==0.12 if: ${{ matrix.config.os == 'macos-latest'}} - - name: Install Build Dependencies + - name: Install HEXRD run: | - python install_build_dependencies.py + pip install . working-directory: hexrd - name: Install requirements-dev.txt @@ -66,14 +70,9 @@ jobs: pip install -r tests/requirements-dev.txt working-directory: hexrd - - name: Install HEXRD - run: | - pip install . - working-directory: hexrd - - name: Run tests env: HEXRD_EXAMPLE_REPO_PATH: ${{ github.workspace }}/examples run: | pytest -s tests/ - working-directory: hexrd \ No newline at end of file + working-directory: hexrd diff --git a/install_build_dependencies.py b/install_build_dependencies.py deleted file mode 100644 index d9fc0eba7..000000000 --- a/install_build_dependencies.py +++ /dev/null @@ -1,88 +0,0 @@ -import hashlib -import os -from pathlib import Path -import shutil -import tarfile -import tempfile -import urllib.request - -# Define where to place the libraries -CONDA_INCLUDE_PATH = os.path.join(os.getenv('CONDA_PREFIX'), 'include') - -def get_file_md5(filepath): - with open(filepath, 'rb') as rf: - return hashlib.md5(rf.read()).hexdigest() - -def download_and_extract_tgz(url, md5sum, path): - temp_dir = tempfile.gettempdir() - temp_file = Path(temp_dir) / '_hexrd_temp_file' - if temp_file.exists(): - temp_file.unlink() - - urllib.request.urlretrieve(url, temp_file) - - file_md5sum = get_file_md5(temp_file) - if file_md5sum != md5sum: - raise Exception( - f'md5sum "{file_md5sum}" of file from "{url}" ' - f'does not match expected md5sum "{md5sum}"' - ) - - os.makedirs(path, exist_ok=True) - with tarfile.open(temp_file, 'r:gz') as tarball: - tarball.extractall(path) - - temp_file.unlink() - -def download_xsimd(path): - url = 'https://github.com/xtensor-stack/xsimd/archive/refs/tags/7.6.0.tar.gz' - md5sum = '6e52c2af8b3cb4688993a0e70825f4e8' - out_dir_name = 'xsimd-7.6.0' - - with tempfile.TemporaryDirectory() as temp_dir: - download_and_extract_tgz(url, md5sum, temp_dir) - - target_path = Path(path) / 'xsimd' - if target_path.exists(): - shutil.rmtree(target_path) - - os.makedirs(path, exist_ok=True) - shutil.move(str(Path(temp_dir) / out_dir_name / 'include/xsimd'), str(Path(path) / 'xsimd/xsimd')) - - return str(target_path) - - -def download_eigen(path): - url = 'https://gitlab.com/libeigen/eigen/-/archive/3.4.0/eigen-3.4.0.tar.gz' - md5sum = '4c527a9171d71a72a9d4186e65bea559' - out_dir_name = 'eigen-3.4.0' - - download_and_extract_tgz(url, md5sum, path) - - target_path = Path(path) / 'eigen3' - if target_path.exists(): - shutil.rmtree(target_path) - - shutil.move(str(Path(path) / out_dir_name), str(Path(path) / 'eigen3')) - - return str(target_path) - -def download_pybind11(path): - url = 'https://github.com/pybind/pybind11/archive/refs/tags/v2.11.0.tar.gz' - md5sum = '90c4946e87c64d8d8fc0ae4edf35d780' - out_dir_name = 'pybind11-2.11.0' - - download_and_extract_tgz(url, md5sum, path) - - target_path = Path(path) / 'pybind11' - if target_path.exists(): - shutil.rmtree(target_path) - - shutil.move(str(Path(path) / out_dir_name), str(Path(path) / 'pybind11')) - - return str(target_path) - -# Call the functions to download and place the libraries -download_eigen(CONDA_INCLUDE_PATH) -download_xsimd(CONDA_INCLUDE_PATH) -download_pybind11(CONDA_INCLUDE_PATH) diff --git a/scripts/install/install_build_dependencies.py b/scripts/install/install_build_dependencies.py new file mode 100755 index 000000000..983013deb --- /dev/null +++ b/scripts/install/install_build_dependencies.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 + +import hashlib +import os +from pathlib import Path +import shutil +import tarfile +import tempfile +import urllib.request + + +def get_file_md5(filepath): + with open(filepath, 'rb') as rf: + return hashlib.md5(rf.read()).hexdigest() + + +def download_and_extract_tgz(url, md5sum, path): + temp_dir = tempfile.gettempdir() + temp_file = Path(temp_dir) / '_hexrd_temp_file' + if temp_file.exists(): + temp_file.unlink() + + urllib.request.urlretrieve(url, temp_file) + + file_md5sum = get_file_md5(temp_file) + if file_md5sum != md5sum: + raise Exception( + f'md5sum "{file_md5sum}" of file from "{url}" ' + f'does not match expected md5sum "{md5sum}"' + ) + + os.makedirs(path, exist_ok=True) + with tarfile.open(temp_file, 'r:gz') as tarball: + tarball.extractall(path) + + temp_file.unlink() + + +def download_xsimd(path): + url = 'https://github.com/xtensor-stack/xsimd/archive/refs/tags/7.6.0.tar.gz' # noqa + md5sum = '6e52c2af8b3cb4688993a0e70825f4e8' + out_dir_name = 'xsimd-7.6.0' + + with tempfile.TemporaryDirectory() as temp_dir: + download_and_extract_tgz(url, md5sum, temp_dir) + + target_path = Path(path) / 'xsimd' + if target_path.exists(): + shutil.rmtree(target_path) + + os.makedirs(path, exist_ok=True) + shutil.move(str(Path(temp_dir) / out_dir_name / 'include/xsimd'), + str(Path(path) / 'xsimd/xsimd')) + + return str(target_path) + + +def download_eigen(path): + url = 'https://gitlab.com/libeigen/eigen/-/archive/3.4.0/eigen-3.4.0.tar.gz' # noqa + md5sum = '4c527a9171d71a72a9d4186e65bea559' + out_dir_name = 'eigen-3.4.0' + + download_and_extract_tgz(url, md5sum, path) + + target_path = Path(path) / 'eigen3' + if target_path.exists(): + shutil.rmtree(target_path) + + shutil.move(str(Path(path) / out_dir_name), str(Path(path) / 'eigen3')) + + return str(target_path) + + +def download_pybind11(path): + url = 'https://github.com/pybind/pybind11/archive/refs/tags/v2.11.0.tar.gz' + md5sum = '90c4946e87c64d8d8fc0ae4edf35d780' + out_dir_name = 'pybind11-2.11.0' + + download_and_extract_tgz(url, md5sum, path) + + target_path = Path(path) / 'pybind11' + if target_path.exists(): + shutil.rmtree(target_path) + + shutil.move(str(Path(path) / out_dir_name), str(Path(path) / 'pybind11')) + + return str(target_path) + + +if __name__ == '__main__': + import sys + + if len(sys.argv) < 2: + sys.exit('