diff --git a/ext/bcmath/libbcmath/src/convert.c b/ext/bcmath/libbcmath/src/convert.c index bf3d9a9a415bf..5438b4c1c44e5 100644 --- a/ext/bcmath/libbcmath/src/convert.c +++ b/ext/bcmath/libbcmath/src/convert.c @@ -17,24 +17,22 @@ #include "bcmath.h" #include "convert.h" #include "private.h" -#ifdef __SSE2__ -# include -#endif +#include "simd.h" char *bc_copy_and_toggle_bcd(char *restrict dest, const char *source, const char *source_end) { const size_t bulk_shift = SWAR_REPEAT('0'); -#ifdef __SSE2__ - /* SIMD SSE2 bulk shift + copy */ - __m128i shift_vector = _mm_set1_epi8('0'); - while (source + sizeof(__m128i) <= source_end) { - __m128i bytes = _mm_loadu_si128((const __m128i *) source); - bytes = _mm_xor_si128(bytes, shift_vector); - _mm_storeu_si128((__m128i *) dest, bytes); +#ifdef HAVE_BC_SIMD_128 + /* SIMD SSE2 or NEON bulk shift + copy */ + bc_simd_128_t shift_vector = bc_simd_set_8x16('0'); + while (source + sizeof(bc_simd_128_t) <= source_end) { + bc_simd_128_t bytes = bc_simd_load_8x16((const bc_simd_128_t *) source); + bytes = bc_simd_xor_8x16(bytes, shift_vector); + bc_simd_store_8x16((bc_simd_128_t *) dest, bytes); - source += sizeof(__m128i); - dest += sizeof(__m128i); + source += sizeof(bc_simd_128_t); + dest += sizeof(bc_simd_128_t); } #endif diff --git a/ext/bcmath/libbcmath/src/simd.h b/ext/bcmath/libbcmath/src/simd.h new file mode 100644 index 0000000000000..af38f8349618c --- /dev/null +++ b/ext/bcmath/libbcmath/src/simd.h @@ -0,0 +1,59 @@ +/* + +----------------------------------------------------------------------+ + | Copyright (c) The PHP Group | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | https://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ + | Authors: Saki Takamachi | + +----------------------------------------------------------------------+ +*/ + + +#ifndef _BCMATH_SIMD_H_ +#define _BCMATH_SIMD_H_ + +#ifdef __SSE2__ +# include + typedef __m128i bc_simd_128_t; +# define HAVE_BC_SIMD_128 +# define bc_simd_set_8x16(x) _mm_set1_epi8(x) +# define bc_simd_load_8x16(ptr) _mm_loadu_si128((const __m128i *) (ptr)) +# define bc_simd_xor_8x16(a, b) _mm_xor_si128(a, b) +# define bc_simd_store_8x16(ptr, val) _mm_storeu_si128((__m128i *) (ptr), val) +# define bc_simd_add_8x16(a, b) _mm_add_epi8(a, b) +# define bc_simd_cmpeq_8x16(a, b) _mm_cmpeq_epi8(a, b) +# define bc_simd_cmplt_8x16(a, b) _mm_cmplt_epi8(a, b) +# define bc_simd_movemask_8x16(a) _mm_movemask_epi8(a) + +#elif defined(__aarch64__) || defined(_M_ARM64) +# include + typedef int8x16_t bc_simd_128_t; +# define HAVE_BC_SIMD_128 +# define bc_simd_set_8x16(x) vdupq_n_s8(x) +# define bc_simd_load_8x16(ptr) vld1q_s8((const int8_t *) (ptr)) +# define bc_simd_xor_8x16(a, b) veorq_s8(a, b) +# define bc_simd_store_8x16(ptr, val) vst1q_s8((int8_t *) (ptr), val) +# define bc_simd_add_8x16(a, b) vaddq_s8(a, b) +# define bc_simd_cmpeq_8x16(a, b) (vreinterpretq_s8_u8(vceqq_s8(a, b))) +# define bc_simd_cmplt_8x16(a, b) (vreinterpretq_s8_u8(vcltq_s8(a, b))) + static inline int bc_simd_movemask_8x16(int8x16_t vec) + { + /** + * based on code from + * https://community.arm.com/arm-community-blogs/b/servers-and-cloud-computing-blog/posts/porting-x86-vector-bitmask-optimizations-to-arm-neon + */ + uint16x8_t high_bits = vreinterpretq_u16_u8(vshrq_n_u8(vreinterpretq_u8_s8(vec), 7)); + uint32x4_t paired16 = vreinterpretq_u32_u16(vsraq_n_u16(high_bits, high_bits, 7)); + uint64x2_t paired32 = vreinterpretq_u64_u32(vsraq_n_u32(paired16, paired16, 14)); + uint8x16_t paired64 = vreinterpretq_u8_u64(vsraq_n_u64(paired32, paired32, 28)); + return vgetq_lane_u8(paired64, 0) | ((int) vgetq_lane_u8(paired64, 8) << 8); + } +#endif + +#endif diff --git a/ext/bcmath/libbcmath/src/str2num.c b/ext/bcmath/libbcmath/src/str2num.c index bd9a44a240503..945de0cf60003 100644 --- a/ext/bcmath/libbcmath/src/str2num.c +++ b/ext/bcmath/libbcmath/src/str2num.c @@ -32,30 +32,28 @@ #include "bcmath.h" #include "convert.h" #include "private.h" +#include "simd.h" #include #include -#ifdef __SSE2__ -# include -#endif /* Convert strings to bc numbers. Base 10 only.*/ -static const char *bc_count_digits(const char *str, const char *end) +static inline const char *bc_count_digits(const char *str, const char *end) { /* Process in bulk */ -#ifdef __SSE2__ - const __m128i offset = _mm_set1_epi8((signed char) (SCHAR_MIN - '0')); +#ifdef HAVE_BC_SIMD_128 + const bc_simd_128_t offset = bc_simd_set_8x16((signed char) (SCHAR_MIN - '0')); /* we use the less than comparator, so add 1 */ - const __m128i threshold = _mm_set1_epi8(SCHAR_MIN + ('9' + 1 - '0')); + const bc_simd_128_t threshold = bc_simd_set_8x16(SCHAR_MIN + ('9' + 1 - '0')); - while (str + sizeof(__m128i) <= end) { - __m128i bytes = _mm_loadu_si128((const __m128i *) str); + while (str + sizeof(bc_simd_128_t) <= end) { + bc_simd_128_t bytes = bc_simd_load_8x16((const bc_simd_128_t *) str); /* Wrapping-add the offset to the bytes, such that all bytes below '0' are positive and others are negative. * More specifically, '0' will be -128 and '9' will be -119. */ - bytes = _mm_add_epi8(bytes, offset); + bytes = bc_simd_add_8x16(bytes, offset); /* Now mark all bytes that are <= '9', i.e. <= -119, i.e. < -118, i.e. the threshold. */ - bytes = _mm_cmplt_epi8(bytes, threshold); + bytes = bc_simd_cmplt_8x16(bytes, threshold); - int mask = _mm_movemask_epi8(bytes); + int mask = bc_simd_movemask_8x16(bytes); if (mask != 0xffff) { /* At least one of the bytes is not within range. Move to the first offending byte. */ #ifdef PHP_HAVE_BUILTIN_CTZL @@ -65,7 +63,7 @@ static const char *bc_count_digits(const char *str, const char *end) #endif } - str += sizeof(__m128i); + str += sizeof(bc_simd_128_t); } #endif @@ -79,19 +77,19 @@ static const char *bc_count_digits(const char *str, const char *end) static inline const char *bc_skip_zero_reverse(const char *scanner, const char *stop) { /* Check in bulk */ -#ifdef __SSE2__ - const __m128i c_zero_repeat = _mm_set1_epi8('0'); - while (scanner - sizeof(__m128i) >= stop) { - scanner -= sizeof(__m128i); - __m128i bytes = _mm_loadu_si128((const __m128i *) scanner); +#ifdef HAVE_BC_SIMD_128 + const bc_simd_128_t c_zero_repeat = bc_simd_set_8x16('0'); + while (scanner - sizeof(bc_simd_128_t) >= stop) { + scanner -= sizeof(bc_simd_128_t); + bc_simd_128_t bytes = bc_simd_load_8x16((const bc_simd_128_t *) scanner); /* Checks if all numeric strings are equal to '0'. */ - bytes = _mm_cmpeq_epi8(bytes, c_zero_repeat); + bytes = bc_simd_cmpeq_8x16(bytes, c_zero_repeat); - int mask = _mm_movemask_epi8(bytes); + int mask = bc_simd_movemask_8x16(bytes); /* The probability of having 16 trailing 0s in a row is very low, so we use EXPECTED. */ if (EXPECTED(mask != 0xffff)) { /* Move the pointer back and check each character in loop. */ - scanner += sizeof(__m128i); + scanner += sizeof(bc_simd_128_t); break; } }