From 6d61d83736ca23c336472e7f2bda59b7bb307bba Mon Sep 17 00:00:00 2001 From: Schrodinger ZHU Yifan Date: Thu, 27 Jun 2024 08:48:13 -0700 Subject: [PATCH] [libc] inline fast path of callonce (#96226) Split from #91572 --------- Co-authored-by: Nick Desaulniers (paternity leave) --- libc/src/__support/threads/callonce.h | 23 ++++++++++++-- .../__support/threads/linux/CMakeLists.txt | 2 ++ libc/src/__support/threads/linux/callonce.cpp | 19 +++--------- libc/src/__support/threads/linux/callonce.h | 31 +++++++++++++++++++ 4 files changed, 58 insertions(+), 17 deletions(-) create mode 100644 libc/src/__support/threads/linux/callonce.h diff --git a/libc/src/__support/threads/callonce.h b/libc/src/__support/threads/callonce.h index b3d6813f7dda9e..67f4c996571072 100644 --- a/libc/src/__support/threads/callonce.h +++ b/libc/src/__support/threads/callonce.h @@ -9,13 +9,32 @@ #ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_CALLONCE_H #define LLVM_LIBC_SRC___SUPPORT_THREADS_CALLONCE_H +#include "src/__support/macros/optimization.h" // LIBC_LIKELY + +// Plaform specific routines, provides: +// - OnceFlag definition +// - callonce_impl::callonce_fastpath for fast path check +// - callonce_impl::callonce_slowpath for slow path execution +#ifdef __linux__ +#include "src/__support/threads/linux/callonce.h" +#else +#error "callonce is not supported on this platform" +#endif + namespace LIBC_NAMESPACE { -struct CallOnceFlag; +// Common definitions using CallOnceCallback = void(void); +namespace callonce_impl { +int callonce_slowpath(CallOnceFlag *flag, CallOnceCallback *callback); +} // namespace callonce_impl -int callonce(CallOnceFlag *flag, CallOnceCallback *callback); +LIBC_INLINE int callonce(CallOnceFlag *flag, CallOnceCallback *callback) { + if (LIBC_LIKELY(callonce_impl::callonce_fastpath(flag))) + return 0; + return callonce_impl::callonce_slowpath(flag, callback); +} } // namespace LIBC_NAMESPACE #endif // LLVM_LIBC_SRC___SUPPORT_THREADS_CALLONCE_H diff --git a/libc/src/__support/threads/linux/CMakeLists.txt b/libc/src/__support/threads/linux/CMakeLists.txt index 95e509b7a825dd..8b7971584e77ec 100644 --- a/libc/src/__support/threads/linux/CMakeLists.txt +++ b/libc/src/__support/threads/linux/CMakeLists.txt @@ -100,8 +100,10 @@ add_object_library( callonce.cpp HDRS ../callonce.h + callonce.h DEPENDS .futex_utils + libc.src.__support.macros.optimization ) add_object_library( diff --git a/libc/src/__support/threads/linux/callonce.cpp b/libc/src/__support/threads/linux/callonce.cpp index b48a514a448753..24d376f447fba1 100644 --- a/libc/src/__support/threads/linux/callonce.cpp +++ b/libc/src/__support/threads/linux/callonce.cpp @@ -7,27 +7,16 @@ //===----------------------------------------------------------------------===// #include "src/__support/threads/callonce.h" -#include "src/__support/macros/optimization.h" +#include "src/__support/threads/linux/callonce.h" #include "src/__support/threads/linux/futex_utils.h" namespace LIBC_NAMESPACE { - -static constexpr FutexWordType NOT_CALLED = 0x0; -static constexpr FutexWordType START = 0x11; -static constexpr FutexWordType WAITING = 0x22; -static constexpr FutexWordType FINISH = 0x33; - -int callonce(CallOnceFlag *flag, CallOnceCallback *func) { +namespace callonce_impl { +int callonce_slowpath(CallOnceFlag *flag, CallOnceCallback *func) { auto *futex_word = reinterpret_cast(flag); FutexWordType not_called = NOT_CALLED; - // Avoid cmpxchg operation if the function has already been called. - // The destination operand of cmpxchg may receive a write cycle without - // regard to the result of the comparison - if (LIBC_LIKELY(futex_word->load(cpp::MemoryOrder::RELAXED) == FINISH)) - return 0; - // The call_once call can return only after the called function |func| // returns. So, we use futexes to synchronize calls with the same flag value. if (futex_word->compare_exchange_strong(not_called, START)) { @@ -46,5 +35,5 @@ int callonce(CallOnceFlag *flag, CallOnceCallback *func) { return 0; } - +} // namespace callonce_impl } // namespace LIBC_NAMESPACE diff --git a/libc/src/__support/threads/linux/callonce.h b/libc/src/__support/threads/linux/callonce.h new file mode 100644 index 00000000000000..315cc6149e9ecf --- /dev/null +++ b/libc/src/__support/threads/linux/callonce.h @@ -0,0 +1,31 @@ +//===-- Linux callonce fastpath -------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_CALLONCE_H +#define LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_CALLONCE_H + +#include "src/__support/threads/linux/futex_utils.h" + +namespace LIBC_NAMESPACE { +using CallOnceFlag = Futex; + +namespace callonce_impl { +static constexpr FutexWordType NOT_CALLED = 0x0; +static constexpr FutexWordType START = 0x11; +static constexpr FutexWordType WAITING = 0x22; +static constexpr FutexWordType FINISH = 0x33; + +// Avoid cmpxchg operation if the function has already been called. +// The destination operand of cmpxchg may receive a write cycle without +// regard to the result of the comparison. +LIBC_INLINE bool callonce_fastpath(CallOnceFlag *flag) { + return flag->load(cpp::MemoryOrder::RELAXED) == FINISH; +} +} // namespace callonce_impl + +} // namespace LIBC_NAMESPACE +#endif // LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_CALLONCE_H