[libc] inline fast path of callonce (llvm#96226)

Split from llvm#91572 --------- Co-authored-by: Nick Desaulniers (paternity leave) <[email protected]>
Xilinx · Jun 27, 2024 · 6d61d83 · 6d61d83
1 parent 2fefc04
commit 6d61d83
Show file tree

Hide file tree

Showing 4 changed files with 58 additions and 17 deletions.
diff --git a/libc/src/__support/threads/callonce.h b/libc/src/__support/threads/callonce.h
@@ -9,13 +9,32 @@
 #ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_CALLONCE_H
 #define LLVM_LIBC_SRC___SUPPORT_THREADS_CALLONCE_H
 
+#include "src/__support/macros/optimization.h" // LIBC_LIKELY
+
+// Plaform specific routines, provides:
+// - OnceFlag definition
+// - callonce_impl::callonce_fastpath for fast path check
+// - callonce_impl::callonce_slowpath for slow path execution
+#ifdef __linux__
+#include "src/__support/threads/linux/callonce.h"
+#else
+#error "callonce is not supported on this platform"
+#endif
+
 namespace LIBC_NAMESPACE {
 
-struct CallOnceFlag;
+// Common definitions
 using CallOnceCallback = void(void);
+namespace callonce_impl {
+int callonce_slowpath(CallOnceFlag *flag, CallOnceCallback *callback);
+} // namespace callonce_impl
 
-int callonce(CallOnceFlag *flag, CallOnceCallback *callback);
+LIBC_INLINE int callonce(CallOnceFlag *flag, CallOnceCallback *callback) {
+  if (LIBC_LIKELY(callonce_impl::callonce_fastpath(flag)))
+    return 0;
 
+  return callonce_impl::callonce_slowpath(flag, callback);
+}
 } // namespace LIBC_NAMESPACE
 
 #endif // LLVM_LIBC_SRC___SUPPORT_THREADS_CALLONCE_H
diff --git a/libc/src/__support/threads/linux/CMakeLists.txt b/libc/src/__support/threads/linux/CMakeLists.txt
@@ -100,8 +100,10 @@ add_object_library(
     callonce.cpp
   HDRS
     ../callonce.h
+    callonce.h
   DEPENDS
     .futex_utils
+    libc.src.__support.macros.optimization
 )
 
 add_object_library(

diff --git a/libc/src/__support/threads/linux/callonce.cpp b/libc/src/__support/threads/linux/callonce.cpp
@@ -7,27 +7,16 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/__support/threads/callonce.h"
-#include "src/__support/macros/optimization.h"
+#include "src/__support/threads/linux/callonce.h"
 #include "src/__support/threads/linux/futex_utils.h"
 
 namespace LIBC_NAMESPACE {
-
-static constexpr FutexWordType NOT_CALLED = 0x0;
-static constexpr FutexWordType START = 0x11;
-static constexpr FutexWordType WAITING = 0x22;
-static constexpr FutexWordType FINISH = 0x33;
-
-int callonce(CallOnceFlag *flag, CallOnceCallback *func) {
+namespace callonce_impl {
+int callonce_slowpath(CallOnceFlag *flag, CallOnceCallback *func) {
   auto *futex_word = reinterpret_cast<Futex *>(flag);
 
   FutexWordType not_called = NOT_CALLED;
 
-  // Avoid cmpxchg operation if the function has already been called.
-  // The destination operand of cmpxchg may receive a write cycle without
-  // regard to the result of the comparison
-  if (LIBC_LIKELY(futex_word->load(cpp::MemoryOrder::RELAXED) == FINISH))
-    return 0;
-
   // The call_once call can return only after the called function |func|
   // returns. So, we use futexes to synchronize calls with the same flag value.
   if (futex_word->compare_exchange_strong(not_called, START)) {
@@ -46,5 +35,5 @@ int callonce(CallOnceFlag *flag, CallOnceCallback *func) {
 
   return 0;
 }
-
+} // namespace callonce_impl
 } // namespace LIBC_NAMESPACE
diff --git a/libc/src/__support/threads/linux/callonce.h b/libc/src/__support/threads/linux/callonce.h
@@ -0,0 +1,31 @@
+//===-- Linux callonce fastpath -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_CALLONCE_H
+#define LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_CALLONCE_H
+
+#include "src/__support/threads/linux/futex_utils.h"
+
+namespace LIBC_NAMESPACE {
+using CallOnceFlag = Futex;
+
+namespace callonce_impl {
+static constexpr FutexWordType NOT_CALLED = 0x0;
+static constexpr FutexWordType START = 0x11;
+static constexpr FutexWordType WAITING = 0x22;
+static constexpr FutexWordType FINISH = 0x33;
+
+// Avoid cmpxchg operation if the function has already been called.
+// The destination operand of cmpxchg may receive a write cycle without
+// regard to the result of the comparison.
+LIBC_INLINE bool callonce_fastpath(CallOnceFlag *flag) {
+  return flag->load(cpp::MemoryOrder::RELAXED) == FINISH;
+}
+} // namespace callonce_impl
+
+} // namespace LIBC_NAMESPACE
+#endif // LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_CALLONCE_H