release 0.6

alibaba · May 8, 2023 · 5c5c47e · 5c5c47e
1 parent 5efcb89
commit 5c5c47e
Show file tree

Hide file tree

Showing 56 changed files with 3,340 additions and 1,483 deletions.
diff --git a/CMake/Findfstack.cmake b/CMake/Findfstack.cmake
@@ -0,0 +1,7 @@
+find_path(FSTACK_INCLUDE_DIRS NAMES ff_api.h PATHS /usr/local/include NO_DEFAULT_PATH)
+
+find_library(FSTACK_LIBRARIES NAMES fstack PATHS /usr/local/lib NO_DEFAULT_PATH)
+
+find_package_handle_standard_args(fstack DEFAULT_MSG FSTACK_LIBRARIES FSTACK_INCLUDE_DIRS)
+
+mark_as_advanced(FSTACK_INCLUDE_DIRS FSTACK_LIBRARIES)
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -2,25 +2,25 @@ cmake_minimum_required(VERSION 3.14 FATAL_ERROR)
 
 project(
         photon
-        VERSION 0.4
+        VERSION 0.6
         LANGUAGES C CXX ASM
 )
 
-# Utilities
+# Utility Modules and Find Modules
 include(FindPackageHandleStandardArgs)
-include(FindPkgConfig)
 include(CheckCXXCompilerFlag)
 include(FetchContent)
 set(FETCHCONTENT_QUIET false)
+find_package(PkgConfig REQUIRED)
 
 # Options
 option(ENABLE_URING "enable io_uring function" OFF)
 option(ENABLE_FUSE "enable fuse function" OFF)
 option(ENABLE_SASL "enable sasl" OFF)
 option(ENABLE_MIMIC_VDSO "enable mimic vdso" OFF)
 option(BUILD_TESTING "enable build testing" OFF)
-option(FETCH_GFLAGS_SOURCE "fetch gflags source" OFF)
-option(FETCH_GTEST_SOURCE "fetch gtest and gmock source" OFF)
+option(FETCH_GTEST_GFLAGS_SOURCE "Fetch gtest, gmock and gflags source code. Link their static libs" OFF)
+option(ENABLE_FSTACK_DPDK "Use f-stack + DPDK as the event engine" OFF)
 
 # Get CPU arch
 execute_process(COMMAND uname -m OUTPUT_VARIABLE ARCH OUTPUT_STRIP_TRAILING_WHITESPACE)
@@ -90,6 +90,7 @@ if (CMAKE_SYSTEM_NAME MATCHES "Linux")
             FetchContent_Declare(
                     liburing
                     GIT_REPOSITORY https://github.com/axboe/liburing.git
+                    # GIT_REPOSITORY https://gitee.com/mirrors_axboe/liburing.git
                     GIT_TAG liburing-2.3
                     UPDATE_COMMAND ./configure
             )
@@ -120,7 +121,7 @@ if (ENABLE_SASL)
     find_package(gsasl REQUIRED)
 endif ()
 
-if (FETCH_GFLAGS_SOURCE)
+if (FETCH_GTEST_GFLAGS_SOURCE)
     FetchContent_Declare(
             gflags
             GIT_REPOSITORY https://github.com/gflags/gflags.git
@@ -129,9 +130,6 @@ if (FETCH_GFLAGS_SOURCE)
             GIT_SUBMODULES ""
     )
     FetchContent_MakeAvailable(gflags)
-endif()
-
-if (FETCH_GTEST_SOURCE)
     FetchContent_Declare(
             googletest
             GIT_REPOSITORY https://github.com/google/googletest.git
@@ -181,7 +179,11 @@ endif ()
 if (ENABLE_SASL)
     list(APPEND PHOTON_SRC net/security-context/sasl-stream.cpp)
 endif ()
+if (ENABLE_FSTACK_DPDK)
+    list(APPEND PHOTON_SRC io/fstack-dpdk.cpp)
+endif ()
 
+# An object library compiles source files but does not archive or link their object files.
 add_library(photon_obj OBJECT ${PHOTON_SRC})
 target_include_directories(photon_obj PUBLIC include ${OPENSSL_INCLUDE_DIR})
 target_compile_definitions(photon_obj PRIVATE _FILE_OFFSET_BITS=64 FUSE_USE_VERSION=29)
@@ -197,7 +199,18 @@ if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 8.0)
     target_compile_options(photon_obj PRIVATE -faligned-new)
 endif()
 
-# Make virtual interface for external dynamic libs
+# Add f-stack and dpdk libs.
+if (ENABLE_FSTACK_DPDK)
+    pkg_check_modules(DPDK REQUIRED libdpdk)
+    find_package(fstack REQUIRED)
+    target_include_directories(photon_obj PRIVATE ${DPDK_INCLUDE_DIRS} ${FSTACK_INCLUDE_DIRS})
+    target_compile_definitions(photon_obj PRIVATE ENABLE_FSTACK_DPDK)
+
+    add_library(fstack_dpdk INTERFACE)
+    target_link_libraries(fstack_dpdk INTERFACE ${DPDK_STATIC_LDFLAGS} ${FSTACK_LIBRARIES})
+endif()
+
+# Make virtual interface for external dynamic libs.
 add_library(external_lib INTERFACE)
 set(EXTERNAL_LIB_ARGS
         ZLIB::ZLIB
@@ -227,27 +240,27 @@ set_target_properties(photon_shared PROPERTIES OUTPUT_NAME photon)
 if (APPLE)
     set(shared_link_libs external_lib -Wl,-force_load easy_weak)
 elseif (ENABLE_URING)
-    set(shared_link_libs -Wl,--whole-archive easy_weak uring -Wl,--no-whole-archive external_lib)
+    set(shared_link_libs -Wl,--whole-archive easy_weak fstack_weak uring -Wl,--no-whole-archive external_lib)
 else()
-    set(shared_link_libs -Wl,--whole-archive easy_weak -Wl,--no-whole-archive external_lib)
+    set(shared_link_libs -Wl,--whole-archive easy_weak fstack_weak -Wl,--no-whole-archive external_lib)
 endif ()
 target_link_libraries(photon_shared ${shared_link_libs})
 
 # Link photon static lib
 add_library(photon_static STATIC $<TARGET_OBJECTS:photon_obj>)
 set_target_properties(photon_static PROPERTIES OUTPUT_NAME photon)
 if (ENABLE_URING)
-    set(static_link_libs easy_weak uring external_lib)
+    set(static_link_libs easy_weak fstack_weak uring external_lib)
 else ()
-    set(static_link_libs easy_weak external_lib)
+    set(static_link_libs easy_weak fstack_weak external_lib)
 endif ()
 target_link_libraries(photon_static ${static_link_libs})
 
 # Build test cases
 if (BUILD_TESTING)
     include(CTest)
 
-if (FETCH_GTEST_SOURCE AND FETCH_GFLAGS_SOURCE)
+if (FETCH_GTEST_GFLAGS_SOURCE)
     set(testing_libs gtest gmock gflags::gflags)
 else()
     find_package(GTest REQUIRED)
@@ -259,6 +272,7 @@ endif()
 
     add_subdirectory(examples)
     add_subdirectory(common/checksum/test)
+    add_subdirectory(common/test)
     add_subdirectory(fs/test)
     add_subdirectory(io/test)
     add_subdirectory(net/test)

diff --git a/common/conststr.h b/common/conststr.h
@@ -87,7 +87,7 @@ struct TString : tstring_base {
     }
     template <char SP, char IGN>
     static constexpr decltype(auto) split() {
-        return TSpliter<SP, IGN, TString>::array;
+        return TSpliter<SP, IGN, TString>::array();
     }
     static constexpr decltype(auto) tsreverse(TString<>) { return TString<>(); }
     template <char ch, char... chs>
@@ -225,15 +225,9 @@ struct TSpliter {
     using Current = decltype(Cut::Head::template strip<IGN>());
     using Next = TSpliter<SP, IGN, typename Cut::Tail>;
     using Array = typename Next::Array::template Prepend<Current>;
-#ifdef __clang__
     static constexpr Current current() { return {}; };
     static constexpr Next next() { return {}; };
     static constexpr Array array() { return {}; };
-#else
-    static constexpr Current current{};
-    static constexpr Next next{};
-    static constexpr Array array{};
-#endif
 };
 
 template <char SP, char IGN>
@@ -242,15 +236,9 @@ struct TSpliter<SP, IGN, TString<>> {
     using Current = TString<>;
     using Next = TSpliter;
     using Array = TStrArray<>;
-#ifdef __clang__
     static constexpr Current current() { return {}; };
     static constexpr Next next() { return {}; };
     static constexpr Array array() { return {}; };
-#else
-    static constexpr Current current{};
-    static constexpr Next next{};
-    static constexpr Array array{};
-#endif
 };
 
 template <char SP, char ch, char... chs>

diff --git a/common/estring.cpp b/common/estring.cpp
@@ -81,7 +81,7 @@ uint64_t estring_view::hex_to_uint64() const
     return ret;
 }
 
-std::string& estring::append(uint64_t x)
+estring& estring::append(uint64_t x)
 {
     auto begin = size();
     do
@@ -94,3 +94,58 @@ std::string& estring::append(uint64_t x)
     std::reverse(ptr + begin, ptr + end);
     return *this;
 }
+
+static_assert(
+    std::is_same<estring&, decltype(std::declval<estring>().append(0ULL))>::value,
+    "estring append uint64 should return estring"
+);
+
+static_assert(
+    std::is_same<estring&, decltype(std::declval<estring>().append('0'))>::value,
+    "estring append char should return estring"
+);
+
+static_assert(
+    std::is_same<estring&, decltype(std::declval<estring>().append("Hello"))>::value,
+    "estring append char* should return estring"
+);
+
+static_assert(
+    std::is_same<estring&, decltype(std::declval<estring>().append(std::declval<char*>(), 5))>::value,
+    "estring append char* and size should return estring"
+);
+
+static_assert(
+    std::is_same<estring&, decltype(std::declval<estring>().append(std::declval<std::string>()))>::value,
+    "estring append std::string should return estring"
+);
+
+static_assert(
+    std::is_same<estring&, decltype(std::declval<estring>().append(std::declval<std::string_view>()))>::value,
+    "estring append std::string_view should return estring"
+);
+
+static_assert(
+    std::is_same<estring&, decltype(std::declval<estring>() += 0ULL)>::value,
+    "estring += uint64 should return estring"
+);
+
+static_assert(
+    std::is_same<estring&, decltype(std::declval<estring>() += '0')>::value,
+    "estring += char should return estring"
+);
+
+static_assert(
+    std::is_same<estring&, decltype(std::declval<estring>() += "Hello")>::value,
+    "estring += char* should return estring"
+);
+
+static_assert(
+    std::is_same<estring&, decltype(std::declval<estring>() += std::declval<std::string>())>::value,
+    "estring += std::string should return estring"
+);
+
+static_assert(
+    std::is_same<estring&, decltype(std::declval<estring>() += std::declval<std::string_view>())>::value,
+    "estring += std::string_view should return estring"
+);
diff --git a/common/estring.h b/common/estring.h
@@ -445,24 +445,33 @@ class estring : public std::string
             n);
     }
 
-    using std::string::operator+=;
-    std::string& operator += (const std::string_view &rhs)
+    // using std::string::operator+=;
+    template<typename T>
+    estring& operator+=(T&& t) {
+        return (estring&)std::string::operator+=(std::forward<T>(t));
+    }
+    estring& operator += (const std::string_view &rhs)
     {
         return append(rhs);
     }
-    using std::string::append;
-    std::string& append(uint64_t x);
-    std::string& append(const std::string_view& sv)
+    // using std::string::append;
+    template <typename... Args>
+    estring& append(Args&&... t) {
+        return (estring&)std::string::append(std::forward<Args>(t)...);
+    }
+    estring& append(uint64_t x);
+    estring& append(const std::string_view& sv)
     {
         return append(sv.data(), sv.size());
     }
 
     template<typename...Ts>
-    std::string& appends(const Ts&...xs)
+    estring& appends(const Ts&...xs)
     {
-        return append(make_cat_list(xs...));
+        return cat(make_cat_list(xs...));
     }
 
+protected:
     template<typename...Ts>
     class CatList : public std::tuple<Ts...>
     {
@@ -495,7 +504,7 @@ class estring : public std::string
 
     template<typename...Ts>
     __attribute__((always_inline))
-    std::string& append(const CatList<Ts...>& cl)
+    estring& cat(const CatList<Ts...>& cl)
     {
         UpperBoundEstimator estimator;
         this->enumerate(cl, estimator);
@@ -506,18 +515,17 @@ class estring : public std::string
     }
 
     template<typename...Ts>
-    std::string& append(const ConditionalCatList<Ts...>& ccl)
+    estring& cat(const ConditionalCatList<Ts...>& ccl)
     {
-        return ccl._cond ? append(ccl._cl) : *this;
+        return ccl._cond ? cat(ccl._cl) : *this;
     }
 
     template<typename A, typename B>
-    std::string& append(const ConditionalCatList2<A, B>& x)
+    estring& cat(const ConditionalCatList2<A, B>& x)
     {
-        return x._cond ? append(x._cl1) : append(x._cl2);
+        return x._cond ? cat(x._cl1) : cat(x._cl2);
     }
 
-protected:
     template<typename...Ts>
     static const ConditionalCatList<Ts...>&
                                  cat_item_filter(const ConditionalCatList<Ts...>& x)
@@ -542,12 +550,12 @@ class estring : public std::string
         return {cond, cat_item_filter(xs)...};
     }
 
-    // template<typename A, typename B> static
-    // auto make_conditional_cat_list2(bool cond, const A& a, const B& b) ->
-    //     ConditionalCatList2<A, B>
-    // {
-    //     return {cond, a, b};
-    // }
+    template<typename ...As, typename ...Bs> static
+    auto make_conditional_cat_list2(bool cond, const CatList<As...>& a, const CatList<Bs...>& b) ->
+        ConditionalCatList2<CatList<As...>, CatList<Bs...>>
+    {
+        return {cond, a, b};
+    }
 
 protected:
     template<size_t I = 0, typename...Ts, typename Mapper>

diff --git a/common/expirecontainer.cpp b/common/expirecontainer.cpp
@@ -41,9 +41,11 @@ ExpireContainerBase::iterator ExpireContainerBase::find(const Item& key_item) {
 }
 
 void ExpireContainerBase::clear() {
-    for (auto x : ({ SCOPED_LOCK(_lock);
-                     _list.node = nullptr;
-                     Set(std::move(_set)); })) {
+    for (auto x : ({
+             SCOPED_LOCK(_lock);
+             _list.node = nullptr;
+             Set(std::move(_set));
+         })) {
         delete x;
     }
 }
@@ -76,7 +78,8 @@ bool ExpireContainerBase::keep_alive(const Item& x, bool insert_if_not_exists) {
 }
 
 ObjectCacheBase::Item* ObjectCacheBase::ref_acquire(const Item& key_item,
-                                                    Delegate<void*> ctor) {
+                                                    Delegate<void*> ctor,
+                                                    uint64_t failure_cooldown) {
     Base::iterator holder;
     Item* item = nullptr;
     expire();
@@ -101,8 +104,10 @@ ObjectCacheBase::Item* ObjectCacheBase::ref_acquire(const Item& key_item,
     } while (!item);
     {
         SCOPED_LOCK(item->_mtx);
-        if (!item->_obj) {
+        if (!item->_obj && (item->_failure <=
+                            photon::sat_sub(photon::now, failure_cooldown))) {
             item->_obj = ctor();
+            if (!item->_obj) item->_failure = photon::now;
         }
     }
     if (!item->_obj) {
@@ -126,6 +131,7 @@ int ObjectCacheBase::ref_release(ItemPtr item, bool recycle) {
             if (item->_recycle) {
                 item->_recycle->signal(1);
             } else {
+                item->_failure = 0;
                 enqueue(item);
             }
         }