diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b4543205f..521d89d21 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,11 +33,12 @@ jobs: shell: bash run: bash build.sh debug -DCONCURRENCY=ON -DENABLE_COVERAGE=ON --make -j4 + # `memtracer_test` unittest runs in `memtracer-test` action. - name: Test shell: bash run: | cd build_debug - make test + ctest -E memtracer_test --verbose - name: lcov shell: bash diff --git a/.github/workflows/clang-format.yml b/.github/workflows/clang-format.yml index 13fceb302..6b216bb8e 100644 --- a/.github/workflows/clang-format.yml +++ b/.github/workflows/clang-format.yml @@ -11,6 +11,6 @@ jobs: with: # sources 和 excludes 最终生成的find查找文件的命令大概是这样的 # find . -type f ! -wholename "./src/observer/sql/parser/lex_sql.*" \( -wholename "./**/*.h" -o -wholename "./**/*.cpp" \) - sources: "**/*.h,**/*.cpp,src/**/*.h,src/**/*.cpp,deps/common/**/*.h,deps/common/**/*.cpp" - excludes: "src/observer/sql/parser/lex_sql.*,src/observer/sql/parser/yacc_sql.*" + sources: "**/*.h,**/*.cpp,src/**/*.h,src/**/*.cpp" + excludes: "src/observer/sql/parser/lex_sql.*,src/observer/sql/parser/yacc_sql.*,deps/3rd/**/*.cpp,deps/3rd/**/*.h" style: "file" \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d3f171a46..ed0c67a5e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -97,8 +97,53 @@ jobs: sudo bash build.sh init bash build.sh release -DCONCURRENCY=ON -DWITH_UNIT_TESTS=OFF -DWITH_BENCHMARK=ON - - name: testing + - name: concurrency-test shell: bash run: | cd build_release/bin/ for file in `find ./ -name "*_concurrency_test" -executable`; do $file; if [ $? -ne 0 ]; then exit 1; fi; done + memtracer-test: + strategy: + matrix: + memtracer: ['LD_PRELOAD=./lib/libmemtracer.so', ''] + runs-on: ubuntu-latest + steps: + - name: Checkout repository and submodules + uses: actions/checkout@v2 + + - name: build + shell: bash + run: | + sudo bash build.sh init + bash build.sh release -DWITH_BENCHMARK=ON -DENABLE_ASAN=OFF -DCONCURRENCY=ON + + - name: memtracer-performance-test + shell: bash + run: | + cd build_release + ${{matrix.memtracer}} ./bin/memtracer_performance_test + - name: memtracer-unittest + shell: bash + run: | + cd build_release + LD_PRELOAD=./lib/libmemtracer.so ./bin/memtracer_test + - name: memtracer-sysbench + shell: bash + run: | + curl -s https://packagecloud.io/install/repositories/akopytov/sysbench/script.deb.sh -o script.deb.sh + sudo bash script.deb.sh + sudo apt -y install sysbench mariadb-client + nohup sh -c '${{matrix.memtracer}} ./build_release/bin/observer -T one-thread-per-connection -s /tmp/miniob.sock -f etc/observer.ini -P mysql -t mvcc -d disk' & + sleep 10 && echo "wake up" + mysql --version + mysql -S /tmp/miniob.sock -e "show tables" + cd test/sysbench + sysbench --mysql-socket=/tmp/miniob.sock --mysql-ignore-errors=41 --threads=10 miniob_insert prepare + sysbench --mysql-socket=/tmp/miniob.sock --mysql-ignore-errors=41 --threads=10 miniob_insert run + killall observer + cd ../.. + nohup ./build_release/bin/observer -T one-thread-per-connection -s /tmp/miniob.sock -f etc/observer.ini -P mysql -t mvcc -d disk & + sleep 10 && echo "wake up" + mysql -S /tmp/miniob.sock -e "show tables" + + diff --git a/CMakeLists.txt b/CMakeLists.txt index a96a309de..a4675ec2f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,6 +15,8 @@ OPTION(ENABLE_TSAN "Build with thread sanitizer" OFF) OPTION(ENABLE_UBSAN "Build with undefined behavior sanitizer" OFF) OPTION(WITH_UNIT_TESTS "Compile miniob with unit tests" ON) OPTION(WITH_BENCHMARK "Compile benchmark" OFF) +# TODO: support MemTracer with sanitizers, currently MemTracer doesn't work with sanitizers. +OPTION(WITH_MEMTRACER "Compile memtracer" ON) OPTION(ENABLE_COVERAGE "Enable unittest coverage" OFF) OPTION(ENABLE_NOPIE "Enable no pie" OFF) OPTION(CONCURRENCY "Support concurrency operations" OFF) diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 0422562d8..a64cfe3cb 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -8,5 +8,9 @@ FOREACH (F ${ALL_SRC}) get_filename_component(prjName ${F} NAME_WE) MESSAGE("Build ${prjName} according to ${F}") ADD_EXECUTABLE(${prjName} ${F}) - TARGET_LINK_LIBRARIES(${prjName} common pthread dl benchmark observer_static) + TARGET_LINK_LIBRARIES(${prjName} common pthread dl benchmark) + if(NOT ${prjName} STREQUAL "memtracer_performance_test") + TARGET_LINK_LIBRARIES(${prjName} observer_static) + endif() + ENDFOREACH (F) diff --git a/benchmark/memtracer_performance_test.cpp b/benchmark/memtracer_performance_test.cpp new file mode 100644 index 000000000..56e2f3649 --- /dev/null +++ b/benchmark/memtracer_performance_test.cpp @@ -0,0 +1,38 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#include + +static void BM_MallocFree(benchmark::State &state) +{ + size_t size = state.range(0); + for (auto _ : state) { + void *ptr = malloc(size); + benchmark::DoNotOptimize(ptr); + free(ptr); + } + state.SetBytesProcessed(static_cast(state.iterations() * size)); +} + +static void BM_NewDelete(benchmark::State &state) +{ + size_t size = state.range(0); + for (auto _ : state) { + char *ptr = new char[size]; + benchmark::DoNotOptimize(ptr); + delete[] ptr; + } + state.SetBytesProcessed(static_cast(state.iterations() * size)); +} + +BENCHMARK(BM_MallocFree)->Arg(8)->Arg(64)->Arg(512)->Arg(1 << 10)->Arg(1 << 20)->Arg(8 << 20)->Arg(1 << 30); +BENCHMARK(BM_NewDelete)->Arg(8)->Arg(64)->Arg(512)->Arg(1 << 10)->Arg(1 << 20)->Arg(8 << 20)->Arg(1 << 30); + +BENCHMARK_MAIN(); \ No newline at end of file diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index 697f2be7c..f8398244a 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -1,2 +1,5 @@ ADD_SUBDIRECTORY(common) +if (WITH_MEMTRACER) + ADD_SUBDIRECTORY(memtracer) +endif() diff --git a/deps/common/CMakeLists.txt b/deps/common/CMakeLists.txt index 85463bc08..7e27ef7e4 100644 --- a/deps/common/CMakeLists.txt +++ b/deps/common/CMakeLists.txt @@ -4,10 +4,6 @@ MESSAGE(STATUS "This is PROJECT_SOURCE_DIR dir " ${PROJECT_SOURCE_DIR}) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) FILE(GLOB_RECURSE ALL_SRC *.cpp) -FOREACH(F ${ALL_SRC}) - SET(SRC_LIST ${SRC_LIST} ${F}) - MESSAGE("Use " ${F}) -ENDFOREACH(F) #SHARED，动态库 #STATIC，静态库 diff --git a/deps/memtracer/CMakeLists.txt b/deps/memtracer/CMakeLists.txt new file mode 100644 index 000000000..6023273a3 --- /dev/null +++ b/deps/memtracer/CMakeLists.txt @@ -0,0 +1,19 @@ +include(CheckCXXCompilerFlag) + +file(GLOB MEMTRACER_SOURCES "*.cpp") + +SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/lib) +add_library(memtracer SHARED ${MEMTRACER_SOURCES}) +#ignore missing attributes +#ref jemalloc: https://github.com/jemalloc/jemalloc/blob/master/configure.ac +CHECK_CXX_COMPILER_FLAG("-Wno-missing-attributes" COMPILER_SUPPORTS_NO_MISSING_ATTRIBUTES) +if(COMPILER_SUPPORTS_NO_MISSING_ATTRIBUTES) + target_compile_options(memtracer PRIVATE -Wno-missing-attributes) +endif() + +# hidden memtracer internal interfaces +set_target_properties(memtracer PROPERTIES + CXX_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN YES +) +target_link_libraries(memtracer pthread) \ No newline at end of file diff --git a/deps/memtracer/allocator.cpp b/deps/memtracer/allocator.cpp new file mode 100644 index 000000000..ba06da297 --- /dev/null +++ b/deps/memtracer/allocator.cpp @@ -0,0 +1,196 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#include "memtracer/allocator.h" +#include + +// `dlsym` calls `calloc` internally, so here use a dummy buffer +// to avoid infinite loop when hook functions initialized. +// ref: +// https://stackoverflow.com/questions/7910666/problems-with-ld-preload-and-calloc-interposition-for-certain-executables +static unsigned char calloc_buffer[8192]; + +// only used internally +inline size_t ptr_size(void *ptr) +{ + if (ptr == NULL) [[unlikely]] { + return 0; + } + return *((size_t *)ptr - 1); +} + +mt_visible void *malloc(size_t size) +{ + MT.init_hook_funcs(); + size_t *ptr = (size_t *)orig_malloc(size + sizeof(size_t)); + if (ptr == NULL) [[unlikely]] { + return NULL; + } + *ptr = size; + MT.alloc(size); + return (void *)(ptr + 1); +} + +mt_visible void *calloc(size_t nelem, size_t size) +{ + if (orig_malloc == NULL) [[unlikely]] { + return calloc_buffer; + } + size_t alloc_size = nelem * size; + void * ptr = malloc(alloc_size); + if (ptr == NULL) [[unlikely]] { + return NULL; + } + memset(ptr, 0, alloc_size); + return ptr; +} + +mt_visible void *realloc(void *ptr, size_t size) +{ + if (ptr == NULL) { + return malloc(size); + } + void *res = NULL; + if (ptr_size(ptr) < size) { + res = malloc(size); + if (res == NULL) [[unlikely]] { + return NULL; + } + memcpy(res, ptr, ptr_size(ptr)); + + free(ptr); + } else { + res = ptr; + } + return res; +} + +mt_visible void free(void *ptr) +{ + MT.init_hook_funcs(); + if (ptr == NULL || ptr == calloc_buffer) [[unlikely]] { + return; + } + MT.free(ptr_size(ptr)); + orig_free((size_t *)ptr - 1); +} + +mt_visible void cfree(void *ptr) { free(ptr); } + +mt_visible void *mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) +{ + MT.init_hook_funcs(); + void *res = orig_mmap(addr, length, prot, flags, fd, offset); + if (res != MAP_FAILED) [[likely]] { + MT.alloc(length); + } + return res; +} + +mt_visible int munmap(void *addr, size_t length) +{ + MT.init_hook_funcs(); + int res = orig_munmap(addr, length); + if (res == 0) [[likely]] { + MT.free(length); + } + return res; +} + +mt_visible char *strdup(const char *s) MT_THROW +{ + size_t len = strlen(s); + char * p = (char *)malloc(len + 1); + if (p == NULL) { + return NULL; + } + memcpy(p, s, len); + p[len] = 0; + return p; +} + +mt_visible char *strndup(const char *s, size_t n) MT_THROW +{ + const char * end = (const char *)memchr(s, 0, n); + const size_t m = (end != NULL ? (size_t)(end - s) : n); + char * t = (char *)malloc(m + 1); + if (t == NULL) + return NULL; + memcpy(t, s, m); + t[m] = 0; + return t; +} + +mt_visible char *realpath(const char *fname, char *resolved_name) +{ + MEMTRACER_LOG("realpath not supported\n"); + exit(-1); +} +mt_visible void *memalign(size_t alignment, size_t size) +{ + MEMTRACER_LOG("memalign not supported\n"); + exit(-1); +} + +mt_visible void *valloc(size_t size) +{ + MEMTRACER_LOG("valloc not supported\n"); + exit(-1); +} + +mt_visible void *pvalloc(size_t size) +{ + MEMTRACER_LOG("valloc not supported\n"); + exit(-1); +} + +mt_visible int posix_memalign(void **memptr, size_t alignment, size_t size) +{ + MEMTRACER_LOG("posix_memalign not supported\n"); + exit(-1); +} + +mt_visible int brk(void *addr) +{ + MEMTRACER_LOG("brk not supported\n"); + exit(-1); +} + +mt_visible void *sbrk(intptr_t increment) +{ + MEMTRACER_LOG("sbrk not supported\n"); + exit(-1); +} + +mt_visible long int syscall(long int __sysno, ...) +{ + MEMTRACER_LOG("syscall not supported\n"); + exit(-1); +} + +mt_visible void *operator new(std::size_t size) { return malloc(size); } + +mt_visible void *operator new[](std::size_t size) { return malloc(size); } + +mt_visible void *operator new(std::size_t size, const std::nothrow_t &) noexcept { return malloc(size); } + +mt_visible void *operator new[](std::size_t size, const std::nothrow_t &) noexcept { return malloc(size); } + +mt_visible void operator delete(void *ptr) noexcept { free(ptr); } + +mt_visible void operator delete[](void *ptr) noexcept { free(ptr); } + +mt_visible void operator delete(void *ptr, const std::nothrow_t &) noexcept { free(ptr); } + +mt_visible void operator delete[](void *ptr, const std::nothrow_t &) noexcept { free(ptr); } + +mt_visible void operator delete(void *ptr, std::size_t size) noexcept { free(ptr); } + +mt_visible void operator delete[](void *ptr, std::size_t size) noexcept { free(ptr); } \ No newline at end of file diff --git a/deps/memtracer/allocator.h b/deps/memtracer/allocator.h new file mode 100644 index 000000000..27480848e --- /dev/null +++ b/deps/memtracer/allocator.h @@ -0,0 +1,72 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once +#include // mmap/munmap + +#include "memtracer/common.h" +#include "memtracer/memtracer.h" + +using namespace memtracer; +#if defined(__linux__) +#define MT_THROW __THROW +#else +#define MT_THROW +#endif + +malloc_func_t orig_malloc = nullptr; +free_func_t orig_free = nullptr; +mmap_func_t orig_mmap = nullptr; +munmap_func_t orig_munmap = nullptr; + +extern "C" mt_visible void *malloc(size_t size); +extern "C" mt_visible void *calloc(size_t nelem, size_t size); +extern "C" mt_visible void *realloc(void *ptr, size_t size); +extern "C" mt_visible void free(void *ptr); +extern "C" mt_visible void cfree(void *ptr); +extern "C" mt_visible void *mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset); +extern "C" mt_visible int munmap(void *addr, size_t length); +extern "C" mt_visible char *strdup(const char *s) MT_THROW; +extern "C" mt_visible char *strndup(const char *s, size_t n) MT_THROW; + +mt_visible void *operator new(std::size_t size); +mt_visible void *operator new[](std::size_t size); +mt_visible void *operator new(std::size_t size, const std::nothrow_t &) noexcept; +mt_visible void *operator new[](std::size_t size, const std::nothrow_t &) noexcept; +mt_visible void operator delete(void *ptr) noexcept; +mt_visible void operator delete[](void *ptr) noexcept; +mt_visible void operator delete(void *ptr, const std::nothrow_t &) noexcept; +mt_visible void operator delete[](void *ptr, const std::nothrow_t &) noexcept; +mt_visible void operator delete(void *ptr, std::size_t size) noexcept; +mt_visible void operator delete[](void *ptr, std::size_t size) noexcept; + +// unsupported libc functions, for simpler memory tracking. +extern "C" mt_visible char * realpath(const char *fname, char *resolved_name); +extern "C" mt_visible void * memalign(size_t alignment, size_t size); +extern "C" mt_visible void * valloc(size_t size); +extern "C" mt_visible void * pvalloc(size_t size); +extern "C" mt_visible int posix_memalign(void **memptr, size_t alignment, size_t size); +extern "C" mt_visible int brk(void *addr); +extern "C" mt_visible void * sbrk(intptr_t increment); +extern "C" mt_visible long int syscall(long int __sysno, ...); + +// forword libc interface +#if defined(__GLIBC__) && defined(__linux__) +extern "C" mt_visible void *__libc_malloc(size_t size) __attribute__((alias("malloc"), used)); +extern "C" mt_visible void *__libc_calloc(size_t nmemb, size_t size) __attribute__((alias("calloc"), used)); +extern "C" mt_visible void *__libc_realloc(void *ptr, size_t size) __attribute__((alias("realloc"), used)); +extern "C" mt_visible void __libc_free(void *ptr) __attribute__((alias("free"), used)); +extern "C" mt_visible void __libc_cfree(void *ptr) __attribute__((alias("cfree"), used)); +extern "C" mt_visible void *__libc_valloc(size_t size) __attribute__((alias("valloc"), used)); +extern "C" mt_visible void *__libc_pvalloc(size_t size) __attribute__((alias("pvalloc"), used)); +extern "C" mt_visible void *__libc_memalign(size_t alignment, size_t size) __attribute__((alias("memalign"), used)); +extern "C" mt_visible int __posix_memalign(void **memptr, size_t alignment, size_t size) + __attribute__((alias("posix_memalign"), used)); +#endif diff --git a/deps/memtracer/common.cpp b/deps/memtracer/common.cpp new file mode 100644 index 000000000..1dc143d6a --- /dev/null +++ b/deps/memtracer/common.cpp @@ -0,0 +1,24 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#include +#include +#include "memtracer/common.h" + +namespace memtracer { +void log_stderr(const char *format, ...) +{ + va_list vl; + va_start(vl, format); + vfprintf(stderr, format, vl); + va_end(vl); +} + +} // namespace memtracer \ No newline at end of file diff --git a/deps/memtracer/common.h b/deps/memtracer/common.h new file mode 100644 index 000000000..d6cb3f4ea --- /dev/null +++ b/deps/memtracer/common.h @@ -0,0 +1,31 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include + +namespace memtracer { +#define mt_visible __attribute__((visibility("default"))) + +using malloc_func_t = void *(*)(size_t); +using free_func_t = void (*)(void *); +using mmap_func_t = void *(*)(void *, size_t, int, int, int, off_t); +using munmap_func_t = int (*)(void *, size_t); + +void log_stderr(const char *format, ...); + +#define MEMTRACER_LOG(format, ...) \ + do { \ + fprintf(stderr, "[MEMTRACER] "); \ + log_stderr(format, ##__VA_ARGS__); \ + } while (0) + +} // namespace memtracer \ No newline at end of file diff --git a/deps/memtracer/memtracer.cpp b/deps/memtracer/memtracer.cpp new file mode 100644 index 000000000..3e6810b5d --- /dev/null +++ b/deps/memtracer/memtracer.cpp @@ -0,0 +1,160 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#include "memtracer.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#define REACH_TIME_INTERVAL(i) \ + ({ \ + bool bret = false; \ + static volatile int64_t last_time = 0; \ + auto now = std::chrono::system_clock::now(); \ + int64_t cur_time = std::chrono::duration_cast(now.time_since_epoch()).count(); \ + if (int64_t(i + last_time) < cur_time) [[unlikely]] { \ + last_time = cur_time; \ + bret = true; \ + } \ + bret; \ + }) +extern memtracer::malloc_func_t orig_malloc; +extern memtracer::free_func_t orig_free; +extern memtracer::mmap_func_t orig_mmap; +extern memtracer::munmap_func_t orig_munmap; + +namespace memtracer { +// used only for getting memory size from `/proc/self/status` +long get_memory_size(const std::string &line) +{ + std::string token; + std::istringstream iss(line); + long size; + // skip token + iss >> token; + iss >> size; + return size; // KB +} + +MemTracer &MemTracer::get_instance() +{ + static MemTracer instance; + return instance; +} + +void MemTracer::init() +{ + MT.init_hook_funcs(); + + // init memory limit + const char *memory_limit_str = std::getenv("MT_MEMORY_LIMIT"); + if (memory_limit_str != nullptr) { + char * end; + unsigned long long value = std::strtoull(memory_limit_str, &end, 10); + if (end != memory_limit_str && *end == '\0') { + MT.set_memory_limit(static_cast(value)); + } else { + MEMTRACER_LOG("Invalid environment variable value for MT_MEMORY_LIMIT: %s\n", memory_limit_str); + } + } + + // init print_interval + const char *print_interval_ms_str = std::getenv("MT_PRINT_INTERVAL_MS"); + if (print_interval_ms_str != nullptr) { + char * end; + unsigned long long value = std::strtoull(print_interval_ms_str, &end, 10); + if (end != print_interval_ms_str && *end == '\0') { + MT.set_print_interval(static_cast(value)); + } else { + MEMTRACER_LOG("Invalid environment variable value for MT_MEMORY_LIMIT: %s\n", print_interval_ms_str); + } + } else { + MT.set_print_interval(1000 * 5); // 5s + } + + // init `text` memory usage + // TODO: support static variables statistic. + size_t text_size = 0; + std::ifstream file("/proc/self/status"); + if (file.is_open()) { + std::string line; + const int KB = 1024; + while (std::getline(file, line)) { + if (line.find("VmExe") != std::string::npos) { + text_size = get_memory_size(line) * KB; + break; + } + } + file.close(); + } + + MT.add_allocated_memory(text_size); + MT.init_stats_thread(); +} + +void MemTracer::init_stats_thread() +{ + if (!t_.joinable()) { + t_ = std::thread(stat); + } +} + +void MemTracer::alloc(size_t size) +{ + if (allocated_memory_.fetch_add(size) + size > memory_limit_) [[unlikely]] { + MEMTRACER_LOG("alloc memory:%lu, allocated_memory: %lu, memory_limit: %lu, Memory limit exceeded!\n", + size, + allocated_memory_.load(), + memory_limit_); + exit(-1); + } + alloc_cnt_.fetch_add(1); +} + +void MemTracer::free(size_t size) +{ + allocated_memory_.fetch_sub(size); + free_cnt_.fetch_add(1); +} + +void MemTracer::destroy() { MT.stop(); } + +void MemTracer::stop() +{ + is_stop_ = true; + t_.join(); +} + +void MemTracer::init_hook_funcs_impl() +{ + orig_malloc = (void *(*)(size_t size))dlsym(RTLD_NEXT, "malloc"); + orig_free = (void (*)(void *ptr))dlsym(RTLD_NEXT, "free"); + orig_mmap = (void *(*)(void *addr, size_t length, int prot, int flags, int fd, off_t offset))dlsym(RTLD_NEXT, "mmap"); + orig_munmap = (int (*)(void *addr, size_t length))dlsym(RTLD_NEXT, "munmap"); +} + +void MemTracer::stat() +{ + const size_t print_interval_ms = MT.print_interval(); + const size_t sleep_interval = 100; // 100 ms + while (!MT.is_stop()) { + if (REACH_TIME_INTERVAL(print_interval_ms)) { + // TODO: optimize the output format + MEMTRACER_LOG("allocated memory: %lu, metadata memory: %lu\n", MT.allocated_memory(), MT.meta_memory()); + } + std::this_thread::sleep_for(std::chrono::milliseconds(sleep_interval)); + } +} +} // namespace memtracer \ No newline at end of file diff --git a/deps/memtracer/memtracer.h b/deps/memtracer/memtracer.h new file mode 100644 index 000000000..e36b6a21f --- /dev/null +++ b/deps/memtracer/memtracer.h @@ -0,0 +1,85 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#pragma once + +#include +#include +#include +#include +#include "memtracer/common.h" + +namespace memtracer { + +#define MT MemTracer::get_instance() + +// MemTracer is used to monitor MiniOB memory usage; +// it is used to run and debug MiniOB under memory-constrained conditions. +// MemTracer statistics memory allocation in MiniOB processes by +// hooking memory alloc/free functions. +// More detail can be found at docs/src/game/miniob-memtracer.md +class MemTracer +{ +public: + static MemTracer &get_instance(); + + MemTracer() = default; + + static void __attribute__((constructor)) init(); + + static void __attribute__((destructor)) destroy(); + + size_t allocated_memory() const { return allocated_memory_.load(); } + + size_t meta_memory() const { return ((alloc_cnt_.load() - free_cnt_.load()) * sizeof(size_t)); } + + size_t print_interval() const { return print_interval_ms_; } + + size_t memory_limit() const { return memory_limit_; } + + bool is_stop() const { return is_stop_; } + + void set_print_interval(size_t print_interval_ms) { print_interval_ms_ = print_interval_ms; } + + inline void add_allocated_memory(size_t size) { allocated_memory_.fetch_add(size); } + + void set_memory_limit(size_t memory_limit) + { + std::call_once(memory_limit_once_, [&]() { memory_limit_ = memory_limit; }); + } + + void alloc(size_t size); + + void free(size_t size); + + inline void init_hook_funcs() { std::call_once(init_hook_funcs_once_, init_hook_funcs_impl); } + +private: + static void init_hook_funcs_impl(); + + void init_stats_thread(); + + void stop(); + + static void stat(); + +private: + bool is_inited_ = false; + bool is_stop_ = false; + std::atomic allocated_memory_{}; + std::atomic alloc_cnt_{}; + std::atomic free_cnt_{}; + std::once_flag init_hook_funcs_once_; + std::once_flag memory_limit_once_; + size_t memory_limit_ = UINT64_MAX; + size_t print_interval_ms_ = 0; + std::thread t_; +}; +} // namespace memtracer \ No newline at end of file diff --git a/deps/memtracer/mt_info.cpp b/deps/memtracer/mt_info.cpp new file mode 100644 index 000000000..b3b35c902 --- /dev/null +++ b/deps/memtracer/mt_info.cpp @@ -0,0 +1,20 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ +#include "memtracer.h" +#include "mt_info.h" + +namespace memtracer { + +mt_visible size_t allocated_memory() { return MT.allocated_memory(); } + +mt_visible size_t meta_memory() { return MT.meta_memory(); } + +mt_visible size_t memory_limit() { return MT.memory_limit(); } +} // namespace memtracer diff --git a/deps/memtracer/mt_info.h b/deps/memtracer/mt_info.h new file mode 100644 index 000000000..6f2d07ee6 --- /dev/null +++ b/deps/memtracer/mt_info.h @@ -0,0 +1,19 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#define mt_visible __attribute__((visibility("default"))) + +namespace memtracer { +mt_visible size_t allocated_memory(); + +mt_visible size_t meta_memory(); + +mt_visible size_t memory_limit(); +} // namespace memtracer \ No newline at end of file diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index aa853523f..577eceb1b 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -36,6 +36,7 @@ - [date 测试说明](./game/miniob-test-comment-date.md) - [date 实现解析](./game/miniob-date-implementation.md) - [drop table 实现解析](./game/miniob-drop-table-implementation.md) + - [内存使用监控相关功能介绍](./game/miniob-memtracer.md) - [数据库基础理论课程](./lectures/index.md) - [版权声明](./lectures/copyright.md) diff --git a/docs/src/game/miniob-memtracer.md b/docs/src/game/miniob-memtracer.md new file mode 100644 index 000000000..25603f73f --- /dev/null +++ b/docs/src/game/miniob-memtracer.md @@ -0,0 +1,125 @@ +# MemTracer + +MemTracer 是一个动态链接库，被用于监控 MiniOB 内存使用；用于在内存受限的条件下，运行和调试 MiniOB。MemTracer 通过 hook 内存分配释放函数，记录 MiniOB 进程中的内存分配情况。 +## 原理介绍 +MemTracer 对内存分配释放函数进行了覆盖（override），以达到对内存动态分配释放（如`malloc/free`, `new/delete`）的监控。除此之外，MemTracer 还会将 MiniOB 进程中代码段等内存占用统计在内。 + +通过在 `LD_PRELOAD` 环境变量中指定 MemTracer 动态库来覆盖 glibc 中的符号，可以实现可插拔方式监控 MiniOB 进程的内存占用。 + +MemTracer 支持设置最大内存限额，当 MiniOB 进程申请超过内存限额的内存时，MemTracer 会调用 `exit(-1)` 使 MiniOB 进程退出。 + +## 使用介绍 +### 编译 +可通过指定 `WITH_MEMTRACER` 控制 MemTracer 的编译（默认编译），MemTracer 动态库默认输出在`${CMAKE_BINARY_DIR}/lib` 目录下。 +下述示例将关闭MemTracer 的编译。 +``` +sudo bash build.sh init +bash build.sh release -DWITH_MEMTRACER=OFF +``` +### 运行 +通过指定 `LD_PRELOAD` 环境变量， 将 MemTracer 动态库加载到 MiniOB 进程中。如： +``` +LD_PRELOAD=./lib/libmemtracer.so ./bin/observer +``` +通过指定`MT_PRINT_INTERVAL_MS` 环境变量，设置内存使用的打印间隔时间，单位为毫秒（ms），默认为 5000 ms（5s）。通过指定`MT_MEMORY_LIMIT` 环境变量，设置内存使用的上限，单位为字节，当超过该值，MiniOB 进程会立即退出。下述示例表明设置内存使用情况的打印间隔为 1000 ms（1s），内存使用上限为1000 字节。 +``` +MT_PRINT_INTERVAL_MS=1000 MT_MEMORY_LIMIT=1000 LD_PRELOAD=./lib/libmemtracer.so ./bin/observer +``` +### 使用场景示例 +1. 通过指定 MiniOB 进程的最大内存限额，可以模拟在内存受限的情况下运行、调试 MiniOB。当超出最大内存限额后，MiniOB 进程会自动退出。 + +指定最大内存限额： +``` +MT_MEMORY_LIMIT=100000000 LD_PRELOAD=./lib/libmemtracer.so ./bin/observer +``` +当由于申请内存超过限额退出，则在退出时会打印相关日志： +``` +[MEMTRACER] alloc memory:24, allocated_memory: 31653580, memory_limit: 31653600, Memory limit exceeded! +``` +2. 通过链接 MemTracer，可以在 MiniOB 进程的任意位置获取当前内存使用情况。 + +步骤1: 链接 libmemtracer.so 到 MiniOB 进程中。 + +步骤2: 在需要获取内存使用情况的位置，调用 `memtracer/mt_info.h` 头文件中的 `memtracer::allocated_memory()` 函数，获取当前内存使用情况。 + +示例代码如下： + +``` +diff --git a/src/observer/CMakeLists.txt b/src/observer/CMakeLists.txt +index c62ac3c..d105fbf 100644 +--- a/src/observer/CMakeLists.txt ++++ b/src/observer/CMakeLists.txt +@@ -20,7 +20,7 @@ FIND_PACKAGE(Libevent CONFIG REQUIRED) + + # JsonCpp cannot work correctly with FIND_PACKAGE + +-SET(LIBRARIES common pthread dl libevent::core libevent::pthreads libjsoncpp.a) ++SET(LIBRARIES common pthread dl libevent::core libevent::pthreads libjsoncpp.a memtracer) + + # 指定目标文件位置 + SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/bin) +diff --git a/src/observer/sql/parser/parse.cpp b/src/observer/sql/parser/parse.cpp +index def0ed1..4f22799 100644 +--- a/src/observer/sql/parser/parse.cpp ++++ b/src/observer/sql/parser/parse.cpp +@@ -15,6 +15,7 @@ See the Mulan PSL v2 for more details. */ + #include "sql/parser/parse.h" + #include "common/log/log.h" + #include "sql/expr/expression.h" ++#include "memtracer/mt_info.h" + + RC parse(char *st, ParsedSqlNode *sqln); + +@@ -41,6 +42,7 @@ int sql_parse(const char *st, ParsedSqlResult *sql_result); + + RC parse(const char *st, ParsedSqlResult *sql_result) + { ++ LOG_ERROR("parse sql `%s`, allocated: %lu\n", st, memtracer::allocated_memory()); + sql_parse(st, sql_result); + return RC::SUCCESS; + } +``` +### 注意 +1. MemTracer 会记录 `mmap` 映射的整个虚拟内存占用, 因此不建议使用 `mmap` 管理内存。 +2. 不允许使用绕过常规内存分配（`malloc`/`free`, `new`/`delete`）的方式申请并使用内存。如使用 `brk/sbrk/syscall` 等。 +3. MemTracer 不支持与 sanitizers （ASAN等）一起使用。 +4. MemTracer 除统计动态内存的申请释放，也会将进程的代码段等内存占用统计在内。 +5. 引入 MemTracer 后，会对内存申请释放函数的性能产生一定影响，在 Github Action 中测试结果如下： +``` +// MemTracer 开启 +----------------------------------------------------------------------------------- +Benchmark Time CPU Iterations UserCounters... +----------------------------------------------------------------------------------- +BM_MallocFree/8 42.1 ns 42.1 ns 17090212 bytes_per_second=181.357M/s +BM_MallocFree/64 39.9 ns 39.9 ns 16763826 bytes_per_second=1.49326G/s +BM_MallocFree/512 40.1 ns 40.1 ns 17553391 bytes_per_second=11.8925G/s +BM_MallocFree/1024 39.9 ns 39.9 ns 17548820 bytes_per_second=23.9052G/s +BM_MallocFree/1048576 53.4 ns 53.4 ns 13150036 bytes_per_second=17.8628T/s +BM_MallocFree/8388608 53.3 ns 53.3 ns 13155704 bytes_per_second=143.106T/s +BM_MallocFree/1073741824 24877 ns 24750 ns 28814 bytes_per_second=39.4565T/s +BM_NewDelete/8 40.6 ns 40.6 ns 17270523 bytes_per_second=187.866M/s +BM_NewDelete/64 40.5 ns 40.5 ns 17279131 bytes_per_second=1.47047G/s +BM_NewDelete/512 40.5 ns 40.5 ns 17276789 bytes_per_second=11.7686G/s +BM_NewDelete/1024 40.5 ns 40.5 ns 17279484 bytes_per_second=23.5455G/s +BM_NewDelete/1048576 53.8 ns 53.8 ns 13003320 bytes_per_second=17.7138T/s +BM_NewDelete/8388608 54.0 ns 54.0 ns 13011425 bytes_per_second=141.183T/s +BM_NewDelete/1073741824 24844 ns 24611 ns 29271 bytes_per_second=39.6796T/s +// MemTracer 关闭 +----------------------------------------------------------------------------------- +Benchmark Time CPU Iterations UserCounters... +----------------------------------------------------------------------------------- +BM_MallocFree/8 10.1 ns 10.1 ns 70194674 bytes_per_second=753.492M/s +BM_MallocFree/64 10.8 ns 10.8 ns 64456192 bytes_per_second=5.50213G/s +BM_MallocFree/512 10.9 ns 10.9 ns 64672143 bytes_per_second=43.8294G/s +BM_MallocFree/1024 10.8 ns 10.8 ns 64616114 bytes_per_second=88.0599G/s +BM_MallocFree/1048576 23.5 ns 23.5 ns 29758318 bytes_per_second=40.5388T/s +BM_MallocFree/8388608 23.5 ns 23.5 ns 29743010 bytes_per_second=324.33T/s +BM_MallocFree/1073741824 20574 ns 20436 ns 34738 bytes_per_second=47.7856T/s +BM_NewDelete/8 13.4 ns 13.4 ns 50775388 bytes_per_second=570.14M/s +BM_NewDelete/64 14.2 ns 14.2 ns 49231328 bytes_per_second=4.18804G/s +BM_NewDelete/512 14.3 ns 14.3 ns 49198936 bytes_per_second=33.4308G/s +BM_NewDelete/1024 14.3 ns 14.3 ns 49215085 bytes_per_second=66.7825G/s +BM_NewDelete/1048576 26.9 ns 26.9 ns 25987611 bytes_per_second=35.4036T/s +BM_NewDelete/8388608 26.9 ns 26.9 ns 25997689 bytes_per_second=283.11T/s +BM_NewDelete/1073741824 20694 ns 20648 ns 34975 bytes_per_second=47.2954T/s +``` diff --git a/src/observer/storage/buffer/double_write_buffer.h b/src/observer/storage/buffer/double_write_buffer.h index 9fc60f9cf..afb0d633c 100644 --- a/src/observer/storage/buffer/double_write_buffer.h +++ b/src/observer/storage/buffer/double_write_buffer.h @@ -46,7 +46,7 @@ class DoubleWriteBuffer struct DoubleWriteBufferHeader { - int32_t page_cnt; + int32_t page_cnt = 0; static const int32_t SIZE; }; diff --git a/unittest/CMakeLists.txt b/unittest/CMakeLists.txt index af11bea98..9eaa0ecdb 100644 --- a/unittest/CMakeLists.txt +++ b/unittest/CMakeLists.txt @@ -6,6 +6,7 @@ INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/src/observer) find_package(GTest CONFIG REQUIRED) include(GoogleTest) + #get_filename_component( FileName # PATH|ABSOLUTE|NAME|EXT|NAME_WE|REALPATH # [CACHE]) @@ -15,7 +16,10 @@ FOREACH (F ${ALL_SRC}) get_filename_component(prjName ${F} NAME_WE) MESSAGE("Build ${prjName} according to ${F}") ADD_EXECUTABLE(${prjName} ${F}) - # 不是所有的单测都需要链接observer_static + # TODO: 不是所有的单测都需要链接observer_static TARGET_LINK_LIBRARIES(${prjName} common pthread dl gtest gtest_main observer_static) - gtest_discover_tests(${prjName}) + add_test(NAME ${prjName} COMMAND ${prjName}) + if (${prjName} STREQUAL "memtracer_test") + TARGET_LINK_LIBRARIES(${prjName} memtracer) + endif() ENDFOREACH (F) diff --git a/unittest/memtracer_test.cpp b/unittest/memtracer_test.cpp new file mode 100644 index 000000000..76024d06f --- /dev/null +++ b/unittest/memtracer_test.cpp @@ -0,0 +1,224 @@ +/* Copyright (c) 2021 OceanBase and/or its affiliates. All rights reserved. +miniob is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. */ + +#include +#include // mmap/munmap +#include +#include "gtest/gtest.h" +#include "memtracer/mt_info.h" + +#ifdef __linux__ +extern "C" void *__libc_malloc(size_t size); +#endif + +class Foo +{ +public: + Foo() : val(1) {} + int val; +}; + +void allocate_and_free_with_new_delete(size_t size) +{ + char *memory = new char[size]; + memset(memory, 0, size); + delete[] memory; +} + +void thread_function(void (*allocator)(size_t), size_t size) { allocator(size); } + +void allocate_and_free_with_malloc_free(size_t size) +{ + char *memory = static_cast(malloc(size)); + memset(memory, 0, size); + free(memory); +} + +void perform_multi_threads_allocation(void (*allocator)(size_t), size_t size, size_t num_threads) +{ + std::vector threads; + for (size_t i = 0; i < num_threads; ++i) { + threads.emplace_back(thread_function, allocator, size); + } + for (auto &t : threads) { + t.join(); + } +} + +TEST(test_mem_tracer, test_mem_tracer_basic) +{ + if (getenv("LD_PRELOAD") == nullptr) { + GTEST_SKIP(); + } + size_t mem_base = memtracer::allocated_memory(); + // malloc/free + { + void *ptr = malloc(1024); + memset(ptr, 0, 1024); + // if no use the memory that allocate by malloc, + // the compiler may optimize it out, + // so we need to use it here (even memset is also optimized) + *(char *)ptr = 'f'; + ASSERT_EQ(memtracer::allocated_memory(), mem_base + 1024); + free(ptr); + ASSERT_EQ(memtracer::allocated_memory(), mem_base); + + ptr = malloc(1024 * 1024 * 1024); + memset(ptr, 0, 1024 * 1024 * 1024); + *(char *)ptr = 'f'; + ASSERT_EQ(memtracer::allocated_memory(), mem_base + 1024 * 1024 * 1024); + free(ptr); + ASSERT_EQ(memtracer::allocated_memory(), mem_base); + + for (int i = 0; i < 1024; ++i) { + ptr = malloc(1); + memset(ptr, 0, 1); + *(char *)ptr = 'f'; + ASSERT_EQ(memtracer::allocated_memory(), mem_base + 1); + free(ptr); + ASSERT_EQ(memtracer::allocated_memory(), mem_base); + } + ASSERT_EQ(memtracer::allocated_memory(), mem_base); + } + + // new/delete + { + char *ptr = new char; + *ptr = 'a'; + ASSERT_EQ(memtracer::allocated_memory(), mem_base + 1); + delete ptr; + ASSERT_EQ(memtracer::allocated_memory(), mem_base); + } + + // new/delete obj + { + Foo *ptr = new Foo; + ASSERT_EQ(memtracer::allocated_memory(), mem_base + sizeof(Foo)); + ASSERT_EQ(1, ptr->val); + delete ptr; + ASSERT_EQ(memtracer::allocated_memory(), mem_base); + } + + // new/delete array + { + char *ptr = new char[1024]; + memset(ptr, 0, 1024); + *ptr = 'f'; + ASSERT_EQ(memtracer::allocated_memory(), mem_base + 1024); + delete[] ptr; + ASSERT_EQ(memtracer::allocated_memory(), mem_base); + } + + // calloc/free + { + void *ptr = calloc(10, 100); + memset(ptr, 0, 10 * 100); + *(char *)ptr = 'a'; + ASSERT_EQ(memtracer::allocated_memory(), mem_base + 10 * 100); + free(ptr); + ASSERT_EQ(memtracer::allocated_memory(), mem_base); + } + + // realloc/free + { + void *ptr = realloc(NULL, 10); + memset(ptr, 0, 10); + *(char *)ptr = 'f'; + ASSERT_EQ(memtracer::allocated_memory(), mem_base + 10); + ptr = realloc(ptr, 1); + memset(ptr, 0, 1); + *(char *)ptr = 'f'; + ASSERT_EQ(memtracer::allocated_memory(), mem_base + 10); + ptr = realloc(ptr, 100); + memset(ptr, 0, 100); + *(char *)ptr = 'f'; + ASSERT_EQ(memtracer::allocated_memory(), mem_base + 100); + free(ptr); + ASSERT_EQ(memtracer::allocated_memory(), mem_base); + } + // mmap/munmap + { + void *ptr = mmap(nullptr, 1024, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + *(char *)ptr = 'f'; + ASSERT_NE(ptr, nullptr); + ASSERT_EQ(memtracer::allocated_memory(), mem_base + 1024); + munmap(ptr, 1024); + ASSERT_EQ(memtracer::allocated_memory(), mem_base); + } + +// __libc_malloc +#ifdef __linux__ + { + void *ptr = __libc_malloc(1024); + memset(ptr, 0, 1024); + *(char *)ptr = 'f'; + ASSERT_EQ(memtracer::allocated_memory(), mem_base + 1024); + free(ptr); + ASSERT_EQ(memtracer::allocated_memory(), mem_base); + } +#endif + + // __builtin_malloc + { + void *ptr = __builtin_malloc(1024); + memset(ptr, 0, 1024); + *(char *)ptr = 'f'; + ASSERT_EQ(memtracer::allocated_memory(), mem_base + 1024); + free(ptr); + ASSERT_EQ(memtracer::allocated_memory(), mem_base); + } +} + +TEST(test_mem_tracer, test_mem_tracer_multi_threads) +{ + if (getenv("LD_PRELOAD") == nullptr) { + GTEST_SKIP(); + } + // Since some of the low-level functions also take up memory, + // for example, pthread will alloc stack via `mmap` and + // the thread's stack will not be released after thread join. + // allocated memory count is not checked here. + { + size_t num_threads = 1; + size_t memory_size = 16; + + perform_multi_threads_allocation(allocate_and_free_with_new_delete, memory_size, num_threads); + perform_multi_threads_allocation(allocate_and_free_with_malloc_free, memory_size, num_threads); + } + { + size_t num_threads = 4; + size_t memory_size = 16; + + perform_multi_threads_allocation(allocate_and_free_with_new_delete, memory_size, num_threads); + perform_multi_threads_allocation(allocate_and_free_with_malloc_free, memory_size, num_threads); + } + + { + size_t num_threads = 16; + size_t memory_size = 1024; + + perform_multi_threads_allocation(allocate_and_free_with_new_delete, memory_size, num_threads); + perform_multi_threads_allocation(allocate_and_free_with_malloc_free, memory_size, num_threads); + } + + { + size_t num_threads = 64; + size_t memory_size = 1024; + + perform_multi_threads_allocation(allocate_and_free_with_new_delete, memory_size, num_threads); + perform_multi_threads_allocation(allocate_and_free_with_malloc_free, memory_size, num_threads); + } +} + +int main(int argc, char **argv) +{ + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file